tinyfan 0.1.1__tar.gz → 0.2.0__tar.gz

{tinyfan-0.1.1 → tinyfan-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinyfan
-Version: 0.1.1
+Version: 0.2.0
 Summary: Tiny Argo Workflows python devkit
 Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
 Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
@@ -31,6 +31,7 @@ Requires-Dist: jsonschema[format]; extra == 'dev'
 Requires-Dist: mypy>=1.14; extra == 'dev'
 Requires-Dist: pytest>=8.3.3; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: types-croniter; extra == 'dev'
 Requires-Dist: types-jsonschema; extra == 'dev'
 Requires-Dist: types-pyyaml; extra == 'dev'
 Provides-Extra: gcs

tinyfan-0.2.0/examples/README.md

@@ -0,0 +1,12 @@
+# How to run
+
+```bash
+# setup k3d
+k3d cluster create tinyfan
+kubectl config use-context k3d-tinyfan
+# install argowf
+kubectl create namespace argo
+kubectl apply -n argo -f "https://github.com/argoproj/argo-workflows/releases/download/v3.6.4/quick-start-minimal.yaml"
+# apply manifests
+tinyfan examples/getting-started.py | kubectl apply -n argo -f -
+```

{tinyfan-0.1.1 → tinyfan-0.2.0}/examples/configs.py

@@ -27,5 +27,5 @@ def trace_configs(
     secretkey: SecretKeyRef,
 ):
     print(
-        f'{constant} - {cliarg} - {configmap.get("otherkey")} - {configmapkey.get_value()} - {secret.get("othersecretkey")} - {secretkey.get_value()}'
+        f"{constant} - {cliarg} - {configmap.get('otherkey')} - {configmapkey.get_value()} - {secret.get('othersecretkey')} - {secretkey.get_value()}"
     )

{tinyfan-0.1.1 → tinyfan-0.2.0}/pyproject.toml

@@ -1,18 +1,11 @@
 [project]
 name = "tinyfan"
-version = "0.1.1"
+version = "0.2.0"
 description = 'Tiny Argo Workflows python devkit'
 readme = "README.md"
 requires-python = ">=3.12"
-keywords = [
-  "argo-workflow",
-  "tinyfan",
-  "workflow",
-  "pipeline",
-]
-authors = [
-  { name = "eunchuldev", email = "eunchulsong@gmail.com" },
-]
+keywords = ["argo-workflow", "tinyfan", "workflow", "pipeline"]
+authors = [{ name = "eunchuldev", email = "eunchulsong@gmail.com" }]
 classifiers = [
   "Operating System :: OS Independent",
   "License :: OSI Approved :: MIT License",
@@ -20,32 +13,16 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
 ]
-dependencies = [
-    "croniter>=3.0.3",
-    "pyyaml>=6.0.2",
-    "typer-slim>=0.15.1",
-]
+dependencies = ["croniter>=3.0.3", "pyyaml>=6.0.2", "typer-slim>=0.15.1"]
 
 [project.scripts]
 tinyfan = "tinyfan.cli:main"
 
 [project.optional-dependencies]
-all = [
-  "tinyfan[gcs]",
-  "tinyfan[bigquery]",
-  "tinyfan[pandas]",
-]
-gcs = [
-  "google-cloud-storage",
-]
-bigquery = [
-  "google-cloud-bigquery[pandas]",
-  "tinyfan[pandas]",
-]
-pandas = [
-  "pandas",
-  "pyarrow",
-]
+all = ["tinyfan[gcs]", "tinyfan[bigquery]", "tinyfan[pandas]"]
+gcs = ["google-cloud-storage"]
+bigquery = ["google-cloud-bigquery[pandas]", "tinyfan[pandas]"]
+pandas = ["pandas", "pyarrow"]
 dev = [
   "pytest >= 8.3.3",
   "jsonpatch>=1",
@@ -54,6 +31,7 @@ dev = [
   "ruff",
   "types-PyYAML",
   "types-jsonschema",
+  "types-croniter",
 ]
 
 [project.urls]

{tinyfan-0.1.1 → tinyfan-0.2.0}/src/tinyfan/codegen.py

@@ -13,8 +13,9 @@ import datetime
 import croniter
 from .argo_typing import ScriptTemplate
 from .utils.embed import embed
-from .utils.merge import deepmerge
+from .utils.merge import deepmerge, dropnone
 from .config import CLI_CONFIG_STATE, Config, ConfigValue, ConfigMapKeyRef, ConfigMapRef, SecretKeyRef, SecretRef
+from .flowrundata import FlowRunData
 
 VALID_CRON_REGEXP = r"^(?:(?:(?:(?:\d+,)+\d+|(?:\d+(?:\/|-|#)\d+)|\d+L?|\*(?:\/\d+)?|L(?:-\d+)?|\?|[A-Z]{3}(?:-[A-Z]{3})?) ?){5,7})|(@hourly|@daily|@midnight|@weekly|@monthly|@yearly|@annually)$"
 VALID_DEPENDS_REGEXP = r"^(?!.*\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+)(?!.*\.(?!Succeeded|Failed|Errored|Skipped|Omitted|Daemoned)\w+)(?!.*(?:&&|\|\|)\s*(?:&&|\|\|))(?!.*!!)[A-Za-z0-9_&|!().\s]+$"
@@ -88,19 +89,21 @@ class AssetNode:
         t1 = cron.get_next(datetime.datetime)
         t2 = cron.get_next(datetime.datetime)
         duration_sec = (t2 - t1).total_seconds()
+        data_interval_start_tmpl = (
+            f"sprig.dateModify('-{duration_sec}s', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime))"
+        )
         return re.sub(
             r'"(\{\{tasks\.[^.]+\.outputs\.parameters\.rundata\}\})"',
             r"\1",
             json.dumps(
                 {
-                    "ds": "{{=sprig.date('2006-01-02', workflow.scheduledTime)}}",
-                    "ts": "{{workflow.scheduledTime}}",
+                    "ds": "{{=sprig.date('2006-01-02', %s)}}" % data_interval_start_tmpl,
+                    "ts": "{{=%s}}" % data_interval_start_tmpl,
                     "data_interval_start": {
-                        DATETIME_ANNOTATION: "{{workflow.scheduledTime}}",
+                        DATETIME_ANNOTATION: "{{=%s}}" % data_interval_start_tmpl,
                     },
                     "data_interval_end": {
-                        DATETIME_ANNOTATION: "{{=sprig.dateModify('%ss', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime))}}"
-                        % duration_sec,
+                        DATETIME_ANNOTATION: "{{workflow.scheduledTime}}",
                     },
                     "parents": {
                         p.asset.name: "{{tasks.%s.outputs.parameters.rundata}}" % p.asset.name for p in self.parents
@@ -136,6 +139,7 @@ class AssetTree:
                 set(self.flow.assets.keys())
                 | set((self.flow.resources or {}).keys())
                 | set((self.flow.configs or {}).keys())
+                | set(inspect.get_annotations(FlowRunData).keys())
             )
             unknown_names = [n for n in func_param_names if n not in known_names]
             if len(unknown_names) > 0:
@@ -169,6 +173,7 @@ class AssetTree:
         self,
         embedded: bool = True,
         container: ScriptTemplate = {},
+        serviceAccountName: str | None = None,
     ) -> str:
         if len(self.nodes) == 0:
             return ""
@@ -184,7 +189,7 @@ class AssetTree:
             root_location = get_root_path(list(self.nodes.values())[0].asset.func)
             source += embed(root_location)
             if not root_location.endswith(".py"):
-                source += "from {{inputs.parameters.module_name}}" " import {{inputs.parameters.asset_name}}\n"
+                source += "from {{inputs.parameters.module_name}} import {{inputs.parameters.asset_name}}\n"
             source += "asset = {{inputs.parameters.asset_name}}.asset\n"
         else:
             source = (
@@ -217,14 +222,15 @@ class AssetTree:
                 "apiVersion": "argoproj.io/v1alpha1",
                 "kind": "CronWorkflow",
                 "metadata": {
-                    "name": f"""{self.flow.name}-{ encode_schedule_to_k8sname(tz, schedule) }""",
-                    "generateName": f"""{self.flow.name}-{ encode_schedule_to_k8sname(tz, schedule) }-""",
+                    "name": f"""{self.flow.name}-{encode_schedule_to_k8sname(tz, schedule)}""",
+                    "generateName": f"""{self.flow.name}-{encode_schedule_to_k8sname(tz, schedule)}-""",
                 },
                 "spec": {
                     "schedule": schedule,
                     "timezone": tz,
                     "workflowSpec": {
                         "entrypoint": "flow",
+                        **dropnone({"serviceAccountName": serviceAccountName or self.flow.serviceAccountName}),
                         "podSpecPatch": yaml_dump(
                             {
                                 "containers": [
@@ -247,6 +253,7 @@ class AssetTree:
                         "templates": [
                             {
                                 "name": node.asset.name.replace("_", "-"),
+                                **dropnone({"serviceAccountName": node.asset.serviceAccountName}),
                                 "script": deepmerge(
                                     {
                                         "image": DEFAULT_IMAGE,
@@ -389,15 +396,20 @@ def codegen(
     location: str | None = None,
     embedded: bool = False,
     container: ScriptTemplate = {},
+    serviceAccountName: str | None = None,
 ) -> str:
     """Generate argocd workflow resource as yaml from tinyfan definitions"""
     CLI_CONFIG_STATE.codegen = True
     if flow:
-        result = AssetTree(flow).compile(embedded, container)
+        result = AssetTree(flow).compile(embedded, container, serviceAccountName)
     elif location:
         import_all_submodules(location)
-        result = "\n---\n".join(AssetTree(flow).compile(embedded, container) for flow in FLOW_REGISTER.values())
+        result = "\n---\n".join(
+            AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
+        )
     else:
-        result = "\n---\n".join(AssetTree(flow).compile(embedded, container) for flow in FLOW_REGISTER.values())
+        result = "\n---\n".join(
+            AssetTree(flow).compile(embedded, container, serviceAccountName) for flow in FLOW_REGISTER.values()
+        )
     CLI_CONFIG_STATE.codegen = False
     return result

{tinyfan-0.1.1 → tinyfan-0.2.0}/src/tinyfan/flow.py

@@ -22,6 +22,7 @@ class Flow:
     assets: dict[str, "Asset[Any, Any, Any]"] = field(default_factory=dict)
     resources: dict[str, ResourceBase] | None = None
     container: ScriptTemplate | None = None
+    serviceAccountName: str | None = None
     store: StoreBase = field(default_factory=NaiveStore)
     configs: dict[str, Config | ConfigValue | str | None] = field(default_factory=dict)
 
@@ -48,6 +49,7 @@ class Asset(Generic[Ret, UMeta, StoreIdx]):
     depends: str | None = None
     name: str = field(init=False)
     container: ScriptTemplate | None = None
+    serviceAccountName: str | None = None
 
     def __post_init__(self):
         self.name = self.func.__name__
@@ -110,6 +112,7 @@ def asset(
     tz: str | None = None,
     metadata: UMeta | None = None,
     container: ScriptTemplate | None = None,
+    serviceAccountName: str | None = None,
 ) -> Callable[..., AssetFunc[Ret, UMeta, StoreIdx]]:
     def wrapper(func: Callable[..., Ret]) -> AssetFunc[Ret, UMeta, StoreIdx]:
         asset = Asset[Ret, UMeta, StoreIdx](
@@ -121,6 +124,7 @@ def asset(
             tz=tz or flow.tz,
             metadata=metadata,
             container=container,
+            serviceAccountName=serviceAccountName,
         )
         return AssetFunc(func, asset)
 

{tinyfan-0.1.1 → tinyfan-0.2.0}/src/tinyfan/utils/merge.py

@@ -15,3 +15,13 @@ def deepmerge(dict1: T, dict2: T, *args: T) -> T:
     for arg in args:
         deepmerge(dict1, arg)
     return dict1
+
+
+def dropnone(d: dict) -> dict:
+    dropping = []
+    for k in d:
+        if d[k] is None:
+            dropping.append(k)
+    for k in dropping:
+        del d[k]
+    return d

{tinyfan-0.1.1 → tinyfan-0.2.0}/tests/test_codegen.py

@@ -13,7 +13,7 @@ def sample_flow() -> Generator:
     flow: Flow = Flow("test")
 
     @asset(flow=flow, schedule="@daily")
-    def asset1():  # type: ignore
+    def asset1(ds: str):  # type: ignore
         print("asset1 is executed")
         return "hello"
 
@@ -127,3 +127,14 @@ def test_codegen_directory(validate_crds):
         if e.json_path != "$.spec.templates[0].script" or e.validator != "required":
             raise e
     assert_code_is_running(actual)
+
+
+def test_rundata_data_interval(sample_flow):
+    tree = AssetTree(sample_flow)
+    node = list(tree.nodes.values())[0]
+    # rundatatmpl = node.rundatatmpl("0 * * * * *")
+    daily = node.rundatatmpl("@daily")
+    assert (
+        daily
+        == """{"ds": "{{=sprig.date('2006-01-02', sprig.dateModify('-86400.0s', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime)))}}", "ts": "{{=sprig.dateModify('-86400.0s', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime))}}", "data_interval_start": {"__tinyfan_datetime__": "{{=sprig.dateModify('-86400.0s', sprig.toDate('2006-01-02T15:04:05Z07:00', workflow.scheduledTime))}}"}, "data_interval_end": {"__tinyfan_datetime__": "{{workflow.scheduledTime}}"}, "parents": {}, "asset_name": "asset1", "flow_name": "test", "module_name": "tests.test_codegen"}"""
+    )

{tinyfan-0.1.1 → tinyfan-0.2.0}/uv.lock

@@ -1,4 +1,5 @@
 version = 1
+revision = 1
 requires-python = ">=3.12"
 
 [[package]]
@@ -774,7 +775,7 @@ wheels = [
 
 [[package]]
 name = "tinyfan"
-version = "0.1.1"
+version = "0.2.0"
 source = { editable = "." }
 dependencies = [
     { name = "croniter" },
@@ -800,6 +801,7 @@ dev = [
     { name = "mypy" },
     { name = "pytest" },
     { name = "ruff" },
+    { name = "types-croniter" },
     { name = "types-jsonschema" },
     { name = "types-pyyaml" },
 ]
@@ -829,9 +831,11 @@ requires-dist = [
     { name = "tinyfan", extras = ["pandas"], marker = "extra == 'all'" },
     { name = "tinyfan", extras = ["pandas"], marker = "extra == 'bigquery'" },
     { name = "typer-slim", specifier = ">=0.15.1" },
+    { name = "types-croniter", marker = "extra == 'dev'" },
     { name = "types-jsonschema", marker = "extra == 'dev'" },
     { name = "types-pyyaml", marker = "extra == 'dev'" },
 ]
+provides-extras = ["all", "gcs", "bigquery", "pandas", "dev"]
 
 [[package]]
 name = "typer-slim"
@@ -846,6 +850,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/1f/7b/032ecd581e2170513bb6dc3cdb2581e20fdb94a272bae70fe93f2bca580b/typer_slim-0.15.1-py3-none-any.whl", hash = "sha256:20233cb89938ea3cca633afee10b906a1b0e7c5330f31ed8c55f4f0779efe6df", size = 44968 },
 ]
 
+[[package]]
+name = "types-croniter"
+version = "5.0.1.20241205"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/77/2f/f3298f148a7983df2a878d53c4a17ef1dd8654a155e11caa8decb7d79670/types_croniter-5.0.1.20241205.tar.gz", hash = "sha256:8a7cb10aa0b487c654a10c14d4e99098c92a5d68ee55c7a92a3df095831a6933", size = 10928 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/dc/ec/ff0222b912fdff040a1908ea226f64ff5a1a384a4062a8714eb8b80ea5d9/types_croniter-5.0.1.20241205-py3-none-any.whl", hash = "sha256:e36e992d6a4600ddf510991a20d3273b888ecea5d77a19078348e00775ba54d3", size = 9375 },
+]
+
 [[package]]
 name = "types-jsonschema"
 version = "4.23.0.20240813"