tinyfan 0.0.1__py3-none-any.whl

tinyfan/__init__.py

@@ -0,0 +1,5 @@
+from .flow import Flow, Asset, asset
+from .stores.base import StoreBase
+from .stores.naive import NaiveStore
+
+__all__ = ["Flow", "Asset", "asset", "StoreBase", "NaiveStore"]

tinyfan/cli.py

@@ -0,0 +1,21 @@
+import typer
+
+from .codegen import codegen, DEFAULT_IMAGE
+
+
+def tinyfan(location: str, embedded: bool = False, image: str = DEFAULT_IMAGE):
+    """
+    Generate argocd workflow resource as yaml from tinyfan definitions
+    """
+    if location.endswith('.py'):
+        print(codegen(location=location, embedded=True, image=image))
+    else:
+        print(codegen(location=location, embedded=embedded, image=image))
+
+
+def main():
+    return typer.run(tinyfan)
+
+
+if __name__ == "__main__":
+    main()

tinyfan/codegen.py

@@ -0,0 +1,304 @@
+from tinyfan.flow import Flow, Asset, FLOW_CATALOG
+import importlib.util
+from typing import Self
+import inspect
+import sys
+import re
+import json
+import yaml
+import os
+import pkgutil
+from .utils.embed import embed
+
+VALID_CRON_REGEXP = r"^(?:(?:(?:(?:\d+,)+\d+|(?:\d+(?:\/|-|#)\d+)|\d+L?|\*(?:\/\d+)?|L(?:-\d+)?|\?|[A-Z]{3}(?:-[A-Z]{3})?) ?){5,7})|(@hourly|@daily|@midnight|@weekly|@monthly|@yearly|@annually)$"
+VALID_DEPENDS_REGEXP = r"^(?!.*\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+)(?!.*\.(?!Succeeded|Failed|Errored|Skipped|Omitted|Daemoned)\w+)(?!.*(?:&&|\|\|)\s*(?:&&|\|\|))(?!.*!!)[A-Za-z0-9_&|!().\s]+$"
+
+EXTRACT_DEPENDS_REGEXP = r"(?<!\.)\b([A-Za-z0-9_]+)\b"
+
+RUNDATA_FILE_PATH = "/tmp/tinyfan/{flow_name}/{asset_name}/rundata.json"
+
+DEFAULT_IMAGE = "python:alpine"
+
+
+def brackets_balanced(code: str) -> bool:
+    code = re.sub(r"[^()]", "", code)
+    while "()" in code:
+        code = code.replace("()", "")
+    return code == ""
+
+
+def get_root_path(func):
+    filename = inspect.getfile(func)
+    path = os.path.abspath(os.path.dirname(filename))
+    if not os.path.exists(os.path.join(path, "__init__.py")):
+        return filename
+    while os.path.exists(os.path.join(os.path.dirname(path), "__init__.py")):
+        path = os.path.dirname(path)
+    return path
+
+
+def encode_schedule_to_k8sname(tz: str, cron: str) -> str:
+    return f"{tz}|{cron}".lower().translate(str.maketrans("* ,-/_+|", "a-cds-p-", "@"))[:58]
+
+
+def import_all_submodules(location: str):
+    try:
+        package = importlib.import_module(location)
+    except ModuleNotFoundError:
+        if not location.endswith(".py"):
+            pkg_name = os.path.dirname(location)
+            location = os.path.join(location, "__init__.py")
+        else:
+            pkg_name = "__tinyfan_repo__"
+        spec = importlib.util.spec_from_file_location(pkg_name, location)
+        if spec is None or spec.loader is None:
+            raise ValueError(f"fail to load module from `{location}.`")
+        package = importlib.util.module_from_spec(spec)
+        sys.modules[spec.name] = package
+        spec.loader.exec_module(package)
+    if package.__name__ != "__tinyfan_repo__":
+        for _, name, _ in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
+            importlib.import_module(name)
+
+
+class AssetNode:
+    asset: Asset
+    children: list[Self]
+    parents: list[Self]
+
+    def __init__(self, asset):
+        self.asset = asset
+        self.children = []
+        self.parents = []
+
+    def __hash__(self):
+        return hash(self.asset.name)
+
+    def asset_module_name(self):
+        mod = inspect.getmodule(self.asset.func)
+        return mod.__name__ if mod else None
+
+    def rundatatmpl(self):
+        return re.sub(
+            r'"(\{\{tasks\.[^.]+\.outputs\.parameters\.rundata\}\})"',
+            r"\1",
+            json.dumps(
+                {
+                    "ds": "{{=sprig.date(workflow.scheduledTime)}}",
+                    "ts": "{{workflow.scheduledTime}}",
+                    "parents": {
+                        p.asset.name: "{{tasks.%s.outputs.parameters.rundata}}" % p.asset.name for p in self.parents
+                    },
+                    "asset_name": self.asset.name,
+                    "flow_name": self.asset.flow.name,
+                    "module_name": self.asset_module_name(),
+                }
+            ),
+        )
+
+    def relatives(self, result: set[Self] | None = None) -> set[Self]:
+        if result is None:
+            result = set[Self]([self])
+        for o in self.parents + self.children:
+            if o not in result:
+                result.add(o)
+                o.relatives(result)
+        return result
+
+
+class AssetTree:
+    flow: Flow
+    nodes: dict[str, AssetNode]
+
+    def __init__(self, flow: Flow):
+        self.flow = flow
+        self.nodes = {name: AssetNode(asset) for name, asset in flow.assets.items()}
+        for node in self.nodes.values():
+            sigs = inspect.signature(node.asset.func)
+            func_param_names = list(sigs.parameters.keys())
+            params_ids = set(name for name in func_param_names if name in self.flow.assets)
+
+            if node.asset.depends is not None:
+                depends_ids = set(re.findall(EXTRACT_DEPENDS_REGEXP, node.asset.depends))
+                node.parents = [self.nodes[n] for n in depends_ids.union(params_ids)]
+                node.asset.depends = f"({node.asset.depends}) && {' && '.join(params_ids - depends_ids)}"
+                for p in node.parents:
+                    p.children.append(node)
+            else:
+                node.parents = [self.nodes[n] for n in params_ids]
+                for p in node.parents:
+                    p.children.append(node)
+                node.asset.depends = " && ".join(params_ids)
+
+    def compile(
+        self,
+        embedded: bool = True,
+        image: str = DEFAULT_IMAGE,
+    ) -> str:
+        if len(self.nodes) == 0:
+            return ""
+
+        if embedded:
+            source = embed("tinyfan", excludes=["codegen.py", "utils/embed.py", "**/__pycache__/*"], minify=False)
+            root_location = get_root_path(list(self.nodes.values())[0].asset.func)
+            source += embed(root_location)
+            if not root_location.endswith(".py"):
+                source += "from {{inputs.parameters.module_name}}" " import {{inputs.parameters.asset_name}}\n"
+            source += "asset = {{inputs.parameters.asset_name}}.asset\n"
+        else:
+            source = (
+                "from {{inputs.parameters.module_name}}"
+                " import {{inputs.parameters.asset_name}}\n"
+                "asset = {{inputs.parameters.asset_name}}.asset\n"
+            )
+        source += (
+            "_, rundata = asset.run({{inputs.parameters.rundata}})\n"
+            "import os\n"
+            "from tinyfan.utils.exjson import dumps\n"
+            f"path = '{RUNDATA_FILE_PATH}'"
+            ".format(flow_name=asset.flow.name, asset_name=asset.name)\n"
+            "os.makedirs(os.path.dirname(path), exist_ok=True)\n"
+            "with open(path, 'w') as f:\n"
+            "    f.write(dumps(rundata))\n"
+        )
+
+        relatives_by_schedules: dict[tuple[str, str], set[AssetNode]] = {}
+        for node in self.nodes.values():
+            if node.asset.schedule is None:
+                continue
+            schedule = (node.asset.tz or self.flow.tz, node.asset.schedule)
+            if schedule not in relatives_by_schedules:
+                relatives_by_schedules[schedule] = set()
+            for o in node.relatives():
+                relatives_by_schedules[schedule].add(o)
+
+        flow_image = self.flow.image or image
+
+        manifests = [
+            {
+                "apiVersion": "argoproj.io/v1alpha1",
+                "kind": "WorkflowTemplate",
+                "metadata": {
+                    "name": self.flow.name,
+                },
+                "spec": {
+                    "templateDefaults": {
+                        "script": {
+                            "name": self.flow.name,
+                            "image": flow_image,
+                            "command": ["python"],
+                            "source": source,
+                        },
+                    },
+                    "templates": [
+                        {
+                            "name": node.asset.name,
+                            "synchronization": {"mutexes": [{"name": f"{self.flow.name}-{node.asset.name}"}]},
+                            **(
+                                {} if node.asset.image == self.flow.image else {
+                                    "script": {
+                                        "name": node.asset.name,
+                                        "image": node.asset.image or flow_image,
+                                        "command": ["python"],
+                                        "source": source,
+                                        # "command": ["python"]
+                                    } 
+                                }
+                            ),
+                            "inputs": {
+                                "parameters": [
+                                    {
+                                        "name": "rundata",
+                                        "value": "{{= nil }}",
+                                    },
+                                    {
+                                        "name": "asset_name",
+                                        "value": "{{= nil }}",
+                                    },
+                                    {
+                                        "name": "module_name",
+                                        "value": "{{= nil }}",
+                                    },
+                                ]
+                            },
+                            "outputs": {
+                                "parameters": [
+                                    {
+                                        "name": "rundata",
+                                        "valueFrom": {
+                                            "path": RUNDATA_FILE_PATH.format(
+                                                flow_name=self.flow.name, asset_name=node.asset.name
+                                            )
+                                        },
+                                    }
+                                ],
+                            },
+                        }
+                        for node in self.nodes.values()
+                    ],
+                },
+            }
+        ] + [
+            {
+                "apiVersion": "argoproj.io/v1alpha1",
+                "kind": "CronWorkflow",
+                "metadata": {
+                    "name": f"""{self.flow.name}-{ encode_schedule_to_k8sname(tz, schedule) }""",
+                    "generateName": f"""{self.flow.name}-{ encode_schedule_to_k8sname(tz, schedule) }-""",
+                },
+                "spec": {
+                    "schedule": schedule,
+                    "timezone": tz,
+                    "workflowSpec": {
+                        "entrypoint": "flow",
+                        "templates": [
+                            {
+                                "name": "flow",
+                                "dag": {
+                                    "tasks": [
+                                        {
+                                            "name": node.asset.name,
+                                            "depends": node.asset.depends,
+                                            "templateRef": {
+                                                "name": self.flow.name,
+                                                "template": node.asset.name,
+                                            },
+                                            "arguments": {
+                                                "parameters": [
+                                                    {
+                                                        "name": "rundata",
+                                                        "value": node.rundatatmpl(),
+                                                    },
+                                                    {
+                                                        "name": "asset_name",
+                                                        "value": node.asset.name,
+                                                    },
+                                                    {
+                                                        "name": "module_name",
+                                                        "value": node.asset_module_name(),
+                                                    },
+                                                ]
+                                            },
+                                        }
+                                        for node in relatives
+                                    ]
+                                },
+                            }
+                        ],
+                    },
+                },
+            }
+            for (tz, schedule), relatives in relatives_by_schedules.items()
+        ]
+        return yaml.dump_all(manifests)
+
+
+def codegen(
+    location: str | None = None,
+    embedded: bool = False,
+    image: str = DEFAULT_IMAGE,
+) -> str:
+    """Generate argocd workflow resource as yaml from tinyfan definitions"""
+    if location:
+        import_all_submodules(location)
+    return "\n---\n".join(AssetTree(flow).compile(embedded, image) for flow in FLOW_CATALOG.values())

tinyfan/flow.py

@@ -0,0 +1,96 @@
+import inspect
+from .stores.base import StoreBase
+from .stores.naive import NaiveStore
+from .flowrundata import FlowRunData, StoreIdx, UMeta
+from dataclasses import dataclass, field
+from typing import Callable, Any, Generic, TypeVar, Mapping
+
+FLOW_CATALOG = {}
+
+Res = TypeVar("Res", bound=Mapping)
+Ret = TypeVar("Ret", bound=Mapping)
+
+
+@dataclass
+class Flow(Generic[Res]):
+    name: str
+    image: str | None = None
+    tz: str = "UTC"
+    assets: dict[str, "Asset[Any, Res, Any, Any]"] = field(default_factory=dict)
+    resources: Res | None = None
+    store: StoreBase = field(default_factory=NaiveStore)
+
+    def __post_init__(self):
+        FLOW_CATALOG[self.name] = self
+
+
+DEFAULT_FLOW: Flow[Any] = Flow("tinyfan", image="python:alpine")
+
+
+@dataclass
+class Asset(Generic[Ret, Res, UMeta, StoreIdx]):
+    flow: Flow[Res]
+    func: Callable[..., Ret]
+    store: StoreBase[Ret, StoreIdx]
+    image: str | None = None
+    schedule: str | None = None
+    tz: str | None = None
+    metadata: UMeta | None = None
+    depends: str | None = None
+    name: str = field(init=False)
+
+    def __post_init__(self):
+        self.name = self.func.__name__
+        self.flow.assets[self.name] = self
+
+    def run(self, rundata: FlowRunData[UMeta, StoreIdx] | None = None) -> tuple[Ret, FlowRunData[UMeta, StoreIdx]]:
+        rundata = rundata or {}
+        sigs = inspect.signature(self.func)
+        func_param_names = list(sigs.parameters.keys())
+        if self.metadata is not None:
+            rundata["metadata"] = self.metadata
+        rundata["asset_name"] = self.name
+        rundata["flow_name"] = self.flow.name
+        mod = inspect.getmodule(self.func)
+        if mod and mod.__name__:
+            rundata["module_name"] = mod.__name__
+        params = {k: v for k, v in rundata.items() if k in func_param_names}
+        if self.flow.resources:
+            for k, v in self.flow.resources:
+                if k in func_param_names:
+                    params[k] = v
+
+        parent_flowrundatas: dict[str, FlowRunData] = rundata.get("parents") or {}
+        for name, prundata in parent_flowrundatas.items():
+            parent = self.flow.assets[name]
+            index = prundata.get("store_entry_idx", None)
+            data = parent.store.retrieve(index, source_rundata=prundata, target_rundata=rundata)
+            if name in func_param_names:
+                params[name] = data
+        ret = self.func(**params)
+        store_entry_idx = self.store.store(ret, rundata)
+        rundata["store_entry_idx"] = store_entry_idx
+        if "parents" in rundata:
+            del rundata["parents"]
+        return (ret, rundata)
+
+
+def asset(
+    flow: Flow = DEFAULT_FLOW,
+    schedule: str | None = None,
+    depends: str | None = None,
+    store: StoreBase | None = None,
+):
+    class wrapper(Generic[Ret]):
+        func: Callable[..., Ret]
+        asset: Asset
+
+        def __init__(self, func: Callable[..., Ret]):
+            self.func = func
+            self.asset = Asset(flow, func=func, depends=depends, schedule=schedule, store=store or flow.store)
+
+        def __call__(self, *args, **kwargs) -> Ret:
+            res = self.func(*args, **kwargs)
+            return res
+
+    return wrapper

tinyfan/flowrundata.py

@@ -0,0 +1,24 @@
+from typing import TypedDict
+from typing import Generic, TypeVar
+from .utils.exjson import SerializableDict
+
+UMeta = TypeVar("UMeta", bound=SerializableDict)
+StoreIdx = TypeVar("StoreIdx", bound=SerializableDict)
+
+
+class FlowRunData(TypedDict, Generic[UMeta, StoreIdx], total=False):
+    # asset bound data
+    metadata: UMeta
+    asset_name: str
+    flow_name: str
+    module_name: str | None
+
+    # runtime data
+    ds: str
+    ts: str
+    parents: dict[str, "FlowRunData"] | None
+
+    # generated data
+    store_entry_idx: StoreIdx | None
+    # data_interval_start: datetime
+

tinyfan/stores/base.py

@@ -0,0 +1,30 @@
+from abc import abstractmethod
+from typing import Generic, TypeVar
+from ..flowrundata import FlowRunData, StoreIdx, UMeta
+
+T = TypeVar("T")
+
+
+class StoreBase(Generic[T, StoreIdx]):
+    @staticmethod
+    @abstractmethod
+    def id() -> str:
+        return "tinyfan.basestore"
+
+    @abstractmethod
+    def store(
+        self,
+        value: T,
+        rundata: FlowRunData,
+    ) -> StoreIdx:
+        """should return store index, which is used when retrieve back"""
+        pass
+
+    @abstractmethod
+    def retrieve(
+        self,
+        index: StoreIdx,
+        source_rundata: FlowRunData[UMeta, StoreIdx],
+        target_rundata: FlowRunData,
+    ) -> T:
+        pass

tinyfan/stores/bigquery.py

@@ -0,0 +1,49 @@
+"""
+from .base import StoreBase, PredefinedParams, Asset
+import pickle
+from typing import TypedDict, Literal
+from google.cloud import bigquery
+from pandas import DataFrame
+
+DataType = Literal["Pandas.DataFrame", "object"]
+
+T = DataFrame
+Idx = TypedDict("Idx", {
+    "project_id": str,
+    "dataset_id": str,
+    "table_id": str,
+})
+Meta = TypedDict("Meta", {
+    "bq_table_id": str,
+})
+
+class BigqueryStore(StoreBase[Idx, Meta, T]):
+    project_id: str
+    dataset_id: str|None
+
+    @staticmethod
+    def id() -> str:
+        return "tinyfan.bigquery"
+
+    def __init__(self, project_id:str, dataset_id:str|None = None):
+        self.project_id = project_id
+        self.dataset_id = dataset_id
+
+    def store(self, data: T, asset: Asset, params: PredefinedParams) -> Idx:
+        client = bigquery.Client()
+        ts = params.get('ts')
+        table_id = asset.name
+        if asset.user_metadata and 'table_id' in asset.user_metadata:
+            table_id = asset.user_metadata['table_id']
+        if '.' not in table_id:
+            if self.dataset_id is None:
+                raise ValueError(f"`{table_id}` is not valid. set the id as `<dataset-id>.<table-id>`, or provide `dataset_id` at store level.")
+            table_id = self.dataset_id + '.' + table_id
+        if isinstance(data, DataFrame):
+            return pandas_gbq.to_gbq(data, table_id, project_id=self.project_id)
+        else:
+            raise ValueError(f"Unsupported data type `{type(data)}` for store `{self.id()}`", )
+
+    def retrieve(self, index: Idx, _: dict | None = None) -> T:
+        return pandas_gbq.from_gbq(index['table_id'], project_id=self.project_id)
+"""

tinyfan/stores/naive.py

@@ -0,0 +1,37 @@
+from .base import StoreBase, FlowRunData
+import pickle
+import zlib
+import base64
+
+
+def encode(s: str | bytes) -> str:
+    b = s.encode() if isinstance(s, str) else s
+    return base64.b64encode(zlib.compress(b, 9)).decode()
+
+
+def decode(s: str) -> bytes:
+    return zlib.decompress(base64.b64decode(s))
+
+
+class NaiveStore(StoreBase[object, str]):
+    @staticmethod
+    def id() -> str:
+        return "tinyfan.naivestore"
+
+    def store(
+        self,
+        value: object,
+        rundata: FlowRunData,
+    ) -> str:
+        del rundata
+        """should return store index, which is used when retrieve back"""
+        return encode(pickle.dumps(value))
+
+    def retrieve(
+        self,
+        index: str,
+        source_rundata: FlowRunData,
+        target_rundata: FlowRunData,
+    ) -> object:
+        del source_rundata, target_rundata
+        return pickle.loads(decode(index))

tinyfan/utils/embed.py

@@ -0,0 +1,129 @@
+import os
+import base64
+import importlib
+import zipfile
+from pathlib import Path
+import io
+import ast
+
+DEFAULT_INCLUDES = ["**/*.*"]
+DEFAULT_EXCLUDES = ["**/__pycache__/*"]
+
+
+class TypeHintRemover(ast.NodeTransformer):
+    def visit_FunctionDef(self, node):
+        node.returns = None
+        if node.args.args:
+            for arg in node.args.args:
+                arg.annotation = None
+        self.generic_visit(node)
+        return node
+
+    def visit_AnnAssign(self, node):
+        if node.value is None:
+            return None
+        return ast.Assign([node.target], node.value)
+
+    def visit_Import(self, node):
+        node.names = [n for n in node.names if n.name != "typing"]
+        return node if node.names else None
+
+    def visit_ImportFrom(self, node):
+        return node if node.module != "typing" else None
+
+
+def remove_type_hint(source: str):
+    parsed_source = ast.parse(source)
+    transformed = TypeHintRemover().visit(parsed_source)
+    return ast.unparse(ast.fix_missing_locations(transformed))
+
+
+def zip_directory(
+    dir_path: str,
+    includes: list[str] = DEFAULT_INCLUDES,
+    excludes: list[str] = DEFAULT_EXCLUDES,
+    minify: bool = False,
+) -> tuple[bytes, str]:
+    dir_path = os.path.abspath(dir_path)
+    parent_dir = os.path.dirname(dir_path)
+    base_name = os.path.basename(dir_path)
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+        path = Path(dir_path)
+        files = set().union(*map(path.glob, includes))
+        files -= set().union(*map(path.glob, excludes))
+        for file_path in map(str, files):
+            arcname = os.path.relpath(file_path, start=parent_dir)
+            with open(file_path, "r") as f:
+                try:
+                    source = f.read()
+                except Exception as e:
+                    raise ValueError(f"fail to zip directory {dir_path}: fail to read file {file_path}") from e
+            if minify:
+                source = remove_type_hint(source)
+            zf.writestr(arcname, source)
+    buf.seek(0)
+    return buf.read(), base_name
+
+
+def embed_singlefile(path: str) -> str:
+    with open(path, "r") as f:
+        sourcecode = f.read()
+    return sourcecode
+
+
+def embed_directory(
+    dir_path: str,
+    main_module: str | None = None,
+    includes: list[str] = DEFAULT_INCLUDES,
+    excludes: list[str] = DEFAULT_EXCLUDES,
+    minify=True,
+) -> str:
+    zip_bytes, base_name = zip_directory(dir_path, includes, excludes, minify)
+    encoded = base64.b64encode(zip_bytes).decode("utf-8")
+    snippet = f"""
+import sys, base64, tempfile
+zip_data = base64.b64decode('{encoded}')
+with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp_file:
+    tmp_file.write(zip_data)
+    sys.path.append(tmp_file.name)
+"""
+    # snippet += f'import {main_module or base_name}\n'
+    return snippet
+
+
+def embed_path(
+    path: str,
+    main_module: str | None = None,
+    includes: list[str] = DEFAULT_INCLUDES,
+    excludes: list[str] = DEFAULT_EXCLUDES,
+) -> str:
+    if os.path.isfile(path):
+        return embed_singlefile(path)
+    else:
+        return embed_directory(path, main_module, includes, excludes)
+
+
+def embed_module(
+    module_name: str,
+    includes: list[str] = DEFAULT_INCLUDES,
+    excludes: list[str] = DEFAULT_EXCLUDES,
+    minify=True,
+) -> str:
+    mod = importlib.import_module(module_name)
+    if mod.__file__ is None:
+        raise ValueError(f"{module_name}.__file__ is None.")
+    pkg_dir = os.path.dirname(mod.__file__)
+    return embed_directory(pkg_dir, module_name, includes, excludes, minify)
+
+
+def embed(
+    path_or_module: str,
+    includes: list[str] = DEFAULT_INCLUDES,
+    excludes: list[str] = DEFAULT_EXCLUDES,
+    minify=False,
+) -> str:
+    try:
+        return embed_module(path_or_module, includes, excludes, minify)
+    except ModuleNotFoundError:
+        return embed_path(path_or_module, None, includes, excludes)

tinyfan/utils/exjson.py

@@ -0,0 +1,30 @@
+from datetime import datetime
+import json
+from typing import Mapping, TypeAlias, Sequence
+
+
+SerializableDict: TypeAlias = (
+    Mapping[str, "SerializableDict"] | Sequence["SerializableDict"] | str | int | float | bool | datetime | None
+)
+
+
+def default(obj):
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    else:
+        return obj
+
+
+def object_hook(obj):
+    try:
+        return datetime.fromisoformat(obj)
+    except ValueError:
+        return obj
+
+
+def dumps(obj: object):
+    return json.dumps(obj, default=default)
+
+
+def loads(obj: str):
+    return json.loads(obj, object_hook=object_hook)

tinyfan-0.0.1.dist-info/METADATA

@@ -0,0 +1,146 @@
+Metadata-Version: 2.1
+Name: tinyfan
+Version: 0.0.1
+Summary: Tiny Argo Workflows python devkit
+Author-email: eunchuldev <eunchulsong@gmail.com>
+Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
+Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
+Project-URL: Source, https://github.com/eunchuldev/tinyfan
+Keywords: argo-workflow,tinyfan,workflow,pipeline
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: croniter>=3.0.3
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: typer-slim>=0.15.1
+Provides-Extra: all
+Requires-Dist: tinyfan[gcp]; extra == "all"
+Requires-Dist: tinyfan[pandas]; extra == "all"
+Provides-Extra: gcp
+Requires-Dist: google-cloud-storage; extra == "gcp"
+Requires-Dist: google-cloud-bigquery[pandas]; extra == "gcp"
+Provides-Extra: pandas
+Requires-Dist: pandas; extra == "pandas"
+Requires-Dist: pyarrow; extra == "pandas"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.3.3; extra == "dev"
+Requires-Dist: jsonpatch>=1; extra == "dev"
+Requires-Dist: jsonschema[format]; extra == "dev"
+Requires-Dist: mypy>=1.12; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-jsonschema; extra == "dev"
+
+# Tinyfan
+
+Tinyfan: Minimalist Data Pipeline Kit - Generate Argo Workflows with Python
+
+# Features
+
+* Generate Argo Workflows manifests from Python data pipeline definitions.
+* Ease of Use and highly extendable
+* Intuitive data model abstraction: let Argo handle orchestration—we focus on the data.
+* Argo Workflows is notably lightweight and powerful – and so are we!
+* Enhanced DevOps Experience: easly testable, Cloud Native and GitOps-ready.
+
+# Our Goal
+
+* **Minimize mental overhead** when building data pipelines.
+
+# Not Our Goal
+
+* **Full-featured orchestration framework:** We don't aim to be a battery-powered, comprehensive data pipeline orchestration solution.  
+  No databases, web servers, or controllers—just a data pipeline compiler. Let's Algo Workflows handle all the complexity.
+
+# Installation
+
+```
+# Requires Python 3.10+
+pipx install tinyfan
+```
+
+# Tiny Example
+
+```python
+# main.py
+
+# Asset definitions
+
+from tinyfan import asset
+
+@asset(schedule="*/3 * * * *")
+def world() -> str:
+    return "world"
+
+@asset()
+def greeting(world: str):
+    print("hello " + world)
+```
+
+```shell
+# Apply the changes to argo workflow
+
+tinyfan template main.py | kubectl apply -f -
+```
+
+# Real World Example (still tiny though!)
+
+```python
+
+from tinyfan import GcsStore, Flow
+import os
+
+# Build time configs:
+# Pod configs are settled on the build time.
+image = os.environ["IMAGE_NAME"]
+
+# Runtime Configs:
+# Other configs are revisited on runetime.
+some_secret = os.environ[""]
+
+# Flow is refer to the Argo Workflow Object.
+# Flow configs are applied to all of child assets
+flow = Flow(
+    name="flow",
+    image=os.environ["IMAGE_NAME"],
+    envSecrets=[""],
+    envConfigMaps=[""],
+    as_default=True,
+)
+image = os.environ['IMAGE_NAME'],
+sa_name = os.environ['SERVICE_ACCOUNT_NAME'],
+gcs_store = GcsStore(),
+naive_store = NaiveStore()
+
+flow = Flow(
+    store=naive_store,
+)
+# authorized by workload-identity-federation
+
+@asset(
+    schedule = "*/10 * * * *",
+    image = "python3:alpine",
+    store = NaiveStore(),
+)
+def target() -> str:
+    return "world"
+
+
+@asset(
+    schedule = "*/10 * * * *",
+    image = "python3:alpine",
+    store = GcsStore(),
+)
+def target() -> str:
+    return "world"
+
+```
+
+
+# License
+
+This project is licensed under the MIT License.

tinyfan-0.0.1.dist-info/RECORD

@@ -0,0 +1,18 @@
+tinyfan/__init__.py,sha256=FqHH3BIb-qKtBuDnEpJMC-KoBxerKSCco3GqvoBlR3o,174
+tinyfan/cli.py,sha256=Jsocb0keH8ywgNdA2TXR40g7BPd6ouU8WDOUwsNRIDs,498
+tinyfan/codegen.py,sha256=vIklPIU9urNF57KVImscpWLoo56Xiruzy3GSi_UPneE,12556
+tinyfan/flow.py,sha256=uoCo4mNculSvBUajBE5LKL0ZSd_J38wD1tdB3WVlMHY,3199
+tinyfan/flowrundata.py,sha256=SKv-aQjXqEQ0AzUGO4AnMTplgbnSSJ4T2FiRkxdTBGc,579
+tinyfan/stores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tinyfan/stores/base.py,sha256=DU_91eU-V-ZRwvm8tNLifeo3fvwyGAo2M1iZxCcpljk,673
+tinyfan/stores/bigquery.py,sha256=bz-ZgXmLvTBT4Yb6ROSrvIB8iNVed4_01haId8c1svc,1648
+tinyfan/stores/gcs.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tinyfan/stores/naive.py,sha256=U_qNWs_Vt9H-gsFWloeG7t8y4QkVzaDuL7O_yTlE_0M,886
+tinyfan/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tinyfan/utils/embed.py,sha256=BgfG8Fmv6RM4boKpBO4GNFG7LpaCqzpNgE3Y-sDfsjo,3911
+tinyfan/utils/exjson.py,sha256=amaiAwNfIqb-GoBQlt0h8fRR9d7i6r-EyRN99pm5I48,616
+tinyfan-0.0.1.dist-info/METADATA,sha256=C1zgBfSHGRatR3HnrDMpg_IWpEDZRXRnjE7050NRc3M,3754
+tinyfan-0.0.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+tinyfan-0.0.1.dist-info/entry_points.txt,sha256=qqB-I0wVkvSRcxuKun-jQ-n8FAD9RKy5SMsaQ83v1Xg,45
+tinyfan-0.0.1.dist-info/top_level.txt,sha256=qNwWOfVkNzotKJHxqQnW3unx0CCL1pwE_2JahjMxDP8,8
+tinyfan-0.0.1.dist-info/RECORD,,

tinyfan-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.6.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tinyfan-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tinyfan = tinyfan.cli:main

tinyfan-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+tinyfan