tinyfan 0.0.2__tar.gz

tinyfan-0.0.2/.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish Pypi
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  publish:
+    name: python
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+      - name: Install the Project
+        run: uv sync --all-extras --dev
+      - name: Run tests
+        run: uv run pytest tests
+      - name: Build Package
+        run: uv build
+      - name: Publish Project
+        run: uv publish
+        env:
+          UV_PUBLISH_PASSWORD: ${{ secrets.UV_PUBLISH_PASSWORD }}
+          UV_PUBLISH_USERNAME: __token__
+

tinyfan-0.0.2/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

tinyfan-0.0.2/.python-version

@@ -0,0 +1 @@
+3.12

tinyfan-0.0.2/PKG-INFO

@@ -0,0 +1,103 @@
+Metadata-Version: 2.4
+Name: tinyfan
+Version: 0.0.2
+Summary: Tiny Argo Workflows python devkit
+Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
+Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
+Project-URL: Source, https://github.com/eunchuldev/tinyfan
+Author-email: eunchuldev <eunchulsong@gmail.com>
+Keywords: argo-workflow,pipeline,tinyfan,workflow
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
+Requires-Dist: croniter>=3.0.3
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: typer-slim>=0.15.1
+Provides-Extra: all
+Requires-Dist: google-cloud-bigquery[pandas]; extra == 'all'
+Requires-Dist: google-cloud-storage; extra == 'all'
+Requires-Dist: pandas; extra == 'all'
+Requires-Dist: pyarrow; extra == 'all'
+Provides-Extra: bigquery
+Requires-Dist: google-cloud-bigquery[pandas]; extra == 'bigquery'
+Requires-Dist: pandas; extra == 'bigquery'
+Requires-Dist: pyarrow; extra == 'bigquery'
+Provides-Extra: dev
+Requires-Dist: jsonpatch>=1; extra == 'dev'
+Requires-Dist: jsonschema[format]; extra == 'dev'
+Requires-Dist: mypy>=1.14; extra == 'dev'
+Requires-Dist: pytest>=8.3.3; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: types-jsonschema; extra == 'dev'
+Requires-Dist: types-pyyaml; extra == 'dev'
+Provides-Extra: gcs
+Requires-Dist: google-cloud-storage; extra == 'gcs'
+Provides-Extra: pandas
+Requires-Dist: pandas; extra == 'pandas'
+Requires-Dist: pyarrow; extra == 'pandas'
+Description-Content-Type: text/markdown
+
+# Tinyfan
+
+Tinyfan: Minimalist Data Pipeline Kit - Generate Argo Workflows with Python
+
+# Features
+
+* Generate Argo Workflows manifests from Python data pipeline definitions.
+* Ease of Use and highly extendable
+* Intuitive data model abstraction: let Argo handle orchestration—we focus on the data.
+* Argo Workflows is notably lightweight and powerful – and so are we!
+* Enhanced DevOps Experience: easly testable, Cloud Native and GitOps-ready.
+
+# Our Goal
+
+* **Minimize mental overhead** when building data pipelines.
+
+# Not Our Goal
+
+* **Full-featured orchestration framework:** We don't aim to be a battery-powered, comprehensive data pipeline orchestration solution.  
+  No databases, web servers, or controllers—just a data pipeline compiler. Let's Algo Workflows handle all the complexity.
+
+# Installation
+
+```
+# Requires Python 3.10+
+pipx install tinyfan
+```
+
+# Tiny Example
+
+```python
+# main.py
+
+# Asset definitions
+
+from tinyfan import asset
+
+@asset(schedule="*/3 * * * *")
+def world() -> str:
+    return "world"
+
+@asset()
+def greeting(world: str):
+    print("hello " + world)
+```
+
+```shell
+# Apply the changes to argo workflow
+
+tinyfan template main.py | kubectl apply -f -
+```
+
+# Real World Example (still tiny though!)
+
+Comming soon
+
+
+
+# License
+
+This project is licensed under the MIT License.

tinyfan-0.0.2/README.md

@@ -0,0 +1,61 @@
+# Tinyfan
+
+Tinyfan: Minimalist Data Pipeline Kit - Generate Argo Workflows with Python
+
+# Features
+
+* Generate Argo Workflows manifests from Python data pipeline definitions.
+* Ease of Use and highly extendable
+* Intuitive data model abstraction: let Argo handle orchestration—we focus on the data.
+* Argo Workflows is notably lightweight and powerful – and so are we!
+* Enhanced DevOps Experience: easly testable, Cloud Native and GitOps-ready.
+
+# Our Goal
+
+* **Minimize mental overhead** when building data pipelines.
+
+# Not Our Goal
+
+* **Full-featured orchestration framework:** We don't aim to be a battery-powered, comprehensive data pipeline orchestration solution.  
+  No databases, web servers, or controllers—just a data pipeline compiler. Let's Algo Workflows handle all the complexity.
+
+# Installation
+
+```
+# Requires Python 3.10+
+pipx install tinyfan
+```
+
+# Tiny Example
+
+```python
+# main.py
+
+# Asset definitions
+
+from tinyfan import asset
+
+@asset(schedule="*/3 * * * *")
+def world() -> str:
+    return "world"
+
+@asset()
+def greeting(world: str):
+    print("hello " + world)
+```
+
+```shell
+# Apply the changes to argo workflow
+
+tinyfan template main.py | kubectl apply -f -
+```
+
+# Real World Example (still tiny though!)
+
+Comming soon
+
+
+
+# License
+
+This project is licensed under the MIT License.

tinyfan-0.0.2/examples/getting-started.py

@@ -0,0 +1,23 @@
+import tinyfan
+from datetime import datetime
+
+
+@tinyfan.asset(
+    schedule="*/1 * * * *",
+)
+def target() -> str:
+    return "world"
+
+
+@tinyfan.asset()
+def greeting(target: str):
+    print("hello " + target)
+
+
+@tinyfan.asset(
+    schedule="*/1 * * * *",
+)
+def check_variables(ds: str, ts: str, data_interval_start: datetime, data_interval_end: datetime) -> None:
+    print(f"ds: {ds}, ts: {ts}")
+    print(f"data_interval_start: {data_interval_start}")
+    print(f"data_interval_end: {data_interval_end}")

tinyfan-0.0.2/examples/realworld/.python-version

@@ -0,0 +1 @@
+3.12

tinyfan-0.0.2/examples/realworld/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "code"
+version = "0.0.1"
+description = "vtown"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []

tinyfan-0.0.2/pyproject.toml

@@ -0,0 +1,82 @@
+[project]
+name = "tinyfan"
+version = "0.0.2"
+description = 'Tiny Argo Workflows python devkit'
+readme = "README.md"
+requires-python = ">=3.12"
+keywords = [
+  "argo-workflow",
+  "tinyfan",
+  "workflow",
+  "pipeline",
+]
+authors = [
+  { name = "eunchuldev", email = "eunchulsong@gmail.com" },
+]
+classifiers = [
+  "Operating System :: OS Independent",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+    "croniter>=3.0.3",
+    "pyyaml>=6.0.2",
+    "typer-slim>=0.15.1",
+]
+
+[project.scripts]
+tinyfan = "tinyfan.cli:main"
+
+[project.optional-dependencies]
+all = [
+  "tinyfan[gcs]",
+  "tinyfan[bigquery]",
+  "tinyfan[pandas]",
+]
+gcs = [
+  "google-cloud-storage",
+]
+bigquery = [
+  "google-cloud-bigquery[pandas]",
+  "tinyfan[pandas]",
+]
+pandas = [
+  "pandas",
+  "pyarrow",
+]
+dev = [
+  "pytest >= 8.3.3",
+  "jsonpatch>=1",
+  "jsonschema[format]",
+  "mypy>=1.14",
+  "ruff",
+  "types-PyYAML",
+  "types-jsonschema",
+]
+
+[project.urls]
+Documentation = "https://github.com/eunchuldev/tinyfan#readme"
+Issues = "https://github.com/eunchuldev/tinyfan/issues"
+Source = "https://github.com/eunchuldev/tinyfan"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.pytest.ini_options]
+addopts = ["--import-mode=importlib"]
+
+[tool.mypy]
+exclude = ['build']
+
+[[tool.mypy.overrides]]
+module = ["lark.*", "jsonpatch.*"]
+ignore_missing_imports = true
+
+[tool.ruff]
+line-length = 120
+
+[tool.uv.workspace]
+members = ["exampels/realworld"]

tinyfan-0.0.2/src/tinyfan/__init__.py

@@ -0,0 +1,5 @@
+from .flow import Flow, Asset, asset
+from .stores.base import StoreBase
+from .stores.naive import NaiveStore
+
+__all__ = ["Flow", "Asset", "asset", "StoreBase", "NaiveStore"]

tinyfan-0.0.2/src/tinyfan/argo_typing.py

@@ -0,0 +1,204 @@
+from typing import TypedDict, NotRequired
+
+
+class ExecAction(TypedDict):
+    command: list[str]
+
+
+class SecretKeySelector(TypedDict):
+    key: str
+    name: str
+    optional: NotRequired[bool]
+
+
+class HTTPHeaderSource(TypedDict):
+    secretKeyRef: SecretKeySelector
+
+
+class HTTPHeader(TypedDict):
+    name: str
+    value: NotRequired[str]
+    valueFrom: NotRequired[HTTPHeaderSource]
+
+
+class HTTPGetAction(TypedDict):
+    host: str
+    httpHeaders: NotRequired[list[HTTPHeader]]
+    path: NotRequired[str]
+    port: NotRequired[str | int]
+    scheme: NotRequired[str]
+
+
+class TCPSocketAction(TypedDict):
+    host: str
+    port: str | int
+
+
+class SleepAction(TypedDict):
+    seconds: int
+
+
+class LifecycleHandler(TypedDict, total=False):
+    exec: ExecAction
+    httpGet: HTTPGetAction
+    sleep: SleepAction
+    tcpSocket: TCPSocketAction
+
+
+class GRPCAction(TypedDict):
+    port: int
+    service: str
+
+
+class Probe(TypedDict, total=False):
+    exec: ExecAction
+    failureThreshold: int
+    grpc: GRPCAction
+    httpGet: HTTPGetAction
+    initialDelaySeconds: int
+    periodSeconds: int
+    successThreshold: int
+    tcpSocket: TCPSocketAction
+    terminationGracePeriodSeconds: int
+    timeoutSeconds: int
+
+
+class ConfigMapKeySelector(TypedDict):
+    key: str
+    name: str
+    optional: NotRequired[bool]
+
+
+class ObjectFieldSelector(TypedDict):
+    apiVersion: str
+    fieldPath: str
+
+
+class ResourceFieldSelector(TypedDict):
+    containerName: str
+    divisor: str
+    resource: str
+
+
+class EnvVarSource(TypedDict, total=False):
+    configMapKeyRef: ConfigMapKeySelector
+    fieldRef: ObjectFieldSelector
+    resourceFieldRef: ResourceFieldSelector
+    secretKeyRef: SecretKeySelector
+
+
+class EnvVar(TypedDict):
+    name: str
+    value: NotRequired[str]
+    valueFrom: NotRequired[EnvVarSource]
+
+
+class Lifecycle(TypedDict, total=False):
+    postStart: LifecycleHandler
+    preStop: LifecycleHandler
+
+
+class ContainerResizePolicy(TypedDict):
+    resourceName: str
+    restartPolicy: str
+
+
+class ResourceQuantity(TypedDict, total=False):
+    memory: str
+    cpu: str
+
+
+class ResourceClaim(TypedDict):
+    name: str
+    request: ResourceQuantity
+
+
+class ResourceRequirements(TypedDict, total=False):
+    claims: list[ResourceClaim]
+    limits: ResourceQuantity
+    requests: ResourceQuantity
+
+
+class AppArmorProfile(TypedDict, total=False):
+    localhostProfile: str
+    type: str
+
+
+class Capabilities(TypedDict, total=False):
+    add: list[str]
+    drop: list[str]
+
+
+class SELinuxOptions(TypedDict, total=False):
+    level: str
+    role: str
+    type: str
+    user: str
+
+
+class SeccompProfile(TypedDict, total=False):
+    localhostProfile: str
+    type: str
+
+
+class WindowOptions(TypedDict, total=False):
+    gmsaCredentialSpec: str
+    gmsaCredentialSpecName: str
+    hostProcess: bool
+    runAsUserName: str
+
+
+class SecurityContext(TypedDict, total=False):
+    allowPrivilegeEscalation: bool
+    appArmorProfile: AppArmorProfile
+    capabilities: Capabilities
+    privileged: bool
+    procMount: str
+    readOnlyRootFilesystem: bool
+    runAsGroup: int
+    runAsNonRoot: bool
+    runAsUser: int
+    seLinuxOptions: SELinuxOptions
+    seccompProfile: SeccompProfile
+    windowOptions: WindowOptions
+
+
+class VolumeDevice(TypedDict):
+    devicePath: str
+    name: str
+
+
+class VolumeMount(TypedDict, total=False):
+    mounstPath: str
+    mountPropagation: str
+    name: str
+    readOnly: bool
+    recursiveReadOnly: str
+    subPath: str
+    subPathExpr: str
+
+
+# Ref: https://argo-workflows.readthedocs.io/en/latest/fields/#scripttemplate
+class ScriptTemplate(TypedDict, total=False):
+    args: list[str]
+    command: list[str]
+    env: list[EnvVar]
+    envFrom: list[EnvVar]
+    image: str
+    imagePullPolicy: str
+    lifecycle: Lifecycle
+    livenessProbe: Probe
+    readinessProbe: Probe
+    resizePolicy: list[ContainerResizePolicy]
+    resources: ResourceRequirements
+    restartPolicy: str
+    securityContext: SecurityContext
+    statusProbe: Probe
+    stdin: bool
+    stdinOnce: bool
+    terminationMessagePath: str
+    terminationMessagePolicy: str
+    tty: bool
+    volumeDevices: list[VolumeDevice]
+    volumeMounts: list[VolumeMount]
+    workingDir: str

tinyfan-0.0.2/src/tinyfan/cli.py

@@ -0,0 +1,24 @@
+import typer
+
+from .codegen import codegen
+
+
+def tinyfan(
+    location: str,
+    embedded: bool = False,
+):
+    """
+    Generate argocd workflow resource as yaml from tinyfan definitions
+    """
+    if location.endswith(".py"):
+        print(codegen(location=location, embedded=True))
+    else:
+        print(codegen(location=location, embedded=embedded))
+
+
+def main():
+    return typer.run(tinyfan)
+
+
+if __name__ == "__main__":
+    main()