tinyfan 0.2.0__tar.gz → 0.2.2__tar.gz

{tinyfan-0.2.0 → tinyfan-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tinyfan
-Version: 0.2.0
+Version: 0.2.2
 Summary: Tiny Argo Workflows python devkit
 Project-URL: Documentation, https://github.com/eunchuldev/tinyfan#readme
 Project-URL: Issues, https://github.com/eunchuldev/tinyfan/issues
@@ -12,6 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
 Requires-Python: >=3.12
 Requires-Dist: croniter>=3.0.3
 Requires-Dist: pyyaml>=6.0.2
@@ -19,6 +20,7 @@ Requires-Dist: typer-slim>=0.15.1
 Provides-Extra: all
 Requires-Dist: google-cloud-bigquery[pandas]; extra == 'all'
 Requires-Dist: google-cloud-storage; extra == 'all'
+Requires-Dist: networkx; extra == 'all'
 Requires-Dist: pandas; extra == 'all'
 Requires-Dist: pyarrow; extra == 'all'
 Provides-Extra: bigquery
@@ -26,9 +28,14 @@ Requires-Dist: google-cloud-bigquery[pandas]; extra == 'bigquery'
 Requires-Dist: pandas; extra == 'bigquery'
 Requires-Dist: pyarrow; extra == 'bigquery'
 Provides-Extra: dev
+Requires-Dist: google-cloud-bigquery[pandas]; extra == 'dev'
+Requires-Dist: google-cloud-storage; extra == 'dev'
 Requires-Dist: jsonpatch>=1; extra == 'dev'
 Requires-Dist: jsonschema[format]; extra == 'dev'
 Requires-Dist: mypy>=1.14; extra == 'dev'
+Requires-Dist: networkx; extra == 'dev'
+Requires-Dist: pandas; extra == 'dev'
+Requires-Dist: pyarrow; extra == 'dev'
 Requires-Dist: pytest>=8.3.3; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'
 Requires-Dist: types-croniter; extra == 'dev'
@@ -39,6 +46,8 @@ Requires-Dist: google-cloud-storage; extra == 'gcs'
 Provides-Extra: pandas
 Requires-Dist: pandas; extra == 'pandas'
 Requires-Dist: pyarrow; extra == 'pandas'
+Provides-Extra: viz
+Requires-Dist: networkx; extra == 'viz'
 Description-Content-Type: text/markdown
 
 # Tinyfan

{tinyfan-0.2.0 → tinyfan-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tinyfan"
-version = "0.2.0"
+version = "0.2.2"
 description = 'Tiny Argo Workflows python devkit'
 readme = "README.md"
 requires-python = ">=3.12"
@@ -12,6 +12,7 @@ classifiers = [
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Typing :: Typed",
 ]
 dependencies = ["croniter>=3.0.3", "pyyaml>=6.0.2", "typer-slim>=0.15.1"]
 
@@ -19,8 +20,9 @@ dependencies = ["croniter>=3.0.3", "pyyaml>=6.0.2", "typer-slim>=0.15.1"]
 tinyfan = "tinyfan.cli:main"
 
 [project.optional-dependencies]
-all = ["tinyfan[gcs]", "tinyfan[bigquery]", "tinyfan[pandas]"]
+all = ["tinyfan[gcs]", "tinyfan[bigquery]", "tinyfan[pandas]", "tinyfan[viz]"]
 gcs = ["google-cloud-storage"]
+viz = ["networkx"]
 bigquery = ["google-cloud-bigquery[pandas]", "tinyfan[pandas]"]
 pandas = ["pandas", "pyarrow"]
 dev = [
@@ -32,6 +34,7 @@ dev = [
   "types-PyYAML",
   "types-jsonschema",
   "types-croniter",
+  "tinyfan[all]",
 ]
 
 [project.urls]

{tinyfan-0.2.0 → tinyfan-0.2.2}/src/tinyfan/codegen.py

@@ -169,6 +169,37 @@ class AssetTree:
             param_refs = [flow_refs[n] for n in func_param_names if n in flow_refs]
             node.refs = list(set(resource_refs + param_refs))
 
+    def relatives_by_schedules(self) -> dict[tuple[str, str], set[AssetNode]]:
+        relatives_by_schedules: dict[tuple[str, str], set[AssetNode]] = {}
+        for node in self.nodes.values():
+            if node.asset.schedule is None:
+                continue
+            schedule = (node.asset.tz or self.flow.tz, node.asset.schedule)
+            if schedule not in relatives_by_schedules:
+                relatives_by_schedules[schedule] = set()
+            for o in node.relatives():
+                relatives_by_schedules[schedule].add(o)
+        return relatives_by_schedules
+
+    def dagviz(self) -> str | None:
+        try:
+            import networkx as nx
+        except ImportError:
+            return None
+
+        from .dagviz import dagviz
+
+        relatives_by_schedules = self.relatives_by_schedules()
+        g: nx.DiGraph = nx.DiGraph()
+        # g.add_node(self.flow.name)
+        # g.add_nodes_from([k for k in relatives_by_schedules.keys()])
+        # g.add_nodes_from(self.nodes.keys())
+        for (_, schedule), relatives in relatives_by_schedules.items():
+            g.add_edge(self.flow.name, schedule)
+            g.add_edges_from([(schedule, r.asset.name) for r in relatives])
+            g.add_edges_from([(r.asset.name, d.asset.name) for r in relatives for d in r.parents])
+        return dagviz(g, round_angle=True)
+
     def compile(
         self,
         embedded: bool = True,
@@ -207,15 +238,7 @@ class AssetTree:
             "    f.write(dumps(rundata))\n"
         )
 
-        relatives_by_schedules: dict[tuple[str, str], set[AssetNode]] = {}
-        for node in self.nodes.values():
-            if node.asset.schedule is None:
-                continue
-            schedule = (node.asset.tz or self.flow.tz, node.asset.schedule)
-            if schedule not in relatives_by_schedules:
-                relatives_by_schedules[schedule] = set()
-            for o in node.relatives():
-                relatives_by_schedules[schedule].add(o)
+        relatives_by_schedules = self.relatives_by_schedules()
 
         manifests = [
             {
@@ -388,7 +411,12 @@ class AssetTree:
             }
             for (tz, schedule), relatives in relatives_by_schedules.items()
         ]
-        return yaml_dump_all(manifests)
+        manifest = ""
+        dagviz = self.dagviz()
+        if dagviz:
+            manifest += "# " + dagviz.replace("\n", "\n# ") + "\n\n"
+        manifest += yaml_dump_all(manifests)
+        return manifest
 
 
 def codegen(

tinyfan-0.2.2/src/tinyfan/dagviz.py

@@ -0,0 +1,54 @@
+# COPIED FROM https://github.com/ctongfei/py-dagviz
+from typing import Hashable, Dict, List
+import array
+import networkx as nx
+
+
+def dagviz(g: nx.DiGraph, round_angle: bool = False) -> str:
+    """
+    Creates a text rendering of a directed acyclic graph (DAG) for visualization purposes in a terminal.
+
+    :param g: A directed acyclic graph, of type `nx.DiGraph`
+    :param round_angle: Whether to use a round-angled box drawing character or not
+    :return: A multi-line string representation of the directed acyclic graph, each line corresponding to a node
+    """
+    assert nx.is_directed_acyclic_graph(g), "Graph contains cycles"
+
+    rows: List[Hashable] = []
+    node_to_row: Dict[Hashable, int] = {}
+    indents: List[int] = []
+
+    def _process_dag(g: nx.DiGraph, indent: int):
+        for sg in nx.weakly_connected_components(g):
+            _process_component(nx.DiGraph(g.subgraph(sg)), indent=indent)
+
+    def _process_component(g: nx.DiGraph, indent: int):
+        sources = [v for v in g.nodes if g.in_degree(v) == 0]
+        for i in range(len(sources)):
+            node_to_row[sources[i]] = len(rows)
+            rows.append(sources[i])
+            indents.append(indent + i)
+        _process_dag(nx.DiGraph(g.subgraph(set(g.nodes).difference(sources))), indent=indent + len(sources))
+
+    _process_dag(g, indent=0)
+    a = [array.array("u", [" "] * indents[i] * 2) for i in range(len(rows))]
+    for i, u in enumerate(rows):
+        successors = sorted(g.successors(u), key=lambda v: node_to_row[v])
+        if len(successors) == 0:
+            continue
+        n = node_to_row[successors[-1]]
+        for j in range(i + 1, n):
+            a[j][indents[i] * 2] = "│"
+        for v in successors[:-1]:
+            j = node_to_row[v]
+            a[j][indents[i] * 2] = "┼" if indents[i] > 0 and a[j][indents[i] * 2 - 1] == "─" else "├"
+            for k in range(indents[i] * 2 + 1, indents[j] * 2):
+                a[j][k] = "─"
+        a[n][indents[i] * 2] = (
+            "┴" if indents[i] > 0 and a[n][indents[i] * 2 - 1] == "─" else ("╰" if round_angle else "└")
+        )
+        for k in range(indents[i] * 2 + 1, indents[n] * 2):
+            a[n][k] = "─"
+
+    lines: List[str] = [x.tounicode() + "• " + str(i).replace("\n", " ") for x, i in zip(a, rows)]
+    return "\n".join(lines)

{tinyfan-0.2.0 → tinyfan-0.2.2}/uv.lock

@@ -429,6 +429,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 },
 ]
 
+[[package]]
+name = "networkx"
+version = "3.4.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263 },
+]
+
 [[package]]
 name = "numpy"
 version = "2.1.2"
@@ -775,7 +784,7 @@ wheels = [
 
 [[package]]
 name = "tinyfan"
-version = "0.2.0"
+version = "0.2.2"
 source = { editable = "." }
 dependencies = [
     { name = "croniter" },
@@ -787,6 +796,7 @@ dependencies = [
 all = [
     { name = "google-cloud-bigquery", extra = ["pandas"] },
     { name = "google-cloud-storage" },
+    { name = "networkx" },
     { name = "pandas" },
     { name = "pyarrow" },
 ]
@@ -796,9 +806,14 @@ bigquery = [
     { name = "pyarrow" },
 ]
 dev = [
+    { name = "google-cloud-bigquery", extra = ["pandas"] },
+    { name = "google-cloud-storage" },
     { name = "jsonpatch" },
     { name = "jsonschema", extra = ["format"] },
     { name = "mypy" },
+    { name = "networkx" },
+    { name = "pandas" },
+    { name = "pyarrow" },
     { name = "pytest" },
     { name = "ruff" },
     { name = "types-croniter" },
@@ -812,6 +827,9 @@ pandas = [
     { name = "pandas" },
     { name = "pyarrow" },
 ]
+viz = [
+    { name = "networkx" },
+]
 
 [package.metadata]
 requires-dist = [
@@ -821,21 +839,24 @@ requires-dist = [
     { name = "jsonpatch", marker = "extra == 'dev'", specifier = ">=1" },
     { name = "jsonschema", extras = ["format"], marker = "extra == 'dev'" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14" },
+    { name = "networkx", marker = "extra == 'viz'" },
     { name = "pandas", marker = "extra == 'pandas'" },
     { name = "pyarrow", marker = "extra == 'pandas'" },
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.3" },
     { name = "pyyaml", specifier = ">=6.0.2" },
     { name = "ruff", marker = "extra == 'dev'" },
+    { name = "tinyfan", extras = ["all"], marker = "extra == 'dev'" },
     { name = "tinyfan", extras = ["bigquery"], marker = "extra == 'all'" },
     { name = "tinyfan", extras = ["gcs"], marker = "extra == 'all'" },
     { name = "tinyfan", extras = ["pandas"], marker = "extra == 'all'" },
     { name = "tinyfan", extras = ["pandas"], marker = "extra == 'bigquery'" },
+    { name = "tinyfan", extras = ["viz"], marker = "extra == 'all'" },
     { name = "typer-slim", specifier = ">=0.15.1" },
     { name = "types-croniter", marker = "extra == 'dev'" },
     { name = "types-jsonschema", marker = "extra == 'dev'" },
     { name = "types-pyyaml", marker = "extra == 'dev'" },
 ]
-provides-extras = ["all", "gcs", "bigquery", "pandas", "dev"]
+provides-extras = ["all", "gcs", "viz", "bigquery", "pandas", "dev"]
 
 [[package]]
 name = "typer-slim"