codeflowhub 0.2.0__tar.gz → 0.2.1__tar.gz

{codeflowhub-0.2.0 → codeflowhub-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeflowhub
-Version: 0.2.0
+Version: 0.2.1
 Summary: workflow development tools
 Author: creaddiscans
 Author-email: creaddiscans@gmail.com

codeflowhub-0.2.1/codeflowhub/airflow/xcom.py

@@ -0,0 +1,140 @@
+"""Airflow REST API client for fetching task data at runtime.
+
+Used inside KubernetesPodOperator containers to fetch xcom/params data
+via Airflow REST API instead of inlining it in shell arguments (ARG_MAX safe).
+
+Environment variables (injected via Airflow Connection):
+  - FLOWHUB_API_URL: Airflow webserver URL (e.g. http://airflow-web:8080)
+  - FLOWHUB_API_USER: Basic auth user
+  - FLOWHUB_API_PASS: Basic auth password
+
+Usage:
+  python3 -m codeflowhub.airflow.xcom xcom <dag_id> <run_id> <task_id> <output_path>
+  python3 -m codeflowhub.airflow.xcom params <dag_id> <run_id> <output_path>
+"""
+import urllib.request
+import json
+import base64
+import os
+import sys
+from urllib.parse import quote
+
+
+def _api_request(path: str):
+    """Send authenticated GET request to Airflow REST API.
+
+    Args:
+        path: API path (e.g. /api/v1/dags/...)
+
+    Returns:
+        parsed JSON response body
+    """
+    required_vars = ['FLOWHUB_API_URL', 'FLOWHUB_API_USER', 'FLOWHUB_API_PASS']
+    missing = [v for v in required_vars if v not in os.environ]
+    if missing:
+        print(f"Error: missing environment variables: {', '.join(missing)}", file=sys.stderr)
+        print("These should be injected via Airflow Connection (FLOWHUB_API_URL, FLOWHUB_API_USER, FLOWHUB_API_PASS)", file=sys.stderr)
+        sys.exit(1)
+
+    auth = base64.b64encode(
+        (os.environ['FLOWHUB_API_USER'] + ':' + os.environ['FLOWHUB_API_PASS']).encode()
+    ).decode()
+
+    url = os.environ['FLOWHUB_API_URL'].rstrip('/') + path
+
+    req = urllib.request.Request(
+        url,
+        headers={
+            'Authorization': 'Basic ' + auth,
+            'Accept': 'application/json',
+        },
+    )
+
+    try:
+        resp = urllib.request.urlopen(req)
+    except urllib.error.HTTPError as e:
+        if e.code == 401:
+            print(f"Error: authentication failed for {url}", file=sys.stderr)
+            print("Check FLOWHUB_API_USER / FLOWHUB_API_PASS in Airflow Connection", file=sys.stderr)
+        elif e.code == 404:
+            print(f"Error: resource not found: {url}", file=sys.stderr)
+        else:
+            print(f"Error: HTTP {e.code} from {url}", file=sys.stderr)
+        sys.exit(1)
+    except urllib.error.URLError as e:
+        print(f"Error: cannot reach Airflow API at {os.environ['FLOWHUB_API_URL']}: {e.reason}", file=sys.stderr)
+        sys.exit(1)
+
+    return json.loads(resp.read())
+
+
+def _write_json(content, output_path: str):
+    """Write JSON content to file."""
+    os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
+    with open(output_path, 'w') as f:
+        if isinstance(content, str):
+            f.write(content)
+        else:
+            json.dump(content, f)
+
+
+def fetch_xcom(dag_id: str, run_id: str, task_id: str, output_path: str):
+    """Fetch xcom value from Airflow REST API and save to file.
+
+    Args:
+        dag_id: Airflow DAG ID
+        run_id: DAG Run ID
+        task_id: upstream task ID
+        output_path: path to save the JSON output
+    """
+    path = (
+        '/api/v1/dags/' + quote(dag_id, safe='')
+        + '/dagRuns/' + quote(run_id, safe='')
+        + '/taskInstances/' + quote(task_id, safe='')
+        + '/xcomEntries/return_value'
+    )
+    body = _api_request(path)
+    _write_json(body['value'], output_path)
+
+
+def fetch_params(dag_id: str, run_id: str, output_path: str):
+    """Fetch DAG run params (conf) from Airflow REST API and save to file.
+
+    Args:
+        dag_id: Airflow DAG ID
+        run_id: DAG Run ID
+        output_path: path to save the JSON output
+    """
+    path = (
+        '/api/v1/dags/' + quote(dag_id, safe='')
+        + '/dagRuns/' + quote(run_id, safe='')
+    )
+    body = _api_request(path)
+    _write_json(body.get('conf', {}), output_path)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("Usage:", file=sys.stderr)
+        print("  python3 -m codeflowhub.airflow.xcom xcom <dag_id> <run_id> <task_id> <output_path>", file=sys.stderr)
+        print("  python3 -m codeflowhub.airflow.xcom params <dag_id> <run_id> <output_path>", file=sys.stderr)
+        sys.exit(1)
+
+    command = sys.argv[1]
+
+    if command == 'xcom':
+        if len(sys.argv) != 6:
+            print("Usage: python3 -m codeflowhub.airflow.xcom xcom <dag_id> <run_id> <task_id> <output_path>", file=sys.stderr)
+            sys.exit(1)
+        fetch_xcom(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])
+
+    elif command == 'params':
+        if len(sys.argv) != 5:
+            print("Usage: python3 -m codeflowhub.airflow.xcom params <dag_id> <run_id> <output_path>", file=sys.stderr)
+            sys.exit(1)
+        fetch_params(sys.argv[2], sys.argv[3], sys.argv[4])
+
+    else:
+        print(f"Unknown command: {command}", file=sys.stderr)
+        print("Available commands: xcom, params", file=sys.stderr)
+        sys.exit(1)

{codeflowhub-0.2.0 → codeflowhub-0.2.1}/codeflowhub/service/airflow_exporter.py

@@ -296,9 +296,9 @@ base_volume_mounts = [
         # failure handler는 retries=0 설정
         retries_code = "\n        retries=0," if trigger_rule == 'one_failed' else ""
 
-        # XCom API fetch를 사용하는 task에 env_vars 주입
+        # XCom API fetch를 사용하는 task에 env_vars 주입 (is_first: params fetch, depend: xcom fetch)
         env_vars_code = ""
-        if self.airflow_connection_id and not is_first and task.depend:
+        if self.airflow_connection_id and (is_first or task.depend):
             env_vars_code = "\n        env_vars=_fh_api_env,"
 
         operator_code = f'''    {task.name} = KubernetesPodOperator(
@@ -315,6 +315,12 @@ base_volume_mounts = [
     def _generate_input_commands(self, task, is_first):
         """Input 데이터 명령어 생성"""
         if is_first:
+            if self.airflow_connection_id:
+                return (
+                    f'python3 -m codeflowhub.airflow.xcom params '
+                    f'{{{{{{{{ dag.dag_id }}}}}}}} {{{{{{{{ dag_run.run_id }}}}}}}} '
+                    f'/app/input/0.json'
+                )
             return '\n'.join(["cat << 'EOF' > /app/input/0.json", '{{{{ params | tojson }}}}', 'EOF'])
         if not task.depend:
             return "echo '{{}}' >> /app/input/0.json"
@@ -322,7 +328,7 @@ base_volume_mounts = [
         if self.airflow_connection_id:
             # XCom API fetch 방식 (ARG_MAX safe)
             commands = [
-                f'python3 -m codeflowhub.airflow.xcom '
+                f'python3 -m codeflowhub.airflow.xcom xcom '
                 f'{{{{{{{{ dag.dag_id }}}}}}}} {{{{{{{{ dag_run.run_id }}}}}}}} '
                 f'{dep.name} /app/input/{i}.json'
                 for i, dep in enumerate(task.depend)

{codeflowhub-0.2.0 → codeflowhub-0.2.1}/codeflowhub.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeflowhub
-Version: 0.2.0
+Version: 0.2.1
 Summary: workflow development tools
 Author: creaddiscans
 Author-email: creaddiscans@gmail.com

{codeflowhub-0.2.0 → codeflowhub-0.2.1}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='codeflowhub',
-    version='0.2.0',
+    version='0.2.1',
     description='workflow development tools',
     author='creaddiscans',
     author_email='creaddiscans@gmail.com',

codeflowhub-0.2.0/codeflowhub/airflow/xcom.py

@@ -1,93 +0,0 @@
-"""Airflow XCom REST API client for fetching task outputs at runtime.
-
-Used inside KubernetesPodOperator containers to fetch upstream task xcom data
-via Airflow REST API instead of inlining it in shell arguments (ARG_MAX safe).
-
-Environment variables (injected via Airflow Connection):
-  - FLOWHUB_API_URL: Airflow webserver URL (e.g. http://airflow-web:8080)
-  - FLOWHUB_API_USER: Basic auth user
-  - FLOWHUB_API_PASS: Basic auth password
-
-Usage:
-  python3 -m codeflowhub.airflow.xcom <dag_id> <run_id> <task_id> <output_path>
-"""
-import urllib.request
-import json
-import base64
-import os
-import sys
-from urllib.parse import quote
-
-
-def fetch_xcom(dag_id: str, run_id: str, task_id: str, output_path: str):
-    """Fetch xcom value from Airflow REST API and save to file.
-
-    Args:
-        dag_id: Airflow DAG ID
-        run_id: DAG Run ID
-        task_id: upstream task ID
-        output_path: path to save the JSON output
-    """
-    required_vars = ['FLOWHUB_API_URL', 'FLOWHUB_API_USER', 'FLOWHUB_API_PASS']
-    missing = [v for v in required_vars if v not in os.environ]
-    if missing:
-        print(f"Error: missing environment variables: {', '.join(missing)}", file=sys.stderr)
-        print("These should be injected via Airflow Connection (FLOWHUB_API_URL, FLOWHUB_API_USER, FLOWHUB_API_PASS)", file=sys.stderr)
-        sys.exit(1)
-
-    auth = base64.b64encode(
-        (os.environ['FLOWHUB_API_USER'] + ':' + os.environ['FLOWHUB_API_PASS']).encode()
-    ).decode()
-
-    url = (
-        os.environ['FLOWHUB_API_URL'].rstrip('/')
-        + '/api/v1/dags/' + quote(dag_id, safe='')
-        + '/dagRuns/' + quote(run_id, safe='')
-        + '/taskInstances/' + quote(task_id, safe='')
-        + '/xcomEntries/return_value'
-    )
-
-    req = urllib.request.Request(
-        url,
-        headers={
-            'Authorization': 'Basic ' + auth,
-            'Accept': 'application/json',
-        },
-    )
-
-    try:
-        resp = urllib.request.urlopen(req)
-    except urllib.error.HTTPError as e:
-        if e.code == 401:
-            print(f"Error: authentication failed for {url}", file=sys.stderr)
-            print("Check FLOWHUB_API_USER / FLOWHUB_API_PASS in Airflow Connection", file=sys.stderr)
-        elif e.code == 404:
-            print(f"Error: xcom not found for task '{task_id}' in dag '{dag_id}' run '{run_id}'", file=sys.stderr)
-            print("Ensure the upstream task completed successfully and pushed xcom", file=sys.stderr)
-        else:
-            print(f"Error: HTTP {e.code} from {url}", file=sys.stderr)
-        sys.exit(1)
-    except urllib.error.URLError as e:
-        print(f"Error: cannot reach Airflow API at {os.environ['FLOWHUB_API_URL']}: {e.reason}", file=sys.stderr)
-        sys.exit(1)
-
-    body = json.loads(resp.read())
-    value = body['value']
-
-    # value can be a string (already JSON) or a parsed object
-    if isinstance(value, str):
-        content = value
-    else:
-        content = json.dumps(value)
-
-    os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
-    with open(output_path, 'w') as f:
-        f.write(content)
-
-
-if __name__ == '__main__':
-    if len(sys.argv) != 5:
-        print(f"Usage: python3 -m codeflowhub.airflow.xcom <dag_id> <run_id> <task_id> <output_path>", file=sys.stderr)
-        sys.exit(1)
-
-    fetch_xcom(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])