codeflowhub 0.1.4__tar.gz → 0.2.1__tar.gz

{codeflowhub-0.1.4 → codeflowhub-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeflowhub
-Version: 0.1.4
+Version: 0.2.1
 Summary: workflow development tools
 Author: creaddiscans
 Author-email: creaddiscans@gmail.com

codeflowhub-0.2.1/codeflowhub/airflow/xcom.py

@@ -0,0 +1,140 @@
+"""Airflow REST API client for fetching task data at runtime.
+
+Used inside KubernetesPodOperator containers to fetch xcom/params data
+via Airflow REST API instead of inlining it in shell arguments (ARG_MAX safe).
+
+Environment variables (injected via Airflow Connection):
+  - FLOWHUB_API_URL: Airflow webserver URL (e.g. http://airflow-web:8080)
+  - FLOWHUB_API_USER: Basic auth user
+  - FLOWHUB_API_PASS: Basic auth password
+
+Usage:
+  python3 -m codeflowhub.airflow.xcom xcom <dag_id> <run_id> <task_id> <output_path>
+  python3 -m codeflowhub.airflow.xcom params <dag_id> <run_id> <output_path>
+"""
+import urllib.request
+import json
+import base64
+import os
+import sys
+from urllib.parse import quote
+
+
+def _api_request(path: str):
+    """Send authenticated GET request to Airflow REST API.
+
+    Args:
+        path: API path (e.g. /api/v1/dags/...)
+
+    Returns:
+        parsed JSON response body
+    """
+    required_vars = ['FLOWHUB_API_URL', 'FLOWHUB_API_USER', 'FLOWHUB_API_PASS']
+    missing = [v for v in required_vars if v not in os.environ]
+    if missing:
+        print(f"Error: missing environment variables: {', '.join(missing)}", file=sys.stderr)
+        print("These should be injected via Airflow Connection (FLOWHUB_API_URL, FLOWHUB_API_USER, FLOWHUB_API_PASS)", file=sys.stderr)
+        sys.exit(1)
+
+    auth = base64.b64encode(
+        (os.environ['FLOWHUB_API_USER'] + ':' + os.environ['FLOWHUB_API_PASS']).encode()
+    ).decode()
+
+    url = os.environ['FLOWHUB_API_URL'].rstrip('/') + path
+
+    req = urllib.request.Request(
+        url,
+        headers={
+            'Authorization': 'Basic ' + auth,
+            'Accept': 'application/json',
+        },
+    )
+
+    try:
+        resp = urllib.request.urlopen(req)
+    except urllib.error.HTTPError as e:
+        if e.code == 401:
+            print(f"Error: authentication failed for {url}", file=sys.stderr)
+            print("Check FLOWHUB_API_USER / FLOWHUB_API_PASS in Airflow Connection", file=sys.stderr)
+        elif e.code == 404:
+            print(f"Error: resource not found: {url}", file=sys.stderr)
+        else:
+            print(f"Error: HTTP {e.code} from {url}", file=sys.stderr)
+        sys.exit(1)
+    except urllib.error.URLError as e:
+        print(f"Error: cannot reach Airflow API at {os.environ['FLOWHUB_API_URL']}: {e.reason}", file=sys.stderr)
+        sys.exit(1)
+
+    return json.loads(resp.read())
+
+
+def _write_json(content, output_path: str):
+    """Write JSON content to file."""
+    os.makedirs(os.path.dirname(output_path) or '.', exist_ok=True)
+    with open(output_path, 'w') as f:
+        if isinstance(content, str):
+            f.write(content)
+        else:
+            json.dump(content, f)
+
+
+def fetch_xcom(dag_id: str, run_id: str, task_id: str, output_path: str):
+    """Fetch xcom value from Airflow REST API and save to file.
+
+    Args:
+        dag_id: Airflow DAG ID
+        run_id: DAG Run ID
+        task_id: upstream task ID
+        output_path: path to save the JSON output
+    """
+    path = (
+        '/api/v1/dags/' + quote(dag_id, safe='')
+        + '/dagRuns/' + quote(run_id, safe='')
+        + '/taskInstances/' + quote(task_id, safe='')
+        + '/xcomEntries/return_value'
+    )
+    body = _api_request(path)
+    _write_json(body['value'], output_path)
+
+
+def fetch_params(dag_id: str, run_id: str, output_path: str):
+    """Fetch DAG run params (conf) from Airflow REST API and save to file.
+
+    Args:
+        dag_id: Airflow DAG ID
+        run_id: DAG Run ID
+        output_path: path to save the JSON output
+    """
+    path = (
+        '/api/v1/dags/' + quote(dag_id, safe='')
+        + '/dagRuns/' + quote(run_id, safe='')
+    )
+    body = _api_request(path)
+    _write_json(body.get('conf', {}), output_path)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("Usage:", file=sys.stderr)
+        print("  python3 -m codeflowhub.airflow.xcom xcom <dag_id> <run_id> <task_id> <output_path>", file=sys.stderr)
+        print("  python3 -m codeflowhub.airflow.xcom params <dag_id> <run_id> <output_path>", file=sys.stderr)
+        sys.exit(1)
+
+    command = sys.argv[1]
+
+    if command == 'xcom':
+        if len(sys.argv) != 6:
+            print("Usage: python3 -m codeflowhub.airflow.xcom xcom <dag_id> <run_id> <task_id> <output_path>", file=sys.stderr)
+            sys.exit(1)
+        fetch_xcom(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])
+
+    elif command == 'params':
+        if len(sys.argv) != 5:
+            print("Usage: python3 -m codeflowhub.airflow.xcom params <dag_id> <run_id> <output_path>", file=sys.stderr)
+            sys.exit(1)
+        fetch_params(sys.argv[2], sys.argv[3], sys.argv[4])
+
+    else:
+        print(f"Unknown command: {command}", file=sys.stderr)
+        print("Available commands: xcom, params", file=sys.stderr)
+        sys.exit(1)

{codeflowhub-0.1.4 → codeflowhub-0.2.1}/codeflowhub/flow.py

@@ -23,13 +23,14 @@ class FlowDecorator(BaseDecorator):
     service_account_name:str  # Kubernetes service account name
     volumes:list  # Kubernetes volumes
     airflow_sidecar_image:str  # Airflow XCom sidecar 이미지
+    airflow_connection_id:str  # Airflow Connection ID for XCom API fetch
     repo:str  # Git repository URL
     path:str  # Git repo 내 작업 경로
     on_failure: 'BaseDecorator' = None  # 모든 task의 기본 failure handler
 
     def __init__(self, *args, namespace='default', env=None, name=None, description=None, params=None,
                  tags=None, annotations=None, service_account_name=None, volumes=None,
-                 airflow_sidecar_image=None, repo=None, path=None, on_failure=None, **kwargs):
+                 airflow_sidecar_image=None, airflow_connection_id=None, repo=None, path=None, on_failure=None, **kwargs):
         # CLI 속성 먼저 초기화 (init()에서 사용됨)
         self._cli_export = None
         self._cli_job_dir = None
@@ -47,6 +48,7 @@ class FlowDecorator(BaseDecorator):
         self.service_account_name = service_account_name
         self.volumes = volumes or []
         self.airflow_sidecar_image = airflow_sidecar_image
+        self.airflow_connection_id = airflow_connection_id
         self.repo = repo
         self.path = path
         self.on_failure = on_failure

{codeflowhub-0.1.4 → codeflowhub-0.2.1}/codeflowhub/service/airflow_exporter.py

@@ -28,6 +28,9 @@ class AirflowExporter:
         # XCom sidecar image: flow의 airflow_sidecar_image를 우선 사용, 없으면 env에서
         self.xcom_sidecar_image = flow_decorator.airflow_sidecar_image or self.env.get('XCOM_SIDECAR_IMAGE', None)
 
+        # Airflow Connection ID for XCom API fetch (None이면 기존 heredoc 방식)
+        self.airflow_connection_id = flow_decorator.airflow_connection_id
+
         # 추가 패키지 경로들 (env에서 설정 가능)
         self.extra_packages = self.env.get('EXTRA_PACKAGES', [])
 
@@ -128,6 +131,18 @@ from datetime import datetime, timedelta
 from airflow import DAG
 from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
 from kubernetes.client import models as k8s
+'''
+
+        # Airflow Connection for XCom API fetch
+        if self.airflow_connection_id:
+            header += f'''
+from airflow.hooks.base import BaseHook
+_conn = BaseHook.get_connection('{self.airflow_connection_id}')
+_fh_api_env = [
+    k8s.V1EnvVar(name='FLOWHUB_API_URL', value=f"{{_conn.schema}}://{{_conn.host}}:{{_conn.port}}"),
+    k8s.V1EnvVar(name='FLOWHUB_API_USER', value=_conn.login),
+    k8s.V1EnvVar(name='FLOWHUB_API_PASS', value=_conn.password),
+]
 '''
 
         # XCom sidecar 설정
@@ -281,10 +296,15 @@ base_volume_mounts = [
         # failure handler는 retries=0 설정
         retries_code = "\n        retries=0," if trigger_rule == 'one_failed' else ""
 
+        # XCom API fetch를 사용하는 task에 env_vars 주입 (is_first: params fetch, depend: xcom fetch)
+        env_vars_code = ""
+        if self.airflow_connection_id and (is_first or task.depend):
+            env_vars_code = "\n        env_vars=_fh_api_env,"
+
         operator_code = f'''    {task.name} = KubernetesPodOperator(
         **common,
         task_id='{task.name}',
-        image='{task_image}',{pool_code}{trigger_rule_code}{retries_code}{tolerations_code}{node_selector_code}{volume_mounts_code}{container_resources_code}{sidecars_code}
+        image='{task_image}',{pool_code}{trigger_rule_code}{retries_code}{env_vars_code}{tolerations_code}{node_selector_code}{volume_mounts_code}{container_resources_code}{sidecars_code}
         arguments=[
             f\'\'\'{arguments}\'\'\'
         ],
@@ -295,13 +315,29 @@ base_volume_mounts = [
     def _generate_input_commands(self, task, is_first):
         """Input 데이터 명령어 생성"""
         if is_first:
+            if self.airflow_connection_id:
+                return (
+                    f'python3 -m codeflowhub.airflow.xcom params '
+                    f'{{{{{{{{ dag.dag_id }}}}}}}} {{{{{{{{ dag_run.run_id }}}}}}}} '
+                    f'/app/input/0.json'
+                )
             return '\n'.join(["cat << 'EOF' > /app/input/0.json", '{{{{ params | tojson }}}}', 'EOF'])
         if not task.depend:
             return "echo '{{}}' >> /app/input/0.json"
 
-        commands = [
-            '\n'.join([f"cat << 'EOF' > /app/input/{i}.json", f'{{{{{{{{ ti.xcom_pull(task_ids=\"{dep.name}\") | tojson }}}}}}}}', 'EOF'])
-            for i, dep in enumerate(task.depend)]
+        if self.airflow_connection_id:
+            # XCom API fetch 방식 (ARG_MAX safe)
+            commands = [
+                f'python3 -m codeflowhub.airflow.xcom xcom '
+                f'{{{{{{{{ dag.dag_id }}}}}}}} {{{{{{{{ dag_run.run_id }}}}}}}} '
+                f'{dep.name} /app/input/{i}.json'
+                for i, dep in enumerate(task.depend)
+            ]
+        else:
+            # 기존 heredoc 방식 (하위 호환)
+            commands = [
+                '\n'.join([f"cat << 'EOF' > /app/input/{i}.json", f'{{{{{{{{ ti.xcom_pull(task_ids=\"{dep.name}\") | tojson }}}}}}}}', 'EOF'])
+                for i, dep in enumerate(task.depend)]
         return "\n                ".join(commands)
 
     def _build_tolerations_code(self, task):

{codeflowhub-0.1.4 → codeflowhub-0.2.1}/codeflowhub.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeflowhub
-Version: 0.1.4
+Version: 0.2.1
 Summary: workflow development tools
 Author: creaddiscans
 Author-email: creaddiscans@gmail.com

{codeflowhub-0.1.4 → codeflowhub-0.2.1}/codeflowhub.egg-info/SOURCES.txt

@@ -11,6 +11,8 @@ codeflowhub.egg-info/PKG-INFO
 codeflowhub.egg-info/SOURCES.txt
 codeflowhub.egg-info/dependency_links.txt
 codeflowhub.egg-info/top_level.txt
+codeflowhub/airflow/__init__.py
+codeflowhub/airflow/xcom.py
 codeflowhub/service/__init__.py
 codeflowhub/service/airflow_exporter.py
 codeflowhub/storage/__init__.py

{codeflowhub-0.1.4 → codeflowhub-0.2.1}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='codeflowhub',
-    version='0.1.4',
+    version='0.2.1',
     description='workflow development tools',
     author='creaddiscans',
     author_email='creaddiscans@gmail.com',