codeflowhub 1.1.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/PKG-INFO +1 -1
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/flow.py +106 -8
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/service/airflow_exporter.py +29 -6
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub.egg-info/PKG-INFO +1 -1
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/setup.py +1 -1
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/LICENSE +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/README.md +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/action.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/airflow/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/airflow/xcom.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/base.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/model.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/service/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/storage/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/storage/local_storage.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/storage/s3_storage.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/storage/storage.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/task.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/analyze_speaker_pkg/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/analyze_speaker_pkg/main.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/extract_voice_pkg/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/extract_voice_pkg/main.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/read_pdf_pkg/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/read_pdf_pkg/main.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/transcript_pkg/__init__.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/transcript_pkg/main.py +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub.egg-info/SOURCES.txt +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub.egg-info/dependency_links.txt +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub.egg-info/top_level.txt +0 -0
- {codeflowhub-1.1.0 → codeflowhub-1.2.0}/setup.cfg +0 -0
|
@@ -3,6 +3,7 @@ import inspect
|
|
|
3
3
|
import json
|
|
4
4
|
import os
|
|
5
5
|
import sys
|
|
6
|
+
from datetime import datetime
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
|
|
8
9
|
from .base import BaseDecorator
|
|
@@ -28,13 +29,16 @@ class FlowDecorator(BaseDecorator):
|
|
|
28
29
|
path:str # Git repo 내 작업 경로
|
|
29
30
|
export_destination:str # Export 시 생성 파일 저장 디렉토리
|
|
30
31
|
default_args:dict # Airflow DAG default_args (owner, retries, retry_delay 등 override)
|
|
32
|
+
schedule:str # Airflow DAG 스케줄 (cron 문자열 / preset / timedelta / None=수동 트리거)
|
|
33
|
+
start_date:datetime # Airflow DAG start_date (None이면 export 시점)
|
|
31
34
|
secrets:list # 모든 task에 공통 주입할 K8s Secret(env) 목록
|
|
32
35
|
on_failure: 'BaseDecorator' = None # 모든 task의 기본 failure handler
|
|
33
36
|
|
|
34
37
|
def __init__(self, *args, namespace='default', env=None, name=None, description=None, params=None,
|
|
35
38
|
tags=None, annotations=None, service_account_name=None, volumes=None,
|
|
36
39
|
airflow_sidecar_image=None, airflow_connection_id=None, repo=None, path=None,
|
|
37
|
-
export_destination='dags', default_args=None, secrets=None, on_failure=None,
|
|
40
|
+
export_destination='dags', default_args=None, secrets=None, on_failure=None,
|
|
41
|
+
schedule=None, start_date=None, schedule_interval=None, **kwargs):
|
|
38
42
|
# CLI 속성 먼저 초기화 (init()에서 사용됨)
|
|
39
43
|
self._cli_export = None
|
|
40
44
|
self._cli_job_dir = None
|
|
@@ -57,6 +61,10 @@ class FlowDecorator(BaseDecorator):
|
|
|
57
61
|
self.path = path
|
|
58
62
|
self.export_destination = export_destination or 'dags'
|
|
59
63
|
self.default_args = default_args or {}
|
|
64
|
+
self.schedule = self._normalize_schedule(
|
|
65
|
+
self._resolve_schedule_alias(schedule, schedule_interval)
|
|
66
|
+
)
|
|
67
|
+
self.start_date = start_date
|
|
60
68
|
# Design Ref: §3.2 — flow-level 공통 Secret(env)
|
|
61
69
|
self.secrets = secrets or []
|
|
62
70
|
self.on_failure = on_failure
|
|
@@ -67,6 +75,80 @@ class FlowDecorator(BaseDecorator):
|
|
|
67
75
|
# super().__init__()는 마지막에 호출 (init()이 실행됨)
|
|
68
76
|
super().__init__(*args, name=name, **kwargs)
|
|
69
77
|
|
|
78
|
+
@staticmethod
|
|
79
|
+
def _resolve_schedule_alias(schedule, schedule_interval):
|
|
80
|
+
"""schedule / schedule_interval(구 이름) 중 하나로 정리
|
|
81
|
+
|
|
82
|
+
Airflow 2.4+에서 DAG 파라미터가 schedule로 통합되었고 Airflow 3에서
|
|
83
|
+
schedule_interval은 제거되었다. 구 이름도 계속 받되 경고한다.
|
|
84
|
+
"""
|
|
85
|
+
if schedule_interval is None:
|
|
86
|
+
return schedule
|
|
87
|
+
if schedule is not None:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
'schedule과 schedule_interval을 동시에 지정할 수 없습니다. '
|
|
90
|
+
'schedule만 사용하세요 (schedule_interval은 구 이름).'
|
|
91
|
+
)
|
|
92
|
+
print(
|
|
93
|
+
"⚠️ schedule_interval은 구 이름입니다. Airflow 2.4+ 표준인 "
|
|
94
|
+
"schedule을 사용하세요.",
|
|
95
|
+
file=sys.stderr
|
|
96
|
+
)
|
|
97
|
+
return schedule_interval
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
def _normalize_schedule(schedule):
|
|
101
|
+
"""schedule 검증 (선언 시점에 조기 실패)
|
|
102
|
+
|
|
103
|
+
허용: None(수동 트리거), cron 문자열('0 2 * * *'), preset('@daily'), timedelta
|
|
104
|
+
"""
|
|
105
|
+
from datetime import timedelta
|
|
106
|
+
|
|
107
|
+
if schedule is None or isinstance(schedule, timedelta):
|
|
108
|
+
return schedule
|
|
109
|
+
|
|
110
|
+
if not isinstance(schedule, str):
|
|
111
|
+
raise TypeError(
|
|
112
|
+
f'schedule은 str(cron/preset), timedelta, None만 가능합니다. '
|
|
113
|
+
f'전달된 타입: {type(schedule).__name__}'
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
value = schedule.strip()
|
|
117
|
+
if not value:
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
presets = {'@once', '@hourly', '@daily', '@weekly', '@monthly',
|
|
121
|
+
'@quarterly', '@yearly', '@annually', '@continuous', '@none'}
|
|
122
|
+
if value.startswith('@'):
|
|
123
|
+
if value not in presets:
|
|
124
|
+
raise ValueError(
|
|
125
|
+
f'알 수 없는 스케줄 preset: "{value}". '
|
|
126
|
+
f'사용 가능: {", ".join(sorted(presets))}'
|
|
127
|
+
)
|
|
128
|
+
return None if value == '@none' else value
|
|
129
|
+
|
|
130
|
+
# cron 표현식은 5개 필드 (분 시 일 월 요일)
|
|
131
|
+
fields = value.split()
|
|
132
|
+
if len(fields) != 5:
|
|
133
|
+
raise ValueError(
|
|
134
|
+
f'cron 표현식은 5개 필드(분 시 일 월 요일)여야 합니다. '
|
|
135
|
+
f'전달된 값: "{value}" ({len(fields)}개 필드)'
|
|
136
|
+
)
|
|
137
|
+
return value
|
|
138
|
+
|
|
139
|
+
def _resolve_start_date(self):
|
|
140
|
+
"""DAG start_date 결정 (Airflow는 스케줄이 있으면 start_date 필수)
|
|
141
|
+
|
|
142
|
+
우선순위: start_date 파라미터 > default_args['start_date'] > 고정 기본값.
|
|
143
|
+
기본값을 datetime.now()가 아닌 고정값으로 두어 재export 시 결정적이도록 한다.
|
|
144
|
+
"""
|
|
145
|
+
if self.start_date is not None:
|
|
146
|
+
return self.start_date
|
|
147
|
+
from_default_args = (self.default_args or {}).get('start_date')
|
|
148
|
+
if from_default_args is not None:
|
|
149
|
+
return from_default_args
|
|
150
|
+
return datetime(2024, 1, 1)
|
|
151
|
+
|
|
70
152
|
def _initialize_env(self, env):
|
|
71
153
|
"""환경 설정 초기화"""
|
|
72
154
|
if env and isinstance(env, dict) and all(isinstance(v, dict) for v in env.values()):
|
|
@@ -258,28 +340,33 @@ class FlowDecorator(BaseDecorator):
|
|
|
258
340
|
print(f' Supported: airflow')
|
|
259
341
|
os._exit(1)
|
|
260
342
|
|
|
261
|
-
from datetime import
|
|
343
|
+
from datetime import timedelta
|
|
262
344
|
|
|
263
345
|
flow_id = self.flow_name or self.name
|
|
264
346
|
dest_dir = self.export_destination or 'dags'
|
|
265
347
|
print(f'🚀 Exporting {flow_id} to Airflow DAG... (destination: {dest_dir})')
|
|
266
348
|
|
|
349
|
+
# 스케줄이 설정된 DAG는 start_date가 필수이므로 default_args에 항상 포함시킨다
|
|
350
|
+
start_date = self._resolve_start_date()
|
|
267
351
|
default_args = {
|
|
268
352
|
'owner': 'flowhub',
|
|
269
353
|
'retries': 1,
|
|
270
354
|
'retry_delay': timedelta(minutes=5),
|
|
271
355
|
**(self.default_args or {}),
|
|
356
|
+
'start_date': start_date,
|
|
272
357
|
}
|
|
273
358
|
|
|
274
359
|
dag_path = self.export_airflow(
|
|
275
360
|
output_path=os.path.join(dest_dir, f'{flow_id}_dag.py'),
|
|
276
|
-
|
|
277
|
-
start_date=
|
|
361
|
+
schedule=self.schedule,
|
|
362
|
+
start_date=start_date,
|
|
278
363
|
default_args=default_args,
|
|
279
364
|
)
|
|
280
365
|
|
|
281
366
|
print(f'✅ Successfully exported to: {dag_path}')
|
|
282
367
|
print(f'📋 DAG ID: {flow_id}')
|
|
368
|
+
print(f'⏰ Schedule: {self.schedule or "None (manual trigger)"}')
|
|
369
|
+
print(f'📅 Start date: {start_date.isoformat()}')
|
|
283
370
|
print(f'🏷️ Tags: [{self.namespace}]')
|
|
284
371
|
os._exit(0)
|
|
285
372
|
|
|
@@ -576,23 +663,34 @@ class FlowDecorator(BaseDecorator):
|
|
|
576
663
|
f'--input-data를 제공하거나 먼저 전체 workflow를 실행하세요.'
|
|
577
664
|
)
|
|
578
665
|
|
|
579
|
-
def export_airflow(self, output_path=None,
|
|
666
|
+
def export_airflow(self, output_path=None, schedule=None, start_date=None, default_args=None,
|
|
667
|
+
schedule_interval=None):
|
|
580
668
|
"""
|
|
581
669
|
Airflow DAG로 export
|
|
582
670
|
|
|
583
671
|
Args:
|
|
584
672
|
output_path: DAG 파일 저장 경로 (기본: dags/{dag_id}.py)
|
|
585
|
-
|
|
586
|
-
start_date: DAG 시작
|
|
673
|
+
schedule: DAG 스케줄. 미지정 시 @flow(schedule=...) 값 사용
|
|
674
|
+
start_date: DAG 시작 날짜. 미지정 시 @flow(start_date=...) 또는 기본값 사용
|
|
587
675
|
default_args: DAG default_args
|
|
676
|
+
schedule_interval: schedule의 구 이름 (deprecated)
|
|
588
677
|
|
|
589
678
|
Returns:
|
|
590
679
|
str: 생성된 DAG 파일 경로
|
|
591
680
|
"""
|
|
681
|
+
# 인자 미지정 시 flow 데코레이터에 선언된 값으로 폴백
|
|
682
|
+
schedule = self._resolve_schedule_alias(schedule, schedule_interval)
|
|
683
|
+
if schedule is None:
|
|
684
|
+
schedule = self.schedule
|
|
685
|
+
else:
|
|
686
|
+
schedule = self._normalize_schedule(schedule)
|
|
687
|
+
if start_date is None:
|
|
688
|
+
start_date = self._resolve_start_date()
|
|
689
|
+
|
|
592
690
|
exporter = AirflowExporter(self)
|
|
593
691
|
return exporter.export(
|
|
594
692
|
output_path=output_path,
|
|
595
|
-
|
|
693
|
+
schedule=schedule,
|
|
596
694
|
start_date=start_date,
|
|
597
695
|
default_args=default_args
|
|
598
696
|
)
|
|
@@ -149,7 +149,7 @@ class AirflowExporter:
|
|
|
149
149
|
|
|
150
150
|
def export(self,
|
|
151
151
|
output_path: str = None,
|
|
152
|
-
|
|
152
|
+
schedule: Optional[str] = None,
|
|
153
153
|
start_date: Optional[datetime] = None,
|
|
154
154
|
default_args: Optional[dict] = None):
|
|
155
155
|
"""Airflow DAG Python 파일로 export"""
|
|
@@ -178,7 +178,7 @@ class AirflowExporter:
|
|
|
178
178
|
configmap_path = self._generate_configmap_yaml(dir_path)
|
|
179
179
|
|
|
180
180
|
# DAG 코드 생성
|
|
181
|
-
dag_code = self._generate_dag_code(
|
|
181
|
+
dag_code = self._generate_dag_code(schedule, start_date, default_args)
|
|
182
182
|
|
|
183
183
|
# 파일 저장
|
|
184
184
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
@@ -192,7 +192,7 @@ class AirflowExporter:
|
|
|
192
192
|
print(f'📋 Apply ConfigMap: kubectl apply -f {configmap_path}')
|
|
193
193
|
return output_path
|
|
194
194
|
|
|
195
|
-
def _generate_dag_code(self,
|
|
195
|
+
def _generate_dag_code(self, schedule, start_date, default_args):
|
|
196
196
|
"""DAG Python 코드 생성 (target.py 스타일)"""
|
|
197
197
|
|
|
198
198
|
flow_name = self.flow.flow_name or self.flow.name
|
|
@@ -273,7 +273,7 @@ SETUP_SCRIPT = """{setup_script}"""
|
|
|
273
273
|
'{self.dag_id}',
|
|
274
274
|
default_args={default_args_str},
|
|
275
275
|
description='{description}',
|
|
276
|
-
|
|
276
|
+
schedule={self._render_schedule(schedule)},
|
|
277
277
|
catchup=False,
|
|
278
278
|
tags={tags_str},{params_line}
|
|
279
279
|
)
|
|
@@ -881,6 +881,17 @@ base_volume_mounts = [
|
|
|
881
881
|
dependencies.append(f"[{deps_str}] >> {task.name}")
|
|
882
882
|
return dependencies
|
|
883
883
|
|
|
884
|
+
def _render_schedule(self, schedule):
|
|
885
|
+
"""schedule → DAG 코드 문자열
|
|
886
|
+
|
|
887
|
+
repr(timedelta)는 'datetime.timedelta(...)'를 만들지만 생성 DAG는
|
|
888
|
+
`from datetime import timedelta`만 import하므로 NameError가 된다.
|
|
889
|
+
"""
|
|
890
|
+
if isinstance(schedule, timedelta):
|
|
891
|
+
total = int(schedule.total_seconds())
|
|
892
|
+
return f'timedelta(seconds={total})'
|
|
893
|
+
return repr(schedule)
|
|
894
|
+
|
|
884
895
|
def _format_dict(self, d, indent=0, base_indent=0):
|
|
885
896
|
"""Dict를 Python 코드 형식으로 포맷팅"""
|
|
886
897
|
return self._format_dict_with_raw_values(d, indent, raw_keys=set(), base_indent=base_indent)
|
|
@@ -908,9 +919,21 @@ base_volume_mounts = [
|
|
|
908
919
|
elif isinstance(value, str):
|
|
909
920
|
items.append(f"'{key}': '{value}'")
|
|
910
921
|
elif isinstance(value, datetime):
|
|
911
|
-
|
|
922
|
+
# 시/분/초가 있으면 함께 렌더 (start_date 정밀도 손실 방지)
|
|
923
|
+
if (value.hour, value.minute, value.second) == (0, 0, 0):
|
|
924
|
+
items.append(f"'{key}': datetime({value.year}, {value.month}, {value.day})")
|
|
925
|
+
else:
|
|
926
|
+
items.append(
|
|
927
|
+
f"'{key}': datetime({value.year}, {value.month}, {value.day}, "
|
|
928
|
+
f"{value.hour}, {value.minute}, {value.second})"
|
|
929
|
+
)
|
|
912
930
|
elif isinstance(value, timedelta):
|
|
913
|
-
|
|
931
|
+
# 분 단위로 나누어떨어지지 않는 값은 초 단위로 렌더 (절삭 방지)
|
|
932
|
+
total = int(value.total_seconds())
|
|
933
|
+
if total % 60 == 0:
|
|
934
|
+
items.append(f"'{key}': timedelta(minutes={total // 60})")
|
|
935
|
+
else:
|
|
936
|
+
items.append(f"'{key}': timedelta(seconds={total})")
|
|
914
937
|
elif isinstance(value, bool):
|
|
915
938
|
items.append(f"'{key}': {value}")
|
|
916
939
|
elif isinstance(value, (int, float)):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{codeflowhub-1.1.0 → codeflowhub-1.2.0}/codeflowhub/template/analyze_speaker_pkg/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|