recurvedata-lib 0.1.487__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of recurvedata-lib might be problematic. Click here for more details.

Files changed (333) hide show
  1. recurvedata/__init__.py +0 -0
  2. recurvedata/__version__.py +1 -0
  3. recurvedata/client/__init__.py +3 -0
  4. recurvedata/client/client.py +150 -0
  5. recurvedata/client/server_client.py +91 -0
  6. recurvedata/config.py +99 -0
  7. recurvedata/connectors/__init__.py +20 -0
  8. recurvedata/connectors/_register.py +46 -0
  9. recurvedata/connectors/base.py +111 -0
  10. recurvedata/connectors/config_schema.py +1575 -0
  11. recurvedata/connectors/connectors/__init__.py +0 -0
  12. recurvedata/connectors/connectors/aliyun_access_key.py +30 -0
  13. recurvedata/connectors/connectors/auth.py +44 -0
  14. recurvedata/connectors/connectors/azure_blob.py +89 -0
  15. recurvedata/connectors/connectors/azure_synapse.py +79 -0
  16. recurvedata/connectors/connectors/bigquery.py +359 -0
  17. recurvedata/connectors/connectors/clickhouse.py +219 -0
  18. recurvedata/connectors/connectors/dingtalk.py +61 -0
  19. recurvedata/connectors/connectors/doris.py +215 -0
  20. recurvedata/connectors/connectors/es.py +62 -0
  21. recurvedata/connectors/connectors/feishu.py +65 -0
  22. recurvedata/connectors/connectors/ftp.py +50 -0
  23. recurvedata/connectors/connectors/generic.py +49 -0
  24. recurvedata/connectors/connectors/google_cloud_storage.py +115 -0
  25. recurvedata/connectors/connectors/google_service_account.py +225 -0
  26. recurvedata/connectors/connectors/hive.py +207 -0
  27. recurvedata/connectors/connectors/impala.py +210 -0
  28. recurvedata/connectors/connectors/jenkins.py +51 -0
  29. recurvedata/connectors/connectors/mail.py +89 -0
  30. recurvedata/connectors/connectors/microsoft_fabric.py +284 -0
  31. recurvedata/connectors/connectors/mongo.py +79 -0
  32. recurvedata/connectors/connectors/mssql.py +131 -0
  33. recurvedata/connectors/connectors/mysql.py +191 -0
  34. recurvedata/connectors/connectors/n8n.py +141 -0
  35. recurvedata/connectors/connectors/oss.py +74 -0
  36. recurvedata/connectors/connectors/owncloud.py +36 -0
  37. recurvedata/connectors/connectors/phoenix.py +36 -0
  38. recurvedata/connectors/connectors/postgres.py +230 -0
  39. recurvedata/connectors/connectors/python.py +50 -0
  40. recurvedata/connectors/connectors/redshift.py +187 -0
  41. recurvedata/connectors/connectors/s3.py +93 -0
  42. recurvedata/connectors/connectors/sftp.py +87 -0
  43. recurvedata/connectors/connectors/slack.py +35 -0
  44. recurvedata/connectors/connectors/spark.py +99 -0
  45. recurvedata/connectors/connectors/starrocks.py +175 -0
  46. recurvedata/connectors/connectors/tencent_cos.py +40 -0
  47. recurvedata/connectors/connectors/tidb.py +49 -0
  48. recurvedata/connectors/const.py +315 -0
  49. recurvedata/connectors/datasource.py +189 -0
  50. recurvedata/connectors/dbapi.py +469 -0
  51. recurvedata/connectors/fs.py +66 -0
  52. recurvedata/connectors/ftp.py +40 -0
  53. recurvedata/connectors/object_store.py +60 -0
  54. recurvedata/connectors/pigeon.py +172 -0
  55. recurvedata/connectors/proxy.py +104 -0
  56. recurvedata/connectors/service.py +223 -0
  57. recurvedata/connectors/utils.py +47 -0
  58. recurvedata/consts.py +49 -0
  59. recurvedata/core/__init__.py +0 -0
  60. recurvedata/core/config.py +46 -0
  61. recurvedata/core/configurable.py +27 -0
  62. recurvedata/core/consts.py +2 -0
  63. recurvedata/core/templating.py +206 -0
  64. recurvedata/core/tracing.py +223 -0
  65. recurvedata/core/transformer.py +186 -0
  66. recurvedata/core/translation.py +91 -0
  67. recurvedata/dbt/client.py +97 -0
  68. recurvedata/dbt/consts.py +99 -0
  69. recurvedata/dbt/cosmos_utils.py +275 -0
  70. recurvedata/dbt/error_codes.py +18 -0
  71. recurvedata/dbt/schemas.py +98 -0
  72. recurvedata/dbt/service.py +451 -0
  73. recurvedata/dbt/utils.py +246 -0
  74. recurvedata/error_codes.py +71 -0
  75. recurvedata/exceptions.py +72 -0
  76. recurvedata/executors/__init__.py +4 -0
  77. recurvedata/executors/cli/__init__.py +7 -0
  78. recurvedata/executors/cli/connector.py +117 -0
  79. recurvedata/executors/cli/dbt.py +118 -0
  80. recurvedata/executors/cli/main.py +82 -0
  81. recurvedata/executors/cli/parameters.py +18 -0
  82. recurvedata/executors/client.py +190 -0
  83. recurvedata/executors/consts.py +50 -0
  84. recurvedata/executors/debug_executor.py +100 -0
  85. recurvedata/executors/executor.py +300 -0
  86. recurvedata/executors/link_executor.py +189 -0
  87. recurvedata/executors/models.py +34 -0
  88. recurvedata/executors/schemas.py +222 -0
  89. recurvedata/executors/service/__init__.py +0 -0
  90. recurvedata/executors/service/connector.py +380 -0
  91. recurvedata/executors/utils.py +172 -0
  92. recurvedata/filestorage/__init__.py +11 -0
  93. recurvedata/filestorage/_factory.py +33 -0
  94. recurvedata/filestorage/backends/__init__.py +0 -0
  95. recurvedata/filestorage/backends/fsspec.py +45 -0
  96. recurvedata/filestorage/backends/local.py +67 -0
  97. recurvedata/filestorage/backends/oss.py +56 -0
  98. recurvedata/filestorage/interface.py +84 -0
  99. recurvedata/operators/__init__.py +10 -0
  100. recurvedata/operators/base.py +28 -0
  101. recurvedata/operators/config.py +21 -0
  102. recurvedata/operators/context.py +255 -0
  103. recurvedata/operators/dbt_operator/__init__.py +2 -0
  104. recurvedata/operators/dbt_operator/model_pipeline_link_operator.py +55 -0
  105. recurvedata/operators/dbt_operator/operator.py +353 -0
  106. recurvedata/operators/link_operator/__init__.py +1 -0
  107. recurvedata/operators/link_operator/operator.py +120 -0
  108. recurvedata/operators/models.py +55 -0
  109. recurvedata/operators/notify_operator/__init__.py +1 -0
  110. recurvedata/operators/notify_operator/operator.py +180 -0
  111. recurvedata/operators/operator.py +119 -0
  112. recurvedata/operators/python_operator/__init__.py +1 -0
  113. recurvedata/operators/python_operator/operator.py +132 -0
  114. recurvedata/operators/sensor_operator/__init__.py +1 -0
  115. recurvedata/operators/sensor_operator/airflow_utils.py +63 -0
  116. recurvedata/operators/sensor_operator/operator.py +172 -0
  117. recurvedata/operators/spark_operator/__init__.py +1 -0
  118. recurvedata/operators/spark_operator/operator.py +200 -0
  119. recurvedata/operators/spark_operator/spark_sample.py +47 -0
  120. recurvedata/operators/sql_operator/__init__.py +1 -0
  121. recurvedata/operators/sql_operator/operator.py +90 -0
  122. recurvedata/operators/task.py +211 -0
  123. recurvedata/operators/transfer_operator/__init__.py +40 -0
  124. recurvedata/operators/transfer_operator/const.py +10 -0
  125. recurvedata/operators/transfer_operator/dump_aliyun_sls.py +82 -0
  126. recurvedata/operators/transfer_operator/dump_sheet_task_base.py +292 -0
  127. recurvedata/operators/transfer_operator/dump_task_cass.py +155 -0
  128. recurvedata/operators/transfer_operator/dump_task_dbapi.py +209 -0
  129. recurvedata/operators/transfer_operator/dump_task_es.py +113 -0
  130. recurvedata/operators/transfer_operator/dump_task_feishu_sheet.py +114 -0
  131. recurvedata/operators/transfer_operator/dump_task_ftp.py +234 -0
  132. recurvedata/operators/transfer_operator/dump_task_google_sheet.py +66 -0
  133. recurvedata/operators/transfer_operator/dump_task_mongodb.py +168 -0
  134. recurvedata/operators/transfer_operator/dump_task_oss.py +285 -0
  135. recurvedata/operators/transfer_operator/dump_task_python.py +212 -0
  136. recurvedata/operators/transfer_operator/dump_task_s3.py +270 -0
  137. recurvedata/operators/transfer_operator/dump_task_sftp.py +229 -0
  138. recurvedata/operators/transfer_operator/load_task_aliyun_oss.py +107 -0
  139. recurvedata/operators/transfer_operator/load_task_azure_blob.py +115 -0
  140. recurvedata/operators/transfer_operator/load_task_azure_synapse.py +90 -0
  141. recurvedata/operators/transfer_operator/load_task_clickhouse.py +167 -0
  142. recurvedata/operators/transfer_operator/load_task_doris.py +164 -0
  143. recurvedata/operators/transfer_operator/load_task_email.py +188 -0
  144. recurvedata/operators/transfer_operator/load_task_es.py +86 -0
  145. recurvedata/operators/transfer_operator/load_task_filebrowser.py +151 -0
  146. recurvedata/operators/transfer_operator/load_task_ftp.py +19 -0
  147. recurvedata/operators/transfer_operator/load_task_google_bigquery.py +90 -0
  148. recurvedata/operators/transfer_operator/load_task_google_cloud_storage.py +127 -0
  149. recurvedata/operators/transfer_operator/load_task_google_sheet.py +130 -0
  150. recurvedata/operators/transfer_operator/load_task_hive.py +158 -0
  151. recurvedata/operators/transfer_operator/load_task_microsoft_fabric.py +105 -0
  152. recurvedata/operators/transfer_operator/load_task_mssql.py +153 -0
  153. recurvedata/operators/transfer_operator/load_task_mysql.py +157 -0
  154. recurvedata/operators/transfer_operator/load_task_owncloud.py +135 -0
  155. recurvedata/operators/transfer_operator/load_task_postgresql.py +109 -0
  156. recurvedata/operators/transfer_operator/load_task_qcloud_cos.py +119 -0
  157. recurvedata/operators/transfer_operator/load_task_recurve_data_prep.py +75 -0
  158. recurvedata/operators/transfer_operator/load_task_redshift.py +95 -0
  159. recurvedata/operators/transfer_operator/load_task_s3.py +150 -0
  160. recurvedata/operators/transfer_operator/load_task_sftp.py +90 -0
  161. recurvedata/operators/transfer_operator/load_task_starrocks.py +169 -0
  162. recurvedata/operators/transfer_operator/load_task_yicrowds.py +97 -0
  163. recurvedata/operators/transfer_operator/mixin.py +31 -0
  164. recurvedata/operators/transfer_operator/operator.py +231 -0
  165. recurvedata/operators/transfer_operator/task.py +223 -0
  166. recurvedata/operators/transfer_operator/utils.py +134 -0
  167. recurvedata/operators/ui.py +80 -0
  168. recurvedata/operators/utils/__init__.py +51 -0
  169. recurvedata/operators/utils/file_factory.py +150 -0
  170. recurvedata/operators/utils/fs.py +10 -0
  171. recurvedata/operators/utils/lineage.py +265 -0
  172. recurvedata/operators/web_init.py +15 -0
  173. recurvedata/pigeon/connector/__init__.py +294 -0
  174. recurvedata/pigeon/connector/_registry.py +17 -0
  175. recurvedata/pigeon/connector/aliyun_oss.py +80 -0
  176. recurvedata/pigeon/connector/awss3.py +123 -0
  177. recurvedata/pigeon/connector/azure_blob.py +176 -0
  178. recurvedata/pigeon/connector/azure_synapse.py +51 -0
  179. recurvedata/pigeon/connector/cass.py +151 -0
  180. recurvedata/pigeon/connector/clickhouse.py +403 -0
  181. recurvedata/pigeon/connector/clickhouse_native.py +351 -0
  182. recurvedata/pigeon/connector/dbapi.py +571 -0
  183. recurvedata/pigeon/connector/doris.py +166 -0
  184. recurvedata/pigeon/connector/es.py +176 -0
  185. recurvedata/pigeon/connector/feishu.py +1135 -0
  186. recurvedata/pigeon/connector/ftp.py +163 -0
  187. recurvedata/pigeon/connector/google_bigquery.py +283 -0
  188. recurvedata/pigeon/connector/google_cloud_storage.py +130 -0
  189. recurvedata/pigeon/connector/hbase_phoenix.py +108 -0
  190. recurvedata/pigeon/connector/hdfs.py +204 -0
  191. recurvedata/pigeon/connector/hive_impala.py +383 -0
  192. recurvedata/pigeon/connector/microsoft_fabric.py +95 -0
  193. recurvedata/pigeon/connector/mongodb.py +56 -0
  194. recurvedata/pigeon/connector/mssql.py +467 -0
  195. recurvedata/pigeon/connector/mysql.py +175 -0
  196. recurvedata/pigeon/connector/owncloud.py +92 -0
  197. recurvedata/pigeon/connector/postgresql.py +267 -0
  198. recurvedata/pigeon/connector/power_bi.py +179 -0
  199. recurvedata/pigeon/connector/qcloud_cos.py +79 -0
  200. recurvedata/pigeon/connector/redshift.py +123 -0
  201. recurvedata/pigeon/connector/sftp.py +73 -0
  202. recurvedata/pigeon/connector/sqlite.py +42 -0
  203. recurvedata/pigeon/connector/starrocks.py +144 -0
  204. recurvedata/pigeon/connector/tableau.py +162 -0
  205. recurvedata/pigeon/const.py +21 -0
  206. recurvedata/pigeon/csv.py +172 -0
  207. recurvedata/pigeon/docs/datasources-example.json +82 -0
  208. recurvedata/pigeon/docs/images/pigeon_design.png +0 -0
  209. recurvedata/pigeon/docs/lightweight-data-sync-solution.md +111 -0
  210. recurvedata/pigeon/dumper/__init__.py +171 -0
  211. recurvedata/pigeon/dumper/aliyun_sls.py +415 -0
  212. recurvedata/pigeon/dumper/base.py +141 -0
  213. recurvedata/pigeon/dumper/cass.py +213 -0
  214. recurvedata/pigeon/dumper/dbapi.py +346 -0
  215. recurvedata/pigeon/dumper/es.py +112 -0
  216. recurvedata/pigeon/dumper/ftp.py +64 -0
  217. recurvedata/pigeon/dumper/mongodb.py +103 -0
  218. recurvedata/pigeon/handler/__init__.py +4 -0
  219. recurvedata/pigeon/handler/base.py +153 -0
  220. recurvedata/pigeon/handler/csv_handler.py +290 -0
  221. recurvedata/pigeon/loader/__init__.py +87 -0
  222. recurvedata/pigeon/loader/base.py +83 -0
  223. recurvedata/pigeon/loader/csv_to_azure_synapse.py +214 -0
  224. recurvedata/pigeon/loader/csv_to_clickhouse.py +152 -0
  225. recurvedata/pigeon/loader/csv_to_doris.py +215 -0
  226. recurvedata/pigeon/loader/csv_to_es.py +51 -0
  227. recurvedata/pigeon/loader/csv_to_google_bigquery.py +169 -0
  228. recurvedata/pigeon/loader/csv_to_hive.py +468 -0
  229. recurvedata/pigeon/loader/csv_to_microsoft_fabric.py +242 -0
  230. recurvedata/pigeon/loader/csv_to_mssql.py +174 -0
  231. recurvedata/pigeon/loader/csv_to_mysql.py +180 -0
  232. recurvedata/pigeon/loader/csv_to_postgresql.py +248 -0
  233. recurvedata/pigeon/loader/csv_to_redshift.py +240 -0
  234. recurvedata/pigeon/loader/csv_to_starrocks.py +233 -0
  235. recurvedata/pigeon/meta.py +116 -0
  236. recurvedata/pigeon/row_factory.py +42 -0
  237. recurvedata/pigeon/schema/__init__.py +124 -0
  238. recurvedata/pigeon/schema/types.py +13 -0
  239. recurvedata/pigeon/sync.py +283 -0
  240. recurvedata/pigeon/transformer.py +146 -0
  241. recurvedata/pigeon/utils/__init__.py +134 -0
  242. recurvedata/pigeon/utils/bloomfilter.py +181 -0
  243. recurvedata/pigeon/utils/date_time.py +323 -0
  244. recurvedata/pigeon/utils/escape.py +15 -0
  245. recurvedata/pigeon/utils/fs.py +266 -0
  246. recurvedata/pigeon/utils/json.py +44 -0
  247. recurvedata/pigeon/utils/keyed_tuple.py +85 -0
  248. recurvedata/pigeon/utils/mp.py +156 -0
  249. recurvedata/pigeon/utils/sql.py +328 -0
  250. recurvedata/pigeon/utils/timing.py +155 -0
  251. recurvedata/provider_manager.py +0 -0
  252. recurvedata/providers/__init__.py +0 -0
  253. recurvedata/providers/dbapi/__init__.py +0 -0
  254. recurvedata/providers/flywheel/__init__.py +0 -0
  255. recurvedata/providers/mysql/__init__.py +0 -0
  256. recurvedata/schedulers/__init__.py +1 -0
  257. recurvedata/schedulers/airflow.py +974 -0
  258. recurvedata/schedulers/airflow_db_process.py +331 -0
  259. recurvedata/schedulers/airflow_operators.py +61 -0
  260. recurvedata/schedulers/airflow_plugin.py +9 -0
  261. recurvedata/schedulers/airflow_trigger_dag_patch.py +117 -0
  262. recurvedata/schedulers/base.py +99 -0
  263. recurvedata/schedulers/cli.py +228 -0
  264. recurvedata/schedulers/client.py +56 -0
  265. recurvedata/schedulers/consts.py +52 -0
  266. recurvedata/schedulers/debug_celery.py +62 -0
  267. recurvedata/schedulers/model.py +63 -0
  268. recurvedata/schedulers/schemas.py +97 -0
  269. recurvedata/schedulers/service.py +20 -0
  270. recurvedata/schedulers/system_dags.py +59 -0
  271. recurvedata/schedulers/task_status.py +279 -0
  272. recurvedata/schedulers/utils.py +73 -0
  273. recurvedata/schema/__init__.py +0 -0
  274. recurvedata/schema/field.py +88 -0
  275. recurvedata/schema/schema.py +55 -0
  276. recurvedata/schema/types.py +17 -0
  277. recurvedata/schema.py +0 -0
  278. recurvedata/server/__init__.py +0 -0
  279. recurvedata/server/app.py +7 -0
  280. recurvedata/server/connector/__init__.py +0 -0
  281. recurvedata/server/connector/api.py +79 -0
  282. recurvedata/server/connector/schemas.py +28 -0
  283. recurvedata/server/data_service/__init__.py +0 -0
  284. recurvedata/server/data_service/api.py +126 -0
  285. recurvedata/server/data_service/client.py +18 -0
  286. recurvedata/server/data_service/consts.py +1 -0
  287. recurvedata/server/data_service/schemas.py +68 -0
  288. recurvedata/server/data_service/service.py +218 -0
  289. recurvedata/server/dbt/__init__.py +0 -0
  290. recurvedata/server/dbt/api.py +116 -0
  291. recurvedata/server/error_code.py +49 -0
  292. recurvedata/server/exceptions.py +19 -0
  293. recurvedata/server/executor/__init__.py +0 -0
  294. recurvedata/server/executor/api.py +37 -0
  295. recurvedata/server/executor/schemas.py +30 -0
  296. recurvedata/server/executor/service.py +220 -0
  297. recurvedata/server/main.py +32 -0
  298. recurvedata/server/schedulers/__init__.py +0 -0
  299. recurvedata/server/schedulers/api.py +252 -0
  300. recurvedata/server/schedulers/schemas.py +50 -0
  301. recurvedata/server/schemas.py +50 -0
  302. recurvedata/utils/__init__.py +15 -0
  303. recurvedata/utils/_typer.py +61 -0
  304. recurvedata/utils/attrdict.py +19 -0
  305. recurvedata/utils/command_helper.py +20 -0
  306. recurvedata/utils/compat.py +12 -0
  307. recurvedata/utils/compression.py +203 -0
  308. recurvedata/utils/crontab.py +42 -0
  309. recurvedata/utils/crypto_util.py +305 -0
  310. recurvedata/utils/dataclass.py +11 -0
  311. recurvedata/utils/date_time.py +464 -0
  312. recurvedata/utils/dispatch.py +114 -0
  313. recurvedata/utils/email_util.py +104 -0
  314. recurvedata/utils/files.py +386 -0
  315. recurvedata/utils/helpers.py +170 -0
  316. recurvedata/utils/httputil.py +117 -0
  317. recurvedata/utils/imports.py +132 -0
  318. recurvedata/utils/json.py +80 -0
  319. recurvedata/utils/log.py +117 -0
  320. recurvedata/utils/log_capture.py +153 -0
  321. recurvedata/utils/mp.py +178 -0
  322. recurvedata/utils/normalizer.py +102 -0
  323. recurvedata/utils/redis_lock.py +474 -0
  324. recurvedata/utils/registry.py +54 -0
  325. recurvedata/utils/shell.py +15 -0
  326. recurvedata/utils/singleton.py +33 -0
  327. recurvedata/utils/sql.py +6 -0
  328. recurvedata/utils/timeout.py +28 -0
  329. recurvedata/utils/tracing.py +14 -0
  330. recurvedata_lib-0.1.487.dist-info/METADATA +605 -0
  331. recurvedata_lib-0.1.487.dist-info/RECORD +333 -0
  332. recurvedata_lib-0.1.487.dist-info/WHEEL +5 -0
  333. recurvedata_lib-0.1.487.dist-info/entry_points.txt +6 -0
@@ -0,0 +1,464 @@
1
+ import datetime
2
+ from typing import Union
3
+
4
+ import croniter
5
+ import dateutil.parser
6
+ import pendulum
7
+
8
+ from recurvedata.utils.registry import register_func
9
+
10
+ _tz_utc = pendulum.timezone("utc")
11
+ _tz_local = pendulum.local_timezone()
12
+
13
+ _DATELIKE = Union[str, datetime.datetime, datetime.date, pendulum.DateTime, pendulum.Date]
14
+ _DATE_OR_DATETIME = Union[datetime.datetime, datetime.date]
15
+ _TZ_TYPE = Union[datetime.tzinfo, str]
16
+
17
+
18
+ def utcnow() -> datetime.datetime:
19
+ """Current datetime in UTC timezone, naive format (without timezone info).
20
+ e.g. datetime.datetime(2022, 10, 8, 9, 52, 13, 489857)
21
+ """
22
+ return datetime.datetime.utcnow()
23
+
24
+
25
+ def utcnow_aware() -> datetime.datetime:
26
+ """Current datetime in UTC timezone, aware format (with timezone info).
27
+ e.g. datetime.datetime(2022, 10, 8, 9, 52, 13, 489857, tzinfo=tzutc())
28
+ """
29
+ return datetime.datetime.utcnow().replace(tzinfo=_tz_utc)
30
+
31
+
32
+ def now() -> datetime.datetime:
33
+ """Current datetime in local timezone, naive format (without timezone info).
34
+ e.g. datetime.datetime(2022, 10, 8, 17, 52, 13, 489857)
35
+ """
36
+ return datetime.datetime.now()
37
+
38
+
39
+ def now_aware() -> datetime.datetime:
40
+ """Current datetime in local timezone, naive format (with timezone info).
41
+ e.g. datetime.datetime(2022, 10, 8, 17, 52, 13, 489857, tzinfo=tzlocal())
42
+ """
43
+ return datetime.datetime.now(tz=_tz_local)
44
+
45
+
46
+ def _ensure_datetime(dttm: _DATELIKE) -> datetime.datetime:
47
+ """Convert a date-like value to a datetime.datetime object, leave the timezone info as-is
48
+
49
+ >>> _ensure_datetime('2022-09-10')
50
+ datetime.datetime(2022, 9, 10, 0, 0)
51
+ >>> _ensure_datetime('2022-09-10 08:00:00+00:00')
52
+ datetime.datetime(2022, 9, 10, 8, 0, tzinfo=tzutc())
53
+ >>> _ensure_datetime(datetime.datetime(2022, 9, 10))
54
+ datetime.datetime(2022, 9, 10, 0, 0)
55
+ >>> _ensure_datetime(pendulum.parse('2022-09-10 08:00:00+00:00'))
56
+ datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('+00:00'))
57
+ """
58
+ if isinstance(dttm, pendulum.DateTime):
59
+ return datetime.datetime.fromtimestamp(dttm.timestamp(), dttm.tz)
60
+ if isinstance(dttm, datetime.datetime):
61
+ return dttm
62
+ if isinstance(dttm, datetime.date):
63
+ return datetime.datetime.combine(dttm, datetime.time.min)
64
+ if isinstance(dttm, str):
65
+ return dateutil.parser.parse(dttm)
66
+ raise TypeError(f"unsupported type {type(dttm)}")
67
+
68
+
69
+ @register_func
70
+ def to_pendulum(dttm: _DATELIKE) -> pendulum.DateTime:
71
+ """Convert a date-like value into pendulum.DateTime, timezone will be set to UTC by default
72
+
73
+ >>> to_pendulum('2022-09-10')
74
+ DateTime(2022, 9, 10, 0, 0, 0, tzinfo=Timezone('UTC'))
75
+ >>> to_pendulum('2022-09-10 12:12:12')
76
+ DateTime(2022, 9, 10, 12, 12, 12, tzinfo=Timezone('UTC'))
77
+ >>> to_pendulum('2022-09-10 12:12:12+08:00')
78
+ DateTime(2022, 9, 10, 12, 12, 12, tzinfo=Timezone('+08:00'))
79
+ >>> to_pendulum(datetime.datetime(2022, 9, 10))
80
+ DateTime(2022, 9, 10, 0, 0, 0, tzinfo=Timezone('UTC'))
81
+ """
82
+ return pendulum.instance(_ensure_datetime(dttm))
83
+
84
+
85
+ def as_local_datetime(dt: _DATELIKE) -> datetime.datetime:
86
+ """Convert a date-like value into local timezone, ignore the original timezone
87
+
88
+ Note those tests only work well in timezon Asia/Shanghai
89
+ >>> as_local_datetime('2022-09-10')
90
+ datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('Asia/Shanghai'))
91
+ >>> as_local_datetime('2022-09-10 12:12:12+08:00')
92
+ datetime.datetime(2022, 9, 10, 12, 12, 12, tzinfo=Timezone('Asia/Shanghai'))
93
+ >>> as_local_datetime(pendulum.parse('2022-09-10 08:00:00+00:00'))
94
+ datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
95
+ """
96
+ return _ensure_datetime(dt).replace(tzinfo=_tz_local)
97
+
98
+
99
+ def _ensure_tz(tz_or_name: _TZ_TYPE) -> datetime.tzinfo:
100
+ if isinstance(tz_or_name, str):
101
+ return pendulum.timezone(tz_or_name)
102
+ return tz_or_name
103
+
104
+
105
+ def astimezone(dt: _DATELIKE, tz: Union[str, datetime.tzinfo]) -> datetime.datetime:
106
+ """Convert or set timezone
107
+
108
+ >>> astimezone('2022-09-10 08:00:00', 'Asia/Shanghai')
109
+ datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
110
+ >>> astimezone('2022-09-10 08:00:00+08:00', 'Asia/Shanghai')
111
+ datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
112
+ >>> astimezone('2022-09-10 08:00:00', 'UTC')
113
+ datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
114
+ >>> astimezone('2022-09-10 08:00:00+00:00', 'Asia/Shanghai')
115
+ datetime.datetime(2022, 9, 10, 16, 0, tzinfo=Timezone('Asia/Shanghai'))
116
+ """
117
+ return _ensure_datetime(dt).astimezone(_ensure_tz(tz))
118
+
119
+
120
+ @register_func
121
+ def convert_tz(dt: _DATELIKE, source: _TZ_TYPE, to: _TZ_TYPE) -> datetime.datetime:
122
+ """Convert timezone.
123
+
124
+ >>> convert_tz('2022-09-10 08:00:00', 'Asia/Shanghai', 'UTC')
125
+ datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
126
+ >>> convert_tz('2022-09-10 00:00:00', 'UTC', 'Asia/Shanghai')
127
+ datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
128
+ >>> convert_tz('2022-09-10 00:00:00', 'Europe/Paris', 'Asia/Shanghai')
129
+ datetime.datetime(2022, 9, 10, 6, 0, tzinfo=Timezone('Asia/Shanghai'))
130
+ """
131
+ return _ensure_datetime(dt).replace(tzinfo=_ensure_tz(source)).astimezone(_ensure_tz(to))
132
+
133
+
134
+ def local_to_utc(dt: _DATELIKE) -> datetime.datetime:
135
+ """Convert a datetime from local to utc
136
+
137
+ >>> local_to_utc('2022-09-10 08:00:00')
138
+ datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
139
+ >>> local_to_utc('2022-09-10 08:00:00+08:00')
140
+ datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
141
+ """
142
+ return convert_tz(dt, source=_tz_local, to=_tz_utc)
143
+
144
+
145
+ def utc_to_local(dt: _DATELIKE) -> datetime.datetime:
146
+ """Convert a datetime from utc to local
147
+
148
+ >>> utc_to_local('2022-09-10 08:00:00')
149
+ datetime.datetime(2022, 9, 10, 16, 0, tzinfo=Timezone('Asia/Shanghai'))
150
+ >>> utc_to_local('2022-09-10 08:00:00+00:00')
151
+ datetime.datetime(2022, 9, 10, 16, 0, tzinfo=Timezone('Asia/Shanghai'))
152
+ """
153
+ return convert_tz(dt, source=_tz_utc, to=_tz_local)
154
+
155
+
156
+ def truncate_second(dttm: _DATELIKE) -> datetime.datetime:
157
+ """Truncate a datetime to **second** resolution
158
+
159
+ >>> truncate_second(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
160
+ datetime.datetime(2022, 9, 10, 8, 1, 2)
161
+ """
162
+ return truncate(dttm, "second")
163
+
164
+
165
+ def truncate_minute(dttm: _DATELIKE) -> datetime.datetime:
166
+ """Truncate a datetime to **minute** resolution
167
+
168
+ >>> truncate_minute(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
169
+ datetime.datetime(2022, 9, 10, 8, 1)
170
+ """
171
+ return truncate(dttm, "minute")
172
+
173
+
174
+ def truncate_hour(dttm: _DATELIKE) -> datetime.datetime:
175
+ """Truncate a datetime to **hour** resolution
176
+
177
+ >>> truncate_hour(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
178
+ datetime.datetime(2022, 9, 10, 8, 0)
179
+ """
180
+ return truncate(dttm, "hour")
181
+
182
+
183
+ def truncate_day(dttm: _DATELIKE) -> datetime.datetime:
184
+ """Truncate a datetime to **date** resolution
185
+
186
+ >>> truncate_day(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
187
+ datetime.datetime(2022, 9, 10, 0, 0)
188
+ """
189
+ return truncate(dttm, "day")
190
+
191
+
192
+ def truncate_week(dttm: _DATELIKE) -> datetime.datetime:
193
+ """Truncate a datetime to **week** resolution, which is the first day of week (Monday)
194
+
195
+ >>> truncate_week(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
196
+ datetime.datetime(2022, 9, 5, 0, 0)
197
+ """
198
+ dttm = truncate(dttm, "day")
199
+ return dttm - datetime.timedelta(days=dttm.isoweekday() - 1)
200
+
201
+
202
+ def truncate_month(dttm: _DATELIKE) -> datetime.datetime:
203
+ """Truncate a datetime to **month** resolution, which is the first day of month
204
+
205
+ >>> truncate_month(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
206
+ datetime.datetime(2022, 9, 1, 0, 0)
207
+ """
208
+ return truncate(dttm, "month")
209
+
210
+
211
+ def truncate_half_month(dttm: _DATELIKE) -> datetime.datetime:
212
+ """Truncate a datetime to **half-week** resolution
213
+
214
+ >>> truncate_half_month(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
215
+ datetime.datetime(2022, 9, 1, 0, 0)
216
+ >>> truncate_half_month(datetime.datetime(2022, 9, 20, 8, 1, 2, 1234))
217
+ datetime.datetime(2022, 9, 15, 0, 0)
218
+ """
219
+
220
+ dttm = truncate_day(dttm)
221
+ if dttm.day >= 15:
222
+ return dttm.replace(day=15)
223
+ return dttm.replace(day=1)
224
+
225
+
226
+ def truncate_quarter(dttm: _DATELIKE) -> datetime.datetime:
227
+ """Truncate a datetime to **quater** resolution
228
+
229
+ >>> truncate_quarter(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
230
+ datetime.datetime(2022, 7, 1, 0, 0)
231
+ """
232
+ dttm = truncate(dttm, "month")
233
+
234
+ month = dttm.month
235
+ if 1 <= month <= 3:
236
+ return dttm.replace(month=1)
237
+ elif 4 <= month <= 6:
238
+ return dttm.replace(month=4)
239
+ elif 7 <= month <= 9:
240
+ return dttm.replace(month=7)
241
+ elif 10 <= month <= 12:
242
+ return dttm.replace(month=10)
243
+
244
+
245
+ def truncate_half_year(dttm: _DATELIKE) -> datetime.datetime:
246
+ """Truncate a datetime to **half-year** resolution
247
+
248
+ >>> truncate_half_year(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
249
+ datetime.datetime(2022, 7, 1, 0, 0)
250
+ >>> truncate_half_year(datetime.datetime(2022, 4, 10, 8, 1, 2, 1234))
251
+ datetime.datetime(2022, 1, 1, 0, 0)
252
+ """
253
+ dttm = truncate(dttm, "month")
254
+ if 1 <= dttm.month <= 6:
255
+ return dttm.replace(month=1)
256
+ return dttm.replace(month=7)
257
+
258
+
259
+ def truncate_year(dttm: _DATELIKE) -> datetime.datetime:
260
+ """Truncate a datetime to **year** resolution
261
+
262
+ >>> truncate_year(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
263
+ datetime.datetime(2022, 1, 1, 0, 0)
264
+ """
265
+ return truncate(dttm, "year")
266
+
267
+
268
+ _PERIODS = {
269
+ "second": dict(microsecond=0),
270
+ "minute": dict(microsecond=0, second=0),
271
+ "hour": dict(microsecond=0, second=0, minute=0),
272
+ "day": dict(
273
+ microsecond=0,
274
+ second=0,
275
+ minute=0,
276
+ hour=0,
277
+ ),
278
+ "month": dict(microsecond=0, second=0, minute=0, hour=0, day=1),
279
+ "year": dict(microsecond=0, second=0, minute=0, hour=0, day=1, month=1),
280
+ }
281
+ _ODD_PERIODS = {"week": truncate_week, "quarter": truncate_quarter, "half_year": truncate_half_year}
282
+
283
+
284
+ def truncate(dttm: _DATELIKE, truncate_to="day") -> datetime.datetime:
285
+ dttm = _ensure_datetime(dttm)
286
+ if truncate_to in _PERIODS:
287
+ return dttm.replace(**_PERIODS[truncate_to])
288
+
289
+ if truncate_to not in _ODD_PERIODS:
290
+ raise ValueError(
291
+ "truncate_to not valid. Valid periods: {}".format(
292
+ ", ".join(list(_PERIODS.keys()) + list(_ODD_PERIODS.keys()))
293
+ )
294
+ )
295
+ return _ODD_PERIODS[truncate_to](dttm)
296
+
297
+
298
+ @register_func
299
+ def date_add(dttm: _DATE_OR_DATETIME, days: int) -> _DATE_OR_DATETIME:
300
+ """Add a specified number of days to a datetime
301
+
302
+ >>> date_add(datetime.datetime(2022, 10, 8), 6)
303
+ datetime.datetime(2022, 10, 14, 0, 0)
304
+ >>> date_add(datetime.datetime(2022, 10, 8, 10), 6)
305
+ datetime.datetime(2022, 10, 14, 10, 0)
306
+ >>> date_add(datetime.datetime(2022, 10, 8), -6)
307
+ datetime.datetime(2022, 10, 2, 0, 0)
308
+ """
309
+ return dttm + datetime.timedelta(days=days)
310
+
311
+
312
+ @register_func
313
+ def month_start(dttm: _DATELIKE) -> datetime.datetime:
314
+ """Get the first day of month, equivalent to `truncate_month`
315
+
316
+ >>> month_start(datetime.datetime(2022, 10, 8))
317
+ datetime.datetime(2022, 10, 1, 0, 0)
318
+ """
319
+ return truncate_month(dttm)
320
+
321
+
322
+ @register_func
323
+ def month_end(dttm: _DATELIKE) -> datetime.datetime:
324
+ """Get the last day of month
325
+
326
+ >>> month_end(datetime.datetime(2022, 10, 8))
327
+ datetime.datetime(2022, 10, 31, 0, 0)
328
+ """
329
+ dt = to_pendulum(dttm).last_of("month")
330
+ return datetime.datetime(dt.year, dt.month, dt.day)
331
+
332
+
333
+ def _get_last_month(dttm: _DATELIKE) -> datetime.datetime:
334
+ return month_start(dttm) - datetime.timedelta(days=1)
335
+
336
+
337
+ def last_month_start(dttm: _DATELIKE) -> datetime.datetime:
338
+ """Get the first day of last month
339
+
340
+ >>> last_month_start(datetime.datetime(2022, 10, 8))
341
+ datetime.datetime(2022, 9, 1, 0, 0)
342
+ """
343
+ return month_start(_get_last_month(dttm))
344
+
345
+
346
+ def last_month_end(dttm: _DATELIKE) -> datetime.datetime:
347
+ """Get the last day of last month
348
+
349
+ >>> last_month_end(datetime.datetime(2022, 10, 8))
350
+ datetime.datetime(2022, 9, 30, 0, 0)
351
+ """
352
+ return month_start(dttm) - datetime.timedelta(days=1)
353
+
354
+
355
+ def _get_last_week(dttm: _DATELIKE) -> datetime.datetime:
356
+ return truncate_week(dttm) - datetime.timedelta(days=7)
357
+
358
+
359
+ def last_week_start(dttm: _DATELIKE) -> datetime.datetime:
360
+ """Get the first day (Monday) of last week
361
+
362
+ >>> last_week_start(datetime.datetime(2022, 10, 8))
363
+ datetime.datetime(2022, 9, 26, 0, 0)
364
+ """
365
+ return truncate_week(dttm) - datetime.timedelta(days=7)
366
+
367
+
368
+ def last_week_end(dttm: _DATELIKE) -> datetime.datetime:
369
+ """Get the first day (Sunday) of last week
370
+
371
+ >>> last_week_end(datetime.datetime(2022, 10, 8))
372
+ datetime.datetime(2022, 10, 2, 0, 0)
373
+ """
374
+ return truncate_week(dttm) - datetime.timedelta(days=1)
375
+
376
+
377
+ def round_time_resolution(dt: datetime.datetime, cron_spec: str) -> datetime.datetime:
378
+ """Truncate a datetime value according to crontab spec, infer the time interval and return the value of `truncate_x`
379
+
380
+ >>> dt = datetime.datetime(2022, 10, 8, 10, 23, 45)
381
+ >>> round_time_resolution(dt, '15 * * * *') # hourly, equivalent to `truncate_hour`
382
+ datetime.datetime(2022, 10, 8, 10, 0)
383
+ >>> round_time_resolution(dt, '0 4 * * *') # daily, equivalent to `truncate_day`
384
+ datetime.datetime(2022, 10, 8, 0, 0)
385
+ >>> round_time_resolution(dt, '0 17 * * TUE') # weekly, equivalent to `truncate_week`
386
+ datetime.datetime(2022, 10, 3, 0, 0)
387
+ >>> round_time_resolution(dt, '0 13 12 * *') # monthly, equivalent to `truncate_month`
388
+ datetime.datetime(2022, 10, 1, 0, 0)
389
+ """
390
+ if not cron_spec or cron_spec == "@once":
391
+ return dt
392
+
393
+ # 这种 schedule interval 会自动处理成整点/自然日/周一/月初
394
+ if cron_spec in ["@hourly", "@daily", "@weekly", "@monthly"]:
395
+ return dt
396
+
397
+ dt = dt.replace(second=0, microsecond=0)
398
+
399
+ # 根据两个 schedule 日期的时间差来反推
400
+ cron = croniter.croniter(cron_spec, start_time=datetime.datetime.now())
401
+ prev_run = cron.get_prev(datetime.datetime)
402
+ next_run = cron.get_next(datetime.datetime)
403
+ interval = next_run - prev_run
404
+
405
+ if interval < datetime.timedelta(hours=1):
406
+ return dt
407
+
408
+ # 每隔 N 个小时(一天内)
409
+ if interval < datetime.timedelta(days=1):
410
+ return truncate_hour(dt)
411
+
412
+ # 每天
413
+ if interval == datetime.timedelta(days=1):
414
+ return truncate_day(dt)
415
+
416
+ # 每周
417
+ if interval == datetime.timedelta(days=7):
418
+ return truncate_week(dt)
419
+
420
+ # 每月,粗暴的把相隔天数是 28~31 当作一个月
421
+ if 28 <= interval.days <= 31:
422
+ return truncate_month(dt)
423
+
424
+ return dt
425
+
426
+
427
+ def month_range(start_date: _DATELIKE, end_date: _DATELIKE) -> list[str]:
428
+ """Get the first day of all months between start_date and end_date
429
+
430
+ >>> month_range('2022-01-02', '2022-05-20')
431
+ ['2022-01-01', '2022-02-01', '2022-03-01', '2022-04-01', '2022-05-01']
432
+ """
433
+ start_date = to_pendulum(start_date).replace(day=1)
434
+ end_date = to_pendulum(end_date).replace(day=1)
435
+ return [x.date().isoformat() for x in pendulum.interval(start_date, end_date).range("months")]
436
+
437
+
438
+ @register_func
439
+ def day_range(start_date: _DATELIKE, end_date: _DATELIKE) -> list[str]:
440
+ """Get all dates between start_date and end_date
441
+
442
+ >>> day_range('2022-01-02', '2022-01-07')
443
+ ['2022-01-02', '2022-01-03', '2022-01-04', '2022-01-05', '2022-01-06', '2022-01-07']
444
+ """
445
+ start_date = to_pendulum(start_date)
446
+ end_date = to_pendulum(end_date)
447
+ return [x.date().isoformat() for x in pendulum.interval(start_date, end_date).range("days")]
448
+
449
+
450
+ def to_local_datetime(value: str) -> datetime.datetime:
451
+ return astimezone(value, _tz_local)
452
+
453
+
454
+ # exports as public
455
+ tz_utc = _tz_utc
456
+ tz_local = _tz_local
457
+ ensure_datetime = _ensure_datetime
458
+ ensure_tz = _ensure_tz
459
+ DATELIKE = _DATELIKE
460
+
461
+ if __name__ == "__main__":
462
+ import doctest
463
+
464
+ doctest.testmod()
@@ -0,0 +1,114 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+
12
+ """
13
+ Helper that implements custom dispatcher logic
14
+ """
15
+
16
+ from functools import update_wrapper
17
+ from types import MappingProxyType
18
+ from typing import Any, Callable, NamedTuple, Optional, Type, TypeVar
19
+
20
+ from pydantic import BaseModel
21
+
22
+ T = TypeVar("T", bound=BaseModel)
23
+
24
+
25
+ class Register(NamedTuple):
26
+ add: Callable
27
+ get: Callable
28
+
29
+
30
+ def enum_register():
31
+ """
32
+ Helps us register custom function for enum values
33
+ """
34
+ registry = {}
35
+
36
+ def add(name: str) -> Callable:
37
+ def inner(fn: Callable) -> Callable:
38
+ registry[name] = fn
39
+ return fn
40
+
41
+ return inner
42
+
43
+ def get(name: str) -> Optional[Callable]:
44
+ return registry.get(name, None)
45
+
46
+ return Register(add, get)
47
+
48
+
49
+ def class_register():
50
+ """
51
+ Helps us register custom functions for classes based on their name
52
+ """
53
+ registry = {}
54
+
55
+ def add(entity_type: Type[T]):
56
+ def inner(fn):
57
+ _name = entity_type.__name__
58
+ registry[_name] = fn
59
+ return fn
60
+
61
+ return inner
62
+
63
+ def get(entity_type: Type[T]) -> Optional[Callable]:
64
+ return registry.get(entity_type.__name__, None)
65
+
66
+ return Register(add, get)
67
+
68
+
69
+ def valuedispatch(func) -> Callable:
70
+ """Value dispatch for methods and functions
71
+
72
+ Args:
73
+ func (_type_): function to run
74
+
75
+ Returns:
76
+ Callable: wrapper
77
+ """
78
+
79
+ registry = {}
80
+
81
+ def _is_valid_dispatch(value):
82
+ return isinstance(value, str)
83
+
84
+ def dispatch(value: str) -> Callable:
85
+ try:
86
+ impl = registry[value]
87
+ except KeyError:
88
+ impl = registry[object]
89
+ return impl
90
+
91
+ def register(value, func=None) -> Callable:
92
+ if _is_valid_dispatch(value):
93
+ if func is None:
94
+ return lambda f: register(value, f)
95
+ else:
96
+ raise TypeError(f"Invalid first argument to register(). {value} is not a string.")
97
+
98
+ registry[value] = func
99
+ return func
100
+
101
+ def wrapper(*args, **kwargs) -> Any:
102
+ if not args:
103
+ raise TypeError(f"{func_name} requires at least 1 argument")
104
+ if isinstance(args[0], (str, bytes)):
105
+ return dispatch(str(args[0]))(*args, **kwargs)
106
+ return dispatch(args[1])(*args, **kwargs)
107
+
108
+ func_name = getattr(func, "__name__", "method value dispatch")
109
+ registry[object] = func
110
+ wrapper.register = register
111
+ wrapper.dispatch = dispatch
112
+ wrapper.registry = MappingProxyType(registry) # making registry read only
113
+ update_wrapper(wrapper, func)
114
+ return wrapper
@@ -0,0 +1,104 @@
1
+ import logging
2
+ import os
3
+ import re
4
+ from typing import Any, Union
5
+
6
+ try:
7
+ import emails
8
+ except ImportError:
9
+ pass
10
+ logger = logging.getLogger(__name__)
11
+ _default_html = """<div><includetail><div style="font:Verdana normal 14px;color:#000;">
12
+ <div style="position:relative;"><blockquote style="margin-Top: 0px; margin-Bottom: 0px; margin-Left: 0.5em">
13
+ <div class="FoxDiv20190108121908737768">
14
+ <div id="mailContentContainer" style=" font-size: 14px; padding: 0px; height: auto; min-height: auto ; ">
15
+ {content}
16
+ </div>
17
+ </div></blockquote>
18
+ </div></div>"""
19
+
20
+
21
+ def send_email(
22
+ mail_to: Union[str, list[str], tuple[str, ...]],
23
+ subject: str,
24
+ html: str = None,
25
+ content: str = None,
26
+ cc: Union[str, list[str], tuple[str, ...]] = None,
27
+ bcc: Union[str, list[str], tuple[str, ...]] = None,
28
+ files: Union[str, list[str], tuple[str, ...], dict[str, str]] = None,
29
+ mail_from: str = "noreply",
30
+ reply_to: str = None,
31
+ smtp_config: dict[str, Any] = None,
32
+ ) -> bool:
33
+ """
34
+ Sends an email.
35
+
36
+ Args:
37
+ mail_to: The recipient of the email. Example: 'e1@example.com' or ['e1@example.com', 'e2@example.com'].
38
+ subject: The subject of the email.
39
+ html: The content of the email with special requirements such as font or background color.
40
+ content: The content of the email in plain text format.
41
+ cc: The CC recipients.
42
+ bcc: The BCC recipients.
43
+ files: The list of attachments. Example:
44
+ '/data/tmp.txt' or ['/data/tmp_1.txt', '/data/tmp_2.txt'] or
45
+ {'category_data.txt':'/data/tmp_1.txt', 'brand_data.txt':'/data/tmp_2.txt'}.
46
+ mail_from: The displayed sender of the email. Default is 'RecurveData SERVICE'.
47
+ reply_to: The default recipient when replying. Default is 'itservice@recurvedata.com'.
48
+ smtp_config: The SMTP server for sending the email.
49
+
50
+ Returns:
51
+ True if the email was sent successfully, False otherwise.
52
+ """
53
+
54
+ if not any((html, content)):
55
+ raise ValueError("At least one of HTML and content is not empty!")
56
+
57
+ if isinstance(files, (list, tuple)):
58
+ attach_files = [(file, os.path.basename(file)) for file in files]
59
+ elif isinstance(files, dict):
60
+ attach_files = [(files[file_name], file_name) for file_name in files]
61
+ elif isinstance(files, str):
62
+ attach_files = [(files, os.path.basename(files))]
63
+ elif not files:
64
+ attach_files = []
65
+ else:
66
+ raise ValueError("The parameter files is only support list、dict or string")
67
+
68
+ for file_path, _ in attach_files:
69
+ if not os.path.exists(file_path):
70
+ raise ValueError(f"The attachment file does not exist! --- {file_path} ")
71
+ if os.path.isdir(file_path):
72
+ raise ValueError(f"Send directory are not supported, please send after compression!--- {file_path}")
73
+
74
+ if not html and content:
75
+ html_content = ""
76
+ for line in content.split("\n"):
77
+ line = line.replace("\t", " " * 4)
78
+ if line:
79
+ result = re.match(r"\s+", line)
80
+ space = result.group(0) if result else ""
81
+ html_content += f'<div style="margin-Left: {len(space)}em">{line.strip()}</div>'
82
+ else:
83
+ html_content += "<div><br></div>"
84
+ html = _default_html.format(content=html_content)
85
+ message = emails.Message(
86
+ subject=subject,
87
+ cc=cc,
88
+ bcc=bcc,
89
+ text="Build passed: {{ project_name }} ...",
90
+ mail_from=(mail_from, smtp_config.get("user")),
91
+ html=html,
92
+ headers={"reply-to": reply_to},
93
+ )
94
+
95
+ for file, file_name in attach_files:
96
+ message.attach(data=open(file, "rb"), filename=file_name)
97
+ response = message.send(to=mail_to, smtp=smtp_config)
98
+ if response.status_code == 250:
99
+ return True
100
+ logger.error(
101
+ f"send email from {message.mail_from} to {mail_to}, "
102
+ f"status_code:{response.status_code}, error_msg:{response._exc}"
103
+ )
104
+ return False