recurvedata-lib 0.1.487__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recurvedata-lib might be problematic. Click here for more details.
- recurvedata/__init__.py +0 -0
- recurvedata/__version__.py +1 -0
- recurvedata/client/__init__.py +3 -0
- recurvedata/client/client.py +150 -0
- recurvedata/client/server_client.py +91 -0
- recurvedata/config.py +99 -0
- recurvedata/connectors/__init__.py +20 -0
- recurvedata/connectors/_register.py +46 -0
- recurvedata/connectors/base.py +111 -0
- recurvedata/connectors/config_schema.py +1575 -0
- recurvedata/connectors/connectors/__init__.py +0 -0
- recurvedata/connectors/connectors/aliyun_access_key.py +30 -0
- recurvedata/connectors/connectors/auth.py +44 -0
- recurvedata/connectors/connectors/azure_blob.py +89 -0
- recurvedata/connectors/connectors/azure_synapse.py +79 -0
- recurvedata/connectors/connectors/bigquery.py +359 -0
- recurvedata/connectors/connectors/clickhouse.py +219 -0
- recurvedata/connectors/connectors/dingtalk.py +61 -0
- recurvedata/connectors/connectors/doris.py +215 -0
- recurvedata/connectors/connectors/es.py +62 -0
- recurvedata/connectors/connectors/feishu.py +65 -0
- recurvedata/connectors/connectors/ftp.py +50 -0
- recurvedata/connectors/connectors/generic.py +49 -0
- recurvedata/connectors/connectors/google_cloud_storage.py +115 -0
- recurvedata/connectors/connectors/google_service_account.py +225 -0
- recurvedata/connectors/connectors/hive.py +207 -0
- recurvedata/connectors/connectors/impala.py +210 -0
- recurvedata/connectors/connectors/jenkins.py +51 -0
- recurvedata/connectors/connectors/mail.py +89 -0
- recurvedata/connectors/connectors/microsoft_fabric.py +284 -0
- recurvedata/connectors/connectors/mongo.py +79 -0
- recurvedata/connectors/connectors/mssql.py +131 -0
- recurvedata/connectors/connectors/mysql.py +191 -0
- recurvedata/connectors/connectors/n8n.py +141 -0
- recurvedata/connectors/connectors/oss.py +74 -0
- recurvedata/connectors/connectors/owncloud.py +36 -0
- recurvedata/connectors/connectors/phoenix.py +36 -0
- recurvedata/connectors/connectors/postgres.py +230 -0
- recurvedata/connectors/connectors/python.py +50 -0
- recurvedata/connectors/connectors/redshift.py +187 -0
- recurvedata/connectors/connectors/s3.py +93 -0
- recurvedata/connectors/connectors/sftp.py +87 -0
- recurvedata/connectors/connectors/slack.py +35 -0
- recurvedata/connectors/connectors/spark.py +99 -0
- recurvedata/connectors/connectors/starrocks.py +175 -0
- recurvedata/connectors/connectors/tencent_cos.py +40 -0
- recurvedata/connectors/connectors/tidb.py +49 -0
- recurvedata/connectors/const.py +315 -0
- recurvedata/connectors/datasource.py +189 -0
- recurvedata/connectors/dbapi.py +469 -0
- recurvedata/connectors/fs.py +66 -0
- recurvedata/connectors/ftp.py +40 -0
- recurvedata/connectors/object_store.py +60 -0
- recurvedata/connectors/pigeon.py +172 -0
- recurvedata/connectors/proxy.py +104 -0
- recurvedata/connectors/service.py +223 -0
- recurvedata/connectors/utils.py +47 -0
- recurvedata/consts.py +49 -0
- recurvedata/core/__init__.py +0 -0
- recurvedata/core/config.py +46 -0
- recurvedata/core/configurable.py +27 -0
- recurvedata/core/consts.py +2 -0
- recurvedata/core/templating.py +206 -0
- recurvedata/core/tracing.py +223 -0
- recurvedata/core/transformer.py +186 -0
- recurvedata/core/translation.py +91 -0
- recurvedata/dbt/client.py +97 -0
- recurvedata/dbt/consts.py +99 -0
- recurvedata/dbt/cosmos_utils.py +275 -0
- recurvedata/dbt/error_codes.py +18 -0
- recurvedata/dbt/schemas.py +98 -0
- recurvedata/dbt/service.py +451 -0
- recurvedata/dbt/utils.py +246 -0
- recurvedata/error_codes.py +71 -0
- recurvedata/exceptions.py +72 -0
- recurvedata/executors/__init__.py +4 -0
- recurvedata/executors/cli/__init__.py +7 -0
- recurvedata/executors/cli/connector.py +117 -0
- recurvedata/executors/cli/dbt.py +118 -0
- recurvedata/executors/cli/main.py +82 -0
- recurvedata/executors/cli/parameters.py +18 -0
- recurvedata/executors/client.py +190 -0
- recurvedata/executors/consts.py +50 -0
- recurvedata/executors/debug_executor.py +100 -0
- recurvedata/executors/executor.py +300 -0
- recurvedata/executors/link_executor.py +189 -0
- recurvedata/executors/models.py +34 -0
- recurvedata/executors/schemas.py +222 -0
- recurvedata/executors/service/__init__.py +0 -0
- recurvedata/executors/service/connector.py +380 -0
- recurvedata/executors/utils.py +172 -0
- recurvedata/filestorage/__init__.py +11 -0
- recurvedata/filestorage/_factory.py +33 -0
- recurvedata/filestorage/backends/__init__.py +0 -0
- recurvedata/filestorage/backends/fsspec.py +45 -0
- recurvedata/filestorage/backends/local.py +67 -0
- recurvedata/filestorage/backends/oss.py +56 -0
- recurvedata/filestorage/interface.py +84 -0
- recurvedata/operators/__init__.py +10 -0
- recurvedata/operators/base.py +28 -0
- recurvedata/operators/config.py +21 -0
- recurvedata/operators/context.py +255 -0
- recurvedata/operators/dbt_operator/__init__.py +2 -0
- recurvedata/operators/dbt_operator/model_pipeline_link_operator.py +55 -0
- recurvedata/operators/dbt_operator/operator.py +353 -0
- recurvedata/operators/link_operator/__init__.py +1 -0
- recurvedata/operators/link_operator/operator.py +120 -0
- recurvedata/operators/models.py +55 -0
- recurvedata/operators/notify_operator/__init__.py +1 -0
- recurvedata/operators/notify_operator/operator.py +180 -0
- recurvedata/operators/operator.py +119 -0
- recurvedata/operators/python_operator/__init__.py +1 -0
- recurvedata/operators/python_operator/operator.py +132 -0
- recurvedata/operators/sensor_operator/__init__.py +1 -0
- recurvedata/operators/sensor_operator/airflow_utils.py +63 -0
- recurvedata/operators/sensor_operator/operator.py +172 -0
- recurvedata/operators/spark_operator/__init__.py +1 -0
- recurvedata/operators/spark_operator/operator.py +200 -0
- recurvedata/operators/spark_operator/spark_sample.py +47 -0
- recurvedata/operators/sql_operator/__init__.py +1 -0
- recurvedata/operators/sql_operator/operator.py +90 -0
- recurvedata/operators/task.py +211 -0
- recurvedata/operators/transfer_operator/__init__.py +40 -0
- recurvedata/operators/transfer_operator/const.py +10 -0
- recurvedata/operators/transfer_operator/dump_aliyun_sls.py +82 -0
- recurvedata/operators/transfer_operator/dump_sheet_task_base.py +292 -0
- recurvedata/operators/transfer_operator/dump_task_cass.py +155 -0
- recurvedata/operators/transfer_operator/dump_task_dbapi.py +209 -0
- recurvedata/operators/transfer_operator/dump_task_es.py +113 -0
- recurvedata/operators/transfer_operator/dump_task_feishu_sheet.py +114 -0
- recurvedata/operators/transfer_operator/dump_task_ftp.py +234 -0
- recurvedata/operators/transfer_operator/dump_task_google_sheet.py +66 -0
- recurvedata/operators/transfer_operator/dump_task_mongodb.py +168 -0
- recurvedata/operators/transfer_operator/dump_task_oss.py +285 -0
- recurvedata/operators/transfer_operator/dump_task_python.py +212 -0
- recurvedata/operators/transfer_operator/dump_task_s3.py +270 -0
- recurvedata/operators/transfer_operator/dump_task_sftp.py +229 -0
- recurvedata/operators/transfer_operator/load_task_aliyun_oss.py +107 -0
- recurvedata/operators/transfer_operator/load_task_azure_blob.py +115 -0
- recurvedata/operators/transfer_operator/load_task_azure_synapse.py +90 -0
- recurvedata/operators/transfer_operator/load_task_clickhouse.py +167 -0
- recurvedata/operators/transfer_operator/load_task_doris.py +164 -0
- recurvedata/operators/transfer_operator/load_task_email.py +188 -0
- recurvedata/operators/transfer_operator/load_task_es.py +86 -0
- recurvedata/operators/transfer_operator/load_task_filebrowser.py +151 -0
- recurvedata/operators/transfer_operator/load_task_ftp.py +19 -0
- recurvedata/operators/transfer_operator/load_task_google_bigquery.py +90 -0
- recurvedata/operators/transfer_operator/load_task_google_cloud_storage.py +127 -0
- recurvedata/operators/transfer_operator/load_task_google_sheet.py +130 -0
- recurvedata/operators/transfer_operator/load_task_hive.py +158 -0
- recurvedata/operators/transfer_operator/load_task_microsoft_fabric.py +105 -0
- recurvedata/operators/transfer_operator/load_task_mssql.py +153 -0
- recurvedata/operators/transfer_operator/load_task_mysql.py +157 -0
- recurvedata/operators/transfer_operator/load_task_owncloud.py +135 -0
- recurvedata/operators/transfer_operator/load_task_postgresql.py +109 -0
- recurvedata/operators/transfer_operator/load_task_qcloud_cos.py +119 -0
- recurvedata/operators/transfer_operator/load_task_recurve_data_prep.py +75 -0
- recurvedata/operators/transfer_operator/load_task_redshift.py +95 -0
- recurvedata/operators/transfer_operator/load_task_s3.py +150 -0
- recurvedata/operators/transfer_operator/load_task_sftp.py +90 -0
- recurvedata/operators/transfer_operator/load_task_starrocks.py +169 -0
- recurvedata/operators/transfer_operator/load_task_yicrowds.py +97 -0
- recurvedata/operators/transfer_operator/mixin.py +31 -0
- recurvedata/operators/transfer_operator/operator.py +231 -0
- recurvedata/operators/transfer_operator/task.py +223 -0
- recurvedata/operators/transfer_operator/utils.py +134 -0
- recurvedata/operators/ui.py +80 -0
- recurvedata/operators/utils/__init__.py +51 -0
- recurvedata/operators/utils/file_factory.py +150 -0
- recurvedata/operators/utils/fs.py +10 -0
- recurvedata/operators/utils/lineage.py +265 -0
- recurvedata/operators/web_init.py +15 -0
- recurvedata/pigeon/connector/__init__.py +294 -0
- recurvedata/pigeon/connector/_registry.py +17 -0
- recurvedata/pigeon/connector/aliyun_oss.py +80 -0
- recurvedata/pigeon/connector/awss3.py +123 -0
- recurvedata/pigeon/connector/azure_blob.py +176 -0
- recurvedata/pigeon/connector/azure_synapse.py +51 -0
- recurvedata/pigeon/connector/cass.py +151 -0
- recurvedata/pigeon/connector/clickhouse.py +403 -0
- recurvedata/pigeon/connector/clickhouse_native.py +351 -0
- recurvedata/pigeon/connector/dbapi.py +571 -0
- recurvedata/pigeon/connector/doris.py +166 -0
- recurvedata/pigeon/connector/es.py +176 -0
- recurvedata/pigeon/connector/feishu.py +1135 -0
- recurvedata/pigeon/connector/ftp.py +163 -0
- recurvedata/pigeon/connector/google_bigquery.py +283 -0
- recurvedata/pigeon/connector/google_cloud_storage.py +130 -0
- recurvedata/pigeon/connector/hbase_phoenix.py +108 -0
- recurvedata/pigeon/connector/hdfs.py +204 -0
- recurvedata/pigeon/connector/hive_impala.py +383 -0
- recurvedata/pigeon/connector/microsoft_fabric.py +95 -0
- recurvedata/pigeon/connector/mongodb.py +56 -0
- recurvedata/pigeon/connector/mssql.py +467 -0
- recurvedata/pigeon/connector/mysql.py +175 -0
- recurvedata/pigeon/connector/owncloud.py +92 -0
- recurvedata/pigeon/connector/postgresql.py +267 -0
- recurvedata/pigeon/connector/power_bi.py +179 -0
- recurvedata/pigeon/connector/qcloud_cos.py +79 -0
- recurvedata/pigeon/connector/redshift.py +123 -0
- recurvedata/pigeon/connector/sftp.py +73 -0
- recurvedata/pigeon/connector/sqlite.py +42 -0
- recurvedata/pigeon/connector/starrocks.py +144 -0
- recurvedata/pigeon/connector/tableau.py +162 -0
- recurvedata/pigeon/const.py +21 -0
- recurvedata/pigeon/csv.py +172 -0
- recurvedata/pigeon/docs/datasources-example.json +82 -0
- recurvedata/pigeon/docs/images/pigeon_design.png +0 -0
- recurvedata/pigeon/docs/lightweight-data-sync-solution.md +111 -0
- recurvedata/pigeon/dumper/__init__.py +171 -0
- recurvedata/pigeon/dumper/aliyun_sls.py +415 -0
- recurvedata/pigeon/dumper/base.py +141 -0
- recurvedata/pigeon/dumper/cass.py +213 -0
- recurvedata/pigeon/dumper/dbapi.py +346 -0
- recurvedata/pigeon/dumper/es.py +112 -0
- recurvedata/pigeon/dumper/ftp.py +64 -0
- recurvedata/pigeon/dumper/mongodb.py +103 -0
- recurvedata/pigeon/handler/__init__.py +4 -0
- recurvedata/pigeon/handler/base.py +153 -0
- recurvedata/pigeon/handler/csv_handler.py +290 -0
- recurvedata/pigeon/loader/__init__.py +87 -0
- recurvedata/pigeon/loader/base.py +83 -0
- recurvedata/pigeon/loader/csv_to_azure_synapse.py +214 -0
- recurvedata/pigeon/loader/csv_to_clickhouse.py +152 -0
- recurvedata/pigeon/loader/csv_to_doris.py +215 -0
- recurvedata/pigeon/loader/csv_to_es.py +51 -0
- recurvedata/pigeon/loader/csv_to_google_bigquery.py +169 -0
- recurvedata/pigeon/loader/csv_to_hive.py +468 -0
- recurvedata/pigeon/loader/csv_to_microsoft_fabric.py +242 -0
- recurvedata/pigeon/loader/csv_to_mssql.py +174 -0
- recurvedata/pigeon/loader/csv_to_mysql.py +180 -0
- recurvedata/pigeon/loader/csv_to_postgresql.py +248 -0
- recurvedata/pigeon/loader/csv_to_redshift.py +240 -0
- recurvedata/pigeon/loader/csv_to_starrocks.py +233 -0
- recurvedata/pigeon/meta.py +116 -0
- recurvedata/pigeon/row_factory.py +42 -0
- recurvedata/pigeon/schema/__init__.py +124 -0
- recurvedata/pigeon/schema/types.py +13 -0
- recurvedata/pigeon/sync.py +283 -0
- recurvedata/pigeon/transformer.py +146 -0
- recurvedata/pigeon/utils/__init__.py +134 -0
- recurvedata/pigeon/utils/bloomfilter.py +181 -0
- recurvedata/pigeon/utils/date_time.py +323 -0
- recurvedata/pigeon/utils/escape.py +15 -0
- recurvedata/pigeon/utils/fs.py +266 -0
- recurvedata/pigeon/utils/json.py +44 -0
- recurvedata/pigeon/utils/keyed_tuple.py +85 -0
- recurvedata/pigeon/utils/mp.py +156 -0
- recurvedata/pigeon/utils/sql.py +328 -0
- recurvedata/pigeon/utils/timing.py +155 -0
- recurvedata/provider_manager.py +0 -0
- recurvedata/providers/__init__.py +0 -0
- recurvedata/providers/dbapi/__init__.py +0 -0
- recurvedata/providers/flywheel/__init__.py +0 -0
- recurvedata/providers/mysql/__init__.py +0 -0
- recurvedata/schedulers/__init__.py +1 -0
- recurvedata/schedulers/airflow.py +974 -0
- recurvedata/schedulers/airflow_db_process.py +331 -0
- recurvedata/schedulers/airflow_operators.py +61 -0
- recurvedata/schedulers/airflow_plugin.py +9 -0
- recurvedata/schedulers/airflow_trigger_dag_patch.py +117 -0
- recurvedata/schedulers/base.py +99 -0
- recurvedata/schedulers/cli.py +228 -0
- recurvedata/schedulers/client.py +56 -0
- recurvedata/schedulers/consts.py +52 -0
- recurvedata/schedulers/debug_celery.py +62 -0
- recurvedata/schedulers/model.py +63 -0
- recurvedata/schedulers/schemas.py +97 -0
- recurvedata/schedulers/service.py +20 -0
- recurvedata/schedulers/system_dags.py +59 -0
- recurvedata/schedulers/task_status.py +279 -0
- recurvedata/schedulers/utils.py +73 -0
- recurvedata/schema/__init__.py +0 -0
- recurvedata/schema/field.py +88 -0
- recurvedata/schema/schema.py +55 -0
- recurvedata/schema/types.py +17 -0
- recurvedata/schema.py +0 -0
- recurvedata/server/__init__.py +0 -0
- recurvedata/server/app.py +7 -0
- recurvedata/server/connector/__init__.py +0 -0
- recurvedata/server/connector/api.py +79 -0
- recurvedata/server/connector/schemas.py +28 -0
- recurvedata/server/data_service/__init__.py +0 -0
- recurvedata/server/data_service/api.py +126 -0
- recurvedata/server/data_service/client.py +18 -0
- recurvedata/server/data_service/consts.py +1 -0
- recurvedata/server/data_service/schemas.py +68 -0
- recurvedata/server/data_service/service.py +218 -0
- recurvedata/server/dbt/__init__.py +0 -0
- recurvedata/server/dbt/api.py +116 -0
- recurvedata/server/error_code.py +49 -0
- recurvedata/server/exceptions.py +19 -0
- recurvedata/server/executor/__init__.py +0 -0
- recurvedata/server/executor/api.py +37 -0
- recurvedata/server/executor/schemas.py +30 -0
- recurvedata/server/executor/service.py +220 -0
- recurvedata/server/main.py +32 -0
- recurvedata/server/schedulers/__init__.py +0 -0
- recurvedata/server/schedulers/api.py +252 -0
- recurvedata/server/schedulers/schemas.py +50 -0
- recurvedata/server/schemas.py +50 -0
- recurvedata/utils/__init__.py +15 -0
- recurvedata/utils/_typer.py +61 -0
- recurvedata/utils/attrdict.py +19 -0
- recurvedata/utils/command_helper.py +20 -0
- recurvedata/utils/compat.py +12 -0
- recurvedata/utils/compression.py +203 -0
- recurvedata/utils/crontab.py +42 -0
- recurvedata/utils/crypto_util.py +305 -0
- recurvedata/utils/dataclass.py +11 -0
- recurvedata/utils/date_time.py +464 -0
- recurvedata/utils/dispatch.py +114 -0
- recurvedata/utils/email_util.py +104 -0
- recurvedata/utils/files.py +386 -0
- recurvedata/utils/helpers.py +170 -0
- recurvedata/utils/httputil.py +117 -0
- recurvedata/utils/imports.py +132 -0
- recurvedata/utils/json.py +80 -0
- recurvedata/utils/log.py +117 -0
- recurvedata/utils/log_capture.py +153 -0
- recurvedata/utils/mp.py +178 -0
- recurvedata/utils/normalizer.py +102 -0
- recurvedata/utils/redis_lock.py +474 -0
- recurvedata/utils/registry.py +54 -0
- recurvedata/utils/shell.py +15 -0
- recurvedata/utils/singleton.py +33 -0
- recurvedata/utils/sql.py +6 -0
- recurvedata/utils/timeout.py +28 -0
- recurvedata/utils/tracing.py +14 -0
- recurvedata_lib-0.1.487.dist-info/METADATA +605 -0
- recurvedata_lib-0.1.487.dist-info/RECORD +333 -0
- recurvedata_lib-0.1.487.dist-info/WHEEL +5 -0
- recurvedata_lib-0.1.487.dist-info/entry_points.txt +6 -0
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
from typing import Union
|
|
3
|
+
|
|
4
|
+
import croniter
|
|
5
|
+
import dateutil.parser
|
|
6
|
+
import pendulum
|
|
7
|
+
|
|
8
|
+
from recurvedata.utils.registry import register_func
|
|
9
|
+
|
|
10
|
+
_tz_utc = pendulum.timezone("utc")
|
|
11
|
+
_tz_local = pendulum.local_timezone()
|
|
12
|
+
|
|
13
|
+
_DATELIKE = Union[str, datetime.datetime, datetime.date, pendulum.DateTime, pendulum.Date]
|
|
14
|
+
_DATE_OR_DATETIME = Union[datetime.datetime, datetime.date]
|
|
15
|
+
_TZ_TYPE = Union[datetime.tzinfo, str]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def utcnow() -> datetime.datetime:
|
|
19
|
+
"""Current datetime in UTC timezone, naive format (without timezone info).
|
|
20
|
+
e.g. datetime.datetime(2022, 10, 8, 9, 52, 13, 489857)
|
|
21
|
+
"""
|
|
22
|
+
return datetime.datetime.utcnow()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def utcnow_aware() -> datetime.datetime:
|
|
26
|
+
"""Current datetime in UTC timezone, aware format (with timezone info).
|
|
27
|
+
e.g. datetime.datetime(2022, 10, 8, 9, 52, 13, 489857, tzinfo=tzutc())
|
|
28
|
+
"""
|
|
29
|
+
return datetime.datetime.utcnow().replace(tzinfo=_tz_utc)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def now() -> datetime.datetime:
|
|
33
|
+
"""Current datetime in local timezone, naive format (without timezone info).
|
|
34
|
+
e.g. datetime.datetime(2022, 10, 8, 17, 52, 13, 489857)
|
|
35
|
+
"""
|
|
36
|
+
return datetime.datetime.now()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def now_aware() -> datetime.datetime:
|
|
40
|
+
"""Current datetime in local timezone, naive format (with timezone info).
|
|
41
|
+
e.g. datetime.datetime(2022, 10, 8, 17, 52, 13, 489857, tzinfo=tzlocal())
|
|
42
|
+
"""
|
|
43
|
+
return datetime.datetime.now(tz=_tz_local)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _ensure_datetime(dttm: _DATELIKE) -> datetime.datetime:
|
|
47
|
+
"""Convert a date-like value to a datetime.datetime object, leave the timezone info as-is
|
|
48
|
+
|
|
49
|
+
>>> _ensure_datetime('2022-09-10')
|
|
50
|
+
datetime.datetime(2022, 9, 10, 0, 0)
|
|
51
|
+
>>> _ensure_datetime('2022-09-10 08:00:00+00:00')
|
|
52
|
+
datetime.datetime(2022, 9, 10, 8, 0, tzinfo=tzutc())
|
|
53
|
+
>>> _ensure_datetime(datetime.datetime(2022, 9, 10))
|
|
54
|
+
datetime.datetime(2022, 9, 10, 0, 0)
|
|
55
|
+
>>> _ensure_datetime(pendulum.parse('2022-09-10 08:00:00+00:00'))
|
|
56
|
+
datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('+00:00'))
|
|
57
|
+
"""
|
|
58
|
+
if isinstance(dttm, pendulum.DateTime):
|
|
59
|
+
return datetime.datetime.fromtimestamp(dttm.timestamp(), dttm.tz)
|
|
60
|
+
if isinstance(dttm, datetime.datetime):
|
|
61
|
+
return dttm
|
|
62
|
+
if isinstance(dttm, datetime.date):
|
|
63
|
+
return datetime.datetime.combine(dttm, datetime.time.min)
|
|
64
|
+
if isinstance(dttm, str):
|
|
65
|
+
return dateutil.parser.parse(dttm)
|
|
66
|
+
raise TypeError(f"unsupported type {type(dttm)}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@register_func
|
|
70
|
+
def to_pendulum(dttm: _DATELIKE) -> pendulum.DateTime:
|
|
71
|
+
"""Convert a date-like value into pendulum.DateTime, timezone will be set to UTC by default
|
|
72
|
+
|
|
73
|
+
>>> to_pendulum('2022-09-10')
|
|
74
|
+
DateTime(2022, 9, 10, 0, 0, 0, tzinfo=Timezone('UTC'))
|
|
75
|
+
>>> to_pendulum('2022-09-10 12:12:12')
|
|
76
|
+
DateTime(2022, 9, 10, 12, 12, 12, tzinfo=Timezone('UTC'))
|
|
77
|
+
>>> to_pendulum('2022-09-10 12:12:12+08:00')
|
|
78
|
+
DateTime(2022, 9, 10, 12, 12, 12, tzinfo=Timezone('+08:00'))
|
|
79
|
+
>>> to_pendulum(datetime.datetime(2022, 9, 10))
|
|
80
|
+
DateTime(2022, 9, 10, 0, 0, 0, tzinfo=Timezone('UTC'))
|
|
81
|
+
"""
|
|
82
|
+
return pendulum.instance(_ensure_datetime(dttm))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def as_local_datetime(dt: _DATELIKE) -> datetime.datetime:
|
|
86
|
+
"""Convert a date-like value into local timezone, ignore the original timezone
|
|
87
|
+
|
|
88
|
+
Note those tests only work well in timezon Asia/Shanghai
|
|
89
|
+
>>> as_local_datetime('2022-09-10')
|
|
90
|
+
datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
91
|
+
>>> as_local_datetime('2022-09-10 12:12:12+08:00')
|
|
92
|
+
datetime.datetime(2022, 9, 10, 12, 12, 12, tzinfo=Timezone('Asia/Shanghai'))
|
|
93
|
+
>>> as_local_datetime(pendulum.parse('2022-09-10 08:00:00+00:00'))
|
|
94
|
+
datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
95
|
+
"""
|
|
96
|
+
return _ensure_datetime(dt).replace(tzinfo=_tz_local)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _ensure_tz(tz_or_name: _TZ_TYPE) -> datetime.tzinfo:
|
|
100
|
+
if isinstance(tz_or_name, str):
|
|
101
|
+
return pendulum.timezone(tz_or_name)
|
|
102
|
+
return tz_or_name
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def astimezone(dt: _DATELIKE, tz: Union[str, datetime.tzinfo]) -> datetime.datetime:
|
|
106
|
+
"""Convert or set timezone
|
|
107
|
+
|
|
108
|
+
>>> astimezone('2022-09-10 08:00:00', 'Asia/Shanghai')
|
|
109
|
+
datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
110
|
+
>>> astimezone('2022-09-10 08:00:00+08:00', 'Asia/Shanghai')
|
|
111
|
+
datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
112
|
+
>>> astimezone('2022-09-10 08:00:00', 'UTC')
|
|
113
|
+
datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
|
|
114
|
+
>>> astimezone('2022-09-10 08:00:00+00:00', 'Asia/Shanghai')
|
|
115
|
+
datetime.datetime(2022, 9, 10, 16, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
116
|
+
"""
|
|
117
|
+
return _ensure_datetime(dt).astimezone(_ensure_tz(tz))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@register_func
|
|
121
|
+
def convert_tz(dt: _DATELIKE, source: _TZ_TYPE, to: _TZ_TYPE) -> datetime.datetime:
|
|
122
|
+
"""Convert timezone.
|
|
123
|
+
|
|
124
|
+
>>> convert_tz('2022-09-10 08:00:00', 'Asia/Shanghai', 'UTC')
|
|
125
|
+
datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
|
|
126
|
+
>>> convert_tz('2022-09-10 00:00:00', 'UTC', 'Asia/Shanghai')
|
|
127
|
+
datetime.datetime(2022, 9, 10, 8, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
128
|
+
>>> convert_tz('2022-09-10 00:00:00', 'Europe/Paris', 'Asia/Shanghai')
|
|
129
|
+
datetime.datetime(2022, 9, 10, 6, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
130
|
+
"""
|
|
131
|
+
return _ensure_datetime(dt).replace(tzinfo=_ensure_tz(source)).astimezone(_ensure_tz(to))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def local_to_utc(dt: _DATELIKE) -> datetime.datetime:
|
|
135
|
+
"""Convert a datetime from local to utc
|
|
136
|
+
|
|
137
|
+
>>> local_to_utc('2022-09-10 08:00:00')
|
|
138
|
+
datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
|
|
139
|
+
>>> local_to_utc('2022-09-10 08:00:00+08:00')
|
|
140
|
+
datetime.datetime(2022, 9, 10, 0, 0, tzinfo=Timezone('UTC'))
|
|
141
|
+
"""
|
|
142
|
+
return convert_tz(dt, source=_tz_local, to=_tz_utc)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def utc_to_local(dt: _DATELIKE) -> datetime.datetime:
|
|
146
|
+
"""Convert a datetime from utc to local
|
|
147
|
+
|
|
148
|
+
>>> utc_to_local('2022-09-10 08:00:00')
|
|
149
|
+
datetime.datetime(2022, 9, 10, 16, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
150
|
+
>>> utc_to_local('2022-09-10 08:00:00+00:00')
|
|
151
|
+
datetime.datetime(2022, 9, 10, 16, 0, tzinfo=Timezone('Asia/Shanghai'))
|
|
152
|
+
"""
|
|
153
|
+
return convert_tz(dt, source=_tz_utc, to=_tz_local)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def truncate_second(dttm: _DATELIKE) -> datetime.datetime:
|
|
157
|
+
"""Truncate a datetime to **second** resolution
|
|
158
|
+
|
|
159
|
+
>>> truncate_second(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
160
|
+
datetime.datetime(2022, 9, 10, 8, 1, 2)
|
|
161
|
+
"""
|
|
162
|
+
return truncate(dttm, "second")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def truncate_minute(dttm: _DATELIKE) -> datetime.datetime:
|
|
166
|
+
"""Truncate a datetime to **minute** resolution
|
|
167
|
+
|
|
168
|
+
>>> truncate_minute(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
169
|
+
datetime.datetime(2022, 9, 10, 8, 1)
|
|
170
|
+
"""
|
|
171
|
+
return truncate(dttm, "minute")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def truncate_hour(dttm: _DATELIKE) -> datetime.datetime:
|
|
175
|
+
"""Truncate a datetime to **hour** resolution
|
|
176
|
+
|
|
177
|
+
>>> truncate_hour(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
178
|
+
datetime.datetime(2022, 9, 10, 8, 0)
|
|
179
|
+
"""
|
|
180
|
+
return truncate(dttm, "hour")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def truncate_day(dttm: _DATELIKE) -> datetime.datetime:
|
|
184
|
+
"""Truncate a datetime to **date** resolution
|
|
185
|
+
|
|
186
|
+
>>> truncate_day(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
187
|
+
datetime.datetime(2022, 9, 10, 0, 0)
|
|
188
|
+
"""
|
|
189
|
+
return truncate(dttm, "day")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def truncate_week(dttm: _DATELIKE) -> datetime.datetime:
|
|
193
|
+
"""Truncate a datetime to **week** resolution, which is the first day of week (Monday)
|
|
194
|
+
|
|
195
|
+
>>> truncate_week(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
196
|
+
datetime.datetime(2022, 9, 5, 0, 0)
|
|
197
|
+
"""
|
|
198
|
+
dttm = truncate(dttm, "day")
|
|
199
|
+
return dttm - datetime.timedelta(days=dttm.isoweekday() - 1)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def truncate_month(dttm: _DATELIKE) -> datetime.datetime:
|
|
203
|
+
"""Truncate a datetime to **month** resolution, which is the first day of month
|
|
204
|
+
|
|
205
|
+
>>> truncate_month(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
206
|
+
datetime.datetime(2022, 9, 1, 0, 0)
|
|
207
|
+
"""
|
|
208
|
+
return truncate(dttm, "month")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def truncate_half_month(dttm: _DATELIKE) -> datetime.datetime:
|
|
212
|
+
"""Truncate a datetime to **half-week** resolution
|
|
213
|
+
|
|
214
|
+
>>> truncate_half_month(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
215
|
+
datetime.datetime(2022, 9, 1, 0, 0)
|
|
216
|
+
>>> truncate_half_month(datetime.datetime(2022, 9, 20, 8, 1, 2, 1234))
|
|
217
|
+
datetime.datetime(2022, 9, 15, 0, 0)
|
|
218
|
+
"""
|
|
219
|
+
|
|
220
|
+
dttm = truncate_day(dttm)
|
|
221
|
+
if dttm.day >= 15:
|
|
222
|
+
return dttm.replace(day=15)
|
|
223
|
+
return dttm.replace(day=1)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def truncate_quarter(dttm: _DATELIKE) -> datetime.datetime:
|
|
227
|
+
"""Truncate a datetime to **quater** resolution
|
|
228
|
+
|
|
229
|
+
>>> truncate_quarter(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
230
|
+
datetime.datetime(2022, 7, 1, 0, 0)
|
|
231
|
+
"""
|
|
232
|
+
dttm = truncate(dttm, "month")
|
|
233
|
+
|
|
234
|
+
month = dttm.month
|
|
235
|
+
if 1 <= month <= 3:
|
|
236
|
+
return dttm.replace(month=1)
|
|
237
|
+
elif 4 <= month <= 6:
|
|
238
|
+
return dttm.replace(month=4)
|
|
239
|
+
elif 7 <= month <= 9:
|
|
240
|
+
return dttm.replace(month=7)
|
|
241
|
+
elif 10 <= month <= 12:
|
|
242
|
+
return dttm.replace(month=10)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def truncate_half_year(dttm: _DATELIKE) -> datetime.datetime:
|
|
246
|
+
"""Truncate a datetime to **half-year** resolution
|
|
247
|
+
|
|
248
|
+
>>> truncate_half_year(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
249
|
+
datetime.datetime(2022, 7, 1, 0, 0)
|
|
250
|
+
>>> truncate_half_year(datetime.datetime(2022, 4, 10, 8, 1, 2, 1234))
|
|
251
|
+
datetime.datetime(2022, 1, 1, 0, 0)
|
|
252
|
+
"""
|
|
253
|
+
dttm = truncate(dttm, "month")
|
|
254
|
+
if 1 <= dttm.month <= 6:
|
|
255
|
+
return dttm.replace(month=1)
|
|
256
|
+
return dttm.replace(month=7)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def truncate_year(dttm: _DATELIKE) -> datetime.datetime:
|
|
260
|
+
"""Truncate a datetime to **year** resolution
|
|
261
|
+
|
|
262
|
+
>>> truncate_year(datetime.datetime(2022, 9, 10, 8, 1, 2, 1234))
|
|
263
|
+
datetime.datetime(2022, 1, 1, 0, 0)
|
|
264
|
+
"""
|
|
265
|
+
return truncate(dttm, "year")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
_PERIODS = {
|
|
269
|
+
"second": dict(microsecond=0),
|
|
270
|
+
"minute": dict(microsecond=0, second=0),
|
|
271
|
+
"hour": dict(microsecond=0, second=0, minute=0),
|
|
272
|
+
"day": dict(
|
|
273
|
+
microsecond=0,
|
|
274
|
+
second=0,
|
|
275
|
+
minute=0,
|
|
276
|
+
hour=0,
|
|
277
|
+
),
|
|
278
|
+
"month": dict(microsecond=0, second=0, minute=0, hour=0, day=1),
|
|
279
|
+
"year": dict(microsecond=0, second=0, minute=0, hour=0, day=1, month=1),
|
|
280
|
+
}
|
|
281
|
+
_ODD_PERIODS = {"week": truncate_week, "quarter": truncate_quarter, "half_year": truncate_half_year}
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def truncate(dttm: _DATELIKE, truncate_to="day") -> datetime.datetime:
|
|
285
|
+
dttm = _ensure_datetime(dttm)
|
|
286
|
+
if truncate_to in _PERIODS:
|
|
287
|
+
return dttm.replace(**_PERIODS[truncate_to])
|
|
288
|
+
|
|
289
|
+
if truncate_to not in _ODD_PERIODS:
|
|
290
|
+
raise ValueError(
|
|
291
|
+
"truncate_to not valid. Valid periods: {}".format(
|
|
292
|
+
", ".join(list(_PERIODS.keys()) + list(_ODD_PERIODS.keys()))
|
|
293
|
+
)
|
|
294
|
+
)
|
|
295
|
+
return _ODD_PERIODS[truncate_to](dttm)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@register_func
|
|
299
|
+
def date_add(dttm: _DATE_OR_DATETIME, days: int) -> _DATE_OR_DATETIME:
|
|
300
|
+
"""Add a specified number of days to a datetime
|
|
301
|
+
|
|
302
|
+
>>> date_add(datetime.datetime(2022, 10, 8), 6)
|
|
303
|
+
datetime.datetime(2022, 10, 14, 0, 0)
|
|
304
|
+
>>> date_add(datetime.datetime(2022, 10, 8, 10), 6)
|
|
305
|
+
datetime.datetime(2022, 10, 14, 10, 0)
|
|
306
|
+
>>> date_add(datetime.datetime(2022, 10, 8), -6)
|
|
307
|
+
datetime.datetime(2022, 10, 2, 0, 0)
|
|
308
|
+
"""
|
|
309
|
+
return dttm + datetime.timedelta(days=days)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@register_func
|
|
313
|
+
def month_start(dttm: _DATELIKE) -> datetime.datetime:
|
|
314
|
+
"""Get the first day of month, equivalent to `truncate_month`
|
|
315
|
+
|
|
316
|
+
>>> month_start(datetime.datetime(2022, 10, 8))
|
|
317
|
+
datetime.datetime(2022, 10, 1, 0, 0)
|
|
318
|
+
"""
|
|
319
|
+
return truncate_month(dttm)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@register_func
|
|
323
|
+
def month_end(dttm: _DATELIKE) -> datetime.datetime:
|
|
324
|
+
"""Get the last day of month
|
|
325
|
+
|
|
326
|
+
>>> month_end(datetime.datetime(2022, 10, 8))
|
|
327
|
+
datetime.datetime(2022, 10, 31, 0, 0)
|
|
328
|
+
"""
|
|
329
|
+
dt = to_pendulum(dttm).last_of("month")
|
|
330
|
+
return datetime.datetime(dt.year, dt.month, dt.day)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _get_last_month(dttm: _DATELIKE) -> datetime.datetime:
|
|
334
|
+
return month_start(dttm) - datetime.timedelta(days=1)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def last_month_start(dttm: _DATELIKE) -> datetime.datetime:
|
|
338
|
+
"""Get the first day of last month
|
|
339
|
+
|
|
340
|
+
>>> last_month_start(datetime.datetime(2022, 10, 8))
|
|
341
|
+
datetime.datetime(2022, 9, 1, 0, 0)
|
|
342
|
+
"""
|
|
343
|
+
return month_start(_get_last_month(dttm))
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def last_month_end(dttm: _DATELIKE) -> datetime.datetime:
|
|
347
|
+
"""Get the last day of last month
|
|
348
|
+
|
|
349
|
+
>>> last_month_end(datetime.datetime(2022, 10, 8))
|
|
350
|
+
datetime.datetime(2022, 9, 30, 0, 0)
|
|
351
|
+
"""
|
|
352
|
+
return month_start(dttm) - datetime.timedelta(days=1)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _get_last_week(dttm: _DATELIKE) -> datetime.datetime:
|
|
356
|
+
return truncate_week(dttm) - datetime.timedelta(days=7)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def last_week_start(dttm: _DATELIKE) -> datetime.datetime:
|
|
360
|
+
"""Get the first day (Monday) of last week
|
|
361
|
+
|
|
362
|
+
>>> last_week_start(datetime.datetime(2022, 10, 8))
|
|
363
|
+
datetime.datetime(2022, 9, 26, 0, 0)
|
|
364
|
+
"""
|
|
365
|
+
return truncate_week(dttm) - datetime.timedelta(days=7)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def last_week_end(dttm: _DATELIKE) -> datetime.datetime:
|
|
369
|
+
"""Get the first day (Sunday) of last week
|
|
370
|
+
|
|
371
|
+
>>> last_week_end(datetime.datetime(2022, 10, 8))
|
|
372
|
+
datetime.datetime(2022, 10, 2, 0, 0)
|
|
373
|
+
"""
|
|
374
|
+
return truncate_week(dttm) - datetime.timedelta(days=1)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def round_time_resolution(dt: datetime.datetime, cron_spec: str) -> datetime.datetime:
|
|
378
|
+
"""Truncate a datetime value according to crontab spec, infer the time interval and return the value of `truncate_x`
|
|
379
|
+
|
|
380
|
+
>>> dt = datetime.datetime(2022, 10, 8, 10, 23, 45)
|
|
381
|
+
>>> round_time_resolution(dt, '15 * * * *') # hourly, equivalent to `truncate_hour`
|
|
382
|
+
datetime.datetime(2022, 10, 8, 10, 0)
|
|
383
|
+
>>> round_time_resolution(dt, '0 4 * * *') # daily, equivalent to `truncate_day`
|
|
384
|
+
datetime.datetime(2022, 10, 8, 0, 0)
|
|
385
|
+
>>> round_time_resolution(dt, '0 17 * * TUE') # weekly, equivalent to `truncate_week`
|
|
386
|
+
datetime.datetime(2022, 10, 3, 0, 0)
|
|
387
|
+
>>> round_time_resolution(dt, '0 13 12 * *') # monthly, equivalent to `truncate_month`
|
|
388
|
+
datetime.datetime(2022, 10, 1, 0, 0)
|
|
389
|
+
"""
|
|
390
|
+
if not cron_spec or cron_spec == "@once":
|
|
391
|
+
return dt
|
|
392
|
+
|
|
393
|
+
# 这种 schedule interval 会自动处理成整点/自然日/周一/月初
|
|
394
|
+
if cron_spec in ["@hourly", "@daily", "@weekly", "@monthly"]:
|
|
395
|
+
return dt
|
|
396
|
+
|
|
397
|
+
dt = dt.replace(second=0, microsecond=0)
|
|
398
|
+
|
|
399
|
+
# 根据两个 schedule 日期的时间差来反推
|
|
400
|
+
cron = croniter.croniter(cron_spec, start_time=datetime.datetime.now())
|
|
401
|
+
prev_run = cron.get_prev(datetime.datetime)
|
|
402
|
+
next_run = cron.get_next(datetime.datetime)
|
|
403
|
+
interval = next_run - prev_run
|
|
404
|
+
|
|
405
|
+
if interval < datetime.timedelta(hours=1):
|
|
406
|
+
return dt
|
|
407
|
+
|
|
408
|
+
# 每隔 N 个小时(一天内)
|
|
409
|
+
if interval < datetime.timedelta(days=1):
|
|
410
|
+
return truncate_hour(dt)
|
|
411
|
+
|
|
412
|
+
# 每天
|
|
413
|
+
if interval == datetime.timedelta(days=1):
|
|
414
|
+
return truncate_day(dt)
|
|
415
|
+
|
|
416
|
+
# 每周
|
|
417
|
+
if interval == datetime.timedelta(days=7):
|
|
418
|
+
return truncate_week(dt)
|
|
419
|
+
|
|
420
|
+
# 每月,粗暴的把相隔天数是 28~31 当作一个月
|
|
421
|
+
if 28 <= interval.days <= 31:
|
|
422
|
+
return truncate_month(dt)
|
|
423
|
+
|
|
424
|
+
return dt
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def month_range(start_date: _DATELIKE, end_date: _DATELIKE) -> list[str]:
|
|
428
|
+
"""Get the first day of all months between start_date and end_date
|
|
429
|
+
|
|
430
|
+
>>> month_range('2022-01-02', '2022-05-20')
|
|
431
|
+
['2022-01-01', '2022-02-01', '2022-03-01', '2022-04-01', '2022-05-01']
|
|
432
|
+
"""
|
|
433
|
+
start_date = to_pendulum(start_date).replace(day=1)
|
|
434
|
+
end_date = to_pendulum(end_date).replace(day=1)
|
|
435
|
+
return [x.date().isoformat() for x in pendulum.interval(start_date, end_date).range("months")]
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
@register_func
|
|
439
|
+
def day_range(start_date: _DATELIKE, end_date: _DATELIKE) -> list[str]:
|
|
440
|
+
"""Get all dates between start_date and end_date
|
|
441
|
+
|
|
442
|
+
>>> day_range('2022-01-02', '2022-01-07')
|
|
443
|
+
['2022-01-02', '2022-01-03', '2022-01-04', '2022-01-05', '2022-01-06', '2022-01-07']
|
|
444
|
+
"""
|
|
445
|
+
start_date = to_pendulum(start_date)
|
|
446
|
+
end_date = to_pendulum(end_date)
|
|
447
|
+
return [x.date().isoformat() for x in pendulum.interval(start_date, end_date).range("days")]
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def to_local_datetime(value: str) -> datetime.datetime:
|
|
451
|
+
return astimezone(value, _tz_local)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
# exports as public
|
|
455
|
+
tz_utc = _tz_utc
|
|
456
|
+
tz_local = _tz_local
|
|
457
|
+
ensure_datetime = _ensure_datetime
|
|
458
|
+
ensure_tz = _ensure_tz
|
|
459
|
+
DATELIKE = _DATELIKE
|
|
460
|
+
|
|
461
|
+
if __name__ == "__main__":
|
|
462
|
+
import doctest
|
|
463
|
+
|
|
464
|
+
doctest.testmod()
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# Copyright 2021 Collate
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
|
|
12
|
+
"""
|
|
13
|
+
Helper that implements custom dispatcher logic
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from functools import update_wrapper
|
|
17
|
+
from types import MappingProxyType
|
|
18
|
+
from typing import Any, Callable, NamedTuple, Optional, Type, TypeVar
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel
|
|
21
|
+
|
|
22
|
+
T = TypeVar("T", bound=BaseModel)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Register(NamedTuple):
|
|
26
|
+
add: Callable
|
|
27
|
+
get: Callable
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def enum_register():
|
|
31
|
+
"""
|
|
32
|
+
Helps us register custom function for enum values
|
|
33
|
+
"""
|
|
34
|
+
registry = {}
|
|
35
|
+
|
|
36
|
+
def add(name: str) -> Callable:
|
|
37
|
+
def inner(fn: Callable) -> Callable:
|
|
38
|
+
registry[name] = fn
|
|
39
|
+
return fn
|
|
40
|
+
|
|
41
|
+
return inner
|
|
42
|
+
|
|
43
|
+
def get(name: str) -> Optional[Callable]:
|
|
44
|
+
return registry.get(name, None)
|
|
45
|
+
|
|
46
|
+
return Register(add, get)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def class_register():
|
|
50
|
+
"""
|
|
51
|
+
Helps us register custom functions for classes based on their name
|
|
52
|
+
"""
|
|
53
|
+
registry = {}
|
|
54
|
+
|
|
55
|
+
def add(entity_type: Type[T]):
|
|
56
|
+
def inner(fn):
|
|
57
|
+
_name = entity_type.__name__
|
|
58
|
+
registry[_name] = fn
|
|
59
|
+
return fn
|
|
60
|
+
|
|
61
|
+
return inner
|
|
62
|
+
|
|
63
|
+
def get(entity_type: Type[T]) -> Optional[Callable]:
|
|
64
|
+
return registry.get(entity_type.__name__, None)
|
|
65
|
+
|
|
66
|
+
return Register(add, get)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def valuedispatch(func) -> Callable:
|
|
70
|
+
"""Value dispatch for methods and functions
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
func (_type_): function to run
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
Callable: wrapper
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
registry = {}
|
|
80
|
+
|
|
81
|
+
def _is_valid_dispatch(value):
|
|
82
|
+
return isinstance(value, str)
|
|
83
|
+
|
|
84
|
+
def dispatch(value: str) -> Callable:
|
|
85
|
+
try:
|
|
86
|
+
impl = registry[value]
|
|
87
|
+
except KeyError:
|
|
88
|
+
impl = registry[object]
|
|
89
|
+
return impl
|
|
90
|
+
|
|
91
|
+
def register(value, func=None) -> Callable:
|
|
92
|
+
if _is_valid_dispatch(value):
|
|
93
|
+
if func is None:
|
|
94
|
+
return lambda f: register(value, f)
|
|
95
|
+
else:
|
|
96
|
+
raise TypeError(f"Invalid first argument to register(). {value} is not a string.")
|
|
97
|
+
|
|
98
|
+
registry[value] = func
|
|
99
|
+
return func
|
|
100
|
+
|
|
101
|
+
def wrapper(*args, **kwargs) -> Any:
|
|
102
|
+
if not args:
|
|
103
|
+
raise TypeError(f"{func_name} requires at least 1 argument")
|
|
104
|
+
if isinstance(args[0], (str, bytes)):
|
|
105
|
+
return dispatch(str(args[0]))(*args, **kwargs)
|
|
106
|
+
return dispatch(args[1])(*args, **kwargs)
|
|
107
|
+
|
|
108
|
+
func_name = getattr(func, "__name__", "method value dispatch")
|
|
109
|
+
registry[object] = func
|
|
110
|
+
wrapper.register = register
|
|
111
|
+
wrapper.dispatch = dispatch
|
|
112
|
+
wrapper.registry = MappingProxyType(registry) # making registry read only
|
|
113
|
+
update_wrapper(wrapper, func)
|
|
114
|
+
return wrapper
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any, Union
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
import emails
|
|
8
|
+
except ImportError:
|
|
9
|
+
pass
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
_default_html = """<div><includetail><div style="font:Verdana normal 14px;color:#000;">
|
|
12
|
+
<div style="position:relative;"><blockquote style="margin-Top: 0px; margin-Bottom: 0px; margin-Left: 0.5em">
|
|
13
|
+
<div class="FoxDiv20190108121908737768">
|
|
14
|
+
<div id="mailContentContainer" style=" font-size: 14px; padding: 0px; height: auto; min-height: auto ; ">
|
|
15
|
+
{content}
|
|
16
|
+
</div>
|
|
17
|
+
</div></blockquote>
|
|
18
|
+
</div></div>"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def send_email(
|
|
22
|
+
mail_to: Union[str, list[str], tuple[str, ...]],
|
|
23
|
+
subject: str,
|
|
24
|
+
html: str = None,
|
|
25
|
+
content: str = None,
|
|
26
|
+
cc: Union[str, list[str], tuple[str, ...]] = None,
|
|
27
|
+
bcc: Union[str, list[str], tuple[str, ...]] = None,
|
|
28
|
+
files: Union[str, list[str], tuple[str, ...], dict[str, str]] = None,
|
|
29
|
+
mail_from: str = "noreply",
|
|
30
|
+
reply_to: str = None,
|
|
31
|
+
smtp_config: dict[str, Any] = None,
|
|
32
|
+
) -> bool:
|
|
33
|
+
"""
|
|
34
|
+
Sends an email.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
mail_to: The recipient of the email. Example: 'e1@example.com' or ['e1@example.com', 'e2@example.com'].
|
|
38
|
+
subject: The subject of the email.
|
|
39
|
+
html: The content of the email with special requirements such as font or background color.
|
|
40
|
+
content: The content of the email in plain text format.
|
|
41
|
+
cc: The CC recipients.
|
|
42
|
+
bcc: The BCC recipients.
|
|
43
|
+
files: The list of attachments. Example:
|
|
44
|
+
'/data/tmp.txt' or ['/data/tmp_1.txt', '/data/tmp_2.txt'] or
|
|
45
|
+
{'category_data.txt':'/data/tmp_1.txt', 'brand_data.txt':'/data/tmp_2.txt'}.
|
|
46
|
+
mail_from: The displayed sender of the email. Default is 'RecurveData SERVICE'.
|
|
47
|
+
reply_to: The default recipient when replying. Default is 'itservice@recurvedata.com'.
|
|
48
|
+
smtp_config: The SMTP server for sending the email.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
True if the email was sent successfully, False otherwise.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
if not any((html, content)):
|
|
55
|
+
raise ValueError("At least one of HTML and content is not empty!")
|
|
56
|
+
|
|
57
|
+
if isinstance(files, (list, tuple)):
|
|
58
|
+
attach_files = [(file, os.path.basename(file)) for file in files]
|
|
59
|
+
elif isinstance(files, dict):
|
|
60
|
+
attach_files = [(files[file_name], file_name) for file_name in files]
|
|
61
|
+
elif isinstance(files, str):
|
|
62
|
+
attach_files = [(files, os.path.basename(files))]
|
|
63
|
+
elif not files:
|
|
64
|
+
attach_files = []
|
|
65
|
+
else:
|
|
66
|
+
raise ValueError("The parameter files is only support list、dict or string")
|
|
67
|
+
|
|
68
|
+
for file_path, _ in attach_files:
|
|
69
|
+
if not os.path.exists(file_path):
|
|
70
|
+
raise ValueError(f"The attachment file does not exist! --- {file_path} ")
|
|
71
|
+
if os.path.isdir(file_path):
|
|
72
|
+
raise ValueError(f"Send directory are not supported, please send after compression!--- {file_path}")
|
|
73
|
+
|
|
74
|
+
if not html and content:
|
|
75
|
+
html_content = ""
|
|
76
|
+
for line in content.split("\n"):
|
|
77
|
+
line = line.replace("\t", " " * 4)
|
|
78
|
+
if line:
|
|
79
|
+
result = re.match(r"\s+", line)
|
|
80
|
+
space = result.group(0) if result else ""
|
|
81
|
+
html_content += f'<div style="margin-Left: {len(space)}em">{line.strip()}</div>'
|
|
82
|
+
else:
|
|
83
|
+
html_content += "<div><br></div>"
|
|
84
|
+
html = _default_html.format(content=html_content)
|
|
85
|
+
message = emails.Message(
|
|
86
|
+
subject=subject,
|
|
87
|
+
cc=cc,
|
|
88
|
+
bcc=bcc,
|
|
89
|
+
text="Build passed: {{ project_name }} ...",
|
|
90
|
+
mail_from=(mail_from, smtp_config.get("user")),
|
|
91
|
+
html=html,
|
|
92
|
+
headers={"reply-to": reply_to},
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
for file, file_name in attach_files:
|
|
96
|
+
message.attach(data=open(file, "rb"), filename=file_name)
|
|
97
|
+
response = message.send(to=mail_to, smtp=smtp_config)
|
|
98
|
+
if response.status_code == 250:
|
|
99
|
+
return True
|
|
100
|
+
logger.error(
|
|
101
|
+
f"send email from {message.mail_from} to {mail_to}, "
|
|
102
|
+
f"status_code:{response.status_code}, error_msg:{response._exc}"
|
|
103
|
+
)
|
|
104
|
+
return False
|