fivetran-connector-sdk 1.5.1__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivetran_connector_sdk/__init__.py +47 -55
- fivetran_connector_sdk/connector_helper.py +148 -55
- fivetran_connector_sdk/helpers.py +28 -13
- fivetran_connector_sdk/operations.py +61 -49
- {fivetran_connector_sdk-1.5.1.dist-info → fivetran_connector_sdk-1.7.0.dist-info}/METADATA +2 -2
- {fivetran_connector_sdk-1.5.1.dist-info → fivetran_connector_sdk-1.7.0.dist-info}/RECORD +9 -9
- {fivetran_connector_sdk-1.5.1.dist-info → fivetran_connector_sdk-1.7.0.dist-info}/WHEEL +0 -0
- {fivetran_connector_sdk-1.5.1.dist-info → fivetran_connector_sdk-1.7.0.dist-info}/entry_points.txt +0 -0
- {fivetran_connector_sdk-1.5.1.dist-info → fivetran_connector_sdk-1.7.0.dist-info}/top_level.txt +0 -0
@@ -19,27 +19,27 @@ from fivetran_connector_sdk.operations import Operations
|
|
19
19
|
from fivetran_connector_sdk import constants
|
20
20
|
from fivetran_connector_sdk.constants import (
|
21
21
|
TESTER_VER, VERSION_FILENAME, UTF_8,
|
22
|
-
VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS,
|
22
|
+
VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, TABLES
|
23
23
|
)
|
24
24
|
from fivetran_connector_sdk.helpers import (
|
25
|
-
print_library_log, reset_local_file_directory, find_connector_object,
|
26
|
-
validate_and_load_state, get_input_from_cli, suggest_correct_command,
|
25
|
+
print_library_log, reset_local_file_directory, find_connector_object, suggest_correct_command,
|
27
26
|
)
|
28
27
|
from fivetran_connector_sdk.connector_helper import (
|
29
28
|
validate_requirements_file, upload_project,
|
30
|
-
update_connection, are_setup_tests_failing,
|
31
|
-
validate_deploy_parameters, get_connection_id,
|
29
|
+
update_connection, are_setup_tests_failing, get_connection_id,
|
32
30
|
handle_failing_tests_message_and_exit, delete_file_if_exists,
|
33
31
|
create_connection, get_os_arch_suffix, get_group_info,
|
34
32
|
java_exe_helper, run_tester, process_tables,
|
35
33
|
update_base_url_if_required, exit_check,
|
36
34
|
get_available_port, tester_root_dir_helper,
|
37
35
|
check_dict, check_newer_version, cleanup_uploaded_project,
|
36
|
+
get_destination_group, get_connection_name, get_api_key, get_state,
|
37
|
+
get_python_version, get_hd_agent_id, get_configuration
|
38
38
|
)
|
39
39
|
|
40
40
|
# Version format: <major_version>.<minor_version>.<patch_version>
|
41
41
|
# (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
|
42
|
-
__version__ = "1.
|
42
|
+
__version__ = "1.7.0"
|
43
43
|
TESTER_VERSION = TESTER_VER
|
44
44
|
MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
|
45
45
|
|
@@ -62,7 +62,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
62
62
|
update_base_url_if_required()
|
63
63
|
|
64
64
|
# Call this method to deploy the connector to Fivetran platform
|
65
|
-
def deploy(self,
|
65
|
+
def deploy(self, project_path: str, deploy_key: str, group: str, connection: str, hd_agent_id: str,
|
66
|
+
configuration: dict = None, config_path = None, python_version: str = None, force: bool = False):
|
66
67
|
"""Deploys the connector to the Fivetran platform.
|
67
68
|
|
68
69
|
Args:
|
@@ -77,8 +78,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
77
78
|
|
78
79
|
print_library_log("We support only `.py` files and a `requirements.txt` file as part of the code upload. *No other code files* are supported or uploaded during the deployment process. Ensure that your code is structured accordingly and all dependencies are listed in `requirements.txt`")
|
79
80
|
|
80
|
-
validate_deploy_parameters(connection, deploy_key)
|
81
|
-
|
82
81
|
check_dict(configuration, True)
|
83
82
|
|
84
83
|
secrets_list = []
|
@@ -92,10 +91,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
92
91
|
"sync_method": "DIRECT"
|
93
92
|
}
|
94
93
|
|
95
|
-
if
|
96
|
-
connection_config["python_version"] =
|
94
|
+
if python_version:
|
95
|
+
connection_config["python_version"] = python_version
|
97
96
|
|
98
|
-
validate_requirements_file(
|
97
|
+
validate_requirements_file(project_path, True, __version__, force)
|
99
98
|
|
100
99
|
group_id, group_name = get_group_info(group, deploy_key)
|
101
100
|
connection_id, service = get_connection_id(connection, group, group_id, deploy_key) or (None, None)
|
@@ -104,23 +103,23 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
104
103
|
if service != 'connector_sdk':
|
105
104
|
print_library_log(
|
106
105
|
f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
|
107
|
-
|
106
|
+
sys.exit(1)
|
108
107
|
else:
|
109
|
-
if
|
108
|
+
if force:
|
110
109
|
confirm = "y"
|
111
|
-
if
|
110
|
+
if configuration:
|
112
111
|
confirm_config = "y"
|
113
112
|
else:
|
114
113
|
confirm = input(
|
115
|
-
f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code. Do you want to proceed with the update? (
|
116
|
-
if confirm.lower() == "y" and
|
117
|
-
confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{
|
114
|
+
f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code. Do you want to proceed with the update? (y/N): ")
|
115
|
+
if confirm.lower() == "y" and configuration:
|
116
|
+
confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{config_path}': key-value pairs not present in the new configuration will be removed; existing keys' values set in the configuration file or in the dashboard will be overwritten with new (empty or non-empty) values; new key-value pairs will be added. Do you want to proceed with the update? (y/N): ")
|
118
117
|
if confirm.lower() == "y" and (not connection_config["secrets_list"] or (confirm_config.lower() == "y")):
|
119
118
|
print_library_log("Updating the connection...\n")
|
120
119
|
upload_project(
|
121
|
-
|
120
|
+
project_path, deploy_key, group_id, group_name, connection)
|
122
121
|
response = update_connection(
|
123
|
-
|
122
|
+
connection_id, connection, group_name, connection_config, deploy_key, hd_agent_id)
|
124
123
|
print("✓")
|
125
124
|
print_library_log(f"Python version {response.json()['data']['config']['python_version']} to be used at runtime.",
|
126
125
|
Logging.Level.INFO)
|
@@ -129,9 +128,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
129
128
|
f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
|
130
129
|
else:
|
131
130
|
print_library_log("Update canceled. The process is now terminating.")
|
132
|
-
|
131
|
+
sys.exit(1)
|
133
132
|
else:
|
134
|
-
upload_project(
|
133
|
+
upload_project(project_path, deploy_key,
|
135
134
|
group_id, group_name, connection)
|
136
135
|
response = create_connection(
|
137
136
|
deploy_key, group_id, connection_config, hd_agent_id)
|
@@ -152,7 +151,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
152
151
|
f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
|
153
152
|
cleanup_uploaded_project(deploy_key,group_id, connection)
|
154
153
|
print_library_log("Please try again with the deploy command after resolving the issue!")
|
155
|
-
|
154
|
+
sys.exit(1)
|
156
155
|
|
157
156
|
# Call this method to run the connector in production
|
158
157
|
def run(self,
|
@@ -377,6 +376,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
377
376
|
print_library_log(error_message, Logging.Level.SEVERE)
|
378
377
|
raise RuntimeError(error_message) from e
|
379
378
|
|
379
|
+
def print_version():
|
380
|
+
print_library_log("fivetran_connector_sdk " + __version__)
|
381
|
+
sys.exit(0)
|
380
382
|
|
381
383
|
def main():
|
382
384
|
"""The main entry point for the script.
|
@@ -389,7 +391,8 @@ def main():
|
|
389
391
|
parser._option_string_actions["-h"].help = "Show this help message and exit"
|
390
392
|
|
391
393
|
# Positional
|
392
|
-
parser.add_argument("
|
394
|
+
parser.add_argument("--version", action="store_true", help="Print the version of the fivetran_connector_sdk and exit")
|
395
|
+
parser.add_argument("command", nargs="?", help="|".join(VALID_COMMANDS))
|
393
396
|
parser.add_argument("project_path", nargs='?', default=os.getcwd(), help="Path to connector project directory")
|
394
397
|
|
395
398
|
# Optional (Not all of these are valid with every mutually exclusive option below)
|
@@ -401,52 +404,41 @@ def main():
|
|
401
404
|
parser.add_argument("-f", "--force", action="store_true", help="Force update an existing connection")
|
402
405
|
parser.add_argument("--python-version", "--python", type=str, help=f"Supported Python versions you can use: {SUPPORTED_PYTHON_VERSIONS}. Defaults to {DEFAULT_PYTHON_VERSION}")
|
403
406
|
parser.add_argument("--hybrid-deployment-agent-id", type=str, help="The Hybrid Deployment agent within the Fivetran system. If nothing is passed, the default agent of the destination is used.")
|
404
|
-
|
405
407
|
args = parser.parse_args()
|
406
408
|
|
409
|
+
if args.version:
|
410
|
+
print_version()
|
411
|
+
|
412
|
+
if not args.command:
|
413
|
+
parser.print_help()
|
414
|
+
sys.exit(1)
|
415
|
+
|
407
416
|
if args.command.lower() == "version":
|
408
|
-
|
409
|
-
return
|
417
|
+
print_version()
|
410
418
|
elif args.command.lower() == "reset":
|
411
419
|
reset_local_file_directory(args)
|
412
|
-
|
420
|
+
sys.exit(0)
|
413
421
|
|
414
422
|
connector_object = find_connector_object(args.project_path)
|
415
423
|
|
416
424
|
if not connector_object:
|
417
425
|
sys.exit(1)
|
418
426
|
|
419
|
-
# Process optional args
|
420
|
-
ft_group = args.destination if args.destination else None
|
421
|
-
ft_connection = args.connection if args.connection else None
|
422
|
-
ft_deploy_key = args.api_key if args.api_key else None
|
423
|
-
hd_agent_id = args.hybrid_deployment_agent_id if args.hybrid_deployment_agent_id else os.getenv(FIVETRAN_HD_AGENT_ID, None)
|
424
|
-
configuration = args.configuration if args.configuration else None
|
425
|
-
state = args.state if args.state else os.getenv('FIVETRAN_STATE', None)
|
426
|
-
|
427
|
-
configuration = validate_and_load_configuration(args, configuration)
|
428
|
-
state = validate_and_load_state(args, state)
|
429
|
-
|
430
|
-
FIVETRAN_BASE_64_ENCODED_API_KEY = os.getenv('FIVETRAN_BASE_64_ENCODED_API_KEY', None)
|
431
|
-
FIVETRAN_DESTINATION_NAME = os.getenv('FIVETRAN_DESTINATION_NAME', None)
|
432
|
-
FIVETRAN_CONNECTION_NAME = os.getenv('FIVETRAN_CONNECTION_NAME', None)
|
433
|
-
|
434
427
|
if args.command.lower() == "deploy":
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
ft_group = get_input_from_cli("Please provide the destination", FIVETRAN_DESTINATION_NAME)
|
443
|
-
|
444
|
-
if not ft_connection:
|
445
|
-
ft_connection = get_input_from_cli("Please provide the connection name",FIVETRAN_CONNECTION_NAME)
|
428
|
+
ft_group = get_destination_group(args)
|
429
|
+
ft_connection = get_connection_name(args)
|
430
|
+
ft_deploy_key = get_api_key(args)
|
431
|
+
python_version = get_python_version(args)
|
432
|
+
hd_agent_id = get_hd_agent_id(args)
|
433
|
+
configuration, config_path = get_configuration(args)
|
434
|
+
get_state(args)
|
446
435
|
|
447
|
-
connector_object.deploy(args, ft_deploy_key, ft_group, ft_connection, hd_agent_id,
|
436
|
+
connector_object.deploy(args.project_path, ft_deploy_key, ft_group, ft_connection, hd_agent_id,
|
437
|
+
configuration, config_path, python_version, args.force)
|
448
438
|
|
449
439
|
elif args.command.lower() == "debug":
|
440
|
+
configuration, config_path = get_configuration(args)
|
441
|
+
state = get_state(args)
|
450
442
|
connector_object.debug(args.project_path, configuration, state)
|
451
443
|
else:
|
452
444
|
if not suggest_correct_command(args.command):
|
@@ -2,6 +2,7 @@ import os
|
|
2
2
|
import re
|
3
3
|
import ast
|
4
4
|
import json
|
5
|
+
import sys
|
5
6
|
import time
|
6
7
|
import socket
|
7
8
|
import platform
|
@@ -17,9 +18,10 @@ from fivetran_connector_sdk.protos import common_pb2
|
|
17
18
|
from fivetran_connector_sdk import constants
|
18
19
|
from fivetran_connector_sdk.logger import Logging
|
19
20
|
from fivetran_connector_sdk.helpers import (
|
20
|
-
print_library_log,
|
21
|
-
|
22
|
-
|
21
|
+
print_library_log, get_renamed_table_name,
|
22
|
+
get_renamed_column_name, get_input_from_cli,
|
23
|
+
validate_and_load_state,
|
24
|
+
validate_and_load_configuration
|
23
25
|
)
|
24
26
|
from fivetran_connector_sdk.constants import (
|
25
27
|
OS_MAP,
|
@@ -48,6 +50,97 @@ from fivetran_connector_sdk.constants import (
|
|
48
50
|
TABLES,
|
49
51
|
)
|
50
52
|
|
53
|
+
def get_destination_group(args):
|
54
|
+
ft_group = args.destination if args.destination else None
|
55
|
+
env_destination_name = os.getenv('FIVETRAN_DESTINATION_NAME', None)
|
56
|
+
if not ft_group:
|
57
|
+
ft_group = get_input_from_cli("Provide the destination name (as displayed in your dashboard destination list)", env_destination_name)
|
58
|
+
return ft_group
|
59
|
+
|
60
|
+
def get_connection_name(args):
|
61
|
+
ft_connection = args.connection if args.connection else None
|
62
|
+
env_connection_name = os.getenv('FIVETRAN_CONNECTION_NAME', None)
|
63
|
+
if not ft_connection:
|
64
|
+
for retrying in range(MAX_RETRIES):
|
65
|
+
ft_connection = get_input_from_cli("Provide the connection name", env_connection_name)
|
66
|
+
if not is_connection_name_valid(ft_connection):
|
67
|
+
if retrying==MAX_RETRIES-1:
|
68
|
+
sys.exit(1)
|
69
|
+
else:
|
70
|
+
print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
|
71
|
+
f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
|
72
|
+
f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
|
73
|
+
print_library_log("Please retry...", Logging.Level.INFO)
|
74
|
+
else:
|
75
|
+
break
|
76
|
+
if not is_connection_name_valid(ft_connection):
|
77
|
+
print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
|
78
|
+
f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
|
79
|
+
f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
|
80
|
+
sys.exit(1)
|
81
|
+
return ft_connection
|
82
|
+
|
83
|
+
def get_api_key(args):
|
84
|
+
ft_deploy_key = args.api_key if args.api_key else None
|
85
|
+
env_api_key = os.getenv('FIVETRAN_API_KEY', None)
|
86
|
+
if not ft_deploy_key:
|
87
|
+
ft_deploy_key = get_input_from_cli("Provide your API Key (Base 64 Encoded)", env_api_key, True)
|
88
|
+
return ft_deploy_key
|
89
|
+
|
90
|
+
def get_python_version(args):
|
91
|
+
python_version = args.python_version if args.python_version else None
|
92
|
+
env_python_version = os.getenv('FIVETRAN_PYTHON_VERSION', None)
|
93
|
+
if env_python_version and not python_version and not args.force:
|
94
|
+
python_version = get_input_from_cli("Provide your python version", env_python_version)
|
95
|
+
return python_version
|
96
|
+
|
97
|
+
def get_hd_agent_id(args):
|
98
|
+
hd_agent_id = args.hybrid_deployment_agent_id if args.hybrid_deployment_agent_id else None
|
99
|
+
env_hd_agent_id = os.getenv('FIVETRAN_HD_AGENT_ID', None)
|
100
|
+
|
101
|
+
if env_hd_agent_id and not hd_agent_id and not args.force:
|
102
|
+
hd_agent_id = get_input_from_cli("Provide the Hybrid Deployment Agent ID", env_hd_agent_id)
|
103
|
+
return hd_agent_id
|
104
|
+
|
105
|
+
def get_state(args):
|
106
|
+
if args.command.lower() == "deploy" and args.state:
|
107
|
+
print_library_log("Unrecognised argument for deploy: --state."
|
108
|
+
"'state' is not set using the 'deploy' command. You can manage the state for deployed connections via "
|
109
|
+
"Fivetran API, https://fivetran.com/docs/connector-sdk/working-with-connector-sdk#workingwithstatejsonfile",
|
110
|
+
Logging.Level.WARNING)
|
111
|
+
sys.exit(1)
|
112
|
+
state = args.state if args.state else os.getenv('FIVETRAN_STATE', None)
|
113
|
+
state = validate_and_load_state(args, state)
|
114
|
+
return state
|
115
|
+
|
116
|
+
def get_configuration(args):
|
117
|
+
configuration = args.configuration if args.configuration else None
|
118
|
+
env_configuration = os.getenv('FIVETRAN_CONFIGURATION', None)
|
119
|
+
if not configuration and not args.force and args.command.lower() == "deploy":
|
120
|
+
json_filepath = os.path.join(args.project_path, "configuration.json")
|
121
|
+
if os.path.exists(json_filepath):
|
122
|
+
print_library_log("configuration.json file detected in the project, "
|
123
|
+
"but no configuration input provided via the command line", Logging.Level.WARNING)
|
124
|
+
env_configuration = env_configuration if env_configuration else "configuration.json"
|
125
|
+
confirm = input(f"Does this debug run/deploy need configuration (y/N):")
|
126
|
+
if confirm.lower()=='y':
|
127
|
+
for retrying in range(MAX_RETRIES):
|
128
|
+
try:
|
129
|
+
configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
|
130
|
+
config_values = validate_and_load_configuration(args.project_path, configuration)
|
131
|
+
return config_values, configuration
|
132
|
+
except ValueError as e:
|
133
|
+
if retrying==MAX_RETRIES-1:
|
134
|
+
print_library_log(f"{e}. Invalid Configuration, Exiting..", Logging.Level.WARNING)
|
135
|
+
sys.exit(1)
|
136
|
+
else:
|
137
|
+
print_library_log(f"{e}. Please retry..", Logging.Level.INFO)
|
138
|
+
else:
|
139
|
+
print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
|
140
|
+
config_values = validate_and_load_configuration(args.project_path, configuration)
|
141
|
+
return config_values, configuration
|
142
|
+
|
143
|
+
|
51
144
|
def check_newer_version(version: str):
|
52
145
|
"""Periodically checks for a newer version of the SDK and notifies the user if one is available."""
|
53
146
|
tester_root_dir = tester_root_dir_helper()
|
@@ -79,7 +172,7 @@ def check_newer_version(version: str):
|
|
79
172
|
except Exception:
|
80
173
|
retry_after = 2 ** index
|
81
174
|
print_library_log(f"Unable to check if a newer version of `fivetran-connector-sdk` is available. "
|
82
|
-
f"Retrying
|
175
|
+
f"Retrying after {retry_after} seconds", Logging.Level.WARNING)
|
83
176
|
time.sleep(retry_after)
|
84
177
|
|
85
178
|
|
@@ -88,7 +181,7 @@ def tester_root_dir_helper() -> str:
|
|
88
181
|
return os.path.join(os.path.expanduser("~"), ROOT_LOCATION)
|
89
182
|
|
90
183
|
def _warn_exit_usage(filename, line_no, func):
|
91
|
-
print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw
|
184
|
+
print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw an error if required " +
|
92
185
|
f"at: {filename}:{line_no}. See the Technical Reference for details: https://fivetran.com/docs/connector-sdk/technical-reference#handlingexceptions",
|
93
186
|
Logging.Level.WARNING)
|
94
187
|
|
@@ -133,14 +226,14 @@ def check_dict(incoming: dict, string_only: bool = False) -> dict:
|
|
133
226
|
|
134
227
|
if not isinstance(incoming, dict):
|
135
228
|
raise ValueError(
|
136
|
-
"Configuration must be provided as a JSON dictionary.
|
229
|
+
"Configuration must be provided as a JSON dictionary. Check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
|
137
230
|
|
138
231
|
if string_only:
|
139
232
|
for k, v in incoming.items():
|
140
233
|
if not isinstance(v, str):
|
141
234
|
print_library_log(
|
142
|
-
"All values in the configuration must be STRING.
|
143
|
-
|
235
|
+
"All values in the configuration must be STRING. Check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
|
236
|
+
sys.exit(1)
|
144
237
|
|
145
238
|
return incoming
|
146
239
|
|
@@ -159,28 +252,11 @@ def is_connection_name_valid(connection: str):
|
|
159
252
|
|
160
253
|
|
161
254
|
def log_unused_deps_error(package_name: str, version: str):
|
162
|
-
print_library_log(f"
|
255
|
+
print_library_log(f"Remove `{package_name}` from requirements.txt."
|
163
256
|
f" The latest version of `{package_name}` is always available when executing your code."
|
164
257
|
f" Current version: {version}", Logging.Level.SEVERE)
|
165
258
|
|
166
259
|
|
167
|
-
def validate_deploy_parameters(connection, deploy_key):
|
168
|
-
if not deploy_key or not connection:
|
169
|
-
print_library_log("The deploy command needs the following parameters:"
|
170
|
-
"\n\tRequired:\n"
|
171
|
-
"\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
|
172
|
-
"\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
|
173
|
-
"\t(Optional):\n"
|
174
|
-
"\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
|
175
|
-
"\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
|
176
|
-
os._exit(1)
|
177
|
-
elif not is_connection_name_valid(connection):
|
178
|
-
print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
|
179
|
-
f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
|
180
|
-
f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
|
181
|
-
os._exit(1)
|
182
|
-
|
183
|
-
|
184
260
|
def is_port_in_use(port: int):
|
185
261
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
186
262
|
return s.connect_ex(('127.0.0.1', port)) == 0
|
@@ -204,7 +280,7 @@ def update_base_url_if_required():
|
|
204
280
|
print_library_log(f"Updating PRODUCTION_BASE_URL to: {base_url}")
|
205
281
|
|
206
282
|
def fetch_requirements_from_file(file_path: str) -> list[str]:
|
207
|
-
"""Reads
|
283
|
+
"""Reads the requirements file and returns a list of dependencies.
|
208
284
|
|
209
285
|
Args:
|
210
286
|
file_path (str): The path to the requirements file.
|
@@ -222,7 +298,7 @@ def fetch_requirements_as_dict(file_path: str) -> dict:
|
|
222
298
|
file_path (str): The path to the requirements file.
|
223
299
|
|
224
300
|
Returns:
|
225
|
-
dict: A dictionary where keys are package names (
|
301
|
+
dict: A dictionary where keys are package names (lowercase) and
|
226
302
|
values are the full dependency strings.
|
227
303
|
"""
|
228
304
|
requirements_dict = {}
|
@@ -345,8 +421,8 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
|
|
345
421
|
if confirm.lower() == "n":
|
346
422
|
if 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
|
347
423
|
print_library_log(
|
348
|
-
f"
|
349
|
-
|
424
|
+
f"Fix your {REQUIREMENTS_TXT} file by removing pre-installed dependencies [fivetran_connector_sdk, requests] to proceed with the deployment.")
|
425
|
+
sys.exit(1)
|
350
426
|
print_library_log(f"Changes identified for unused libraries have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
|
351
427
|
elif confirm.lower() == "y":
|
352
428
|
update_unused_requirements = True
|
@@ -375,15 +451,15 @@ def handle_unused_deps(unused_deps, version):
|
|
375
451
|
if 'fivetran_connector_sdk' in unused_deps:
|
376
452
|
log_unused_deps_error("fivetran_connector_sdk", version)
|
377
453
|
if 'requests' in unused_deps:
|
378
|
-
log_unused_deps_error("requests", "2.32.
|
454
|
+
log_unused_deps_error("requests", "2.32.4")
|
379
455
|
print_library_log("The following dependencies are not needed, "
|
380
|
-
f"they are
|
456
|
+
f"they are already installed or not in use. Remove them from {REQUIREMENTS_TXT}:", Logging.Level.WARNING)
|
381
457
|
print(*unused_deps)
|
382
458
|
|
383
459
|
def handle_missing_deps(missing_deps):
|
384
|
-
print_library_log(f"
|
460
|
+
print_library_log(f"Include the following dependency libraries in {REQUIREMENTS_TXT}, to be used by "
|
385
461
|
"Fivetran production. "
|
386
|
-
"For more information,
|
462
|
+
"For more information, see our docs: "
|
387
463
|
"https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
|
388
464
|
"#workingwithrequirementstxtfile", Logging.Level.SEVERE)
|
389
465
|
print(*list(missing_deps.values()))
|
@@ -420,15 +496,15 @@ def upload_project(project_path: str, deploy_key: str, group_id: str, group_name
|
|
420
496
|
upload_file_path, deploy_key, group_id, connection)
|
421
497
|
delete_file_if_exists(upload_file_path)
|
422
498
|
if not upload_result:
|
423
|
-
|
499
|
+
sys.exit(1)
|
424
500
|
|
425
501
|
|
426
502
|
def cleanup_uploaded_project(deploy_key: str, group_id: str, connection: str):
|
427
503
|
cleanup_result = cleanup_uploaded_code(deploy_key, group_id, connection)
|
428
504
|
if not cleanup_result:
|
429
|
-
|
505
|
+
sys.exit(1)
|
430
506
|
|
431
|
-
def update_connection(
|
507
|
+
def update_connection(id: str, name: str, group: str, config: dict, deploy_key: str, hd_agent_id: str):
|
432
508
|
"""Updates the connection with the given ID, name, group, configuration, and deployment key.
|
433
509
|
|
434
510
|
Args:
|
@@ -440,7 +516,7 @@ def update_connection(args: dict, id: str, name: str, group: str, config: dict,
|
|
440
516
|
deploy_key (str): The deployment key.
|
441
517
|
hd_agent_id (str): The hybrid deployment agent ID within the Fivetran system.
|
442
518
|
"""
|
443
|
-
if not
|
519
|
+
if not config.get("secrets_list"):
|
444
520
|
del config["secrets_list"]
|
445
521
|
|
446
522
|
json_payload = {
|
@@ -467,7 +543,7 @@ def update_connection(args: dict, id: str, name: str, group: str, config: dict,
|
|
467
543
|
print_library_log(
|
468
544
|
f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{response.json()['message']}'.",
|
469
545
|
Logging.Level.SEVERE)
|
470
|
-
|
546
|
+
sys.exit(1)
|
471
547
|
return response
|
472
548
|
|
473
549
|
def handle_failing_tests_message_and_exit(resp, log_message):
|
@@ -475,8 +551,8 @@ def handle_failing_tests_message_and_exit(resp, log_message):
|
|
475
551
|
print_failing_setup_tests(resp)
|
476
552
|
connection_id = resp.json().get('data', {}).get('id')
|
477
553
|
print_library_log(f"Connection ID: {connection_id}")
|
478
|
-
print_library_log("
|
479
|
-
|
554
|
+
print_library_log("Try again with the deploy command after resolving the issue!")
|
555
|
+
sys.exit(1)
|
480
556
|
|
481
557
|
def are_setup_tests_failing(response) -> bool:
|
482
558
|
"""Checks for failed setup tests in the response and returns True if any test has failed, otherwise False."""
|
@@ -496,7 +572,7 @@ def print_failing_setup_tests(response):
|
|
496
572
|
test.get("status") == "FAILED" or test.get("status") == "JOB_FAILED"]
|
497
573
|
|
498
574
|
if failed_tests:
|
499
|
-
print_library_log("
|
575
|
+
print_library_log("The following setup tests have failed!", Logging.Level.WARNING)
|
500
576
|
for test in failed_tests:
|
501
577
|
print_library_log(f"Test: {test.get('title')}", Logging.Level.WARNING)
|
502
578
|
print_library_log(f"Status: {test.get('status')}", Logging.Level.WARNING)
|
@@ -520,7 +596,7 @@ def get_connection_id(name: str, group: str, group_id: str, deploy_key: str) ->
|
|
520
596
|
if not resp.ok:
|
521
597
|
print_library_log(
|
522
598
|
f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
|
523
|
-
|
599
|
+
sys.exit(1)
|
524
600
|
|
525
601
|
if resp.json()['data']['items']:
|
526
602
|
return resp.json()['data']['items'][0]['id'], resp.json()['data']['items'][0]['service']
|
@@ -597,13 +673,13 @@ def zip_folder(project_path: str) -> str:
|
|
597
673
|
|
598
674
|
if not connector_file_exists:
|
599
675
|
print_library_log(
|
600
|
-
"The 'connector.py' file is missing.
|
676
|
+
"The 'connector.py' file is missing. Ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase. All custom connectors require this file because Fivetran calls it to start a sync.",
|
601
677
|
Logging.Level.SEVERE)
|
602
|
-
|
678
|
+
sys.exit(1)
|
603
679
|
|
604
680
|
if custom_drivers_exists and not custom_driver_installation_script_exists:
|
605
681
|
print_library_log(INSTALLATION_SCRIPT_MISSING_MESSAGE, Logging.Level.SEVERE)
|
606
|
-
|
682
|
+
sys.exit(1)
|
607
683
|
|
608
684
|
return upload_filepath
|
609
685
|
|
@@ -717,16 +793,16 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
|
|
717
793
|
|
718
794
|
if not resp.ok:
|
719
795
|
print_library_log(
|
720
|
-
f"The request failed with status code: {resp.status_code}.
|
796
|
+
f"The request failed with status code: {resp.status_code}. Ensure you're using a valid base64-encoded API key and try again.",
|
721
797
|
Logging.Level.SEVERE)
|
722
|
-
|
798
|
+
sys.exit(1)
|
723
799
|
|
724
800
|
data = resp.json().get("data", {})
|
725
801
|
groups = data.get("items")
|
726
802
|
|
727
803
|
if not groups:
|
728
804
|
print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
|
729
|
-
|
805
|
+
sys.exit(1)
|
730
806
|
|
731
807
|
if not group:
|
732
808
|
if len(groups) == 1:
|
@@ -735,7 +811,7 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
|
|
735
811
|
print_library_log(
|
736
812
|
"Destination name is required when there are multiple destinations in the account",
|
737
813
|
Logging.Level.SEVERE)
|
738
|
-
|
814
|
+
sys.exit(1)
|
739
815
|
else:
|
740
816
|
while True:
|
741
817
|
for grp in groups:
|
@@ -752,8 +828,8 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
|
|
752
828
|
groups = data.get("items", [])
|
753
829
|
|
754
830
|
print_library_log(
|
755
|
-
f"
|
756
|
-
|
831
|
+
f"We couldn't find the specified destination '{group}' in your account.", Logging.Level.SEVERE)
|
832
|
+
sys.exit(1)
|
757
833
|
|
758
834
|
def java_exe_helper(location: str, os_arch_suffix: str) -> str:
|
759
835
|
"""Returns the path to the Java executable.
|
@@ -878,13 +954,22 @@ def process_columns(columns, entry):
|
|
878
954
|
|
879
955
|
elif isinstance(type, dict):
|
880
956
|
if type['type'].upper() != "DECIMAL":
|
881
|
-
|
957
|
+
error_message = (
|
958
|
+
f"Expecting DECIMAL data type for dictionary column entry, but got: {type['type']} in entry: {entry} "
|
959
|
+
f"for column: {column_name}. "
|
960
|
+
"Dictionary type is only allowed for DECIMAL columns with 'precision' and 'scale' fields, "
|
961
|
+
"as in: {'type': 'DECIMAL', 'precision': <int>, 'scale': <int>}. "
|
962
|
+
"For all other data types, use a string as the column type."
|
963
|
+
)
|
964
|
+
raise ValueError(error_message)
|
882
965
|
column.type = common_pb2.DataType.DECIMAL
|
883
966
|
column.decimal.precision = type['precision']
|
884
967
|
column.decimal.scale = type['scale']
|
885
968
|
|
886
969
|
else:
|
887
|
-
raise ValueError(
|
970
|
+
raise ValueError(
|
971
|
+
f"Unrecognized column type for column: {column_name} in entry: {entry}. Got: {str(type)}"
|
972
|
+
)
|
888
973
|
|
889
974
|
if "primary_key" in entry and name in entry["primary_key"]:
|
890
975
|
column.primary_key = True
|
@@ -901,7 +986,15 @@ def process_data_type(column, type):
|
|
901
986
|
elif type.upper() == "LONG":
|
902
987
|
column.type = common_pb2.DataType.LONG
|
903
988
|
elif type.upper() == "DECIMAL":
|
904
|
-
raise ValueError(
|
989
|
+
raise ValueError(
|
990
|
+
"DECIMAL data type missing precision and scale. "
|
991
|
+
"Use a dictionary for DECIMAL column type like: "
|
992
|
+
'''"col_name": { # Decimal data type with precision and scale.\n'''
|
993
|
+
''' "type": "DECIMAL",\n'''
|
994
|
+
''' "precision": 15,\n'''
|
995
|
+
''' "scale": 2\n'''
|
996
|
+
'''}'''
|
997
|
+
)
|
905
998
|
elif type.upper() == "FLOAT":
|
906
999
|
column.type = common_pb2.DataType.FLOAT
|
907
1000
|
elif type.upper() == "DOUBLE":
|
@@ -29,19 +29,23 @@ from fivetran_connector_sdk.constants import (
|
|
29
29
|
RENAMED_TABLE_NAMES = {}
|
30
30
|
RENAMED_COL_NAMES = {}
|
31
31
|
|
32
|
-
def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
|
32
|
+
def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO, dev_log: bool = False):
|
33
33
|
"""Logs a library message with the specified logging level.
|
34
34
|
|
35
35
|
Args:
|
36
36
|
level (Logging.Level): The logging level.
|
37
37
|
message (str): The message to log.
|
38
|
+
dev_log (bool): Boolean value to check if it is dev log and shouldn't be visible to customer
|
38
39
|
"""
|
39
40
|
if constants.DEBUGGING or constants.EXECUTED_VIA_CLI:
|
41
|
+
if dev_log:
|
42
|
+
return
|
40
43
|
current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
|
41
44
|
print(f"{Logging.get_color(level)}{current_time} {level.name} {LOGGING_PREFIX}: {message} {Logging.reset_color()}")
|
42
45
|
else:
|
46
|
+
message_origin = "library_dev" if dev_log else "library"
|
43
47
|
escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
|
44
|
-
log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "
|
48
|
+
log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "{message_origin}"}}'
|
45
49
|
print(log_message)
|
46
50
|
|
47
51
|
def is_special(c):
|
@@ -252,12 +256,16 @@ def edit_distance(first_string: str, second_string: str) -> int:
|
|
252
256
|
return previous_row[second_string_length]
|
253
257
|
|
254
258
|
|
255
|
-
def get_input_from_cli(prompt: str, default_value: str) -> str:
|
259
|
+
def get_input_from_cli(prompt: str, default_value: str, hide_value = False) -> str:
|
256
260
|
"""
|
257
261
|
Prompts the user for input.
|
258
262
|
"""
|
259
263
|
if default_value:
|
260
|
-
|
264
|
+
if hide_value:
|
265
|
+
default_value_hidden = default_value[0:8] + "********"
|
266
|
+
value = input(f"{prompt} [Default : {default_value_hidden}]: ").strip() or default_value
|
267
|
+
else:
|
268
|
+
value = input(f"{prompt} [Default : {default_value}]: ").strip() or default_value
|
261
269
|
else:
|
262
270
|
value = input(f"{prompt}: ").strip()
|
263
271
|
|
@@ -265,29 +273,36 @@ def get_input_from_cli(prompt: str, default_value: str) -> str:
|
|
265
273
|
raise ValueError("Missing required input: Expected a value but received None")
|
266
274
|
return value
|
267
275
|
|
268
|
-
def validate_and_load_configuration(
|
276
|
+
def validate_and_load_configuration(project_path, configuration):
|
269
277
|
if configuration:
|
270
|
-
|
278
|
+
configuration = os.path.expanduser(configuration)
|
279
|
+
if os.path.isabs(configuration):
|
280
|
+
json_filepath = os.path.abspath(configuration)
|
281
|
+
else:
|
282
|
+
relative_path = os.path.join(project_path, configuration)
|
283
|
+
json_filepath = os.path.abspath(str(relative_path))
|
271
284
|
if os.path.isfile(json_filepath):
|
272
285
|
with open(json_filepath, 'r', encoding=UTF_8) as fi:
|
273
|
-
|
286
|
+
try:
|
287
|
+
configuration = json.load(fi)
|
288
|
+
except:
|
289
|
+
raise ValueError(
|
290
|
+
"Configuration must be provided as a JSON file. Please check your input. Reference: "
|
291
|
+
"https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
|
274
292
|
if len(configuration) > MAX_CONFIG_FIELDS:
|
275
293
|
raise ValueError(f"Configuration field count exceeds maximum of {MAX_CONFIG_FIELDS}. Reduce the field count.")
|
276
294
|
else:
|
277
295
|
raise ValueError(
|
278
|
-
"Configuration
|
279
|
-
"https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
|
296
|
+
f"Configuration path is incorrect, cannot find file at the location {json_filepath}")
|
280
297
|
else:
|
281
|
-
|
282
|
-
if os.path.exists(json_filepath):
|
283
|
-
print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
|
298
|
+
print_library_log("No configuration file passed.", Logging.Level.INFO)
|
284
299
|
configuration = {}
|
285
300
|
return configuration
|
286
301
|
|
287
302
|
|
288
303
|
def validate_and_load_state(args, state):
|
289
304
|
if state:
|
290
|
-
json_filepath =
|
305
|
+
json_filepath = os.path.abspath(os.path.join(args.project_path, args.state))
|
291
306
|
else:
|
292
307
|
json_filepath = os.path.join(args.project_path, "files", "state.json")
|
293
308
|
|
@@ -1,6 +1,6 @@
|
|
1
|
-
import os
|
2
1
|
import json
|
3
2
|
import inspect
|
3
|
+
import sys
|
4
4
|
|
5
5
|
from datetime import datetime
|
6
6
|
from google.protobuf import timestamp_pb2
|
@@ -18,6 +18,12 @@ from fivetran_connector_sdk.helpers import (
|
|
18
18
|
from fivetran_connector_sdk.logger import Logging
|
19
19
|
from fivetran_connector_sdk.protos import connector_sdk_pb2, common_pb2
|
20
20
|
|
21
|
+
_LOG_DATA_TYPE_INFERENCE = {
|
22
|
+
"boolean": True,
|
23
|
+
"binary": True,
|
24
|
+
"json": True
|
25
|
+
}
|
26
|
+
|
21
27
|
class Operations:
|
22
28
|
@staticmethod
|
23
29
|
def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
|
@@ -174,7 +180,7 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
|
|
174
180
|
if v is None:
|
175
181
|
mapped_data[key] = common_pb2.ValueType(null=True)
|
176
182
|
elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
|
177
|
-
map_defined_data_type(columns, key, mapped_data, v)
|
183
|
+
map_defined_data_type(columns[key].type, key, mapped_data, v)
|
178
184
|
else:
|
179
185
|
map_inferred_data_type(key, mapped_data, v)
|
180
186
|
return mapped_data
|
@@ -189,13 +195,22 @@ def map_inferred_data_type(k, mapped_data, v):
|
|
189
195
|
elif isinstance(v, float):
|
190
196
|
mapped_data[k] = common_pb2.ValueType(float=v)
|
191
197
|
elif isinstance(v, bool):
|
198
|
+
if _LOG_DATA_TYPE_INFERENCE["boolean"]:
|
199
|
+
print_library_log("Fivetran: Boolean Datatype has been inferred", Logging.Level.INFO, True)
|
200
|
+
_LOG_DATA_TYPE_INFERENCE["boolean"] = False
|
192
201
|
mapped_data[k] = common_pb2.ValueType(bool=v)
|
193
202
|
elif isinstance(v, bytes):
|
203
|
+
if _LOG_DATA_TYPE_INFERENCE["binary"]:
|
204
|
+
print_library_log("Fivetran: Binary Datatype has been inferred", Logging.Level.INFO, True)
|
205
|
+
_LOG_DATA_TYPE_INFERENCE["binary"] = False
|
194
206
|
mapped_data[k] = common_pb2.ValueType(binary=v)
|
195
207
|
elif isinstance(v, list):
|
196
208
|
raise ValueError(
|
197
209
|
"Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
|
198
210
|
elif isinstance(v, dict):
|
211
|
+
if _LOG_DATA_TYPE_INFERENCE["json"]:
|
212
|
+
print_library_log("Fivetran: JSON Datatype has been inferred", Logging.Level.INFO, True)
|
213
|
+
_LOG_DATA_TYPE_INFERENCE["json"] = False
|
199
214
|
mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
|
200
215
|
elif isinstance(v, str):
|
201
216
|
mapped_data[k] = common_pb2.ValueType(string=v)
|
@@ -203,53 +218,50 @@ def map_inferred_data_type(k, mapped_data, v):
|
|
203
218
|
# Convert arbitrary objects to string
|
204
219
|
mapped_data[k] = common_pb2.ValueType(string=str(v))
|
205
220
|
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
mapped_data[k] =
|
227
|
-
elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
|
228
|
-
if '.' not in v: v = v + ".0"
|
229
|
-
timestamp = timestamp_pb2.Timestamp()
|
230
|
-
dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
|
231
|
-
timestamp.FromDatetime(dt)
|
232
|
-
mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
|
233
|
-
elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
|
234
|
-
timestamp = timestamp_pb2.Timestamp()
|
235
|
-
dt = v if isinstance(v, datetime) else _parse_datetime_str(v)
|
236
|
-
timestamp.FromDatetime(dt)
|
237
|
-
mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
|
238
|
-
elif columns[k].type == common_pb2.DataType.BINARY:
|
239
|
-
mapped_data[k] = common_pb2.ValueType(binary=v)
|
240
|
-
elif columns[k].type == common_pb2.DataType.XML:
|
241
|
-
mapped_data[k] = common_pb2.ValueType(xml=v)
|
242
|
-
elif columns[k].type == common_pb2.DataType.STRING:
|
243
|
-
incoming = v if isinstance(v, str) else str(v)
|
244
|
-
mapped_data[k] = common_pb2.ValueType(string=incoming)
|
245
|
-
elif columns[k].type == common_pb2.DataType.JSON:
|
246
|
-
mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
|
221
|
+
_TYPE_HANDLERS = {
|
222
|
+
common_pb2.DataType.BOOLEAN: lambda val: common_pb2.ValueType(bool=val),
|
223
|
+
common_pb2.DataType.SHORT: lambda val: common_pb2.ValueType(short=val),
|
224
|
+
common_pb2.DataType.INT: lambda val: common_pb2.ValueType(int=val),
|
225
|
+
common_pb2.DataType.LONG: lambda val: common_pb2.ValueType(long=val),
|
226
|
+
common_pb2.DataType.DECIMAL: lambda val: common_pb2.ValueType(decimal=val),
|
227
|
+
common_pb2.DataType.FLOAT: lambda val: common_pb2.ValueType(float=val),
|
228
|
+
common_pb2.DataType.DOUBLE: lambda val: common_pb2.ValueType(double=val),
|
229
|
+
common_pb2.DataType.NAIVE_DATE: lambda val: common_pb2.ValueType(naive_date= _parse_naive_date_str(val)),
|
230
|
+
common_pb2.DataType.NAIVE_DATETIME: lambda val: common_pb2.ValueType(naive_datetime= _parse_naive_datetime_str(val)),
|
231
|
+
common_pb2.DataType.UTC_DATETIME: lambda val: common_pb2.ValueType(utc_datetime= _parse_utc_datetime_str(val)),
|
232
|
+
common_pb2.DataType.BINARY: lambda val: common_pb2.ValueType(binary=val),
|
233
|
+
common_pb2.DataType.XML: lambda val: common_pb2.ValueType(xml=val),
|
234
|
+
common_pb2.DataType.STRING: lambda val: common_pb2.ValueType(string=val if isinstance(val, str) else str(val)),
|
235
|
+
common_pb2.DataType.JSON: lambda val: common_pb2.ValueType(json=json.dumps(val))
|
236
|
+
}
|
237
|
+
|
238
|
+
def map_defined_data_type(data_type, k, mapped_data, v):
|
239
|
+
handler = _TYPE_HANDLERS.get(data_type)
|
240
|
+
if handler:
|
241
|
+
mapped_data[k] = handler(v)
|
247
242
|
else:
|
248
|
-
raise ValueError(f"Unsupported data type encountered: {
|
249
|
-
|
250
|
-
def
|
251
|
-
|
252
|
-
|
243
|
+
raise ValueError(f"Unsupported data type encountered: {data_type}. Please use valid data types.")
|
244
|
+
|
245
|
+
def _parse_utc_datetime_str(v):
|
246
|
+
timestamp = timestamp_pb2.Timestamp()
|
247
|
+
dt = v
|
248
|
+
if not isinstance(v, datetime):
|
249
|
+
dt = datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
|
250
|
+
timestamp.FromDatetime(dt)
|
251
|
+
return timestamp
|
252
|
+
|
253
|
+
def _parse_naive_datetime_str(v):
|
254
|
+
if '.' not in v: v = v + ".0"
|
255
|
+
timestamp = timestamp_pb2.Timestamp()
|
256
|
+
dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
|
257
|
+
timestamp.FromDatetime(dt)
|
258
|
+
return timestamp
|
259
|
+
|
260
|
+
def _parse_naive_date_str(v):
|
261
|
+
timestamp = timestamp_pb2.Timestamp()
|
262
|
+
dt = datetime.strptime(v, "%Y-%m-%d")
|
263
|
+
timestamp.FromDatetime(dt)
|
264
|
+
return timestamp
|
253
265
|
|
254
266
|
def _yield_check(stack):
|
255
267
|
"""Checks for the presence of 'yield' in the calling code.
|
@@ -270,7 +282,7 @@ def _yield_check(stack):
|
|
270
282
|
if 'yield' not in calling_code:
|
271
283
|
print_library_log(
|
272
284
|
f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
|
273
|
-
|
285
|
+
sys.exit(1)
|
274
286
|
else:
|
275
287
|
# This should never happen
|
276
288
|
raise RuntimeError(
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: fivetran_connector_sdk
|
3
|
-
Version: 1.
|
3
|
+
Version: 1.7.0
|
4
4
|
Summary: Build custom connectors on Fivetran platform
|
5
5
|
Author-email: Fivetran <developers@fivetran.com>
|
6
6
|
Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
|
@@ -13,7 +13,7 @@ Requires-Python: >=3.9
|
|
13
13
|
Description-Content-Type: text/markdown
|
14
14
|
Requires-Dist: grpcio==1.71.0
|
15
15
|
Requires-Dist: grpcio-tools==1.71.0
|
16
|
-
Requires-Dist: requests==2.32.
|
16
|
+
Requires-Dist: requests==2.32.4
|
17
17
|
Requires-Dist: pipreqs==0.5.0
|
18
18
|
Requires-Dist: unidecode==1.4.0
|
19
19
|
|
@@ -1,9 +1,9 @@
|
|
1
|
-
fivetran_connector_sdk/__init__.py,sha256=
|
2
|
-
fivetran_connector_sdk/connector_helper.py,sha256=
|
1
|
+
fivetran_connector_sdk/__init__.py,sha256=bmTxQskIeTFFK-0ei94514eS5JWo4SxSaA4hZQLgUPU,20382
|
2
|
+
fivetran_connector_sdk/connector_helper.py,sha256=eezyCVOAVJ1f1m2BwAsNfgHRksFmp4y5Eo4CUZoxqrQ,43997
|
3
3
|
fivetran_connector_sdk/constants.py,sha256=tY8-fwB7O11orifFcDIELxfvYNz9-bNUhh6f5cXXws0,2287
|
4
|
-
fivetran_connector_sdk/helpers.py,sha256=
|
4
|
+
fivetran_connector_sdk/helpers.py,sha256=ZA9vU-bgMlw9QcMd-bDEUixsQ80U4bsCRmQPca_o1CA,13115
|
5
5
|
fivetran_connector_sdk/logger.py,sha256=arI6eNUfm_AvE3EV_PmbSRzrpTVpZKJ8pZMtDn93TKU,3018
|
6
|
-
fivetran_connector_sdk/operations.py,sha256=
|
6
|
+
fivetran_connector_sdk/operations.py,sha256=_O1btg_-dncQuZkBg9TrEMIyS6OVs75bSWzqAb6-Ovk,11515
|
7
7
|
fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
8
|
fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
|
9
9
|
fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
|
@@ -11,8 +11,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
|
|
11
11
|
fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
|
12
12
|
fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
|
13
13
|
fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
|
14
|
-
fivetran_connector_sdk-1.
|
15
|
-
fivetran_connector_sdk-1.
|
16
|
-
fivetran_connector_sdk-1.
|
17
|
-
fivetran_connector_sdk-1.
|
18
|
-
fivetran_connector_sdk-1.
|
14
|
+
fivetran_connector_sdk-1.7.0.dist-info/METADATA,sha256=vqeFr54ujVw_IElK32RB1nKD-Q2maTtiLJIhHfWWodE,3150
|
15
|
+
fivetran_connector_sdk-1.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
16
|
+
fivetran_connector_sdk-1.7.0.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
|
17
|
+
fivetran_connector_sdk-1.7.0.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
|
18
|
+
fivetran_connector_sdk-1.7.0.dist-info/RECORD,,
|
File without changes
|
{fivetran_connector_sdk-1.5.1.dist-info → fivetran_connector_sdk-1.7.0.dist-info}/entry_points.txt
RENAMED
File without changes
|
{fivetran_connector_sdk-1.5.1.dist-info → fivetran_connector_sdk-1.7.0.dist-info}/top_level.txt
RENAMED
File without changes
|