fivetran-connector-sdk 1.4.5__py3-none-any.whl → 1.4.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,744 +1,48 @@
1
- import argparse
2
- from typing import Optional, Tuple
3
-
1
+ import os
2
+ import sys
4
3
  import grpc
5
- import importlib.util
6
- import inspect
7
4
  import json
8
- import os
9
- import unicodedata
10
- from unidecode import unidecode
11
- import platform
12
- import requests as rq
13
5
  import shutil
14
- import subprocess
15
- import sys
16
- import time
6
+ import argparse
17
7
  import traceback
18
- import re
19
- import socket
20
- import ast
21
-
22
- from concurrent import futures
23
- from datetime import datetime
24
- from enum import IntEnum
25
- from google.protobuf import timestamp_pb2
26
- from zipfile import ZipFile, ZIP_DEFLATED
8
+ import requests as rq
27
9
  from http import HTTPStatus
10
+ from zipfile import ZipFile
11
+ from concurrent import futures
28
12
 
29
13
  from fivetran_connector_sdk.protos import common_pb2
30
14
  from fivetran_connector_sdk.protos import connector_sdk_pb2
31
15
  from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
32
16
 
17
+ from fivetran_connector_sdk.logger import Logging
18
+ from fivetran_connector_sdk.operations import Operations
19
+ from fivetran_connector_sdk import constants
20
+ from fivetran_connector_sdk.constants import (
21
+ TESTER_VER, VERSION_FILENAME, UTF_8,
22
+ VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, FIVETRAN_HD_AGENT_ID, TABLES
23
+ )
24
+ from fivetran_connector_sdk.helpers import (
25
+ print_library_log, reset_local_file_directory, find_connector_object, validate_and_load_configuration,
26
+ validate_and_load_state, get_input_from_cli, suggest_correct_command,
27
+ )
28
+ from fivetran_connector_sdk.connector_helper import (
29
+ validate_requirements_file, upload_project,
30
+ update_connection, are_setup_tests_failing,
31
+ validate_deploy_parameters, get_connection_id,
32
+ handle_failing_tests_message_and_exit, delete_file_if_exists,
33
+ create_connection, get_os_arch_suffix, get_group_info,
34
+ java_exe_helper, run_tester, process_tables,
35
+ update_base_url_if_required, exit_check,
36
+ get_available_port, tester_root_dir_helper,
37
+ check_dict, check_newer_version, cleanup_uploaded_project,
38
+ )
39
+
33
40
  # Version format: <major_version>.<minor_version>.<patch_version>
34
41
  # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
35
- __version__ = "1.4.5"
36
-
37
- WIN_OS = "windows"
38
- ARM_64 = "arm64"
39
- X64 = "x64"
40
-
41
- OS_MAP = {
42
- "darwin": "mac",
43
- "linux": "linux",
44
- WIN_OS: WIN_OS
45
- }
46
-
47
- ARCH_MAP = {
48
- "x86_64": X64,
49
- "amd64": X64,
50
- ARM_64: ARM_64,
51
- "aarch64": ARM_64
52
- }
53
-
54
- TESTER_VERSION = "0.25.0521.001"
55
- TESTER_FILENAME = "run_sdk_tester.jar"
56
- VERSION_FILENAME = "version.txt"
57
- UPLOAD_FILENAME = "code.zip"
58
- LAST_VERSION_CHECK_FILE = "_last_version_check"
59
- ROOT_LOCATION = ".ft_sdk_connector_tester"
60
- CONFIG_FILE = "_config.json"
61
- OUTPUT_FILES_DIR = "files"
62
- REQUIREMENTS_TXT = "requirements.txt"
63
- PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
64
- ONE_DAY_IN_SEC = 24 * 60 * 60
65
- MAX_RETRIES = 3
66
- LOGGING_PREFIX = "Fivetran-Connector-SDK"
67
- LOGGING_DELIMITER = ": "
68
- VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
69
- ROOT_FILENAME = "connector.py"
70
-
71
- # Compile patterns used in the implementation
72
- WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')
73
- NON_WORD_PATTERN = re.compile(r'\W')
74
- WORD_OR_DOLLAR_PATTERN = re.compile(r'[\w$]')
75
- DROP_LEADING_UNDERSCORE = re.compile(r'_+([a-zA-Z]\w*)')
76
- WORD_PATTERN = re.compile(r'\w')
77
-
78
- EXCLUDED_DIRS = ["__pycache__", "lib", "include", OUTPUT_FILES_DIR]
79
- EXCLUDED_PIPREQS_DIRS = ["bin,etc,include,lib,Lib,lib64,Scripts,share"]
80
- VALID_COMMANDS = ["debug", "deploy", "reset", "version"]
81
- MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND = 3
82
- COMMANDS_AND_SYNONYMS = {
83
- "debug": {"test", "verify", "diagnose", "check"},
84
- "deploy": {"upload", "ship", "launch", "release"},
85
- "reset": {"reinitialize", "reinitialise", "re-initialize", "re-initialise", "restart", "restore"},
86
- }
87
-
88
- CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
89
- DEBUGGING = False
90
- EXECUTED_VIA_CLI = False
91
- PRODUCTION_BASE_URL = "https://api.fivetran.com"
92
- TABLES = {}
93
- RENAMED_TABLE_NAMES = {}
94
- RENAMED_COL_NAMES = {}
95
- INSTALLATION_SCRIPT_MISSING_MESSAGE = "The 'installation.sh' file is missing in the 'drivers' directory. Please ensure that 'installation.sh' is present to properly configure drivers."
96
- INSTALLATION_SCRIPT = "installation.sh"
97
- DRIVERS = "drivers"
98
- JAVA_LONG_MAX_VALUE = 9223372036854775807
99
- MAX_CONFIG_FIELDS = 100
100
- SUPPORTED_PYTHON_VERSIONS = ["3.12", "3.11", "3.10", "3.9"]
101
- DEFAULT_PYTHON_VERSION = "3.12"
102
- FIVETRAN_HD_AGENT_ID = "FIVETRAN_HD_AGENT_ID"
103
- UTF_8 = "utf-8"
104
-
105
-
106
- class Logging:
107
- class Level(IntEnum):
108
- FINE = 1
109
- INFO = 2
110
- WARNING = 3
111
- SEVERE = 4
112
-
113
- LOG_LEVEL = None
114
-
115
- @staticmethod
116
- def __log(level: Level, message: str):
117
- """Logs a message with the specified logging level.
118
-
119
- Args:
120
- level (Logging.Level): The logging level.
121
- message (str): The message to log.
122
- """
123
- if DEBUGGING:
124
- current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
125
- escaped_message = json.dumps(message).strip('"')
126
- print(f"{Logging._get_color(level)}{current_time} {level.name}: {escaped_message} {Logging._reset_color()}")
127
- else:
128
- escaped_message = json.dumps(message)
129
- log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "connector_sdk"}}'
130
- print(log_message)
131
-
132
- @staticmethod
133
- def _get_color(level):
134
- if level == Logging.Level.WARNING:
135
- return "\033[93m" # Yellow
136
- elif level == Logging.Level.SEVERE:
137
- return "\033[91m" # Red
138
- return ""
139
-
140
- @staticmethod
141
- def _reset_color():
142
- return "\033[0m"
143
-
144
- @staticmethod
145
- def fine(message: str):
146
- """Logs a fine-level message.
147
-
148
- Args:
149
- message (str): The message to log.
150
- """
151
- if DEBUGGING and Logging.LOG_LEVEL == Logging.Level.FINE:
152
- Logging.__log(Logging.Level.FINE, message)
153
-
154
- @staticmethod
155
- def info(message: str):
156
- """Logs an info-level message.
157
-
158
- Args:
159
- message (str): The message to log.
160
- """
161
- if Logging.LOG_LEVEL <= Logging.Level.INFO:
162
- Logging.__log(Logging.Level.INFO, message)
163
-
164
- @staticmethod
165
- def warning(message: str):
166
- """Logs a warning-level message.
167
-
168
- Args:
169
- message (str): The message to log.
170
- """
171
- if Logging.LOG_LEVEL <= Logging.Level.WARNING:
172
- Logging.__log(Logging.Level.WARNING, message)
173
-
174
- @staticmethod
175
- def severe(message: str, exception: Exception = None):
176
- """Logs a severe-level message.
177
-
178
- Args:
179
- message (str): The message to log.
180
- exception (Exception, optional): Exception to be logged if provided.
181
- """
182
- if Logging.LOG_LEVEL <= Logging.Level.SEVERE:
183
- Logging.__log(Logging.Level.SEVERE, message)
184
-
185
- if exception:
186
- exc_type, exc_value, exc_traceback = type(exception), exception, exception.__traceback__
187
- tb_str = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, limit=1))
188
-
189
- for error in tb_str.split("\n"):
190
- Logging.__log(Logging.Level.SEVERE, error)
191
-
192
-
193
- class Operations:
194
- @staticmethod
195
- def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
196
- """Updates records with the same primary key if already present in the destination. Inserts new records if not already present in the destination.
197
-
198
- Args:
199
- table (str): The name of the table.
200
- data (dict): The data to upsert.
201
-
202
- Returns:
203
- list[connector_sdk_pb2.UpdateResponse]: A list of update responses.
204
- """
205
- if DEBUGGING:
206
- _yield_check(inspect.stack())
207
-
208
- responses = []
209
-
210
- table = get_renamed_table_name(table)
211
- columns = _get_columns(table)
212
- if not columns:
213
- global TABLES
214
- for field in data.keys():
215
- field_name = get_renamed_column_name(field)
216
- columns[field_name] = common_pb2.Column(
217
- name=field_name, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
218
- new_table = common_pb2.Table(name=table, columns=columns.values())
219
- TABLES[table] = new_table
220
-
221
- mapped_data = _map_data_to_columns(data, columns)
222
- record = connector_sdk_pb2.Record(
223
- schema_name=None,
224
- table_name=table,
225
- type=common_pb2.OpType.UPSERT,
226
- data=mapped_data
227
- )
228
-
229
- responses.append(
230
- connector_sdk_pb2.UpdateResponse(
231
- operation=connector_sdk_pb2.Operation(record=record)))
232
-
233
- return responses
234
-
235
- @staticmethod
236
- def update(table: str, modified: dict) -> connector_sdk_pb2.UpdateResponse:
237
- """Performs an update operation on the specified table with the given modified data.
238
-
239
- Args:
240
- table (str): The name of the table.
241
- modified (dict): The modified data.
242
-
243
- Returns:
244
- connector_sdk_pb2.UpdateResponse: The update response.
245
- """
246
- if DEBUGGING:
247
- _yield_check(inspect.stack())
248
-
249
- table = get_renamed_table_name(table)
250
- columns = _get_columns(table)
251
- mapped_data = _map_data_to_columns(modified, columns)
252
- record = connector_sdk_pb2.Record(
253
- schema_name=None,
254
- table_name=table,
255
- type=common_pb2.OpType.UPDATE,
256
- data=mapped_data
257
- )
258
-
259
- return connector_sdk_pb2.UpdateResponse(
260
- operation=connector_sdk_pb2.Operation(record=record))
261
-
262
- @staticmethod
263
- def delete(table: str, keys: dict) -> connector_sdk_pb2.UpdateResponse:
264
- """Performs a soft delete operation on the specified table with the given keys.
265
-
266
- Args:
267
- table (str): The name of the table.
268
- keys (dict): The keys to delete.
269
-
270
- Returns:
271
- connector_sdk_pb2.UpdateResponse: The delete response.
272
- """
273
- if DEBUGGING:
274
- _yield_check(inspect.stack())
275
-
276
- table = get_renamed_table_name(table)
277
- columns = _get_columns(table)
278
- mapped_data = _map_data_to_columns(keys, columns)
279
- record = connector_sdk_pb2.Record(
280
- schema_name=None,
281
- table_name=table,
282
- type=common_pb2.OpType.DELETE,
283
- data=mapped_data
284
- )
285
-
286
- return connector_sdk_pb2.UpdateResponse(
287
- operation=connector_sdk_pb2.Operation(record=record))
288
-
289
- @staticmethod
290
- def checkpoint(state: dict) -> connector_sdk_pb2.UpdateResponse:
291
- """Checkpoint saves the connector's state. State is a dict which stores information to continue the
292
- sync from where it left off in the previous sync. For example, you may choose to have a field called
293
- "cursor" with a timestamp value to indicate up to when the data has been synced. This makes it possible
294
- for the next sync to fetch data incrementally from that time forward. See below for a few example fields
295
- which act as parameters for use by the connector code.\n
296
- {
297
- "initialSync": true,\n
298
- "cursor": "1970-01-01T00:00:00.00Z",\n
299
- "last_resync": "1970-01-01T00:00:00.00Z",\n
300
- "thread_count": 5,\n
301
- "api_quota_left": 5000000
302
- }
303
-
304
- Args:
305
- state (dict): The state to checkpoint/save.
306
-
307
- Returns:
308
- connector_sdk_pb2.UpdateResponse: The checkpoint response.
309
- """
310
- if DEBUGGING:
311
- _yield_check(inspect.stack())
312
-
313
- return connector_sdk_pb2.UpdateResponse(
314
- operation=connector_sdk_pb2.Operation(checkpoint=connector_sdk_pb2.Checkpoint(
315
- state_json=json.dumps(state))))
316
-
317
-
318
- def check_newer_version():
319
- """Periodically checks for a newer version of the SDK and notifies the user if one is available."""
320
- tester_root_dir = _tester_root_dir()
321
- last_check_file_path = os.path.join(tester_root_dir, LAST_VERSION_CHECK_FILE)
322
- if not os.path.isdir(tester_root_dir):
323
- os.makedirs(tester_root_dir, exist_ok=True)
324
-
325
- if os.path.isfile(last_check_file_path):
326
- # Is it time to check again?
327
- with open(last_check_file_path, 'r', encoding=UTF_8) as f_in:
328
- timestamp = int(f_in.read())
329
- if (int(time.time()) - timestamp) < ONE_DAY_IN_SEC:
330
- return
331
-
332
- for index in range(MAX_RETRIES):
333
- try:
334
- # check version and save current time
335
- response = rq.get(PYPI_PACKAGE_DETAILS_URL)
336
- response.raise_for_status()
337
- data = json.loads(response.text)
338
- latest_version = data["info"]["version"]
339
- if __version__ < latest_version:
340
- print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}")
341
- print_library_log("[notice] To update, run: pip install --upgrade fivetran-connector-sdk")
342
-
343
- with open(last_check_file_path, 'w', encoding=UTF_8) as f_out:
344
- f_out.write(f"{int(time.time())}")
345
- break
346
- except Exception:
347
- retry_after = 2 ** index
348
- print_library_log(f"Unable to check if a newer version of `fivetran-connector-sdk` is available. Retrying again after {retry_after} seconds", Logging.Level.WARNING)
349
- time.sleep(retry_after)
350
-
351
-
352
- def _tester_root_dir() -> str:
353
- """Returns the root directory for the tester."""
354
- return os.path.join(os.path.expanduser("~"), ROOT_LOCATION)
355
-
356
-
357
- def _get_columns(table: str) -> dict:
358
- """Retrieves the columns for the specified table.
359
-
360
- Args:
361
- table (str): The name of the table.
362
-
363
- Returns:
364
- dict: The columns for the table.
365
- """
366
- columns = {}
367
- if table in TABLES:
368
- for column in TABLES[table].columns:
369
- columns[column.name] = column
370
-
371
- return columns
372
-
373
-
374
- def _map_data_to_columns(data: dict, columns: dict) -> dict:
375
- """Maps data to the specified columns.
376
-
377
- Args:
378
- data (dict): The data to map.
379
- columns (dict): The columns to map the data to.
380
-
381
- Returns:
382
- dict: The mapped data.
383
- """
384
- mapped_data = {}
385
- for k, v in data.items():
386
- key = get_renamed_column_name(k)
387
- if v is None:
388
- mapped_data[key] = common_pb2.ValueType(null=True)
389
- elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
390
- map_defined_data_type(columns, key, mapped_data, v)
391
- else:
392
- map_inferred_data_type(key, mapped_data, v)
393
- return mapped_data
394
-
395
-
396
- def map_inferred_data_type(k, mapped_data, v):
397
- # We can infer type from the value
398
- if isinstance(v, int):
399
- if abs(v) > JAVA_LONG_MAX_VALUE:
400
- mapped_data[k] = common_pb2.ValueType(float=v)
401
- else:
402
- mapped_data[k] = common_pb2.ValueType(long=v)
403
- elif isinstance(v, float):
404
- mapped_data[k] = common_pb2.ValueType(float=v)
405
- elif isinstance(v, bool):
406
- mapped_data[k] = common_pb2.ValueType(bool=v)
407
- elif isinstance(v, bytes):
408
- mapped_data[k] = common_pb2.ValueType(binary=v)
409
- elif isinstance(v, list):
410
- raise ValueError(
411
- "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
412
- elif isinstance(v, dict):
413
- mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
414
- elif isinstance(v, str):
415
- mapped_data[k] = common_pb2.ValueType(string=v)
416
- else:
417
- # Convert arbitrary objects to string
418
- mapped_data[k] = common_pb2.ValueType(string=str(v))
419
-
420
-
421
- def map_defined_data_type(columns, k, mapped_data, v):
422
- if columns[k].type == common_pb2.DataType.BOOLEAN:
423
- mapped_data[k] = common_pb2.ValueType(bool=v)
424
- elif columns[k].type == common_pb2.DataType.SHORT:
425
- mapped_data[k] = common_pb2.ValueType(short=v)
426
- elif columns[k].type == common_pb2.DataType.INT:
427
- mapped_data[k] = common_pb2.ValueType(int=v)
428
- elif columns[k].type == common_pb2.DataType.LONG:
429
- mapped_data[k] = common_pb2.ValueType(long=v)
430
- elif columns[k].type == common_pb2.DataType.DECIMAL:
431
- mapped_data[k] = common_pb2.ValueType(decimal=v)
432
- elif columns[k].type == common_pb2.DataType.FLOAT:
433
- mapped_data[k] = common_pb2.ValueType(float=v)
434
- elif columns[k].type == common_pb2.DataType.DOUBLE:
435
- mapped_data[k] = common_pb2.ValueType(double=v)
436
- elif columns[k].type == common_pb2.DataType.NAIVE_DATE:
437
- timestamp = timestamp_pb2.Timestamp()
438
- dt = datetime.strptime(v, "%Y-%m-%d")
439
- timestamp.FromDatetime(dt)
440
- mapped_data[k] = common_pb2.ValueType(naive_date=timestamp)
441
- elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
442
- if '.' not in v: v = v + ".0"
443
- timestamp = timestamp_pb2.Timestamp()
444
- dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
445
- timestamp.FromDatetime(dt)
446
- mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
447
- elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
448
- timestamp = timestamp_pb2.Timestamp()
449
- dt = v if isinstance(v, datetime) else _parse_datetime_str(v)
450
- timestamp.FromDatetime(dt)
451
- mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
452
- elif columns[k].type == common_pb2.DataType.BINARY:
453
- mapped_data[k] = common_pb2.ValueType(binary=v)
454
- elif columns[k].type == common_pb2.DataType.XML:
455
- mapped_data[k] = common_pb2.ValueType(xml=v)
456
- elif columns[k].type == common_pb2.DataType.STRING:
457
- incoming = v if isinstance(v, str) else str(v)
458
- mapped_data[k] = common_pb2.ValueType(string=incoming)
459
- elif columns[k].type == common_pb2.DataType.JSON:
460
- mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
461
- else:
462
- raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
463
-
464
- def _warn_exit_usage(filename, line_no, func):
465
- print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw a error if required " +
466
- f"at: {filename}:{line_no}. See the Technical Reference for details: https://fivetran.com/docs/connector-sdk/technical-reference#handlingexceptions",
467
- Logging.Level.WARNING)
468
-
469
- def _exit_check(project_path):
470
- """Checks for the presence of 'exit()' in the calling code.
471
- Args:
472
- project_path: The absolute project_path to check exit in the connector.py file in the project.
473
- """
474
- # We expect the connector.py to catch errors or throw exceptions
475
- # This is a warning shown to let the customer know that we expect either the yield call or error thrown
476
- # exit() or sys.exit() in between some yields can cause the connector to be stuck without processing further upsert calls
477
-
478
- filepath = os.path.join(project_path, ROOT_FILENAME)
479
- with open(filepath, "r", encoding=UTF_8) as f:
480
- try:
481
- tree = ast.parse(f.read())
482
- for node in ast.walk(tree):
483
- if isinstance(node, ast.Call):
484
- if isinstance(node.func, ast.Name) and node.func.id == "exit":
485
- _warn_exit_usage(ROOT_FILENAME, node.lineno, "exit()")
486
- elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
487
- if node.func.attr == "_exit" and node.func.value.id == "os":
488
- _warn_exit_usage(ROOT_FILENAME, node.lineno, "os._exit()")
489
- if node.func.attr == "exit" and node.func.value.id == "sys":
490
- _warn_exit_usage(ROOT_FILENAME, node.lineno, "sys.exit()")
491
- except SyntaxError as e:
492
- print_library_log(f"SyntaxError in {ROOT_FILENAME}: {e}", Logging.Level.SEVERE)
493
-
494
-
495
- def _parse_datetime_str(dt):
496
- return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
497
-
498
-
499
- def _yield_check(stack):
500
- """Checks for the presence of 'yield' in the calling code.
501
- Args:
502
- stack: The stack frame to check.
503
- """
504
-
505
- # Known issue with inspect.getmodule() and yield behavior in a frozen application.
506
- # When using inspect.getmodule() on stack frames obtained by inspect.stack(), it fails
507
- # to resolve the modules in a frozen application due to incompatible assumptions about
508
- # the file paths. This can lead to unexpected behavior, such as yield returning None or
509
- # the failure to retrieve the module inside a frozen app
510
- # (Reference: https://github.com/pyinstaller/pyinstaller/issues/5963)
511
-
512
- called_method = stack[0].function
513
- calling_code = stack[1].code_context[0]
514
- if f"{called_method}(" in calling_code:
515
- if 'yield' not in calling_code:
516
- print_library_log(
517
- f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
518
- os._exit(1)
519
- else:
520
- # This should never happen
521
- raise RuntimeError(
522
- f"The '{called_method}' function is missing in the connector calling code '{calling_code}'. Please ensure that the '{called_method}' function is properly defined in your code to proceed. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsmethods")
523
-
524
-
525
- def _check_dict(incoming: dict, string_only: bool = False) -> dict:
526
- """Validates the incoming dictionary.
527
- Args:
528
- incoming (dict): The dictionary to validate.
529
- string_only (bool): Whether to allow only string values.
530
-
531
- Returns:
532
- dict: The validated dictionary.
533
- """
534
-
535
- if not incoming:
536
- return {}
537
-
538
- if not isinstance(incoming, dict):
539
- raise ValueError(
540
- "Configuration must be provided as a JSON dictionary. Please check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
541
-
542
- if string_only:
543
- for k, v in incoming.items():
544
- if not isinstance(v, str):
545
- print_library_log(
546
- "All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
547
- os._exit(1)
548
-
549
- return incoming
550
-
551
-
552
- def is_connection_name_valid(connection: str):
553
- """Validates if the incoming connection schema name is valid or not.
554
- Args:
555
- connection (str): The connection schema name being validated.
556
-
557
- Returns:
558
- bool: True if connection name is valid.
559
- """
560
-
561
- pattern = re.compile(CONNECTION_SCHEMA_NAME_PATTERN)
562
- return pattern.match(connection)
563
-
564
-
565
- def log_unused_deps_error(package_name: str, version: str):
566
- print_library_log(f"Please remove `{package_name}` from requirements.txt."
567
- f" The latest version of `{package_name}` is always available when executing your code."
568
- f" Current version: {version}", Logging.Level.SEVERE)
569
-
570
-
571
- def validate_deploy_parameters(connection, deploy_key):
572
- if not deploy_key or not connection:
573
- print_library_log("The deploy command needs the following parameters:"
574
- "\n\tRequired:\n"
575
- "\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
576
- "\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
577
- "\t(Optional):\n"
578
- "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
579
- "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
580
- os._exit(1)
581
- elif not is_connection_name_valid(connection):
582
- print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
583
- f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
584
- f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
585
- os._exit(1)
586
-
587
- def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
588
- """Logs a library message with the specified logging level.
589
-
590
- Args:
591
- level (Logging.Level): The logging level.
592
- message (str): The message to log.
593
- """
594
- if DEBUGGING or EXECUTED_VIA_CLI:
595
- current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
596
- escaped_message = json.dumps(message).strip('"')
597
- print(f"{Logging._get_color(level)}{current_time} {level.name} {LOGGING_PREFIX}: {escaped_message} {Logging._reset_color()}")
598
- else:
599
- escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
600
- log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
601
- print(log_message)
602
-
603
-
604
- def is_port_in_use(port: int):
605
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
606
- return s.connect_ex(('127.0.0.1', port)) == 0
607
-
608
-
609
- def get_available_port():
610
- for port in range(50051, 50061):
611
- if not is_port_in_use(port):
612
- return port
613
- return None
614
-
615
-
616
- def update_base_url_if_required():
617
- config_file_path = os.path.join(_tester_root_dir(), CONFIG_FILE)
618
- if os.path.isfile(config_file_path):
619
- with open(config_file_path, 'r', encoding=UTF_8) as f:
620
- data = json.load(f)
621
- base_url = data.get('production_base_url')
622
- if base_url is not None:
623
- global PRODUCTION_BASE_URL
624
- PRODUCTION_BASE_URL = base_url
625
- print_library_log(f"Updating PRODUCTION_BASE_URL to: {base_url}")
626
-
627
-
628
- def is_special(c):
629
- """Check if the character is a special character."""
630
- return not WORD_OR_DOLLAR_PATTERN.fullmatch(c)
631
-
632
-
633
- def starts_word(previous, current):
634
- """
635
- Check if the current character starts a new word based on the previous character.
636
- """
637
- return (previous and previous.islower() and current.isupper()) or (
638
- previous and previous.isdigit() != current.isdigit()
639
- )
640
-
641
-
642
- def underscore_invalid_leading_character(name, valid_leading_regex):
643
- """
644
- Ensure the name starts with a valid leading character.
645
- """
646
- if name and not valid_leading_regex.match(name[0]):
647
- name = f'_{name}'
648
- return name
649
-
650
-
651
- def single_underscore_case(name):
652
- """
653
- Convert the input name to single underscore case, replacing special characters and spaces.
654
- """
655
- acc = []
656
- previous = None
657
-
658
- for char_index, c in enumerate(name):
659
- if char_index == 0 and c == '$':
660
- acc.append('_')
661
- elif is_special(c):
662
- acc.append('_')
663
- elif c == ' ':
664
- acc.append('_')
665
- elif starts_word(previous, c):
666
- acc.append('_')
667
- acc.append(c.lower())
668
- else:
669
- acc.append(c.lower())
670
-
671
- previous = c
672
-
673
- name = ''.join(acc)
674
- return re.sub(r'_+', '_', name)
42
+ __version__ = "1.4.6"
43
+ TESTER_VERSION = TESTER_VER
675
44
 
676
-
677
- def contains_only_word_dash_dot(name):
678
- """
679
- Check if the name contains only word characters, dashes, and dots.
680
- """
681
- return bool(WORD_DASH_DOT_PATTERN.fullmatch(name))
682
-
683
-
684
- def transliterate(name):
685
- """
686
- Transliterate the input name if it contains non-word, dash, or dot characters.
687
- """
688
- if contains_only_word_dash_dot(name):
689
- return name
690
- # Step 1: Normalize the name to NFD form (decomposed form)
691
- normalized_name = unicodedata.normalize('NFD', name)
692
- # Step 2: Remove combining characters (diacritics, accents, etc.)
693
- normalized_name = ''.join(char for char in normalized_name if not unicodedata.combining(char))
694
- # Step 3: Normalize back to NFC form (composed form)
695
- normalized_name = unicodedata.normalize('NFC', normalized_name)
696
- # Step 4: Convert the string to ASCII using `unidecode` (removes any remaining non-ASCII characters)
697
- normalized_name = unidecode(normalized_name)
698
- # Step 5: Return the normalized name
699
- return normalized_name
700
-
701
-
702
- def redshift_safe(name):
703
- """
704
- Make the name safe for use in Redshift.
705
- """
706
- name = transliterate(name)
707
- name = NON_WORD_PATTERN.sub('_', name)
708
- name = single_underscore_case(name)
709
- name = underscore_invalid_leading_character(name, WORD_PATTERN)
710
- return name
711
-
712
-
713
- def safe_drop_underscores(name):
714
- """
715
- Drop leading underscores if the name starts with valid characters after sanitization.
716
- """
717
- safe_name = redshift_safe(name)
718
- match = DROP_LEADING_UNDERSCORE.match(safe_name)
719
- if match:
720
- return match.group(1)
721
- return safe_name
722
-
723
-
724
- def get_renamed_table_name(source_table):
725
- """
726
- Process a source table name to ensure it conforms to naming rules.
727
- """
728
- if source_table not in RENAMED_TABLE_NAMES:
729
- RENAMED_TABLE_NAMES[source_table] = safe_drop_underscores(source_table)
730
-
731
- return RENAMED_TABLE_NAMES[source_table]
732
-
733
-
734
- def get_renamed_column_name(source_column):
735
- """
736
- Process a source column name to ensure it conforms to naming rules.
737
- """
738
- if source_column not in RENAMED_COL_NAMES:
739
- RENAMED_COL_NAMES[source_column] = redshift_safe(source_column)
740
-
741
- return RENAMED_COL_NAMES[source_column]
45
+ __all__ = [cls.__name__ for cls in [Logging, Operations]]
742
46
 
743
47
  class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
744
48
  def __init__(self, update, schema=None):
@@ -756,211 +60,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
756
60
 
757
61
  update_base_url_if_required()
758
62
 
759
- @staticmethod
760
- def fetch_requirements_from_file(file_path: str) -> list[str]:
761
- """Reads a requirements file and returns a list of dependencies.
762
-
763
- Args:
764
- file_path (str): The path to the requirements file.
765
-
766
- Returns:
767
- list[str]: A list of dependencies as strings.
768
- """
769
- with open(file_path, 'r', encoding=UTF_8) as f:
770
- return f.read().splitlines()
771
-
772
- @staticmethod
773
- def fetch_requirements_as_dict(self, file_path: str) -> dict:
774
- """Converts a list of dependencies from the requirements file into a dictionary.
775
-
776
- Args:
777
- file_path (str): The path to the requirements file.
778
-
779
- Returns:
780
- dict: A dictionary where keys are package names (lowercased) and
781
- values are the full dependency strings.
782
- """
783
- requirements_dict = {}
784
- if not os.path.exists(file_path):
785
- return requirements_dict
786
- for requirement in self.fetch_requirements_from_file(file_path):
787
- requirement = requirement.strip()
788
- if not requirement or requirement.startswith("#"): # Skip empty lines and comments
789
- continue
790
- try:
791
- key = re.split(r"==|>=|<=|>|<", requirement)[0]
792
- requirements_dict[key.lower().replace('-', '_')] = requirement.lower()
793
- except ValueError:
794
- print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
795
- return requirements_dict
796
-
797
- def validate_requirements_file(self, project_path: str, is_deploy: bool, force: bool = False):
798
- """Validates the `requirements.txt` file against the project's actual dependencies.
799
-
800
- This method generates a temporary requirements file using `pipreqs`, compares
801
- it with the existing `requirements.txt`, and checks for version mismatches,
802
- missing dependencies, and unused dependencies. It will issue warnings, errors,
803
- or even terminate the process depending on whether it's being run for deployment.
804
-
805
- Args:
806
- project_path (str): The path to the project directory containing the `requirements.txt`.
807
- is_deploy (bool): If `True`, the method will exit the process on critical errors.
808
- force (bool): Force update an existing connection.
809
-
810
- """
811
- # Detect and exclude virtual environment directories
812
- venv_dirs = [name for name in os.listdir(project_path)
813
- if os.path.isdir(os.path.join(project_path, name)) and
814
- VIRTUAL_ENV_CONFIG in os.listdir(os.path.join(project_path, name))]
815
-
816
- ignored_dirs = EXCLUDED_PIPREQS_DIRS + venv_dirs if venv_dirs else EXCLUDED_PIPREQS_DIRS
817
-
818
- # tmp_requirements is only generated when pipreqs command is successful
819
- requirements_file_path = os.path.join(project_path, REQUIREMENTS_TXT)
820
- tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
821
- # copying packages of requirements file to tmp file to handle pipreqs fail use-case
822
- self.copy_requirements_file_to_tmp_requirements_file(os.path.join(project_path, REQUIREMENTS_TXT), tmp_requirements_file_path)
823
- # Run the pipreqs command and capture stderr
824
- attempt = 0
825
- while attempt < MAX_RETRIES:
826
- attempt += 1
827
- result = subprocess.run(
828
- ["pipreqs", project_path, "--savepath", tmp_requirements_file_path, "--ignore", ",".join(ignored_dirs)],
829
- stderr=subprocess.PIPE,
830
- text=True # Ensures output is in string format
831
- )
832
-
833
- if result.returncode == 0:
834
- break
835
-
836
- print_library_log(f"Attempt {attempt}: pipreqs check failed.", Logging.Level.WARNING)
837
-
838
- if attempt < MAX_RETRIES:
839
- retry_after = 3 ** attempt
840
- print_library_log(f"Retrying in {retry_after} seconds...", Logging.Level.SEVERE)
841
- time.sleep(retry_after)
842
- else:
843
- print_library_log(f"pipreqs failed after {MAX_RETRIES} attempts with:", Logging.Level.SEVERE)
844
- print_library_log(result.stderr, Logging.Level.SEVERE)
845
- print_library_log(f"Skipping validation of requirements.txt due to error connecting to PyPI (Python Package Index) APIs. Continuing with {'deploy' if is_deploy else 'debug'}...", Logging.Level.SEVERE)
846
-
847
- tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
848
- self.remove_unwanted_packages(tmp_requirements)
849
- os.remove(tmp_requirements_file_path)
850
-
851
- # remove corrupt requirements listed by pipreqs
852
- corrupt_requirements = [key for key in tmp_requirements if key.startswith("~")]
853
- for requirement in corrupt_requirements:
854
- del tmp_requirements[requirement]
855
-
856
- update_version_requirements = False
857
- update_missing_requirements = False
858
- update_unused_requirements = False
859
- if len(tmp_requirements) > 0:
860
- requirements = self.load_or_add_requirements_file(requirements_file_path)
861
-
862
- version_mismatch_deps = {key: tmp_requirements[key] for key in
863
- (requirements.keys() & tmp_requirements.keys())
864
- if requirements[key] != tmp_requirements[key]}
865
- if version_mismatch_deps:
866
- print_library_log("We recommend using the current stable version for the following:", Logging.Level.WARNING)
867
- print(version_mismatch_deps)
868
- if is_deploy and not force:
869
- confirm = input(
870
- f"Would you like us to update {REQUIREMENTS_TXT} to the current stable versions of the dependent libraries? (Y/N):")
871
- if confirm.lower() == "y":
872
- update_version_requirements = True
873
- for requirement in version_mismatch_deps:
874
- requirements[requirement] = tmp_requirements[requirement]
875
- elif confirm.lower() == "n":
876
- print_library_log(f"Ignored the identified dependency version conflicts. These changes are NOT made to {REQUIREMENTS_TXT}")
877
-
878
- missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
879
- if missing_deps:
880
- self.handle_missing_deps(missing_deps)
881
- if is_deploy and not force:
882
- confirm = input(
883
- f"Would you like us to update {REQUIREMENTS_TXT} to add missing dependent libraries? (Y/N):")
884
- if confirm.lower() == "n":
885
- print_library_log(f"Ignored dependencies identified as needed. These changes are NOT made to {REQUIREMENTS_TXT}. Please review the requirements as this can fail after deploy.")
886
- elif confirm.lower() == "y":
887
- update_missing_requirements = True
888
- for requirement in missing_deps:
889
- requirements[requirement] = tmp_requirements[requirement]
890
-
891
- unused_deps = list(requirements.keys() - tmp_requirements.keys())
892
- if unused_deps:
893
- self.handle_unused_deps(unused_deps)
894
- if is_deploy and not force:
895
- confirm = input(f"Would you like us to update {REQUIREMENTS_TXT} to remove the unused libraries? (Y/N):")
896
- if confirm.lower() == "n":
897
- if 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
898
- print_library_log(f"Please fix your {REQUIREMENTS_TXT} file by removing pre-installed dependencies to proceed with deployment.")
899
- os._exit(1)
900
- print_library_log(f"Ignored libraries identified as unused. These changes are NOT made to {REQUIREMENTS_TXT}")
901
- elif confirm.lower() == "y":
902
- update_unused_requirements = True
903
- for requirement in unused_deps:
904
- del requirements[requirement]
905
-
906
-
907
- if update_version_requirements or update_missing_requirements or update_unused_requirements:
908
- with open(requirements_file_path, "w", encoding=UTF_8) as file:
909
- file.write("\n".join(requirements.values()))
910
- print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully.")
911
-
912
- else:
913
- if os.path.exists(requirements_file_path):
914
- print_library_log(f"{REQUIREMENTS_TXT} is not required as no additional "
915
- "Python libraries are required or all required libraries for "
916
- "your code are pre-installed.", Logging.Level.WARNING)
917
- with open(requirements_file_path, 'w') as file:
918
- file.write("")
919
-
920
-
921
- if is_deploy: print_library_log(f"Validation of {REQUIREMENTS_TXT} completed.")
922
-
923
- def handle_unused_deps(self, unused_deps):
924
- if 'fivetran_connector_sdk' in unused_deps:
925
- log_unused_deps_error("fivetran_connector_sdk", __version__)
926
- if 'requests' in unused_deps:
927
- log_unused_deps_error("requests", "2.32.3")
928
- print_library_log("The following dependencies are not needed, "
929
- f"they are not used or already installed. Please remove them from {REQUIREMENTS_TXT}:", Logging.Level.WARNING)
930
- print(*unused_deps)
931
-
932
- def handle_missing_deps(self, missing_deps):
933
- print_library_log(f"Please include the following dependency libraries in {REQUIREMENTS_TXT}, to be used by "
934
- "Fivetran production. "
935
- "For more information, please visit: "
936
- "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
937
- "#workingwithrequirementstxtfile", Logging.Level.SEVERE)
938
- print(*list(missing_deps.values()))
939
-
940
- def load_or_add_requirements_file(self, requirements_file_path):
941
- if os.path.exists(requirements_file_path):
942
- requirements = self.fetch_requirements_as_dict(self, requirements_file_path)
943
- else:
944
- with open(requirements_file_path, 'w', encoding=UTF_8):
945
- pass
946
- requirements = {}
947
- print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
948
- return requirements
949
-
950
- def copy_requirements_file_to_tmp_requirements_file(self, requirements_file_path: str, tmp_requirements_file_path):
951
- if os.path.exists(requirements_file_path):
952
- requirements_file_content = self.fetch_requirements_from_file(requirements_file_path)
953
- with open(tmp_requirements_file_path, 'w') as file:
954
- file.write("\n".join(requirements_file_content))
955
-
956
- @staticmethod
957
- def remove_unwanted_packages(requirements: dict):
958
- # remove the `fivetran_connector_sdk` and `requests` packages from requirements as we already pre-installed them.
959
- if requirements.get("fivetran_connector_sdk") is not None:
960
- requirements.pop("fivetran_connector_sdk")
961
- if requirements.get('requests') is not None:
962
- requirements.pop("requests")
963
-
964
63
  # Call this method to deploy the connector to Fivetran platform
965
64
  def deploy(self, args: dict, deploy_key: str, group: str, connection: str, hd_agent_id: str, configuration: dict = None):
966
65
  """Deploys the connector to the Fivetran platform.
@@ -973,14 +72,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
973
72
  hd_agent_id (str): The hybrid deployment agent ID within the Fivetran system.
974
73
  configuration (dict): The configuration dictionary.
975
74
  """
976
- global EXECUTED_VIA_CLI
977
- EXECUTED_VIA_CLI = True
75
+ constants.EXECUTED_VIA_CLI = True
978
76
 
979
77
  print_library_log("We support only `.py` files and a `requirements.txt` file as part of the code upload. *No other code files* are supported or uploaded during the deployment process. Ensure that your code is structured accordingly and all dependencies are listed in `requirements.txt`")
980
78
 
981
79
  validate_deploy_parameters(connection, deploy_key)
982
80
 
983
- _check_dict(configuration, True)
81
+ check_dict(configuration, True)
984
82
 
985
83
  secrets_list = []
986
84
  if configuration:
@@ -996,10 +94,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
996
94
  if args.python_version:
997
95
  connection_config["python_version"] = args.python_version
998
96
 
999
- self.validate_requirements_file(args.project_path, True, args.force)
97
+ validate_requirements_file(args.project_path, True, __version__, args.force)
1000
98
 
1001
- group_id, group_name = self.__get_group_info(group, deploy_key)
1002
- connection_id, service = self.__get_connection_id(connection, group, group_id, deploy_key) or (None, None)
99
+ group_id, group_name = get_group_info(group, deploy_key)
100
+ connection_id, service = get_connection_id(connection, group, group_id, deploy_key) or (None, None)
1003
101
 
1004
102
  if connection_id:
1005
103
  if service != 'connector_sdk':
@@ -1018,9 +116,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1018
116
  confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{args.configuration}': key-value pairs not present in the new configuration will be removed; existing keys' values set in the cofiguration file or in the dashboard will be overwritten with new (empty or non-empty) values; new key-value pairs will be added. Do you want to proceed with the update? (Y/N): ")
1019
117
  if confirm.lower() == "y" and (not connection_config["secrets_list"] or (confirm_config.lower() == "y")):
1020
118
  print_library_log("Updating the connection...\n")
1021
- self.__upload_project(
119
+ upload_project(
1022
120
  args.project_path, deploy_key, group_id, group_name, connection)
1023
- response = self.__update_connection(
121
+ response = update_connection(
1024
122
  args, connection_id, connection, group_name, connection_config, deploy_key, hd_agent_id)
1025
123
  print("✓")
1026
124
  print_library_log(f"Python version {response.json()['data']['config']['python_version']} to be used at runtime.",
@@ -1032,13 +130,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1032
130
  print_library_log("Update canceled. The process is now terminating.")
1033
131
  os._exit(1)
1034
132
  else:
1035
- self.__upload_project(args.project_path, deploy_key,
133
+ upload_project(args.project_path, deploy_key,
1036
134
  group_id, group_name, connection)
1037
- response = self.__create_connection(
135
+ response = create_connection(
1038
136
  deploy_key, group_id, connection_config, hd_agent_id)
1039
137
  if response.ok and response.status_code == HTTPStatus.CREATED:
1040
- if Connector.__are_setup_tests_failing(response):
1041
- Connector.__handle_failing_tests_message_and_exit(response, "The connection was created, but setup tests failed!")
138
+ if are_setup_tests_failing(response):
139
+ handle_failing_tests_message_and_exit(response, "The connection was created, but setup tests failed!")
1042
140
  else:
1043
141
  print_library_log(
1044
142
  f"The connection '{connection}' has been created successfully.\n")
@@ -1051,355 +149,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1051
149
  else:
1052
150
  print_library_log(
1053
151
  f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
1054
- self.__cleanup_uploaded_project(deploy_key,group_id, connection)
152
+ cleanup_uploaded_project(deploy_key,group_id, connection)
1055
153
  print_library_log("Please try again with the deploy command after resolving the issue!")
1056
154
  os._exit(1)
1057
155
 
1058
- def __upload_project(self, project_path: str, deploy_key: str, group_id: str, group_name: str, connection: str):
1059
- print_library_log(
1060
- f"Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
1061
- upload_file_path = self.__create_upload_file(project_path)
1062
- upload_result = self.__upload(
1063
- upload_file_path, deploy_key, group_id, connection)
1064
- os.remove(upload_file_path)
1065
- if not upload_result:
1066
- os._exit(1)
1067
-
1068
- def __cleanup_uploaded_project(self, deploy_key: str, group_id: str, connection: str):
1069
- cleanup_result = self.__cleanup_uploaded_code(deploy_key, group_id, connection)
1070
- if not cleanup_result:
1071
- os._exit(1)
1072
-
1073
- @staticmethod
1074
- def __update_connection(args: dict, id: str, name: str, group: str, config: dict, deploy_key: str, hd_agent_id: str):
1075
- """Updates the connection with the given ID, name, group, configuration, and deployment key.
1076
-
1077
- Args:
1078
- args (dict): The command arguments.
1079
- id (str): The connection ID.
1080
- name (str): The connection name.
1081
- group (str): The group name.
1082
- config (dict): The configuration dictionary.
1083
- deploy_key (str): The deployment key.
1084
- hd_agent_id (str): The hybrid deployment agent ID within the Fivetran system.
1085
- """
1086
- if not args.configuration:
1087
- del config["secrets_list"]
1088
-
1089
- json_payload = {
1090
- "config": config,
1091
- "run_setup_tests": True
1092
- }
1093
-
1094
- # hybrid_deployment_agent_id is optional when redeploying your connection.
1095
- # Customer can use it to change existing hybrid_deployment_agent_id.
1096
- if hd_agent_id:
1097
- json_payload["hybrid_deployment_agent_id"] = hd_agent_id
1098
-
1099
- response = rq.patch(f"{PRODUCTION_BASE_URL}/v1/connectors/{id}",
1100
- headers={"Authorization": f"Basic {deploy_key}"},
1101
- json=json_payload)
1102
-
1103
- if response.ok and response.status_code == HTTPStatus.OK:
1104
- if Connector.__are_setup_tests_failing(response):
1105
- Connector.__handle_failing_tests_message_and_exit(response, "The connection was updated, but setup tests failed!")
1106
- else:
1107
- print_library_log(f"Connection '{name}' in group '{group}' updated successfully.", Logging.Level.INFO)
1108
-
1109
- else:
1110
- print_library_log(
1111
- f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{response.json()['message']}'.", Logging.Level.SEVERE)
1112
- os._exit(1)
1113
- return response
1114
-
1115
- @staticmethod
1116
- def __handle_failing_tests_message_and_exit(resp, log_message):
1117
- print_library_log(log_message, Logging.Level.SEVERE)
1118
- Connector.__print_failing_setup_tests(resp)
1119
- connection_id = resp.json().get('data', {}).get('id')
1120
- print_library_log(f"Connection ID: {connection_id}")
1121
- print_library_log("Please try again with the deploy command after resolving the issue!")
1122
- os._exit(1)
1123
-
1124
- @staticmethod
1125
- def __are_setup_tests_failing(response) -> bool:
1126
- """Checks for failed setup tests in the response and returns True if any test has failed, otherwise False."""
1127
- response_json = response.json()
1128
- setup_tests = response_json.get("data", {}).get("setup_tests", [])
1129
-
1130
- # Return True if any test has "FAILED" status, otherwise False
1131
- return any(test.get("status") == "FAILED" or test.get("status") == "JOB_FAILED" for test in setup_tests)
1132
-
1133
-
1134
- @staticmethod
1135
- def __print_failing_setup_tests(response):
1136
- """Checks for failed setup tests in the response and print errors."""
1137
- response_json = response.json()
1138
- setup_tests = response_json.get("data", {}).get("setup_tests", [])
1139
-
1140
- # Collect failed setup tests
1141
- failed_tests = [test for test in setup_tests if test.get("status") == "FAILED" or test.get("status") == "JOB_FAILED"]
1142
-
1143
- if failed_tests:
1144
- print_library_log("Following setup tests have failed!", Logging.Level.WARNING)
1145
- for test in failed_tests:
1146
- print_library_log(f"Test: {test.get('title')}", Logging.Level.WARNING)
1147
- print_library_log(f"Status: {test.get('status')}", Logging.Level.WARNING)
1148
- print_library_log(f"Message: {test.get('message')}", Logging.Level.WARNING)
1149
-
1150
-
1151
- @staticmethod
1152
- def __get_connection_id(name: str, group: str, group_id: str, deploy_key: str) -> Optional[Tuple[str, str]]:
1153
- """Retrieves the connection ID for the specified connection schema name, group, and deployment key.
1154
-
1155
- Args:
1156
- name (str): The connection name.
1157
- group (str): The group name.
1158
- group_id (str): The group ID.
1159
- deploy_key (str): The deployment key.
1160
-
1161
- Returns:
1162
- str: The connection ID, or None
1163
- """
1164
- resp = rq.get(f"{PRODUCTION_BASE_URL}/v1/groups/{group_id}/connectors",
1165
- headers={"Authorization": f"Basic {deploy_key}"},
1166
- params={"schema": name})
1167
- if not resp.ok:
1168
- print_library_log(
1169
- f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
1170
- os._exit(1)
1171
-
1172
- if resp.json()['data']['items']:
1173
- return resp.json()['data']['items'][0]['id'], resp.json()['data']['items'][0]['service']
1174
-
1175
- return None
1176
-
1177
- @staticmethod
1178
- def __create_connection(deploy_key: str, group_id: str, config: dict, hd_agent_id: str) -> rq.Response:
1179
- """Creates a new connection with the given deployment key, group ID, and configuration.
1180
-
1181
- Args:
1182
- deploy_key (str): The deployment key.
1183
- group_id (str): The group ID.
1184
- config (dict): The configuration dictionary.
1185
- hd_agent_id (str): The hybrid deployment agent ID within the Fivetran system.
1186
-
1187
- Returns:
1188
- rq.Response: The response object.
1189
- """
1190
- response = rq.post(f"{PRODUCTION_BASE_URL}/v1/connectors",
1191
- headers={"Authorization": f"Basic {deploy_key}"},
1192
- json={
1193
- "group_id": group_id,
1194
- "service": "connector_sdk",
1195
- "config": config,
1196
- "paused": True,
1197
- "run_setup_tests": True,
1198
- "sync_frequency": "360",
1199
- "hybrid_deployment_agent_id": hd_agent_id
1200
- })
1201
- return response
1202
-
1203
- def __create_upload_file(self, project_path: str) -> str:
1204
- """Creates an upload file for the given project path.
1205
-
1206
- Args:
1207
- project_path (str): The path to the project.
1208
-
1209
- Returns:
1210
- str: The path to the upload file.
1211
- """
1212
- print_library_log("Packaging your project for upload...")
1213
- zip_file_path = self.__zip_folder(project_path)
1214
- print("✓")
1215
- return zip_file_path
1216
-
1217
- def __zip_folder(self, project_path: str) -> str:
1218
- """Zips the folder at the given project path.
1219
-
1220
- Args:
1221
- project_path (str): The path to the project.
1222
-
1223
- Returns:
1224
- str: The path to the zip file.
1225
- """
1226
- upload_filepath = os.path.join(project_path, UPLOAD_FILENAME)
1227
- connector_file_exists = False
1228
- custom_drivers_exists = False
1229
- custom_driver_installation_script_exists = False
1230
-
1231
- with ZipFile(upload_filepath, 'w', ZIP_DEFLATED) as zipf:
1232
- for root, files in self.__dir_walker(project_path):
1233
- if os.path.basename(root) == DRIVERS:
1234
- custom_drivers_exists = True
1235
- if INSTALLATION_SCRIPT in files:
1236
- custom_driver_installation_script_exists = True
1237
- for file in files:
1238
- if file == ROOT_FILENAME:
1239
- connector_file_exists = True
1240
- file_path = os.path.join(root, file)
1241
- arcname = os.path.relpath(file_path, project_path)
1242
- zipf.write(file_path, arcname)
1243
-
1244
- if not connector_file_exists:
1245
- print_library_log(
1246
- "The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.", Logging.Level.SEVERE)
1247
- os._exit(1)
1248
-
1249
- if custom_drivers_exists and not custom_driver_installation_script_exists:
1250
- print_library_log(INSTALLATION_SCRIPT_MISSING_MESSAGE, Logging.Level.SEVERE)
1251
- os._exit(1)
1252
-
1253
- return upload_filepath
1254
-
1255
- def __dir_walker(self, top):
1256
- """Walks the directory tree starting at the given top directory.
1257
-
1258
- Args:
1259
- top (str): The top directory to start the walk.
1260
-
1261
- Yields:
1262
- tuple: A tuple containing the current directory path and a list of files.
1263
- """
1264
- dirs, files = [], []
1265
- for name in os.listdir(top):
1266
- path = os.path.join(top, name)
1267
- if os.path.isdir(path):
1268
- if (name not in EXCLUDED_DIRS) and (not name.startswith(".")):
1269
- if VIRTUAL_ENV_CONFIG not in os.listdir(path): # Check for virtual env indicator
1270
- dirs.append(name)
1271
- else:
1272
- # Include all files if in `drivers` folder
1273
- if os.path.basename(top) == DRIVERS:
1274
- files.append(name)
1275
- if name.endswith(".py") or name == "requirements.txt":
1276
- files.append(name)
1277
-
1278
- yield top, files
1279
- for name in dirs:
1280
- new_path = os.path.join(top, name)
1281
- for x in self.__dir_walker(new_path):
1282
- yield x
1283
-
1284
- @staticmethod
1285
- def __upload(local_path: str, deploy_key: str, group_id: str, connection: str) -> bool:
1286
- """Uploads the local code file for the specified group and connection.
1287
-
1288
- Args:
1289
- local_path (str): The local file path.
1290
- deploy_key (str): The deployment key.
1291
- group_id (str): The group ID.
1292
- connection (str): The connection name.
1293
-
1294
- Returns:
1295
- bool: True if the upload was successful, False otherwise.
1296
- """
1297
- print_library_log("Uploading your project...")
1298
- response = rq.post(f"{PRODUCTION_BASE_URL}/v1/deploy/{group_id}/{connection}",
1299
- files={'file': open(local_path, 'rb')},
1300
- headers={"Authorization": f"Basic {deploy_key}"})
1301
- if response.ok:
1302
- print("✓")
1303
- return True
1304
-
1305
- print_library_log(f"Unable to upload the project, failed with error: {response.reason}", Logging.Level.SEVERE)
1306
- return False
1307
-
1308
- @staticmethod
1309
- def __cleanup_uploaded_code(deploy_key: str, group_id: str, connection: str) -> bool:
1310
- """Cleans up the uploaded code file for the specified group and connection, if creation fails.
1311
-
1312
- Args:
1313
- deploy_key (str): The deployment key.
1314
- group_id (str): The group ID.
1315
- connection (str): The connection name.
1316
-
1317
- Returns:
1318
- bool: True if the cleanup was successful, False otherwise.
1319
- """
1320
- print_library_log("INFO: Cleaning up your uploaded project ")
1321
- response = rq.post(f"{PRODUCTION_BASE_URL}/v1/cleanup_code/{group_id}/{connection}",
1322
- headers={"Authorization": f"Basic {deploy_key}"})
1323
- if response.ok:
1324
- print("✓")
1325
- return True
1326
-
1327
- print_library_log(f"SEVERE: Unable to cleanup the project, failed with error: {response.reason}", Logging.Level.SEVERE)
1328
- return False
1329
-
1330
- @staticmethod
1331
- def __get_os_arch_suffix() -> str:
1332
- """
1333
- Returns the operating system and architecture suffix for the current operating system.
1334
- """
1335
- system = platform.system().lower()
1336
- machine = platform.machine().lower()
1337
-
1338
- if system not in OS_MAP:
1339
- raise RuntimeError(f"Unsupported OS: {system}")
1340
-
1341
- plat = OS_MAP[system]
1342
-
1343
- if machine not in ARCH_MAP or (plat == WIN_OS and ARCH_MAP[machine] != X64):
1344
- raise RuntimeError(f"Unsupported architecture '{machine}' for {plat}")
1345
-
1346
- return f"{plat}-{ARCH_MAP[machine]}"
1347
-
1348
- @staticmethod
1349
- def __get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
1350
- """Retrieves the group information for the specified group and deployment key.
1351
-
1352
- Args:
1353
- group (str): The group name.
1354
- deploy_key (str): The deployment key.
1355
-
1356
- Returns:
1357
- tuple[str, str]: A tuple containing the group ID and group name.
1358
- """
1359
- groups_url = f"{PRODUCTION_BASE_URL}/v1/groups"
1360
-
1361
- params = {"limit": 500}
1362
- headers = {"Authorization": f"Basic {deploy_key}"}
1363
- resp = rq.get(groups_url, headers=headers, params=params)
1364
-
1365
- if not resp.ok:
1366
- print_library_log(
1367
- f"The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
1368
- os._exit(1)
1369
-
1370
- data = resp.json().get("data", {})
1371
- groups = data.get("items")
1372
-
1373
- if not groups:
1374
- print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
1375
- os._exit(1)
1376
-
1377
- if not group:
1378
- if len(groups) == 1:
1379
- return groups[0]['id'], groups[0]['name']
1380
- else:
1381
- print_library_log(
1382
- "Destination name is required when there are multiple destinations in the account", Logging.Level.SEVERE)
1383
- os._exit(1)
1384
- else:
1385
- while True:
1386
- for grp in groups:
1387
- if grp['name'] == group:
1388
- return grp['id'], grp['name']
1389
-
1390
- next_cursor = data.get("next_cursor")
1391
- if not next_cursor:
1392
- break
1393
-
1394
- params = {"cursor": next_cursor, "limit": 500}
1395
- resp = rq.get(groups_url, headers=headers, params=params)
1396
- data = resp.json().get("data", {})
1397
- groups = data.get("items", [])
1398
-
1399
- print_library_log(
1400
- f"The specified destination '{group}' was not found in your account.", Logging.Level.SEVERE)
1401
- os._exit(1)
1402
-
1403
156
  # Call this method to run the connector in production
1404
157
  def run(self,
1405
158
  port: int = 50051,
@@ -1417,18 +170,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1417
170
  Returns:
1418
171
  grpc.Server: The gRPC server instance.
1419
172
  """
1420
- self.configuration = _check_dict(configuration, True)
1421
- self.state = _check_dict(state)
173
+ self.configuration = check_dict(configuration, True)
174
+ self.state = check_dict(state)
1422
175
  Logging.LOG_LEVEL = log_level
1423
176
 
1424
- if not DEBUGGING:
177
+ if not constants.DEBUGGING:
1425
178
  print_library_log(f"Running on fivetran_connector_sdk: {__version__}")
1426
179
 
1427
180
  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
1428
181
  connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
1429
182
  server.add_insecure_port("[::]:" + str(port))
1430
183
  server.start()
1431
- if DEBUGGING:
184
+ if constants.DEBUGGING:
1432
185
  return server
1433
186
  server.wait_for_termination()
1434
187
 
@@ -1448,15 +201,14 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1448
201
  state (dict): The state dictionary, same as state.json if present.
1449
202
  log_level (Logging.Level): The logging level.
1450
203
  """
1451
- global DEBUGGING
1452
- DEBUGGING = True
204
+ constants.DEBUGGING = True
1453
205
 
1454
- check_newer_version()
206
+ check_newer_version(__version__)
1455
207
 
1456
208
  Logging.LOG_LEVEL = log_level
1457
- os_arch_suffix = self.__get_os_arch_suffix()
1458
- tester_root_dir = _tester_root_dir()
1459
- java_exe = self.__java_exe(tester_root_dir, os_arch_suffix)
209
+ os_arch_suffix = get_os_arch_suffix()
210
+ tester_root_dir = tester_root_dir_helper()
211
+ java_exe = java_exe_helper(tester_root_dir, os_arch_suffix)
1460
212
  install_tester = False
1461
213
  version_file = os.path.join(tester_root_dir, VERSION_FILENAME)
1462
214
  if os.path.isfile(version_file):
@@ -1494,7 +246,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1494
246
  with ZipFile(download_filepath, 'r') as z_object:
1495
247
  z_object.extractall(path=tester_root_dir)
1496
248
  # delete zip file
1497
- os.remove(download_filepath)
249
+ delete_file_if_exists(download_filepath)
1498
250
  # make java binary executable
1499
251
  import stat
1500
252
  st = os.stat(java_exe)
@@ -1505,10 +257,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1505
257
  raise RuntimeError(f"\nSEVERE: Failed to install the connector tester. Error details: {traceback.format_exc()}")
1506
258
 
1507
259
  project_path = os.getcwd() if project_path is None else project_path
1508
- self.validate_requirements_file(project_path, False)
260
+ validate_requirements_file(project_path, False, __version__)
1509
261
  print_library_log(f"Debugging connector at: {project_path}")
1510
262
  available_port = get_available_port()
1511
- _exit_check(project_path)
263
+ exit_check(project_path)
1512
264
 
1513
265
  if available_port is None:
1514
266
  raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
@@ -1521,85 +273,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1521
273
 
1522
274
  try:
1523
275
  print_library_log("Running connector tester...")
1524
- for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port, json.dumps(self.state), json.dumps(self.configuration)):
276
+ for log_msg in run_tester(java_exe, tester_root_dir, project_path, available_port, json.dumps(self.state), json.dumps(self.configuration)):
1525
277
  print(log_msg, end="")
1526
278
  except:
1527
279
  print(traceback.format_exc())
1528
280
  finally:
1529
281
  server.stop(grace=2.0)
1530
282
 
1531
- @staticmethod
1532
- def __java_exe(location: str, os_arch_suffix: str) -> str:
1533
- """Returns the path to the Java executable.
1534
-
1535
- Args:
1536
- location (str): The location of the Java executable.
1537
- os_arch_suffix (str): The name of the operating system and architecture
1538
-
1539
- Returns:
1540
- str: The path to the Java executable.
1541
- """
1542
- java_exe_base = os.path.join(location, "bin", "java")
1543
- return f"{java_exe_base}.exe" if os_arch_suffix == f"{WIN_OS}-{X64}" else java_exe_base
1544
-
1545
- @staticmethod
1546
- def process_stream(stream):
1547
- """Processes a stream of text lines, replacing occurrences of a specified pattern.
1548
-
1549
- This method reads each line from the provided stream, searches for occurrences of
1550
- a predefined pattern, and replaces them with a specified replacement string.
1551
-
1552
- Args:
1553
- stream (iterable): An iterable stream of text lines, typically from a file or another input source.
1554
-
1555
- Yields:
1556
- str: Each line from the stream after replacing the matched pattern with the replacement string.
1557
- """
1558
- pattern = r'com\.fivetran\.partner_sdk.*\.tools\.testers\.\S+'
1559
-
1560
- for line in iter(stream.readline, ""):
1561
- if not re.search(pattern, line):
1562
- yield line
1563
-
1564
- @staticmethod
1565
- def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int, state_json: str, configuration_json: str):
1566
- """Runs the connector tester.
1567
-
1568
- Args:
1569
- java_exe (str): The path to the Java executable.
1570
- root_dir (str): The root directory.
1571
- project_path (str): The path to the project.
1572
-
1573
- Yields:
1574
- str: The log messages from the tester.
1575
- """
1576
- working_dir = os.path.join(project_path, OUTPUT_FILES_DIR)
1577
- try:
1578
- os.mkdir(working_dir)
1579
- except FileExistsError:
1580
- pass
1581
-
1582
- cmd = [java_exe,
1583
- "-jar",
1584
- os.path.join(root_dir, TESTER_FILENAME),
1585
- "--connector-sdk=true",
1586
- f"--port={port}",
1587
- f"--working-dir={working_dir}",
1588
- "--tester-type=source",
1589
- f"--state={state_json}",
1590
- f"--configuration={configuration_json}"]
1591
-
1592
- popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
1593
- for line in Connector.process_stream(popen.stderr):
1594
- yield Connector._maybe_colorize_jar_output(line)
1595
-
1596
- for line in Connector.process_stream(popen.stdout):
1597
- yield Connector._maybe_colorize_jar_output(line)
1598
- popen.stdout.close()
1599
- return_code = popen.wait()
1600
- if return_code:
1601
- raise subprocess.CalledProcessError(return_code, cmd)
1602
-
1603
283
  # -- Methods below override ConnectorServicer methods
1604
284
  def ConfigurationForm(self, request, context):
1605
285
  """Overrides the ConfigurationForm method from ConnectorServicer.
@@ -1639,7 +319,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1639
319
  Returns:
1640
320
  connector_sdk_pb2.SchemaResponse: The schema response.
1641
321
  """
1642
- global TABLES
322
+
1643
323
  table_list = {}
1644
324
 
1645
325
  if not self.schema_method:
@@ -1649,7 +329,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1649
329
  configuration = self.configuration if self.configuration else request.configuration
1650
330
  print_library_log("Initiating the 'schema' method call...", Logging.Level.INFO)
1651
331
  response = self.schema_method(configuration)
1652
- self.process_tables(response, table_list)
332
+ process_tables(response, table_list)
1653
333
  return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
1654
334
 
1655
335
  except Exception as e:
@@ -1658,91 +338,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1658
338
  print_library_log(error_message, Logging.Level.SEVERE)
1659
339
  raise RuntimeError(error_message) from e
1660
340
 
1661
- def process_tables(self, response, table_list):
1662
- for entry in response:
1663
- if 'table' not in entry:
1664
- raise ValueError("Entry missing table name: " + entry)
1665
-
1666
- table_name = get_renamed_table_name(entry['table'])
1667
-
1668
- if table_name in table_list:
1669
- raise ValueError("Table already defined: " + table_name)
1670
-
1671
- table = common_pb2.Table(name=table_name)
1672
- columns = {}
1673
-
1674
- if "primary_key" in entry:
1675
- self.process_primary_keys(columns, entry)
1676
-
1677
- if "columns" in entry:
1678
- self.process_columns(columns, entry)
1679
-
1680
- table.columns.extend(columns.values())
1681
- TABLES[table_name] = table
1682
- table_list[table_name] = table
1683
-
1684
- def process_primary_keys(self, columns, entry):
1685
- for pkey_name in entry["primary_key"]:
1686
- column_name = get_renamed_column_name(pkey_name)
1687
- column = columns[column_name] if column_name in columns else common_pb2.Column(name=column_name)
1688
- column.primary_key = True
1689
- columns[column_name] = column
1690
-
1691
- def process_columns(self, columns, entry):
1692
- for name, type in entry["columns"].items():
1693
- column_name = get_renamed_column_name(name)
1694
- column = columns[column_name] if column_name in columns else common_pb2.Column(name=column_name)
1695
-
1696
- if isinstance(type, str):
1697
- self.process_data_type(column, type)
1698
-
1699
- elif isinstance(type, dict):
1700
- if type['type'].upper() != "DECIMAL":
1701
- raise ValueError("Expecting DECIMAL data type")
1702
- column.type = common_pb2.DataType.DECIMAL
1703
- column.decimal.precision = type['precision']
1704
- column.decimal.scale = type['scale']
1705
-
1706
- else:
1707
- raise ValueError("Unrecognized column type: ", str(type))
1708
-
1709
- if "primary_key" in entry and name in entry["primary_key"]:
1710
- column.primary_key = True
1711
-
1712
- columns[column_name] = column
1713
-
1714
- def process_data_type(self, column, type):
1715
- if type.upper() == "BOOLEAN":
1716
- column.type = common_pb2.DataType.BOOLEAN
1717
- elif type.upper() == "SHORT":
1718
- column.type = common_pb2.DataType.SHORT
1719
- elif type.upper() == "INT":
1720
- column.type = common_pb2.DataType.INT
1721
- elif type.upper() == "LONG":
1722
- column.type = common_pb2.DataType.LONG
1723
- elif type.upper() == "DECIMAL":
1724
- raise ValueError("DECIMAL data type missing precision and scale")
1725
- elif type.upper() == "FLOAT":
1726
- column.type = common_pb2.DataType.FLOAT
1727
- elif type.upper() == "DOUBLE":
1728
- column.type = common_pb2.DataType.DOUBLE
1729
- elif type.upper() == "NAIVE_DATE":
1730
- column.type = common_pb2.DataType.NAIVE_DATE
1731
- elif type.upper() == "NAIVE_DATETIME":
1732
- column.type = common_pb2.DataType.NAIVE_DATETIME
1733
- elif type.upper() == "UTC_DATETIME":
1734
- column.type = common_pb2.DataType.UTC_DATETIME
1735
- elif type.upper() == "BINARY":
1736
- column.type = common_pb2.DataType.BINARY
1737
- elif type.upper() == "XML":
1738
- column.type = common_pb2.DataType.XML
1739
- elif type.upper() == "STRING":
1740
- column.type = common_pb2.DataType.STRING
1741
- elif type.upper() == "JSON":
1742
- column.type = common_pb2.DataType.JSON
1743
- else:
1744
- raise ValueError("Unrecognized column type encountered:: ", str(type))
1745
-
1746
341
  def Update(self, request, context):
1747
342
  """Overrides the Update method from ConnectorServicer.
1748
343
 
@@ -1775,129 +370,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1775
370
  print_library_log(error_message, Logging.Level.SEVERE)
1776
371
  raise RuntimeError(error_message) from e
1777
372
 
1778
- @staticmethod
1779
- def _maybe_colorize_jar_output(line: str) -> str:
1780
- if not DEBUGGING:
1781
- return line
1782
-
1783
- if "SEVERE" in line or "ERROR" in line or "Exception" in line or "FAILED" in line:
1784
- return f"\033[91m{line}\033[0m" # Red
1785
- elif "WARN" in line or "WARNING" in line:
1786
- return f"\033[93m{line}\033[0m" # Yellow
1787
- return line
1788
-
1789
-
1790
- def find_connector_object(project_path) -> Optional[Connector]:
1791
- """Finds the connector object in the given project path.
1792
- Args:
1793
- project_path (str): The path to the project.
1794
-
1795
- Returns:
1796
- Optional[Connector]: The connector object or None if not found.
1797
- """
1798
-
1799
- sys.path.append(project_path) # Allows python interpreter to search for modules in this path
1800
- module_name = "connector_connector_code"
1801
- connector_py = os.path.join(project_path, ROOT_FILENAME)
1802
- try:
1803
- spec = importlib.util.spec_from_file_location(module_name, connector_py)
1804
- module = importlib.util.module_from_spec(spec)
1805
- sys.modules[module_name] = module
1806
- spec.loader.exec_module(module)
1807
- for obj in dir(module):
1808
- if not obj.startswith('__'): # Exclude built-in attributes
1809
- obj_attr = getattr(module, obj)
1810
- if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
1811
- return obj_attr
1812
- except FileNotFoundError:
1813
- print_library_log(
1814
- "The connector object is missing in the current directory. Please ensure that you are running the command from correct directory or that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
1815
- return None
1816
-
1817
- print_library_log(
1818
- "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
1819
- return None
1820
-
1821
-
1822
- def suggest_correct_command(input_command: str) -> bool:
1823
- # for typos
1824
- # calculate the edit distance of the input command (lowercased) with each of the valid commands
1825
- edit_distances_of_commands = sorted(
1826
- [(command, edit_distance(command, input_command.lower())) for command in VALID_COMMANDS], key=lambda x: x[1])
1827
-
1828
- if edit_distances_of_commands[0][1] <= MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND:
1829
- # if the closest command is within the max allowed edit distance, we suggest that command
1830
- # threshold is kept to prevent suggesting a valid command for an obvious wrong command like `fivetran iknowthisisntacommandbuttryanyway`
1831
- print_suggested_command_message(edit_distances_of_commands[0][0], input_command)
1832
- return True
1833
-
1834
- # for synonyms
1835
- for (command, synonyms) in COMMANDS_AND_SYNONYMS.items():
1836
- # check if the input command (lowercased) is a recognised synonym of any of the valid commands, if yes, suggest that command
1837
- if input_command.lower() in synonyms:
1838
- print_suggested_command_message(command, input_command)
1839
- return True
1840
-
1841
- return False
1842
-
1843
-
1844
- def print_suggested_command_message(valid_command: str, input_command: str) -> None:
1845
- print_library_log(f"`fivetran {input_command}` is not a valid command.", Logging.Level.SEVERE)
1846
- print_library_log(f"Did you mean `fivetran {valid_command}`?", Logging.Level.SEVERE)
1847
- print_library_log("Use `fivetran --help` for more details.", Logging.Level.SEVERE)
1848
-
1849
-
1850
- def edit_distance(first_string: str, second_string: str) -> int:
1851
- first_string_length: int = len(first_string)
1852
- second_string_length: int = len(second_string)
1853
-
1854
- # Initialize the previous row of distances (for the base case of an empty first string) 'previous_row[j]' holds
1855
- # the edit distance between an empty prefix of 'first_string' and the first 'j' characters of 'second_string'.
1856
- # The first row is filled with values [0, 1, 2, ..., second_string_length]
1857
- previous_row: list[int] = list(range(second_string_length + 1))
1858
-
1859
- # Rest of the rows
1860
- for first_string_index in range(1, first_string_length + 1):
1861
- # Start the current row with the distance for an empty second string
1862
- current_row: list[int] = [first_string_index]
1863
-
1864
- # Iterate over each character in the second string
1865
- for second_string_index in range(1, second_string_length + 1):
1866
- if first_string[first_string_index - 1] == second_string[second_string_index - 1]:
1867
- # If characters match, no additional cost
1868
- current_row.append(previous_row[second_string_index - 1])
1869
- else:
1870
- # Minimum cost of insertion, deletion, or substitution
1871
- current_row.append(
1872
- 1 + min(current_row[-1], previous_row[second_string_index], previous_row[second_string_index - 1]))
1873
-
1874
- # Move to the next row
1875
- previous_row = current_row
1876
-
1877
- # The last value in the last row is the edit distance
1878
- return previous_row[second_string_length]
1879
-
1880
-
1881
- def get_input_from_cli(prompt : str, default_value: str) -> str:
1882
- """
1883
- Prompts the user for input.
1884
- """
1885
- if default_value:
1886
- value = input(f"{prompt} [Default : {default_value}]: ").strip() or default_value
1887
- else:
1888
- value = input(f"{prompt}: ").strip()
1889
-
1890
- if not value:
1891
- raise ValueError("Missing required input: Expected a value but received None")
1892
- return value
1893
-
1894
373
 
1895
374
  def main():
1896
375
  """The main entry point for the script.
1897
376
  Parses command line arguments and passes them to connector object methods
1898
377
  """
1899
- global EXECUTED_VIA_CLI
1900
- EXECUTED_VIA_CLI = True
378
+
379
+ constants.EXECUTED_VIA_CLI = True
1901
380
 
1902
381
  parser = argparse.ArgumentParser(allow_abbrev=False, add_help=True)
1903
382
  parser._option_string_actions["-h"].help = "Show this help message and exit"
@@ -1967,58 +446,5 @@ def main():
1967
446
  raise NotImplementedError(f"Invalid command: {args.command}, see `fivetran --help`")
1968
447
 
1969
448
 
1970
- def validate_and_load_configuration(args, configuration):
1971
- if configuration:
1972
- json_filepath = os.path.join(args.project_path, args.configuration)
1973
- if os.path.isfile(json_filepath):
1974
- with open(json_filepath, 'r', encoding=UTF_8) as fi:
1975
- configuration = json.load(fi)
1976
- if len(configuration) > MAX_CONFIG_FIELDS:
1977
- raise ValueError(f"Configuration field count exceeds maximum of {MAX_CONFIG_FIELDS}. Reduce the field count.")
1978
- else:
1979
- raise ValueError(
1980
- "Configuration must be provided as a JSON file. Please check your input. Reference: "
1981
- "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
1982
- else:
1983
- json_filepath = os.path.join(args.project_path, "configuration.json")
1984
- if os.path.exists(json_filepath):
1985
- print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
1986
- configuration = {}
1987
- return configuration
1988
-
1989
-
1990
- def validate_and_load_state(args, state):
1991
- if state:
1992
- json_filepath = os.path.join(args.project_path, args.state)
1993
- else:
1994
- json_filepath = os.path.join(args.project_path, "files", "state.json")
1995
-
1996
- if os.path.exists(json_filepath):
1997
- if os.path.isfile(json_filepath):
1998
- with open(json_filepath, 'r', encoding=UTF_8) as fi:
1999
- state = json.load(fi)
2000
- elif state.lstrip().startswith("{"):
2001
- state = json.loads(state)
2002
- else:
2003
- state = {}
2004
- return state
2005
-
2006
-
2007
- def reset_local_file_directory(args):
2008
- files_path = os.path.join(args.project_path, OUTPUT_FILES_DIR)
2009
- confirm = input(
2010
- "This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
2011
- if confirm.lower() != "y":
2012
- print_library_log("Reset canceled")
2013
- else:
2014
- try:
2015
- if os.path.exists(files_path) and os.path.isdir(files_path):
2016
- shutil.rmtree(files_path)
2017
- print_library_log("Reset Successful")
2018
- except Exception as e:
2019
- print_library_log("Reset Failed", Logging.Level.SEVERE)
2020
- raise e
2021
-
2022
-
2023
449
  if __name__ == "__main__":
2024
450
  main()