fivetran-connector-sdk 0.11.21.1__py3-none-any.whl → 0.12.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivetran_connector_sdk/__init__.py +163 -81
- {fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/METADATA +1 -1
- {fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/RECORD +6 -6
- {fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/WHEEL +0 -0
- {fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/entry_points.txt +0 -0
- {fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/top_level.txt +0 -0
@@ -14,6 +14,7 @@ import sys
|
|
14
14
|
import time
|
15
15
|
import traceback
|
16
16
|
import re
|
17
|
+
import socket
|
17
18
|
|
18
19
|
from concurrent import futures
|
19
20
|
from datetime import datetime
|
@@ -25,7 +26,7 @@ from fivetran_connector_sdk.protos import common_pb2
|
|
25
26
|
from fivetran_connector_sdk.protos import connector_sdk_pb2
|
26
27
|
from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
|
27
28
|
|
28
|
-
__version__ = "0.
|
29
|
+
__version__ = "0.12.09.2"
|
29
30
|
|
30
31
|
MAC_OS = "mac"
|
31
32
|
WIN_OS = "windows"
|
@@ -55,8 +56,11 @@ COMMANDS_AND_SYNONYMS = {
|
|
55
56
|
|
56
57
|
CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
|
57
58
|
DEBUGGING = False
|
59
|
+
EXECUTED_VIA_CLI = False
|
58
60
|
TABLES = {}
|
59
61
|
|
62
|
+
JAVA_LONG_MAX_VALUE = 9223372036854775807
|
63
|
+
|
60
64
|
|
61
65
|
class Logging:
|
62
66
|
class Level(IntEnum):
|
@@ -76,9 +80,10 @@ class Logging:
|
|
76
80
|
message (str): The message to log.
|
77
81
|
"""
|
78
82
|
if DEBUGGING:
|
79
|
-
|
83
|
+
current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
|
84
|
+
print(f"{current_time} {level.name}: {message}")
|
80
85
|
else:
|
81
|
-
print(f'{{"level":"{level.name}", "message": "{message}", "
|
86
|
+
print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "connector_sdk"}}')
|
82
87
|
|
83
88
|
@staticmethod
|
84
89
|
def fine(message: str):
|
@@ -257,7 +262,7 @@ def check_newer_version():
|
|
257
262
|
data = json.loads(response.text)
|
258
263
|
latest_version = data["info"]["version"]
|
259
264
|
if __version__ < latest_version:
|
260
|
-
|
265
|
+
print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
|
261
266
|
f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
|
262
267
|
|
263
268
|
with open(last_check_file_path, 'w') as f_out:
|
@@ -265,7 +270,7 @@ def check_newer_version():
|
|
265
270
|
break
|
266
271
|
except Exception:
|
267
272
|
retry_after = 2 ** index
|
268
|
-
|
273
|
+
print_library_log(f"Unable to check if a newer version of `fivetran-connector-sdk` is available. Retrying again after {retry_after} seconds", Logging.Level.WARNING)
|
269
274
|
time.sleep(retry_after)
|
270
275
|
|
271
276
|
|
@@ -316,10 +321,10 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
|
|
316
321
|
def map_inferred_data_type(k, mapped_data, v):
|
317
322
|
# We can infer type from the value
|
318
323
|
if isinstance(v, int):
|
319
|
-
if abs(v) >
|
320
|
-
mapped_data[k] = common_pb2.ValueType(
|
324
|
+
if abs(v) > JAVA_LONG_MAX_VALUE:
|
325
|
+
mapped_data[k] = common_pb2.ValueType(float=v)
|
321
326
|
else:
|
322
|
-
mapped_data[k] = common_pb2.ValueType(
|
327
|
+
mapped_data[k] = common_pb2.ValueType(long=v)
|
323
328
|
elif isinstance(v, float):
|
324
329
|
mapped_data[k] = common_pb2.ValueType(float=v)
|
325
330
|
elif isinstance(v, bool):
|
@@ -404,8 +409,8 @@ def _yield_check(stack):
|
|
404
409
|
calling_code = stack[1].code_context[0]
|
405
410
|
if f"{called_method}(" in calling_code:
|
406
411
|
if 'yield' not in calling_code:
|
407
|
-
|
408
|
-
f"
|
412
|
+
print_library_log(
|
413
|
+
f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
|
409
414
|
os._exit(1)
|
410
415
|
else:
|
411
416
|
# This should never happen
|
@@ -433,8 +438,8 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
|
|
433
438
|
if string_only:
|
434
439
|
for k, v in incoming.items():
|
435
440
|
if not isinstance(v, str):
|
436
|
-
|
437
|
-
"
|
441
|
+
print_library_log(
|
442
|
+
"All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
|
438
443
|
os._exit(1)
|
439
444
|
|
440
445
|
return incoming
|
@@ -454,28 +459,53 @@ def is_connection_name_valid(connection: str):
|
|
454
459
|
|
455
460
|
|
456
461
|
def log_unused_deps_error(package_name: str, version: str):
|
457
|
-
|
462
|
+
print_library_log(f"Please remove `{package_name}` from requirements.txt."
|
458
463
|
f" The latest version of `{package_name}` is always available when executing your code."
|
459
|
-
f" Current version: {version}")
|
464
|
+
f" Current version: {version}", Logging.Level.SEVERE)
|
460
465
|
os._exit(1)
|
461
466
|
|
462
467
|
|
463
468
|
def validate_deploy_parameters(connection, deploy_key):
|
464
469
|
if not deploy_key or not connection:
|
465
|
-
|
470
|
+
print_library_log("The deploy command needs the following parameters:"
|
466
471
|
"\n\tRequired:\n"
|
467
472
|
"\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
|
468
473
|
"\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
|
469
474
|
"\t(Optional):\n"
|
470
475
|
"\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
|
471
|
-
"\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)")
|
476
|
+
"\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
|
472
477
|
os._exit(1)
|
473
478
|
if not is_connection_name_valid(connection):
|
474
|
-
|
479
|
+
print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
|
475
480
|
f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
|
476
|
-
f"letters, or digits (0-9). Uppercase characters are not allowed.")
|
481
|
+
f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
|
477
482
|
os._exit(1)
|
478
483
|
|
484
|
+
def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
|
485
|
+
"""Logs a library message with the specified logging level.
|
486
|
+
|
487
|
+
Args:
|
488
|
+
level (Logging.Level): The logging level.
|
489
|
+
message (str): The message to log.
|
490
|
+
"""
|
491
|
+
if DEBUGGING or EXECUTED_VIA_CLI:
|
492
|
+
current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
|
493
|
+
print(f'{current_time} {level.name}: {message}')
|
494
|
+
else:
|
495
|
+
print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "library"}}')
|
496
|
+
|
497
|
+
|
498
|
+
def is_port_in_use(port: int):
|
499
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
500
|
+
return s.connect_ex(('127.0.0.1', port)) == 0
|
501
|
+
|
502
|
+
|
503
|
+
def get_available_port():
|
504
|
+
for port in range(50051, 50061):
|
505
|
+
if not is_port_in_use(port):
|
506
|
+
return port
|
507
|
+
return None
|
508
|
+
|
479
509
|
|
480
510
|
class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
481
511
|
def __init__(self, update, schema=None):
|
@@ -524,7 +554,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
524
554
|
key = re.split(r"==|>=|<=|>|<", requirement)[0]
|
525
555
|
requirements_dict[key.lower()] = requirement.lower()
|
526
556
|
except ValueError:
|
527
|
-
|
557
|
+
print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
|
528
558
|
return requirements_dict
|
529
559
|
|
530
560
|
def validate_requirements_file(self, project_path: str, is_deploy: bool):
|
@@ -557,7 +587,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
557
587
|
(requirements.keys() & tmp_requirements.keys())
|
558
588
|
if requirements[key] != tmp_requirements[key]}
|
559
589
|
if version_mismatch_deps:
|
560
|
-
|
590
|
+
print_library_log("We recommend using the current stable version for the following:", Logging.Level.WARNING)
|
561
591
|
print(version_mismatch_deps)
|
562
592
|
|
563
593
|
missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
|
@@ -569,11 +599,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
569
599
|
self.handle_unused_deps(unused_deps)
|
570
600
|
else:
|
571
601
|
if os.path.exists(REQUIREMENTS_TXT):
|
572
|
-
|
602
|
+
print_library_log("`requirements.txt` is not required as no additional "
|
573
603
|
"Python libraries are required or all required libraries for "
|
574
|
-
"your code are pre-installed.")
|
604
|
+
"your code are pre-installed.", Logging.Level.WARNING)
|
575
605
|
|
576
|
-
if is_deploy:
|
606
|
+
if is_deploy: print_library_log("Successful validation of requirements.txt")
|
577
607
|
|
578
608
|
def handle_unused_deps(self, unused_deps):
|
579
609
|
if 'fivetran_connector_sdk' in unused_deps:
|
@@ -581,18 +611,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
581
611
|
elif 'requests' in unused_deps:
|
582
612
|
log_unused_deps_error("requests", "2.32.3")
|
583
613
|
else:
|
584
|
-
|
614
|
+
print_library_log("The following dependencies are not needed, "
|
585
615
|
"they are not used or already installed. Please remove them from requirements.txt:")
|
586
616
|
print(*unused_deps)
|
587
617
|
|
588
618
|
def handle_missing_deps(self, is_deploy, missing_deps):
|
589
|
-
|
590
|
-
print(log_level +
|
591
|
-
": Please include the following dependency libraries in requirements.txt, to be used by "
|
619
|
+
print_library_log("Please include the following dependency libraries in requirements.txt, to be used by "
|
592
620
|
"Fivetran production. "
|
593
621
|
"For more information, please visit: "
|
594
622
|
"https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
|
595
|
-
"#workingwithrequirementstxtfile")
|
623
|
+
"#workingwithrequirementstxtfile", Logging.Level.SEVERE)
|
596
624
|
print(*list(missing_deps.values()))
|
597
625
|
if is_deploy:
|
598
626
|
os._exit(1)
|
@@ -604,7 +632,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
604
632
|
with open(REQUIREMENTS_TXT, 'w'):
|
605
633
|
pass
|
606
634
|
requirements = {}
|
607
|
-
|
635
|
+
print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
|
608
636
|
return requirements
|
609
637
|
|
610
638
|
# Call this method to deploy the connector to Fivetran platform
|
@@ -619,6 +647,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
619
647
|
connection (str): The connection name.
|
620
648
|
configuration (dict): The configuration dictionary.
|
621
649
|
"""
|
650
|
+
global EXECUTED_VIA_CLI
|
651
|
+
EXECUTED_VIA_CLI = True
|
652
|
+
|
622
653
|
validate_deploy_parameters(connection, deploy_key)
|
623
654
|
|
624
655
|
_check_dict(configuration, True)
|
@@ -642,8 +673,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
642
673
|
|
643
674
|
if connection_id:
|
644
675
|
if service != 'connector_sdk':
|
645
|
-
|
646
|
-
f"
|
676
|
+
print_library_log(
|
677
|
+
f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
|
647
678
|
os._exit(1)
|
648
679
|
else:
|
649
680
|
if force:
|
@@ -652,17 +683,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
652
683
|
confirm = input(
|
653
684
|
f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code and configuration. Do you want to proceed with the update? (Y/N): ")
|
654
685
|
if confirm.lower() == "y":
|
655
|
-
|
686
|
+
print_library_log("Updating the connection...\n")
|
656
687
|
self.__upload_project(
|
657
688
|
project_path, deploy_key, group_id, group_name, connection)
|
658
689
|
self.__update_connection(
|
659
690
|
connection_id, connection, group_name, connection_config, deploy_key)
|
660
691
|
print("✓")
|
661
|
-
|
662
|
-
|
663
|
-
f"
|
692
|
+
print_library_log(f"Connector ID: {connection_id}")
|
693
|
+
print_library_log(
|
694
|
+
f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
|
664
695
|
else:
|
665
|
-
|
696
|
+
print_library_log("Update canceled. The process is now terminating.")
|
666
697
|
os._exit(1)
|
667
698
|
else:
|
668
699
|
self.__upload_project(project_path, deploy_key,
|
@@ -670,20 +701,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
670
701
|
response = self.__create_connection(
|
671
702
|
deploy_key, group_id, connection_config)
|
672
703
|
if response.ok:
|
673
|
-
|
674
|
-
f"
|
704
|
+
print_library_log(
|
705
|
+
f"The connection '{connection}' has been created successfully.\n")
|
675
706
|
connection_id = response.json()['data']['id']
|
676
|
-
|
677
|
-
|
678
|
-
f"
|
707
|
+
print_library_log(f"Connector ID: {connection_id}")
|
708
|
+
print_library_log(
|
709
|
+
f"Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
|
679
710
|
else:
|
680
|
-
|
681
|
-
f"
|
711
|
+
print_library_log(
|
712
|
+
f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
|
713
|
+
self.__cleanup_uploaded_project(deploy_key,group_id, connection)
|
714
|
+
print_library_log("Please try again with the deploy command after resolving the issue!")
|
682
715
|
os._exit(1)
|
683
716
|
|
684
717
|
def __upload_project(self, project_path: str, deploy_key: str, group_id: str, group_name: str, connection: str):
|
685
|
-
|
686
|
-
f"
|
718
|
+
print_library_log(
|
719
|
+
f"Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
|
687
720
|
upload_file_path = self.__create_upload_file(project_path)
|
688
721
|
upload_result = self.__upload(
|
689
722
|
upload_file_path, deploy_key, group_id, connection)
|
@@ -691,6 +724,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
691
724
|
if not upload_result:
|
692
725
|
os._exit(1)
|
693
726
|
|
727
|
+
def __cleanup_uploaded_project(self, deploy_key: str, group_id: str, connection: str):
|
728
|
+
cleanup_result = self.__cleanup_uploaded_code(deploy_key, group_id, connection)
|
729
|
+
if not cleanup_result:
|
730
|
+
os._exit(1)
|
731
|
+
|
694
732
|
@staticmethod
|
695
733
|
def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
|
696
734
|
"""Updates the connection with the given ID, name, group, configuration, and deployment key.
|
@@ -713,8 +751,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
713
751
|
})
|
714
752
|
|
715
753
|
if not resp.ok:
|
716
|
-
|
717
|
-
f"
|
754
|
+
print_library_log(
|
755
|
+
f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.", Logging.Level.SEVERE)
|
718
756
|
os._exit(1)
|
719
757
|
|
720
758
|
@staticmethod
|
@@ -734,8 +772,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
734
772
|
headers={"Authorization": f"Basic {deploy_key}"},
|
735
773
|
params={"schema": name})
|
736
774
|
if not resp.ok:
|
737
|
-
|
738
|
-
f"
|
775
|
+
print_library_log(
|
776
|
+
f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
|
739
777
|
os._exit(1)
|
740
778
|
|
741
779
|
if resp.json()['data']['items']:
|
@@ -776,7 +814,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
776
814
|
Returns:
|
777
815
|
str: The path to the upload file.
|
778
816
|
"""
|
779
|
-
|
817
|
+
print_library_log("Packaging your project for upload...")
|
780
818
|
zip_file_path = self.__zip_folder(project_path)
|
781
819
|
print("✓")
|
782
820
|
return zip_file_path
|
@@ -803,8 +841,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
803
841
|
zipf.write(file_path, arcname)
|
804
842
|
|
805
843
|
if not connector_file_exists:
|
806
|
-
|
807
|
-
"
|
844
|
+
print_library_log(
|
845
|
+
"The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.", Logging.Level.SEVERE)
|
808
846
|
os._exit(1)
|
809
847
|
return upload_filepath
|
810
848
|
|
@@ -846,15 +884,37 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
846
884
|
Returns:
|
847
885
|
bool: True if the upload was successful, False otherwise.
|
848
886
|
"""
|
849
|
-
|
850
|
-
response = rq.post(f"https://api.fivetran.com/
|
887
|
+
print_library_log("Uploading your project...")
|
888
|
+
response = rq.post(f"https://api.fivetran.com/v1/deploy/{group_id}/{connection}",
|
851
889
|
files={'file': open(local_path, 'rb')},
|
852
890
|
headers={"Authorization": f"Basic {deploy_key}"})
|
853
891
|
if response.ok:
|
854
892
|
print("✓")
|
855
893
|
return True
|
856
894
|
|
857
|
-
|
895
|
+
print_library_log(f"Unable to upload the project, failed with error: {response.reason}", Logging.Level.SEVERE)
|
896
|
+
return False
|
897
|
+
|
898
|
+
@staticmethod
|
899
|
+
def __cleanup_uploaded_code(deploy_key: str, group_id: str, connection: str) -> bool:
|
900
|
+
"""Cleans up the uploaded code file for the specified group and connection, if creation fails.
|
901
|
+
|
902
|
+
Args:
|
903
|
+
deploy_key (str): The deployment key.
|
904
|
+
group_id (str): The group ID.
|
905
|
+
connection (str): The connection name.
|
906
|
+
|
907
|
+
Returns:
|
908
|
+
bool: True if the cleanup was successful, False otherwise.
|
909
|
+
"""
|
910
|
+
print_library_log("INFO: Cleaning up your uploaded project ")
|
911
|
+
response = rq.post(f"https://api.fivetran.com/v1/cleanup_code/{group_id}/{connection}",
|
912
|
+
headers={"Authorization": f"Basic {deploy_key}"})
|
913
|
+
if response.ok:
|
914
|
+
print("✓")
|
915
|
+
return True
|
916
|
+
|
917
|
+
print_library_log(f"SEVERE: Unable to cleanup the project, failed with error: {response.reason}", Logging.Level.SEVERE)
|
858
918
|
return False
|
859
919
|
|
860
920
|
@staticmethod
|
@@ -891,23 +951,23 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
891
951
|
resp = rq.get(groups_url, headers=headers, params=params)
|
892
952
|
|
893
953
|
if not resp.ok:
|
894
|
-
|
895
|
-
f"
|
954
|
+
print_library_log(
|
955
|
+
f"Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
|
896
956
|
os._exit(1)
|
897
957
|
|
898
958
|
data = resp.json().get("data", {})
|
899
959
|
groups = data.get("items")
|
900
960
|
|
901
961
|
if not groups:
|
902
|
-
|
962
|
+
print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
|
903
963
|
os._exit(1)
|
904
964
|
|
905
965
|
if not group:
|
906
966
|
if len(groups) == 1:
|
907
967
|
return groups[0]['id'], groups[0]['name']
|
908
968
|
else:
|
909
|
-
|
910
|
-
"
|
969
|
+
print_library_log(
|
970
|
+
"Destination name is required when there are multiple destinations in the account", Logging.Level.SEVERE)
|
911
971
|
os._exit(1)
|
912
972
|
else:
|
913
973
|
while True:
|
@@ -924,8 +984,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
924
984
|
data = resp.json().get("data", {})
|
925
985
|
groups = data.get("items", [])
|
926
986
|
|
927
|
-
|
928
|
-
f"
|
987
|
+
print_library_log(
|
988
|
+
f"The specified destination '{group}' was not found in your account.", Logging.Level.SEVERE)
|
929
989
|
os._exit(1)
|
930
990
|
|
931
991
|
# Call this method to run the connector in production
|
@@ -950,7 +1010,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
950
1010
|
Logging.LOG_LEVEL = log_level
|
951
1011
|
|
952
1012
|
if not DEBUGGING:
|
953
|
-
|
1013
|
+
print_library_log(f"Running on fivetran_connector_sdk: {__version__}")
|
954
1014
|
|
955
1015
|
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
|
956
1016
|
connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
|
@@ -1006,7 +1066,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1006
1066
|
download_filename = f"sdk-connector-tester-{os_name}-{TESTER_VERSION}.zip"
|
1007
1067
|
download_filepath = os.path.join(tester_root_dir, download_filename)
|
1008
1068
|
try:
|
1009
|
-
|
1069
|
+
print_library_log(f"Downloading connector tester version: {TESTER_VERSION} ")
|
1010
1070
|
download_url = f"https://github.com/fivetran/fivetran_sdk_tools/releases/download/{TESTER_VERSION}/{download_filename}"
|
1011
1071
|
r = rq.get(download_url)
|
1012
1072
|
if r.ok:
|
@@ -1038,15 +1098,21 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1038
1098
|
|
1039
1099
|
project_path = os.getcwd() if project_path is None else project_path
|
1040
1100
|
self.validate_requirements_file(project_path, False)
|
1041
|
-
|
1042
|
-
|
1101
|
+
print_library_log(f"Debugging connector at: {project_path}")
|
1102
|
+
available_port = get_available_port()
|
1103
|
+
|
1104
|
+
if available_port is None:
|
1105
|
+
raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
|
1106
|
+
"Please ensure a port is available and try again")
|
1107
|
+
|
1108
|
+
server = self.run(available_port, configuration, state, log_level=log_level)
|
1043
1109
|
|
1044
1110
|
# Uncomment this to run the tester manually
|
1045
1111
|
# server.wait_for_termination()
|
1046
1112
|
|
1047
1113
|
try:
|
1048
|
-
|
1049
|
-
for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path,
|
1114
|
+
print_library_log(f"Running connector tester...")
|
1115
|
+
for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port):
|
1050
1116
|
print(log_msg, end="")
|
1051
1117
|
except:
|
1052
1118
|
print(traceback.format_exc())
|
@@ -1088,7 +1154,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1088
1154
|
yield modified_line
|
1089
1155
|
|
1090
1156
|
@staticmethod
|
1091
|
-
def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int
|
1157
|
+
def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int):
|
1092
1158
|
"""Runs the connector tester.
|
1093
1159
|
|
1094
1160
|
Args:
|
@@ -1168,10 +1234,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1168
1234
|
if not self.schema_method:
|
1169
1235
|
return connector_sdk_pb2.SchemaResponse(schema_response_not_supported=True)
|
1170
1236
|
else:
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1237
|
+
try:
|
1238
|
+
configuration = self.configuration if self.configuration else request.configuration
|
1239
|
+
response = self.schema_method(configuration)
|
1240
|
+
self.process_tables(response)
|
1241
|
+
return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
|
1242
|
+
|
1243
|
+
except Exception as e:
|
1244
|
+
tb = traceback.format_exc()
|
1245
|
+
error_message = f"Error: {str(e)}\n{tb}"
|
1246
|
+
raise RuntimeError(error_message) from e
|
1175
1247
|
|
1176
1248
|
def process_tables(self, response):
|
1177
1249
|
for entry in response:
|
@@ -1229,7 +1301,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1229
1301
|
elif type.upper() == "SHORT":
|
1230
1302
|
column.type = common_pb2.DataType.SHORT
|
1231
1303
|
elif type.upper() == "INT":
|
1232
|
-
column.type = common_pb2.DataType.
|
1304
|
+
column.type = common_pb2.DataType.INT
|
1233
1305
|
elif type.upper() == "LONG":
|
1234
1306
|
column.type = common_pb2.DataType.LONG
|
1235
1307
|
elif type.upper() == "DECIMAL":
|
@@ -1280,6 +1352,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1280
1352
|
if str(e) != "'NoneType' object is not iterable":
|
1281
1353
|
raise e
|
1282
1354
|
|
1355
|
+
except Exception as e:
|
1356
|
+
tb = traceback.format_exc()
|
1357
|
+
error_message = f"Error: {str(e)}\n{tb}"
|
1358
|
+
raise RuntimeError(error_message) from e
|
1359
|
+
|
1283
1360
|
|
1284
1361
|
def find_connector_object(project_path) -> Connector:
|
1285
1362
|
"""Finds the connector object in the given project path.
|
@@ -1303,8 +1380,8 @@ def find_connector_object(project_path) -> Connector:
|
|
1303
1380
|
if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
|
1304
1381
|
return obj_attr
|
1305
1382
|
|
1306
|
-
|
1307
|
-
"
|
1383
|
+
print_library_log(
|
1384
|
+
"The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
|
1308
1385
|
sys.exit(1)
|
1309
1386
|
|
1310
1387
|
|
@@ -1371,6 +1448,8 @@ def main():
|
|
1371
1448
|
"""The main entry point for the script.
|
1372
1449
|
Parses command line arguments and passes them to connector object methods
|
1373
1450
|
"""
|
1451
|
+
global EXECUTED_VIA_CLI
|
1452
|
+
EXECUTED_VIA_CLI = True
|
1374
1453
|
|
1375
1454
|
parser = argparse.ArgumentParser(allow_abbrev=False)
|
1376
1455
|
|
@@ -1389,7 +1468,7 @@ def main():
|
|
1389
1468
|
args = parser.parse_args()
|
1390
1469
|
|
1391
1470
|
if args.command.lower() == "version":
|
1392
|
-
|
1471
|
+
print_library_log("fivetran_connector_sdk " + __version__)
|
1393
1472
|
return
|
1394
1473
|
|
1395
1474
|
connector_object = find_connector_object(args.project_path)
|
@@ -1406,7 +1485,7 @@ def main():
|
|
1406
1485
|
|
1407
1486
|
if args.command.lower() == "deploy":
|
1408
1487
|
if args.state:
|
1409
|
-
|
1488
|
+
print_library_log("'state' parameter is not used for 'deploy' command", Logging.Level.WARNING)
|
1410
1489
|
connector_object.deploy(args.project_path, args.force, ft_deploy_key, ft_group, ft_connection, configuration)
|
1411
1490
|
|
1412
1491
|
elif args.command.lower() == "debug":
|
@@ -1430,6 +1509,9 @@ def validate_and_load_configuration(args, configuration):
|
|
1430
1509
|
"Configuration must be provided as a JSON file. Please check your input. Reference: "
|
1431
1510
|
"https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
|
1432
1511
|
else:
|
1512
|
+
json_filepath = os.path.join(args.project_path, "configuration.json")
|
1513
|
+
if os.path.exists(json_filepath):
|
1514
|
+
print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
|
1433
1515
|
configuration = {}
|
1434
1516
|
return configuration
|
1435
1517
|
|
@@ -1452,14 +1534,14 @@ def reset_local_file_directory(args):
|
|
1452
1534
|
confirm = input(
|
1453
1535
|
"This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
|
1454
1536
|
if confirm.lower() != "y":
|
1455
|
-
|
1537
|
+
print_library_log("Reset canceled")
|
1456
1538
|
else:
|
1457
1539
|
try:
|
1458
1540
|
if os.path.exists(files_path) and os.path.isdir(files_path):
|
1459
1541
|
shutil.rmtree(files_path)
|
1460
|
-
|
1542
|
+
print_library_log("Reset Successful")
|
1461
1543
|
except Exception as e:
|
1462
|
-
|
1544
|
+
print_library_log("Reset Failed", Logging.Level.SEVERE)
|
1463
1545
|
raise e
|
1464
1546
|
|
1465
1547
|
|
{fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fivetran_connector_sdk
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.12.9.2
|
4
4
|
Summary: Build custom connectors on Fivetran platform
|
5
5
|
Author-email: Fivetran <developers@fivetran.com>
|
6
6
|
Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
|
{fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/RECORD
RENAMED
@@ -1,4 +1,4 @@
|
|
1
|
-
fivetran_connector_sdk/__init__.py,sha256=
|
1
|
+
fivetran_connector_sdk/__init__.py,sha256=2k64NC7j9Q8HeF4y7_NBeyO2fqtexxgPQeuHfEAIslQ,62965
|
2
2
|
fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
3
|
fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
|
4
4
|
fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
|
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
|
|
6
6
|
fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
|
7
7
|
fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
|
8
8
|
fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
|
9
|
-
fivetran_connector_sdk-0.
|
10
|
-
fivetran_connector_sdk-0.
|
11
|
-
fivetran_connector_sdk-0.
|
12
|
-
fivetran_connector_sdk-0.
|
13
|
-
fivetran_connector_sdk-0.
|
9
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/METADATA,sha256=3yH28B6Qmc0TIL1tqPwmltqetrH3ewwBxMMfeJI35n0,2938
|
10
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
11
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
|
12
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
|
13
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/RECORD,,
|
{fivetran_connector_sdk-0.11.21.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|
File without changes
|