fivetran-connector-sdk 0.11.14.1__py3-none-any.whl → 0.12.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivetran_connector_sdk/__init__.py +163 -79
- {fivetran_connector_sdk-0.11.14.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/METADATA +6 -5
- {fivetran_connector_sdk-0.11.14.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/RECORD +6 -6
- {fivetran_connector_sdk-0.11.14.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/WHEEL +1 -1
- {fivetran_connector_sdk-0.11.14.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/entry_points.txt +0 -0
- {fivetran_connector_sdk-0.11.14.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/top_level.txt +0 -0
@@ -14,6 +14,7 @@ import sys
|
|
14
14
|
import time
|
15
15
|
import traceback
|
16
16
|
import re
|
17
|
+
import socket
|
17
18
|
|
18
19
|
from concurrent import futures
|
19
20
|
from datetime import datetime
|
@@ -25,7 +26,7 @@ from fivetran_connector_sdk.protos import common_pb2
|
|
25
26
|
from fivetran_connector_sdk.protos import connector_sdk_pb2
|
26
27
|
from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
|
27
28
|
|
28
|
-
__version__ = "0.
|
29
|
+
__version__ = "0.12.09.2"
|
29
30
|
|
30
31
|
MAC_OS = "mac"
|
31
32
|
WIN_OS = "windows"
|
@@ -55,8 +56,11 @@ COMMANDS_AND_SYNONYMS = {
|
|
55
56
|
|
56
57
|
CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
|
57
58
|
DEBUGGING = False
|
59
|
+
EXECUTED_VIA_CLI = False
|
58
60
|
TABLES = {}
|
59
61
|
|
62
|
+
JAVA_LONG_MAX_VALUE = 9223372036854775807
|
63
|
+
|
60
64
|
|
61
65
|
class Logging:
|
62
66
|
class Level(IntEnum):
|
@@ -76,9 +80,10 @@ class Logging:
|
|
76
80
|
message (str): The message to log.
|
77
81
|
"""
|
78
82
|
if DEBUGGING:
|
79
|
-
|
83
|
+
current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
|
84
|
+
print(f"{current_time} {level.name}: {message}")
|
80
85
|
else:
|
81
|
-
print(f'{{"level":"{level.name}", "message": "{message}", "
|
86
|
+
print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "connector_sdk"}}')
|
82
87
|
|
83
88
|
@staticmethod
|
84
89
|
def fine(message: str):
|
@@ -257,7 +262,7 @@ def check_newer_version():
|
|
257
262
|
data = json.loads(response.text)
|
258
263
|
latest_version = data["info"]["version"]
|
259
264
|
if __version__ < latest_version:
|
260
|
-
|
265
|
+
print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
|
261
266
|
f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
|
262
267
|
|
263
268
|
with open(last_check_file_path, 'w') as f_out:
|
@@ -265,7 +270,7 @@ def check_newer_version():
|
|
265
270
|
break
|
266
271
|
except Exception:
|
267
272
|
retry_after = 2 ** index
|
268
|
-
|
273
|
+
print_library_log(f"Unable to check if a newer version of `fivetran-connector-sdk` is available. Retrying again after {retry_after} seconds", Logging.Level.WARNING)
|
269
274
|
time.sleep(retry_after)
|
270
275
|
|
271
276
|
|
@@ -316,10 +321,10 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
|
|
316
321
|
def map_inferred_data_type(k, mapped_data, v):
|
317
322
|
# We can infer type from the value
|
318
323
|
if isinstance(v, int):
|
319
|
-
if abs(v) >
|
320
|
-
mapped_data[k] = common_pb2.ValueType(
|
324
|
+
if abs(v) > JAVA_LONG_MAX_VALUE:
|
325
|
+
mapped_data[k] = common_pb2.ValueType(float=v)
|
321
326
|
else:
|
322
|
-
mapped_data[k] = common_pb2.ValueType(
|
327
|
+
mapped_data[k] = common_pb2.ValueType(long=v)
|
323
328
|
elif isinstance(v, float):
|
324
329
|
mapped_data[k] = common_pb2.ValueType(float=v)
|
325
330
|
elif isinstance(v, bool):
|
@@ -404,8 +409,8 @@ def _yield_check(stack):
|
|
404
409
|
calling_code = stack[1].code_context[0]
|
405
410
|
if f"{called_method}(" in calling_code:
|
406
411
|
if 'yield' not in calling_code:
|
407
|
-
|
408
|
-
f"
|
412
|
+
print_library_log(
|
413
|
+
f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
|
409
414
|
os._exit(1)
|
410
415
|
else:
|
411
416
|
# This should never happen
|
@@ -433,8 +438,8 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
|
|
433
438
|
if string_only:
|
434
439
|
for k, v in incoming.items():
|
435
440
|
if not isinstance(v, str):
|
436
|
-
|
437
|
-
"
|
441
|
+
print_library_log(
|
442
|
+
"All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
|
438
443
|
os._exit(1)
|
439
444
|
|
440
445
|
return incoming
|
@@ -454,28 +459,53 @@ def is_connection_name_valid(connection: str):
|
|
454
459
|
|
455
460
|
|
456
461
|
def log_unused_deps_error(package_name: str, version: str):
|
457
|
-
|
462
|
+
print_library_log(f"Please remove `{package_name}` from requirements.txt."
|
458
463
|
f" The latest version of `{package_name}` is always available when executing your code."
|
459
|
-
f" Current version: {version}")
|
464
|
+
f" Current version: {version}", Logging.Level.SEVERE)
|
460
465
|
os._exit(1)
|
461
466
|
|
462
467
|
|
463
468
|
def validate_deploy_parameters(connection, deploy_key):
|
464
469
|
if not deploy_key or not connection:
|
465
|
-
|
470
|
+
print_library_log("The deploy command needs the following parameters:"
|
466
471
|
"\n\tRequired:\n"
|
467
472
|
"\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
|
468
473
|
"\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
|
469
474
|
"\t(Optional):\n"
|
470
475
|
"\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
|
471
|
-
"\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)")
|
476
|
+
"\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
|
472
477
|
os._exit(1)
|
473
478
|
if not is_connection_name_valid(connection):
|
474
|
-
|
479
|
+
print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
|
475
480
|
f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
|
476
|
-
f"letters, or digits (0-9). Uppercase characters are not allowed.")
|
481
|
+
f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
|
477
482
|
os._exit(1)
|
478
483
|
|
484
|
+
def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
|
485
|
+
"""Logs a library message with the specified logging level.
|
486
|
+
|
487
|
+
Args:
|
488
|
+
level (Logging.Level): The logging level.
|
489
|
+
message (str): The message to log.
|
490
|
+
"""
|
491
|
+
if DEBUGGING or EXECUTED_VIA_CLI:
|
492
|
+
current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
|
493
|
+
print(f'{current_time} {level.name}: {message}')
|
494
|
+
else:
|
495
|
+
print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "library"}}')
|
496
|
+
|
497
|
+
|
498
|
+
def is_port_in_use(port: int):
|
499
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
500
|
+
return s.connect_ex(('127.0.0.1', port)) == 0
|
501
|
+
|
502
|
+
|
503
|
+
def get_available_port():
|
504
|
+
for port in range(50051, 50061):
|
505
|
+
if not is_port_in_use(port):
|
506
|
+
return port
|
507
|
+
return None
|
508
|
+
|
479
509
|
|
480
510
|
class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
481
511
|
def __init__(self, update, schema=None):
|
@@ -524,7 +554,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
524
554
|
key = re.split(r"==|>=|<=|>|<", requirement)[0]
|
525
555
|
requirements_dict[key.lower()] = requirement.lower()
|
526
556
|
except ValueError:
|
527
|
-
|
557
|
+
print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
|
528
558
|
return requirements_dict
|
529
559
|
|
530
560
|
def validate_requirements_file(self, project_path: str, is_deploy: bool):
|
@@ -557,7 +587,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
557
587
|
(requirements.keys() & tmp_requirements.keys())
|
558
588
|
if requirements[key] != tmp_requirements[key]}
|
559
589
|
if version_mismatch_deps:
|
560
|
-
|
590
|
+
print_library_log("We recommend using the current stable version for the following:", Logging.Level.WARNING)
|
561
591
|
print(version_mismatch_deps)
|
562
592
|
|
563
593
|
missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
|
@@ -569,11 +599,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
569
599
|
self.handle_unused_deps(unused_deps)
|
570
600
|
else:
|
571
601
|
if os.path.exists(REQUIREMENTS_TXT):
|
572
|
-
|
602
|
+
print_library_log("`requirements.txt` is not required as no additional "
|
573
603
|
"Python libraries are required or all required libraries for "
|
574
|
-
"your code are pre-installed.")
|
604
|
+
"your code are pre-installed.", Logging.Level.WARNING)
|
575
605
|
|
576
|
-
if is_deploy:
|
606
|
+
if is_deploy: print_library_log("Successful validation of requirements.txt")
|
577
607
|
|
578
608
|
def handle_unused_deps(self, unused_deps):
|
579
609
|
if 'fivetran_connector_sdk' in unused_deps:
|
@@ -581,18 +611,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
581
611
|
elif 'requests' in unused_deps:
|
582
612
|
log_unused_deps_error("requests", "2.32.3")
|
583
613
|
else:
|
584
|
-
|
614
|
+
print_library_log("The following dependencies are not needed, "
|
585
615
|
"they are not used or already installed. Please remove them from requirements.txt:")
|
586
616
|
print(*unused_deps)
|
587
617
|
|
588
618
|
def handle_missing_deps(self, is_deploy, missing_deps):
|
589
|
-
|
590
|
-
print(log_level +
|
591
|
-
": Please include the following dependency libraries in requirements.txt, to be used by "
|
619
|
+
print_library_log("Please include the following dependency libraries in requirements.txt, to be used by "
|
592
620
|
"Fivetran production. "
|
593
621
|
"For more information, please visit: "
|
594
622
|
"https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
|
595
|
-
"#workingwithrequirementstxtfile")
|
623
|
+
"#workingwithrequirementstxtfile", Logging.Level.SEVERE)
|
596
624
|
print(*list(missing_deps.values()))
|
597
625
|
if is_deploy:
|
598
626
|
os._exit(1)
|
@@ -604,7 +632,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
604
632
|
with open(REQUIREMENTS_TXT, 'w'):
|
605
633
|
pass
|
606
634
|
requirements = {}
|
607
|
-
|
635
|
+
print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
|
608
636
|
return requirements
|
609
637
|
|
610
638
|
# Call this method to deploy the connector to Fivetran platform
|
@@ -619,6 +647,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
619
647
|
connection (str): The connection name.
|
620
648
|
configuration (dict): The configuration dictionary.
|
621
649
|
"""
|
650
|
+
global EXECUTED_VIA_CLI
|
651
|
+
EXECUTED_VIA_CLI = True
|
652
|
+
|
622
653
|
validate_deploy_parameters(connection, deploy_key)
|
623
654
|
|
624
655
|
_check_dict(configuration, True)
|
@@ -642,8 +673,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
642
673
|
|
643
674
|
if connection_id:
|
644
675
|
if service != 'connector_sdk':
|
645
|
-
|
646
|
-
f"
|
676
|
+
print_library_log(
|
677
|
+
f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
|
647
678
|
os._exit(1)
|
648
679
|
else:
|
649
680
|
if force:
|
@@ -652,16 +683,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
652
683
|
confirm = input(
|
653
684
|
f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code and configuration. Do you want to proceed with the update? (Y/N): ")
|
654
685
|
if confirm.lower() == "y":
|
655
|
-
|
686
|
+
print_library_log("Updating the connection...\n")
|
656
687
|
self.__upload_project(
|
657
688
|
project_path, deploy_key, group_id, group_name, connection)
|
658
689
|
self.__update_connection(
|
659
690
|
connection_id, connection, group_name, connection_config, deploy_key)
|
660
691
|
print("✓")
|
661
|
-
|
662
|
-
|
692
|
+
print_library_log(f"Connector ID: {connection_id}")
|
693
|
+
print_library_log(
|
694
|
+
f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
|
663
695
|
else:
|
664
|
-
|
696
|
+
print_library_log("Update canceled. The process is now terminating.")
|
665
697
|
os._exit(1)
|
666
698
|
else:
|
667
699
|
self.__upload_project(project_path, deploy_key,
|
@@ -669,19 +701,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
669
701
|
response = self.__create_connection(
|
670
702
|
deploy_key, group_id, connection_config)
|
671
703
|
if response.ok:
|
672
|
-
|
673
|
-
f"
|
704
|
+
print_library_log(
|
705
|
+
f"The connection '{connection}' has been created successfully.\n")
|
674
706
|
connection_id = response.json()['data']['id']
|
675
|
-
|
676
|
-
|
707
|
+
print_library_log(f"Connector ID: {connection_id}")
|
708
|
+
print_library_log(
|
709
|
+
f"Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
|
677
710
|
else:
|
678
|
-
|
679
|
-
f"
|
711
|
+
print_library_log(
|
712
|
+
f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
|
713
|
+
self.__cleanup_uploaded_project(deploy_key,group_id, connection)
|
714
|
+
print_library_log("Please try again with the deploy command after resolving the issue!")
|
680
715
|
os._exit(1)
|
681
716
|
|
682
717
|
def __upload_project(self, project_path: str, deploy_key: str, group_id: str, group_name: str, connection: str):
|
683
|
-
|
684
|
-
f"
|
718
|
+
print_library_log(
|
719
|
+
f"Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
|
685
720
|
upload_file_path = self.__create_upload_file(project_path)
|
686
721
|
upload_result = self.__upload(
|
687
722
|
upload_file_path, deploy_key, group_id, connection)
|
@@ -689,6 +724,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
689
724
|
if not upload_result:
|
690
725
|
os._exit(1)
|
691
726
|
|
727
|
+
def __cleanup_uploaded_project(self, deploy_key: str, group_id: str, connection: str):
|
728
|
+
cleanup_result = self.__cleanup_uploaded_code(deploy_key, group_id, connection)
|
729
|
+
if not cleanup_result:
|
730
|
+
os._exit(1)
|
731
|
+
|
692
732
|
@staticmethod
|
693
733
|
def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
|
694
734
|
"""Updates the connection with the given ID, name, group, configuration, and deployment key.
|
@@ -711,8 +751,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
711
751
|
})
|
712
752
|
|
713
753
|
if not resp.ok:
|
714
|
-
|
715
|
-
f"
|
754
|
+
print_library_log(
|
755
|
+
f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.", Logging.Level.SEVERE)
|
716
756
|
os._exit(1)
|
717
757
|
|
718
758
|
@staticmethod
|
@@ -732,8 +772,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
732
772
|
headers={"Authorization": f"Basic {deploy_key}"},
|
733
773
|
params={"schema": name})
|
734
774
|
if not resp.ok:
|
735
|
-
|
736
|
-
f"
|
775
|
+
print_library_log(
|
776
|
+
f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
|
737
777
|
os._exit(1)
|
738
778
|
|
739
779
|
if resp.json()['data']['items']:
|
@@ -774,7 +814,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
774
814
|
Returns:
|
775
815
|
str: The path to the upload file.
|
776
816
|
"""
|
777
|
-
|
817
|
+
print_library_log("Packaging your project for upload...")
|
778
818
|
zip_file_path = self.__zip_folder(project_path)
|
779
819
|
print("✓")
|
780
820
|
return zip_file_path
|
@@ -801,8 +841,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
801
841
|
zipf.write(file_path, arcname)
|
802
842
|
|
803
843
|
if not connector_file_exists:
|
804
|
-
|
805
|
-
"
|
844
|
+
print_library_log(
|
845
|
+
"The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.", Logging.Level.SEVERE)
|
806
846
|
os._exit(1)
|
807
847
|
return upload_filepath
|
808
848
|
|
@@ -844,15 +884,37 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
844
884
|
Returns:
|
845
885
|
bool: True if the upload was successful, False otherwise.
|
846
886
|
"""
|
847
|
-
|
848
|
-
response = rq.post(f"https://api.fivetran.com/
|
887
|
+
print_library_log("Uploading your project...")
|
888
|
+
response = rq.post(f"https://api.fivetran.com/v1/deploy/{group_id}/{connection}",
|
849
889
|
files={'file': open(local_path, 'rb')},
|
850
890
|
headers={"Authorization": f"Basic {deploy_key}"})
|
851
891
|
if response.ok:
|
852
892
|
print("✓")
|
853
893
|
return True
|
854
894
|
|
855
|
-
|
895
|
+
print_library_log(f"Unable to upload the project, failed with error: {response.reason}", Logging.Level.SEVERE)
|
896
|
+
return False
|
897
|
+
|
898
|
+
@staticmethod
|
899
|
+
def __cleanup_uploaded_code(deploy_key: str, group_id: str, connection: str) -> bool:
|
900
|
+
"""Cleans up the uploaded code file for the specified group and connection, if creation fails.
|
901
|
+
|
902
|
+
Args:
|
903
|
+
deploy_key (str): The deployment key.
|
904
|
+
group_id (str): The group ID.
|
905
|
+
connection (str): The connection name.
|
906
|
+
|
907
|
+
Returns:
|
908
|
+
bool: True if the cleanup was successful, False otherwise.
|
909
|
+
"""
|
910
|
+
print_library_log("INFO: Cleaning up your uploaded project ")
|
911
|
+
response = rq.post(f"https://api.fivetran.com/v1/cleanup_code/{group_id}/{connection}",
|
912
|
+
headers={"Authorization": f"Basic {deploy_key}"})
|
913
|
+
if response.ok:
|
914
|
+
print("✓")
|
915
|
+
return True
|
916
|
+
|
917
|
+
print_library_log(f"SEVERE: Unable to cleanup the project, failed with error: {response.reason}", Logging.Level.SEVERE)
|
856
918
|
return False
|
857
919
|
|
858
920
|
@staticmethod
|
@@ -889,23 +951,23 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
889
951
|
resp = rq.get(groups_url, headers=headers, params=params)
|
890
952
|
|
891
953
|
if not resp.ok:
|
892
|
-
|
893
|
-
f"
|
954
|
+
print_library_log(
|
955
|
+
f"Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
|
894
956
|
os._exit(1)
|
895
957
|
|
896
958
|
data = resp.json().get("data", {})
|
897
959
|
groups = data.get("items")
|
898
960
|
|
899
961
|
if not groups:
|
900
|
-
|
962
|
+
print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
|
901
963
|
os._exit(1)
|
902
964
|
|
903
965
|
if not group:
|
904
966
|
if len(groups) == 1:
|
905
967
|
return groups[0]['id'], groups[0]['name']
|
906
968
|
else:
|
907
|
-
|
908
|
-
"
|
969
|
+
print_library_log(
|
970
|
+
"Destination name is required when there are multiple destinations in the account", Logging.Level.SEVERE)
|
909
971
|
os._exit(1)
|
910
972
|
else:
|
911
973
|
while True:
|
@@ -922,8 +984,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
922
984
|
data = resp.json().get("data", {})
|
923
985
|
groups = data.get("items", [])
|
924
986
|
|
925
|
-
|
926
|
-
f"
|
987
|
+
print_library_log(
|
988
|
+
f"The specified destination '{group}' was not found in your account.", Logging.Level.SEVERE)
|
927
989
|
os._exit(1)
|
928
990
|
|
929
991
|
# Call this method to run the connector in production
|
@@ -948,7 +1010,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
948
1010
|
Logging.LOG_LEVEL = log_level
|
949
1011
|
|
950
1012
|
if not DEBUGGING:
|
951
|
-
|
1013
|
+
print_library_log(f"Running on fivetran_connector_sdk: {__version__}")
|
952
1014
|
|
953
1015
|
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
|
954
1016
|
connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
|
@@ -1004,7 +1066,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1004
1066
|
download_filename = f"sdk-connector-tester-{os_name}-{TESTER_VERSION}.zip"
|
1005
1067
|
download_filepath = os.path.join(tester_root_dir, download_filename)
|
1006
1068
|
try:
|
1007
|
-
|
1069
|
+
print_library_log(f"Downloading connector tester version: {TESTER_VERSION} ")
|
1008
1070
|
download_url = f"https://github.com/fivetran/fivetran_sdk_tools/releases/download/{TESTER_VERSION}/{download_filename}"
|
1009
1071
|
r = rq.get(download_url)
|
1010
1072
|
if r.ok:
|
@@ -1036,15 +1098,21 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1036
1098
|
|
1037
1099
|
project_path = os.getcwd() if project_path is None else project_path
|
1038
1100
|
self.validate_requirements_file(project_path, False)
|
1039
|
-
|
1040
|
-
|
1101
|
+
print_library_log(f"Debugging connector at: {project_path}")
|
1102
|
+
available_port = get_available_port()
|
1103
|
+
|
1104
|
+
if available_port is None:
|
1105
|
+
raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
|
1106
|
+
"Please ensure a port is available and try again")
|
1107
|
+
|
1108
|
+
server = self.run(available_port, configuration, state, log_level=log_level)
|
1041
1109
|
|
1042
1110
|
# Uncomment this to run the tester manually
|
1043
1111
|
# server.wait_for_termination()
|
1044
1112
|
|
1045
1113
|
try:
|
1046
|
-
|
1047
|
-
for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path,
|
1114
|
+
print_library_log(f"Running connector tester...")
|
1115
|
+
for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port):
|
1048
1116
|
print(log_msg, end="")
|
1049
1117
|
except:
|
1050
1118
|
print(traceback.format_exc())
|
@@ -1086,7 +1154,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1086
1154
|
yield modified_line
|
1087
1155
|
|
1088
1156
|
@staticmethod
|
1089
|
-
def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int
|
1157
|
+
def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int):
|
1090
1158
|
"""Runs the connector tester.
|
1091
1159
|
|
1092
1160
|
Args:
|
@@ -1166,10 +1234,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1166
1234
|
if not self.schema_method:
|
1167
1235
|
return connector_sdk_pb2.SchemaResponse(schema_response_not_supported=True)
|
1168
1236
|
else:
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1237
|
+
try:
|
1238
|
+
configuration = self.configuration if self.configuration else request.configuration
|
1239
|
+
response = self.schema_method(configuration)
|
1240
|
+
self.process_tables(response)
|
1241
|
+
return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
|
1242
|
+
|
1243
|
+
except Exception as e:
|
1244
|
+
tb = traceback.format_exc()
|
1245
|
+
error_message = f"Error: {str(e)}\n{tb}"
|
1246
|
+
raise RuntimeError(error_message) from e
|
1173
1247
|
|
1174
1248
|
def process_tables(self, response):
|
1175
1249
|
for entry in response:
|
@@ -1227,7 +1301,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1227
1301
|
elif type.upper() == "SHORT":
|
1228
1302
|
column.type = common_pb2.DataType.SHORT
|
1229
1303
|
elif type.upper() == "INT":
|
1230
|
-
column.type = common_pb2.DataType.
|
1304
|
+
column.type = common_pb2.DataType.INT
|
1231
1305
|
elif type.upper() == "LONG":
|
1232
1306
|
column.type = common_pb2.DataType.LONG
|
1233
1307
|
elif type.upper() == "DECIMAL":
|
@@ -1278,6 +1352,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
|
|
1278
1352
|
if str(e) != "'NoneType' object is not iterable":
|
1279
1353
|
raise e
|
1280
1354
|
|
1355
|
+
except Exception as e:
|
1356
|
+
tb = traceback.format_exc()
|
1357
|
+
error_message = f"Error: {str(e)}\n{tb}"
|
1358
|
+
raise RuntimeError(error_message) from e
|
1359
|
+
|
1281
1360
|
|
1282
1361
|
def find_connector_object(project_path) -> Connector:
|
1283
1362
|
"""Finds the connector object in the given project path.
|
@@ -1301,8 +1380,8 @@ def find_connector_object(project_path) -> Connector:
|
|
1301
1380
|
if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
|
1302
1381
|
return obj_attr
|
1303
1382
|
|
1304
|
-
|
1305
|
-
"
|
1383
|
+
print_library_log(
|
1384
|
+
"The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
|
1306
1385
|
sys.exit(1)
|
1307
1386
|
|
1308
1387
|
|
@@ -1369,6 +1448,8 @@ def main():
|
|
1369
1448
|
"""The main entry point for the script.
|
1370
1449
|
Parses command line arguments and passes them to connector object methods
|
1371
1450
|
"""
|
1451
|
+
global EXECUTED_VIA_CLI
|
1452
|
+
EXECUTED_VIA_CLI = True
|
1372
1453
|
|
1373
1454
|
parser = argparse.ArgumentParser(allow_abbrev=False)
|
1374
1455
|
|
@@ -1387,7 +1468,7 @@ def main():
|
|
1387
1468
|
args = parser.parse_args()
|
1388
1469
|
|
1389
1470
|
if args.command.lower() == "version":
|
1390
|
-
|
1471
|
+
print_library_log("fivetran_connector_sdk " + __version__)
|
1391
1472
|
return
|
1392
1473
|
|
1393
1474
|
connector_object = find_connector_object(args.project_path)
|
@@ -1404,7 +1485,7 @@ def main():
|
|
1404
1485
|
|
1405
1486
|
if args.command.lower() == "deploy":
|
1406
1487
|
if args.state:
|
1407
|
-
|
1488
|
+
print_library_log("'state' parameter is not used for 'deploy' command", Logging.Level.WARNING)
|
1408
1489
|
connector_object.deploy(args.project_path, args.force, ft_deploy_key, ft_group, ft_connection, configuration)
|
1409
1490
|
|
1410
1491
|
elif args.command.lower() == "debug":
|
@@ -1428,6 +1509,9 @@ def validate_and_load_configuration(args, configuration):
|
|
1428
1509
|
"Configuration must be provided as a JSON file. Please check your input. Reference: "
|
1429
1510
|
"https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
|
1430
1511
|
else:
|
1512
|
+
json_filepath = os.path.join(args.project_path, "configuration.json")
|
1513
|
+
if os.path.exists(json_filepath):
|
1514
|
+
print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
|
1431
1515
|
configuration = {}
|
1432
1516
|
return configuration
|
1433
1517
|
|
@@ -1450,14 +1534,14 @@ def reset_local_file_directory(args):
|
|
1450
1534
|
confirm = input(
|
1451
1535
|
"This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
|
1452
1536
|
if confirm.lower() != "y":
|
1453
|
-
|
1537
|
+
print_library_log("Reset canceled")
|
1454
1538
|
else:
|
1455
1539
|
try:
|
1456
1540
|
if os.path.exists(files_path) and os.path.isdir(files_path):
|
1457
1541
|
shutil.rmtree(files_path)
|
1458
|
-
|
1542
|
+
print_library_log("Reset Successful")
|
1459
1543
|
except Exception as e:
|
1460
|
-
|
1544
|
+
print_library_log("Reset Failed", Logging.Level.SEVERE)
|
1461
1545
|
raise e
|
1462
1546
|
|
1463
1547
|
|
{fivetran_connector_sdk-0.11.14.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/METADATA
RENAMED
@@ -1,19 +1,20 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fivetran_connector_sdk
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.12.9.2
|
4
4
|
Summary: Build custom connectors on Fivetran platform
|
5
5
|
Author-email: Fivetran <developers@fivetran.com>
|
6
6
|
Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
|
7
7
|
Project-URL: Github, https://github.com/fivetran/fivetran_connector_sdk
|
8
|
+
Project-URL: Changelog, https://fivetran.com/docs/connectors/connector-sdk/changelog
|
8
9
|
Classifier: Programming Language :: Python :: 3
|
9
10
|
Classifier: License :: OSI Approved :: MIT License
|
10
11
|
Classifier: Operating System :: OS Independent
|
11
12
|
Requires-Python: >=3.9
|
12
13
|
Description-Content-Type: text/markdown
|
13
|
-
Requires-Dist: grpcio
|
14
|
-
Requires-Dist: grpcio-tools
|
15
|
-
Requires-Dist: requests
|
16
|
-
Requires-Dist: pipreqs
|
14
|
+
Requires-Dist: grpcio==1.60.1
|
15
|
+
Requires-Dist: grpcio-tools==1.60.1
|
16
|
+
Requires-Dist: requests==2.32.3
|
17
|
+
Requires-Dist: pipreqs==0.5.0
|
17
18
|
|
18
19
|
# **fivetran-connector-sdk**
|
19
20
|
[](https://pepy.tech/project/fivetran-connector-sdk)
|
{fivetran_connector_sdk-0.11.14.1.dist-info → fivetran_connector_sdk-0.12.9.2.dist-info}/RECORD
RENAMED
@@ -1,4 +1,4 @@
|
|
1
|
-
fivetran_connector_sdk/__init__.py,sha256=
|
1
|
+
fivetran_connector_sdk/__init__.py,sha256=2k64NC7j9Q8HeF4y7_NBeyO2fqtexxgPQeuHfEAIslQ,62965
|
2
2
|
fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
3
|
fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
|
4
4
|
fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
|
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
|
|
6
6
|
fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
|
7
7
|
fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
|
8
8
|
fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
|
9
|
-
fivetran_connector_sdk-0.
|
10
|
-
fivetran_connector_sdk-0.
|
11
|
-
fivetran_connector_sdk-0.
|
12
|
-
fivetran_connector_sdk-0.
|
13
|
-
fivetran_connector_sdk-0.
|
9
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/METADATA,sha256=3yH28B6Qmc0TIL1tqPwmltqetrH3ewwBxMMfeJI35n0,2938
|
10
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
11
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
|
12
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
|
13
|
+
fivetran_connector_sdk-0.12.9.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|