fivetran-connector-sdk 0.11.21.1__py3-none-any.whl → 0.12.9.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,7 @@ import sys
14
14
  import time
15
15
  import traceback
16
16
  import re
17
+ import socket
17
18
 
18
19
  from concurrent import futures
19
20
  from datetime import datetime
@@ -25,7 +26,7 @@ from fivetran_connector_sdk.protos import common_pb2
25
26
  from fivetran_connector_sdk.protos import connector_sdk_pb2
26
27
  from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
27
28
 
28
- __version__ = "0.11.21.1"
29
+ __version__ = "0.12.09.2"
29
30
 
30
31
  MAC_OS = "mac"
31
32
  WIN_OS = "windows"
@@ -55,8 +56,11 @@ COMMANDS_AND_SYNONYMS = {
55
56
 
56
57
  CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
57
58
  DEBUGGING = False
59
+ EXECUTED_VIA_CLI = False
58
60
  TABLES = {}
59
61
 
62
+ JAVA_LONG_MAX_VALUE = 9223372036854775807
63
+
60
64
 
61
65
  class Logging:
62
66
  class Level(IntEnum):
@@ -76,9 +80,10 @@ class Logging:
76
80
  message (str): The message to log.
77
81
  """
78
82
  if DEBUGGING:
79
- print(message)
83
+ current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
84
+ print(f"{current_time} {level.name}: {message}")
80
85
  else:
81
- print(f'{{"level":"{level.name}", "message": "{message}", "message-origin": "connector_sdk"}}')
86
+ print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "connector_sdk"}}')
82
87
 
83
88
  @staticmethod
84
89
  def fine(message: str):
@@ -257,7 +262,7 @@ def check_newer_version():
257
262
  data = json.loads(response.text)
258
263
  latest_version = data["info"]["version"]
259
264
  if __version__ < latest_version:
260
- print(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
265
+ print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
261
266
  f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
262
267
 
263
268
  with open(last_check_file_path, 'w') as f_out:
@@ -265,7 +270,7 @@ def check_newer_version():
265
270
  break
266
271
  except Exception:
267
272
  retry_after = 2 ** index
268
- print(f"WARNING: Unable to check if a newer version of `fivetran-connector-sdk` is available. Retrying again after {retry_after} seconds")
273
+ print_library_log(f"Unable to check if a newer version of `fivetran-connector-sdk` is available. Retrying again after {retry_after} seconds", Logging.Level.WARNING)
269
274
  time.sleep(retry_after)
270
275
 
271
276
 
@@ -316,10 +321,10 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
316
321
  def map_inferred_data_type(k, mapped_data, v):
317
322
  # We can infer type from the value
318
323
  if isinstance(v, int):
319
- if abs(v) > 2147483647:
320
- mapped_data[k] = common_pb2.ValueType(long=v)
324
+ if abs(v) > JAVA_LONG_MAX_VALUE:
325
+ mapped_data[k] = common_pb2.ValueType(float=v)
321
326
  else:
322
- mapped_data[k] = common_pb2.ValueType(int=v)
327
+ mapped_data[k] = common_pb2.ValueType(long=v)
323
328
  elif isinstance(v, float):
324
329
  mapped_data[k] = common_pb2.ValueType(float=v)
325
330
  elif isinstance(v, bool):
@@ -404,8 +409,8 @@ def _yield_check(stack):
404
409
  calling_code = stack[1].code_context[0]
405
410
  if f"{called_method}(" in calling_code:
406
411
  if 'yield' not in calling_code:
407
- print(
408
- f"SEVERE: Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'")
412
+ print_library_log(
413
+ f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
409
414
  os._exit(1)
410
415
  else:
411
416
  # This should never happen
@@ -433,8 +438,8 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
433
438
  if string_only:
434
439
  for k, v in incoming.items():
435
440
  if not isinstance(v, str):
436
- print(
437
- "SEVERE: All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.")
441
+ print_library_log(
442
+ "All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
438
443
  os._exit(1)
439
444
 
440
445
  return incoming
@@ -454,28 +459,53 @@ def is_connection_name_valid(connection: str):
454
459
 
455
460
 
456
461
  def log_unused_deps_error(package_name: str, version: str):
457
- print(f"ERROR: Please remove `{package_name}` from requirements.txt."
462
+ print_library_log(f"Please remove `{package_name}` from requirements.txt."
458
463
  f" The latest version of `{package_name}` is always available when executing your code."
459
- f" Current version: {version}")
464
+ f" Current version: {version}", Logging.Level.SEVERE)
460
465
  os._exit(1)
461
466
 
462
467
 
463
468
  def validate_deploy_parameters(connection, deploy_key):
464
469
  if not deploy_key or not connection:
465
- print("SEVERE: The deploy command needs the following parameters:"
470
+ print_library_log("The deploy command needs the following parameters:"
466
471
  "\n\tRequired:\n"
467
472
  "\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
468
473
  "\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
469
474
  "\t(Optional):\n"
470
475
  "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
471
- "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)")
476
+ "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
472
477
  os._exit(1)
473
478
  if not is_connection_name_valid(connection):
474
- print(f"SEVERE: Connection name: {connection} is invalid!\n The connection name should start with an "
479
+ print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
475
480
  f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
476
- f"letters, or digits (0-9). Uppercase characters are not allowed.")
481
+ f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
477
482
  os._exit(1)
478
483
 
484
+ def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
485
+ """Logs a library message with the specified logging level.
486
+
487
+ Args:
488
+ level (Logging.Level): The logging level.
489
+ message (str): The message to log.
490
+ """
491
+ if DEBUGGING or EXECUTED_VIA_CLI:
492
+ current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
493
+ print(f'{current_time} {level.name}: {message}')
494
+ else:
495
+ print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "library"}}')
496
+
497
+
498
+ def is_port_in_use(port: int):
499
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
500
+ return s.connect_ex(('127.0.0.1', port)) == 0
501
+
502
+
503
+ def get_available_port():
504
+ for port in range(50051, 50061):
505
+ if not is_port_in_use(port):
506
+ return port
507
+ return None
508
+
479
509
 
480
510
  class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
481
511
  def __init__(self, update, schema=None):
@@ -524,7 +554,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
524
554
  key = re.split(r"==|>=|<=|>|<", requirement)[0]
525
555
  requirements_dict[key.lower()] = requirement.lower()
526
556
  except ValueError:
527
- print(f"Error: Invalid requirement format: '{requirement}'")
557
+ print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
528
558
  return requirements_dict
529
559
 
530
560
  def validate_requirements_file(self, project_path: str, is_deploy: bool):
@@ -557,7 +587,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
557
587
  (requirements.keys() & tmp_requirements.keys())
558
588
  if requirements[key] != tmp_requirements[key]}
559
589
  if version_mismatch_deps:
560
- print("WARNING: We recommend using the current stable version for the following:")
590
+ print_library_log("We recommend using the current stable version for the following:", Logging.Level.WARNING)
561
591
  print(version_mismatch_deps)
562
592
 
563
593
  missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
@@ -569,11 +599,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
569
599
  self.handle_unused_deps(unused_deps)
570
600
  else:
571
601
  if os.path.exists(REQUIREMENTS_TXT):
572
- print("WARNING: `requirements.txt` is not required as no additional "
602
+ print_library_log("`requirements.txt` is not required as no additional "
573
603
  "Python libraries are required or all required libraries for "
574
- "your code are pre-installed.")
604
+ "your code are pre-installed.", Logging.Level.WARNING)
575
605
 
576
- if is_deploy: print("Successful validation of requirements.txt")
606
+ if is_deploy: print_library_log("Successful validation of requirements.txt")
577
607
 
578
608
  def handle_unused_deps(self, unused_deps):
579
609
  if 'fivetran_connector_sdk' in unused_deps:
@@ -581,18 +611,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
581
611
  elif 'requests' in unused_deps:
582
612
  log_unused_deps_error("requests", "2.32.3")
583
613
  else:
584
- print("INFO: The following dependencies are not needed, "
614
+ print_library_log("The following dependencies are not needed, "
585
615
  "they are not used or already installed. Please remove them from requirements.txt:")
586
616
  print(*unused_deps)
587
617
 
588
618
  def handle_missing_deps(self, is_deploy, missing_deps):
589
- log_level = "ERROR" if is_deploy else "WARNING"
590
- print(log_level +
591
- ": Please include the following dependency libraries in requirements.txt, to be used by "
619
+ print_library_log("Please include the following dependency libraries in requirements.txt, to be used by "
592
620
  "Fivetran production. "
593
621
  "For more information, please visit: "
594
622
  "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
595
- "#workingwithrequirementstxtfile")
623
+ "#workingwithrequirementstxtfile", Logging.Level.SEVERE)
596
624
  print(*list(missing_deps.values()))
597
625
  if is_deploy:
598
626
  os._exit(1)
@@ -604,7 +632,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
604
632
  with open(REQUIREMENTS_TXT, 'w'):
605
633
  pass
606
634
  requirements = {}
607
- print("WARNING: Adding `requirements.txt` file to your project folder.")
635
+ print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
608
636
  return requirements
609
637
 
610
638
  # Call this method to deploy the connector to Fivetran platform
@@ -619,6 +647,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
619
647
  connection (str): The connection name.
620
648
  configuration (dict): The configuration dictionary.
621
649
  """
650
+ global EXECUTED_VIA_CLI
651
+ EXECUTED_VIA_CLI = True
652
+
622
653
  validate_deploy_parameters(connection, deploy_key)
623
654
 
624
655
  _check_dict(configuration, True)
@@ -642,8 +673,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
642
673
 
643
674
  if connection_id:
644
675
  if service != 'connector_sdk':
645
- print(
646
- f"SEVERE: The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.")
676
+ print_library_log(
677
+ f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
647
678
  os._exit(1)
648
679
  else:
649
680
  if force:
@@ -652,17 +683,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
652
683
  confirm = input(
653
684
  f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code and configuration. Do you want to proceed with the update? (Y/N): ")
654
685
  if confirm.lower() == "y":
655
- print("INFO: Updating the connection...\n")
686
+ print_library_log("Updating the connection...\n")
656
687
  self.__upload_project(
657
688
  project_path, deploy_key, group_id, group_name, connection)
658
689
  self.__update_connection(
659
690
  connection_id, connection, group_name, connection_config, deploy_key)
660
691
  print("✓")
661
- print(f"INFO: Connector ID: {connection_id}")
662
- print(
663
- f"INFO: Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
692
+ print_library_log(f"Connector ID: {connection_id}")
693
+ print_library_log(
694
+ f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
664
695
  else:
665
- print("INFO: Update canceled. The process is now terminating.")
696
+ print_library_log("Update canceled. The process is now terminating.")
666
697
  os._exit(1)
667
698
  else:
668
699
  self.__upload_project(project_path, deploy_key,
@@ -670,20 +701,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
670
701
  response = self.__create_connection(
671
702
  deploy_key, group_id, connection_config)
672
703
  if response.ok:
673
- print(
674
- f"INFO: The connection '{connection}' has been created successfully.\n")
704
+ print_library_log(
705
+ f"The connection '{connection}' has been created successfully.\n")
675
706
  connection_id = response.json()['data']['id']
676
- print(f"INFO: Connector ID: {connection_id}")
677
- print(
678
- f"INFO: Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
707
+ print_library_log(f"Connector ID: {connection_id}")
708
+ print_library_log(
709
+ f"Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
679
710
  else:
680
- print(
681
- f"SEVERE: Unable to create a new connection, failed with error: {response.json()['message']}")
711
+ print_library_log(
712
+ f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
713
+ self.__cleanup_uploaded_project(deploy_key,group_id, connection)
714
+ print_library_log("Please try again with the deploy command after resolving the issue!")
682
715
  os._exit(1)
683
716
 
684
717
  def __upload_project(self, project_path: str, deploy_key: str, group_id: str, group_name: str, connection: str):
685
- print(
686
- f"INFO: Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
718
+ print_library_log(
719
+ f"Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
687
720
  upload_file_path = self.__create_upload_file(project_path)
688
721
  upload_result = self.__upload(
689
722
  upload_file_path, deploy_key, group_id, connection)
@@ -691,6 +724,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
691
724
  if not upload_result:
692
725
  os._exit(1)
693
726
 
727
+ def __cleanup_uploaded_project(self, deploy_key: str, group_id: str, connection: str):
728
+ cleanup_result = self.__cleanup_uploaded_code(deploy_key, group_id, connection)
729
+ if not cleanup_result:
730
+ os._exit(1)
731
+
694
732
  @staticmethod
695
733
  def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
696
734
  """Updates the connection with the given ID, name, group, configuration, and deployment key.
@@ -713,8 +751,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
713
751
  })
714
752
 
715
753
  if not resp.ok:
716
- print(
717
- f"SEVERE: Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.")
754
+ print_library_log(
755
+ f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.", Logging.Level.SEVERE)
718
756
  os._exit(1)
719
757
 
720
758
  @staticmethod
@@ -734,8 +772,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
734
772
  headers={"Authorization": f"Basic {deploy_key}"},
735
773
  params={"schema": name})
736
774
  if not resp.ok:
737
- print(
738
- f"SEVERE: Unable to fetch connection list in destination '{group}'")
775
+ print_library_log(
776
+ f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
739
777
  os._exit(1)
740
778
 
741
779
  if resp.json()['data']['items']:
@@ -776,7 +814,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
776
814
  Returns:
777
815
  str: The path to the upload file.
778
816
  """
779
- print("INFO: Packaging your project for upload...")
817
+ print_library_log("Packaging your project for upload...")
780
818
  zip_file_path = self.__zip_folder(project_path)
781
819
  print("✓")
782
820
  return zip_file_path
@@ -803,8 +841,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
803
841
  zipf.write(file_path, arcname)
804
842
 
805
843
  if not connector_file_exists:
806
- print(
807
- "SEVERE: The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.")
844
+ print_library_log(
845
+ "The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.", Logging.Level.SEVERE)
808
846
  os._exit(1)
809
847
  return upload_filepath
810
848
 
@@ -846,15 +884,37 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
846
884
  Returns:
847
885
  bool: True if the upload was successful, False otherwise.
848
886
  """
849
- print("INFO: Uploading your project...", end="", flush=True)
850
- response = rq.post(f"https://api.fivetran.com/v2/deploy/{group_id}/{connection}",
887
+ print_library_log("Uploading your project...")
888
+ response = rq.post(f"https://api.fivetran.com/v1/deploy/{group_id}/{connection}",
851
889
  files={'file': open(local_path, 'rb')},
852
890
  headers={"Authorization": f"Basic {deploy_key}"})
853
891
  if response.ok:
854
892
  print("✓")
855
893
  return True
856
894
 
857
- print("SEVERE: Unable to upload the project, failed with error: ", response.reason)
895
+ print_library_log(f"Unable to upload the project, failed with error: {response.reason}", Logging.Level.SEVERE)
896
+ return False
897
+
898
+ @staticmethod
899
+ def __cleanup_uploaded_code(deploy_key: str, group_id: str, connection: str) -> bool:
900
+ """Cleans up the uploaded code file for the specified group and connection, if creation fails.
901
+
902
+ Args:
903
+ deploy_key (str): The deployment key.
904
+ group_id (str): The group ID.
905
+ connection (str): The connection name.
906
+
907
+ Returns:
908
+ bool: True if the cleanup was successful, False otherwise.
909
+ """
910
+ print_library_log("INFO: Cleaning up your uploaded project ")
911
+ response = rq.post(f"https://api.fivetran.com/v1/cleanup_code/{group_id}/{connection}",
912
+ headers={"Authorization": f"Basic {deploy_key}"})
913
+ if response.ok:
914
+ print("✓")
915
+ return True
916
+
917
+ print_library_log(f"SEVERE: Unable to cleanup the project, failed with error: {response.reason}", Logging.Level.SEVERE)
858
918
  return False
859
919
 
860
920
  @staticmethod
@@ -891,23 +951,23 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
891
951
  resp = rq.get(groups_url, headers=headers, params=params)
892
952
 
893
953
  if not resp.ok:
894
- print(
895
- f"SEVERE: Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.")
954
+ print_library_log(
955
+ f"Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
896
956
  os._exit(1)
897
957
 
898
958
  data = resp.json().get("data", {})
899
959
  groups = data.get("items")
900
960
 
901
961
  if not groups:
902
- print("SEVERE: No destinations defined in the account")
962
+ print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
903
963
  os._exit(1)
904
964
 
905
965
  if not group:
906
966
  if len(groups) == 1:
907
967
  return groups[0]['id'], groups[0]['name']
908
968
  else:
909
- print(
910
- "SEVERE: Destination name is required when there are multiple destinations in the account")
969
+ print_library_log(
970
+ "Destination name is required when there are multiple destinations in the account", Logging.Level.SEVERE)
911
971
  os._exit(1)
912
972
  else:
913
973
  while True:
@@ -924,8 +984,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
924
984
  data = resp.json().get("data", {})
925
985
  groups = data.get("items", [])
926
986
 
927
- print(
928
- f"SEVERE: The specified destination '{group}' was not found in your account.")
987
+ print_library_log(
988
+ f"The specified destination '{group}' was not found in your account.", Logging.Level.SEVERE)
929
989
  os._exit(1)
930
990
 
931
991
  # Call this method to run the connector in production
@@ -950,7 +1010,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
950
1010
  Logging.LOG_LEVEL = log_level
951
1011
 
952
1012
  if not DEBUGGING:
953
- print(f"Running on fivetran_connector_sdk: {__version__}")
1013
+ print_library_log(f"Running on fivetran_connector_sdk: {__version__}")
954
1014
 
955
1015
  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
956
1016
  connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
@@ -1006,7 +1066,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1006
1066
  download_filename = f"sdk-connector-tester-{os_name}-{TESTER_VERSION}.zip"
1007
1067
  download_filepath = os.path.join(tester_root_dir, download_filename)
1008
1068
  try:
1009
- print(f"INFO: Downloading connector tester version: {TESTER_VERSION} ", end="", flush=True)
1069
+ print_library_log(f"Downloading connector tester version: {TESTER_VERSION} ")
1010
1070
  download_url = f"https://github.com/fivetran/fivetran_sdk_tools/releases/download/{TESTER_VERSION}/{download_filename}"
1011
1071
  r = rq.get(download_url)
1012
1072
  if r.ok:
@@ -1038,15 +1098,21 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1038
1098
 
1039
1099
  project_path = os.getcwd() if project_path is None else project_path
1040
1100
  self.validate_requirements_file(project_path, False)
1041
- print(f"INFO: Debugging connector at: {project_path}")
1042
- server = self.run(50051, configuration, state, log_level=log_level)
1101
+ print_library_log(f"Debugging connector at: {project_path}")
1102
+ available_port = get_available_port()
1103
+
1104
+ if available_port is None:
1105
+ raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
1106
+ "Please ensure a port is available and try again")
1107
+
1108
+ server = self.run(available_port, configuration, state, log_level=log_level)
1043
1109
 
1044
1110
  # Uncomment this to run the tester manually
1045
1111
  # server.wait_for_termination()
1046
1112
 
1047
1113
  try:
1048
- print(f"INFO: Running connector tester...")
1049
- for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, 50051):
1114
+ print_library_log(f"Running connector tester...")
1115
+ for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port):
1050
1116
  print(log_msg, end="")
1051
1117
  except:
1052
1118
  print(traceback.format_exc())
@@ -1088,7 +1154,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1088
1154
  yield modified_line
1089
1155
 
1090
1156
  @staticmethod
1091
- def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int = 50051):
1157
+ def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int):
1092
1158
  """Runs the connector tester.
1093
1159
 
1094
1160
  Args:
@@ -1168,10 +1234,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1168
1234
  if not self.schema_method:
1169
1235
  return connector_sdk_pb2.SchemaResponse(schema_response_not_supported=True)
1170
1236
  else:
1171
- configuration = self.configuration if self.configuration else request.configuration
1172
- response = self.schema_method(configuration)
1173
- self.process_tables(response)
1174
- return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
1237
+ try:
1238
+ configuration = self.configuration if self.configuration else request.configuration
1239
+ response = self.schema_method(configuration)
1240
+ self.process_tables(response)
1241
+ return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
1242
+
1243
+ except Exception as e:
1244
+ tb = traceback.format_exc()
1245
+ error_message = f"Error: {str(e)}\n{tb}"
1246
+ raise RuntimeError(error_message) from e
1175
1247
 
1176
1248
  def process_tables(self, response):
1177
1249
  for entry in response:
@@ -1229,7 +1301,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1229
1301
  elif type.upper() == "SHORT":
1230
1302
  column.type = common_pb2.DataType.SHORT
1231
1303
  elif type.upper() == "INT":
1232
- column.type = common_pb2.DataType.SHORT
1304
+ column.type = common_pb2.DataType.INT
1233
1305
  elif type.upper() == "LONG":
1234
1306
  column.type = common_pb2.DataType.LONG
1235
1307
  elif type.upper() == "DECIMAL":
@@ -1280,6 +1352,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1280
1352
  if str(e) != "'NoneType' object is not iterable":
1281
1353
  raise e
1282
1354
 
1355
+ except Exception as e:
1356
+ tb = traceback.format_exc()
1357
+ error_message = f"Error: {str(e)}\n{tb}"
1358
+ raise RuntimeError(error_message) from e
1359
+
1283
1360
 
1284
1361
  def find_connector_object(project_path) -> Connector:
1285
1362
  """Finds the connector object in the given project path.
@@ -1303,8 +1380,8 @@ def find_connector_object(project_path) -> Connector:
1303
1380
  if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
1304
1381
  return obj_attr
1305
1382
 
1306
- print(
1307
- "SEVERE: The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector")
1383
+ print_library_log(
1384
+ "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
1308
1385
  sys.exit(1)
1309
1386
 
1310
1387
 
@@ -1371,6 +1448,8 @@ def main():
1371
1448
  """The main entry point for the script.
1372
1449
  Parses command line arguments and passes them to connector object methods
1373
1450
  """
1451
+ global EXECUTED_VIA_CLI
1452
+ EXECUTED_VIA_CLI = True
1374
1453
 
1375
1454
  parser = argparse.ArgumentParser(allow_abbrev=False)
1376
1455
 
@@ -1389,7 +1468,7 @@ def main():
1389
1468
  args = parser.parse_args()
1390
1469
 
1391
1470
  if args.command.lower() == "version":
1392
- print("fivetran_connector_sdk " + __version__)
1471
+ print_library_log("fivetran_connector_sdk " + __version__)
1393
1472
  return
1394
1473
 
1395
1474
  connector_object = find_connector_object(args.project_path)
@@ -1406,7 +1485,7 @@ def main():
1406
1485
 
1407
1486
  if args.command.lower() == "deploy":
1408
1487
  if args.state:
1409
- print("WARNING: 'state' parameter is not used for 'deploy' command")
1488
+ print_library_log("'state' parameter is not used for 'deploy' command", Logging.Level.WARNING)
1410
1489
  connector_object.deploy(args.project_path, args.force, ft_deploy_key, ft_group, ft_connection, configuration)
1411
1490
 
1412
1491
  elif args.command.lower() == "debug":
@@ -1430,6 +1509,9 @@ def validate_and_load_configuration(args, configuration):
1430
1509
  "Configuration must be provided as a JSON file. Please check your input. Reference: "
1431
1510
  "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
1432
1511
  else:
1512
+ json_filepath = os.path.join(args.project_path, "configuration.json")
1513
+ if os.path.exists(json_filepath):
1514
+ print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
1433
1515
  configuration = {}
1434
1516
  return configuration
1435
1517
 
@@ -1452,14 +1534,14 @@ def reset_local_file_directory(args):
1452
1534
  confirm = input(
1453
1535
  "This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
1454
1536
  if confirm.lower() != "y":
1455
- print("INFO: Reset canceled")
1537
+ print_library_log("Reset canceled")
1456
1538
  else:
1457
1539
  try:
1458
1540
  if os.path.exists(files_path) and os.path.isdir(files_path):
1459
1541
  shutil.rmtree(files_path)
1460
- print("INFO: Reset Successful")
1542
+ print_library_log("Reset Successful")
1461
1543
  except Exception as e:
1462
- print("ERROR: Reset Failed")
1544
+ print_library_log("Reset Failed", Logging.Level.SEVERE)
1463
1545
  raise e
1464
1546
 
1465
1547
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: fivetran_connector_sdk
3
- Version: 0.11.21.1
3
+ Version: 0.12.9.2
4
4
  Summary: Build custom connectors on Fivetran platform
5
5
  Author-email: Fivetran <developers@fivetran.com>
6
6
  Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -1,4 +1,4 @@
1
- fivetran_connector_sdk/__init__.py,sha256=P3Qolsvkfvrj1fAHAvkirN1dFQyZTA1sHYrLGYQUdn4,58853
1
+ fivetran_connector_sdk/__init__.py,sha256=2k64NC7j9Q8HeF4y7_NBeyO2fqtexxgPQeuHfEAIslQ,62965
2
2
  fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
3
  fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
4
4
  fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
6
6
  fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
7
7
  fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
8
8
  fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
9
- fivetran_connector_sdk-0.11.21.1.dist-info/METADATA,sha256=VaZkhj5RbDVaazn5dBWHNkJJk2jvDx96FOokLyLo7Zc,2939
10
- fivetran_connector_sdk-0.11.21.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
11
- fivetran_connector_sdk-0.11.21.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
12
- fivetran_connector_sdk-0.11.21.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
13
- fivetran_connector_sdk-0.11.21.1.dist-info/RECORD,,
9
+ fivetran_connector_sdk-0.12.9.2.dist-info/METADATA,sha256=3yH28B6Qmc0TIL1tqPwmltqetrH3ewwBxMMfeJI35n0,2938
10
+ fivetran_connector_sdk-0.12.9.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
11
+ fivetran_connector_sdk-0.12.9.2.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
12
+ fivetran_connector_sdk-0.12.9.2.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
13
+ fivetran_connector_sdk-0.12.9.2.dist-info/RECORD,,