fivetran-connector-sdk 0.11.14.1__py3-none-any.whl → 0.12.9.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,7 @@ import sys
14
14
  import time
15
15
  import traceback
16
16
  import re
17
+ import socket
17
18
 
18
19
  from concurrent import futures
19
20
  from datetime import datetime
@@ -25,7 +26,7 @@ from fivetran_connector_sdk.protos import common_pb2
25
26
  from fivetran_connector_sdk.protos import connector_sdk_pb2
26
27
  from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
27
28
 
28
- __version__ = "0.11.14.1"
29
+ __version__ = "0.12.09.2"
29
30
 
30
31
  MAC_OS = "mac"
31
32
  WIN_OS = "windows"
@@ -55,8 +56,11 @@ COMMANDS_AND_SYNONYMS = {
55
56
 
56
57
  CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
57
58
  DEBUGGING = False
59
+ EXECUTED_VIA_CLI = False
58
60
  TABLES = {}
59
61
 
62
+ JAVA_LONG_MAX_VALUE = 9223372036854775807
63
+
60
64
 
61
65
  class Logging:
62
66
  class Level(IntEnum):
@@ -76,9 +80,10 @@ class Logging:
76
80
  message (str): The message to log.
77
81
  """
78
82
  if DEBUGGING:
79
- print(message)
83
+ current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
84
+ print(f"{current_time} {level.name}: {message}")
80
85
  else:
81
- print(f'{{"level":"{level.name}", "message": "{message}", "message-origin": "connector_sdk"}}')
86
+ print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "connector_sdk"}}')
82
87
 
83
88
  @staticmethod
84
89
  def fine(message: str):
@@ -257,7 +262,7 @@ def check_newer_version():
257
262
  data = json.loads(response.text)
258
263
  latest_version = data["info"]["version"]
259
264
  if __version__ < latest_version:
260
- print(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
265
+ print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
261
266
  f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
262
267
 
263
268
  with open(last_check_file_path, 'w') as f_out:
@@ -265,7 +270,7 @@ def check_newer_version():
265
270
  break
266
271
  except Exception:
267
272
  retry_after = 2 ** index
268
- print(f"WARNING: Unable to check if a newer version of `fivetran-connector-sdk` is available. Retrying again after {retry_after} seconds")
273
+ print_library_log(f"Unable to check if a newer version of `fivetran-connector-sdk` is available. Retrying again after {retry_after} seconds", Logging.Level.WARNING)
269
274
  time.sleep(retry_after)
270
275
 
271
276
 
@@ -316,10 +321,10 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
316
321
  def map_inferred_data_type(k, mapped_data, v):
317
322
  # We can infer type from the value
318
323
  if isinstance(v, int):
319
- if abs(v) > 2147483647:
320
- mapped_data[k] = common_pb2.ValueType(long=v)
324
+ if abs(v) > JAVA_LONG_MAX_VALUE:
325
+ mapped_data[k] = common_pb2.ValueType(float=v)
321
326
  else:
322
- mapped_data[k] = common_pb2.ValueType(int=v)
327
+ mapped_data[k] = common_pb2.ValueType(long=v)
323
328
  elif isinstance(v, float):
324
329
  mapped_data[k] = common_pb2.ValueType(float=v)
325
330
  elif isinstance(v, bool):
@@ -404,8 +409,8 @@ def _yield_check(stack):
404
409
  calling_code = stack[1].code_context[0]
405
410
  if f"{called_method}(" in calling_code:
406
411
  if 'yield' not in calling_code:
407
- print(
408
- f"SEVERE: Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'")
412
+ print_library_log(
413
+ f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
409
414
  os._exit(1)
410
415
  else:
411
416
  # This should never happen
@@ -433,8 +438,8 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
433
438
  if string_only:
434
439
  for k, v in incoming.items():
435
440
  if not isinstance(v, str):
436
- print(
437
- "SEVERE: All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.")
441
+ print_library_log(
442
+ "All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
438
443
  os._exit(1)
439
444
 
440
445
  return incoming
@@ -454,28 +459,53 @@ def is_connection_name_valid(connection: str):
454
459
 
455
460
 
456
461
  def log_unused_deps_error(package_name: str, version: str):
457
- print(f"ERROR: Please remove `{package_name}` from requirements.txt."
462
+ print_library_log(f"Please remove `{package_name}` from requirements.txt."
458
463
  f" The latest version of `{package_name}` is always available when executing your code."
459
- f" Current version: {version}")
464
+ f" Current version: {version}", Logging.Level.SEVERE)
460
465
  os._exit(1)
461
466
 
462
467
 
463
468
  def validate_deploy_parameters(connection, deploy_key):
464
469
  if not deploy_key or not connection:
465
- print("SEVERE: The deploy command needs the following parameters:"
470
+ print_library_log("The deploy command needs the following parameters:"
466
471
  "\n\tRequired:\n"
467
472
  "\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
468
473
  "\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
469
474
  "\t(Optional):\n"
470
475
  "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
471
- "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)")
476
+ "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
472
477
  os._exit(1)
473
478
  if not is_connection_name_valid(connection):
474
- print(f"SEVERE: Connection name: {connection} is invalid!\n The connection name should start with an "
479
+ print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
475
480
  f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
476
- f"letters, or digits (0-9). Uppercase characters are not allowed.")
481
+ f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
477
482
  os._exit(1)
478
483
 
484
+ def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
485
+ """Logs a library message with the specified logging level.
486
+
487
+ Args:
488
+ level (Logging.Level): The logging level.
489
+ message (str): The message to log.
490
+ """
491
+ if DEBUGGING or EXECUTED_VIA_CLI:
492
+ current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
493
+ print(f'{current_time} {level.name}: {message}')
494
+ else:
495
+ print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "library"}}')
496
+
497
+
498
+ def is_port_in_use(port: int):
499
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
500
+ return s.connect_ex(('127.0.0.1', port)) == 0
501
+
502
+
503
+ def get_available_port():
504
+ for port in range(50051, 50061):
505
+ if not is_port_in_use(port):
506
+ return port
507
+ return None
508
+
479
509
 
480
510
  class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
481
511
  def __init__(self, update, schema=None):
@@ -524,7 +554,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
524
554
  key = re.split(r"==|>=|<=|>|<", requirement)[0]
525
555
  requirements_dict[key.lower()] = requirement.lower()
526
556
  except ValueError:
527
- print(f"Error: Invalid requirement format: '{requirement}'")
557
+ print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
528
558
  return requirements_dict
529
559
 
530
560
  def validate_requirements_file(self, project_path: str, is_deploy: bool):
@@ -557,7 +587,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
557
587
  (requirements.keys() & tmp_requirements.keys())
558
588
  if requirements[key] != tmp_requirements[key]}
559
589
  if version_mismatch_deps:
560
- print("WARNING: We recommend using the current stable version for the following:")
590
+ print_library_log("We recommend using the current stable version for the following:", Logging.Level.WARNING)
561
591
  print(version_mismatch_deps)
562
592
 
563
593
  missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
@@ -569,11 +599,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
569
599
  self.handle_unused_deps(unused_deps)
570
600
  else:
571
601
  if os.path.exists(REQUIREMENTS_TXT):
572
- print("WARNING: `requirements.txt` is not required as no additional "
602
+ print_library_log("`requirements.txt` is not required as no additional "
573
603
  "Python libraries are required or all required libraries for "
574
- "your code are pre-installed.")
604
+ "your code are pre-installed.", Logging.Level.WARNING)
575
605
 
576
- if is_deploy: print("Successful validation of requirements.txt")
606
+ if is_deploy: print_library_log("Successful validation of requirements.txt")
577
607
 
578
608
  def handle_unused_deps(self, unused_deps):
579
609
  if 'fivetran_connector_sdk' in unused_deps:
@@ -581,18 +611,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
581
611
  elif 'requests' in unused_deps:
582
612
  log_unused_deps_error("requests", "2.32.3")
583
613
  else:
584
- print("INFO: The following dependencies are not needed, "
614
+ print_library_log("The following dependencies are not needed, "
585
615
  "they are not used or already installed. Please remove them from requirements.txt:")
586
616
  print(*unused_deps)
587
617
 
588
618
  def handle_missing_deps(self, is_deploy, missing_deps):
589
- log_level = "ERROR" if is_deploy else "WARNING"
590
- print(log_level +
591
- ": Please include the following dependency libraries in requirements.txt, to be used by "
619
+ print_library_log("Please include the following dependency libraries in requirements.txt, to be used by "
592
620
  "Fivetran production. "
593
621
  "For more information, please visit: "
594
622
  "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
595
- "#workingwithrequirementstxtfile")
623
+ "#workingwithrequirementstxtfile", Logging.Level.SEVERE)
596
624
  print(*list(missing_deps.values()))
597
625
  if is_deploy:
598
626
  os._exit(1)
@@ -604,7 +632,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
604
632
  with open(REQUIREMENTS_TXT, 'w'):
605
633
  pass
606
634
  requirements = {}
607
- print("WARNING: Adding `requirements.txt` file to your project folder.")
635
+ print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
608
636
  return requirements
609
637
 
610
638
  # Call this method to deploy the connector to Fivetran platform
@@ -619,6 +647,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
619
647
  connection (str): The connection name.
620
648
  configuration (dict): The configuration dictionary.
621
649
  """
650
+ global EXECUTED_VIA_CLI
651
+ EXECUTED_VIA_CLI = True
652
+
622
653
  validate_deploy_parameters(connection, deploy_key)
623
654
 
624
655
  _check_dict(configuration, True)
@@ -642,8 +673,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
642
673
 
643
674
  if connection_id:
644
675
  if service != 'connector_sdk':
645
- print(
646
- f"SEVERE: The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.")
676
+ print_library_log(
677
+ f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
647
678
  os._exit(1)
648
679
  else:
649
680
  if force:
@@ -652,16 +683,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
652
683
  confirm = input(
653
684
  f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code and configuration. Do you want to proceed with the update? (Y/N): ")
654
685
  if confirm.lower() == "y":
655
- print("INFO: Updating the connection...\n")
686
+ print_library_log("Updating the connection...\n")
656
687
  self.__upload_project(
657
688
  project_path, deploy_key, group_id, group_name, connection)
658
689
  self.__update_connection(
659
690
  connection_id, connection, group_name, connection_config, deploy_key)
660
691
  print("✓")
661
- print(
662
- f"INFO: Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
692
+ print_library_log(f"Connector ID: {connection_id}")
693
+ print_library_log(
694
+ f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
663
695
  else:
664
- print("INFO: Update canceled. The process is now terminating.")
696
+ print_library_log("Update canceled. The process is now terminating.")
665
697
  os._exit(1)
666
698
  else:
667
699
  self.__upload_project(project_path, deploy_key,
@@ -669,19 +701,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
669
701
  response = self.__create_connection(
670
702
  deploy_key, group_id, connection_config)
671
703
  if response.ok:
672
- print(
673
- f"INFO: The connection '{connection}' has been created successfully.\n")
704
+ print_library_log(
705
+ f"The connection '{connection}' has been created successfully.\n")
674
706
  connection_id = response.json()['data']['id']
675
- print(
676
- f"INFO: Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
707
+ print_library_log(f"Connector ID: {connection_id}")
708
+ print_library_log(
709
+ f"Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
677
710
  else:
678
- print(
679
- f"SEVERE: Unable to create a new connection, failed with error: {response.json()['message']}")
711
+ print_library_log(
712
+ f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
713
+ self.__cleanup_uploaded_project(deploy_key,group_id, connection)
714
+ print_library_log("Please try again with the deploy command after resolving the issue!")
680
715
  os._exit(1)
681
716
 
682
717
  def __upload_project(self, project_path: str, deploy_key: str, group_id: str, group_name: str, connection: str):
683
- print(
684
- f"INFO: Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
718
+ print_library_log(
719
+ f"Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
685
720
  upload_file_path = self.__create_upload_file(project_path)
686
721
  upload_result = self.__upload(
687
722
  upload_file_path, deploy_key, group_id, connection)
@@ -689,6 +724,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
689
724
  if not upload_result:
690
725
  os._exit(1)
691
726
 
727
+ def __cleanup_uploaded_project(self, deploy_key: str, group_id: str, connection: str):
728
+ cleanup_result = self.__cleanup_uploaded_code(deploy_key, group_id, connection)
729
+ if not cleanup_result:
730
+ os._exit(1)
731
+
692
732
  @staticmethod
693
733
  def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
694
734
  """Updates the connection with the given ID, name, group, configuration, and deployment key.
@@ -711,8 +751,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
711
751
  })
712
752
 
713
753
  if not resp.ok:
714
- print(
715
- f"SEVERE: Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.")
754
+ print_library_log(
755
+ f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.", Logging.Level.SEVERE)
716
756
  os._exit(1)
717
757
 
718
758
  @staticmethod
@@ -732,8 +772,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
732
772
  headers={"Authorization": f"Basic {deploy_key}"},
733
773
  params={"schema": name})
734
774
  if not resp.ok:
735
- print(
736
- f"SEVERE: Unable to fetch connection list in destination '{group}'")
775
+ print_library_log(
776
+ f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
737
777
  os._exit(1)
738
778
 
739
779
  if resp.json()['data']['items']:
@@ -774,7 +814,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
774
814
  Returns:
775
815
  str: The path to the upload file.
776
816
  """
777
- print("INFO: Packaging your project for upload...")
817
+ print_library_log("Packaging your project for upload...")
778
818
  zip_file_path = self.__zip_folder(project_path)
779
819
  print("✓")
780
820
  return zip_file_path
@@ -801,8 +841,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
801
841
  zipf.write(file_path, arcname)
802
842
 
803
843
  if not connector_file_exists:
804
- print(
805
- "SEVERE: The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.")
844
+ print_library_log(
845
+ "The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.", Logging.Level.SEVERE)
806
846
  os._exit(1)
807
847
  return upload_filepath
808
848
 
@@ -844,15 +884,37 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
844
884
  Returns:
845
885
  bool: True if the upload was successful, False otherwise.
846
886
  """
847
- print("INFO: Uploading your project...", end="", flush=True)
848
- response = rq.post(f"https://api.fivetran.com/v2/deploy/{group_id}/{connection}",
887
+ print_library_log("Uploading your project...")
888
+ response = rq.post(f"https://api.fivetran.com/v1/deploy/{group_id}/{connection}",
849
889
  files={'file': open(local_path, 'rb')},
850
890
  headers={"Authorization": f"Basic {deploy_key}"})
851
891
  if response.ok:
852
892
  print("✓")
853
893
  return True
854
894
 
855
- print("SEVERE: Unable to upload the project, failed with error: ", response.reason)
895
+ print_library_log(f"Unable to upload the project, failed with error: {response.reason}", Logging.Level.SEVERE)
896
+ return False
897
+
898
+ @staticmethod
899
+ def __cleanup_uploaded_code(deploy_key: str, group_id: str, connection: str) -> bool:
900
+ """Cleans up the uploaded code file for the specified group and connection, if creation fails.
901
+
902
+ Args:
903
+ deploy_key (str): The deployment key.
904
+ group_id (str): The group ID.
905
+ connection (str): The connection name.
906
+
907
+ Returns:
908
+ bool: True if the cleanup was successful, False otherwise.
909
+ """
910
+ print_library_log("INFO: Cleaning up your uploaded project ")
911
+ response = rq.post(f"https://api.fivetran.com/v1/cleanup_code/{group_id}/{connection}",
912
+ headers={"Authorization": f"Basic {deploy_key}"})
913
+ if response.ok:
914
+ print("✓")
915
+ return True
916
+
917
+ print_library_log(f"SEVERE: Unable to cleanup the project, failed with error: {response.reason}", Logging.Level.SEVERE)
856
918
  return False
857
919
 
858
920
  @staticmethod
@@ -889,23 +951,23 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
889
951
  resp = rq.get(groups_url, headers=headers, params=params)
890
952
 
891
953
  if not resp.ok:
892
- print(
893
- f"SEVERE: Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.")
954
+ print_library_log(
955
+ f"Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
894
956
  os._exit(1)
895
957
 
896
958
  data = resp.json().get("data", {})
897
959
  groups = data.get("items")
898
960
 
899
961
  if not groups:
900
- print("SEVERE: No destinations defined in the account")
962
+ print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
901
963
  os._exit(1)
902
964
 
903
965
  if not group:
904
966
  if len(groups) == 1:
905
967
  return groups[0]['id'], groups[0]['name']
906
968
  else:
907
- print(
908
- "SEVERE: Destination name is required when there are multiple destinations in the account")
969
+ print_library_log(
970
+ "Destination name is required when there are multiple destinations in the account", Logging.Level.SEVERE)
909
971
  os._exit(1)
910
972
  else:
911
973
  while True:
@@ -922,8 +984,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
922
984
  data = resp.json().get("data", {})
923
985
  groups = data.get("items", [])
924
986
 
925
- print(
926
- f"SEVERE: The specified destination '{group}' was not found in your account.")
987
+ print_library_log(
988
+ f"The specified destination '{group}' was not found in your account.", Logging.Level.SEVERE)
927
989
  os._exit(1)
928
990
 
929
991
  # Call this method to run the connector in production
@@ -948,7 +1010,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
948
1010
  Logging.LOG_LEVEL = log_level
949
1011
 
950
1012
  if not DEBUGGING:
951
- print(f"Running on fivetran_connector_sdk: {__version__}")
1013
+ print_library_log(f"Running on fivetran_connector_sdk: {__version__}")
952
1014
 
953
1015
  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
954
1016
  connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
@@ -1004,7 +1066,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1004
1066
  download_filename = f"sdk-connector-tester-{os_name}-{TESTER_VERSION}.zip"
1005
1067
  download_filepath = os.path.join(tester_root_dir, download_filename)
1006
1068
  try:
1007
- print(f"INFO: Downloading connector tester version: {TESTER_VERSION} ", end="", flush=True)
1069
+ print_library_log(f"Downloading connector tester version: {TESTER_VERSION} ")
1008
1070
  download_url = f"https://github.com/fivetran/fivetran_sdk_tools/releases/download/{TESTER_VERSION}/{download_filename}"
1009
1071
  r = rq.get(download_url)
1010
1072
  if r.ok:
@@ -1036,15 +1098,21 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1036
1098
 
1037
1099
  project_path = os.getcwd() if project_path is None else project_path
1038
1100
  self.validate_requirements_file(project_path, False)
1039
- print(f"INFO: Debugging connector at: {project_path}")
1040
- server = self.run(50051, configuration, state, log_level=log_level)
1101
+ print_library_log(f"Debugging connector at: {project_path}")
1102
+ available_port = get_available_port()
1103
+
1104
+ if available_port is None:
1105
+ raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
1106
+ "Please ensure a port is available and try again")
1107
+
1108
+ server = self.run(available_port, configuration, state, log_level=log_level)
1041
1109
 
1042
1110
  # Uncomment this to run the tester manually
1043
1111
  # server.wait_for_termination()
1044
1112
 
1045
1113
  try:
1046
- print(f"INFO: Running connector tester...")
1047
- for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, 50051):
1114
+ print_library_log(f"Running connector tester...")
1115
+ for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port):
1048
1116
  print(log_msg, end="")
1049
1117
  except:
1050
1118
  print(traceback.format_exc())
@@ -1086,7 +1154,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1086
1154
  yield modified_line
1087
1155
 
1088
1156
  @staticmethod
1089
- def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int = 50051):
1157
+ def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int):
1090
1158
  """Runs the connector tester.
1091
1159
 
1092
1160
  Args:
@@ -1166,10 +1234,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1166
1234
  if not self.schema_method:
1167
1235
  return connector_sdk_pb2.SchemaResponse(schema_response_not_supported=True)
1168
1236
  else:
1169
- configuration = self.configuration if self.configuration else request.configuration
1170
- response = self.schema_method(configuration)
1171
- self.process_tables(response)
1172
- return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
1237
+ try:
1238
+ configuration = self.configuration if self.configuration else request.configuration
1239
+ response = self.schema_method(configuration)
1240
+ self.process_tables(response)
1241
+ return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
1242
+
1243
+ except Exception as e:
1244
+ tb = traceback.format_exc()
1245
+ error_message = f"Error: {str(e)}\n{tb}"
1246
+ raise RuntimeError(error_message) from e
1173
1247
 
1174
1248
  def process_tables(self, response):
1175
1249
  for entry in response:
@@ -1227,7 +1301,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1227
1301
  elif type.upper() == "SHORT":
1228
1302
  column.type = common_pb2.DataType.SHORT
1229
1303
  elif type.upper() == "INT":
1230
- column.type = common_pb2.DataType.SHORT
1304
+ column.type = common_pb2.DataType.INT
1231
1305
  elif type.upper() == "LONG":
1232
1306
  column.type = common_pb2.DataType.LONG
1233
1307
  elif type.upper() == "DECIMAL":
@@ -1278,6 +1352,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
1278
1352
  if str(e) != "'NoneType' object is not iterable":
1279
1353
  raise e
1280
1354
 
1355
+ except Exception as e:
1356
+ tb = traceback.format_exc()
1357
+ error_message = f"Error: {str(e)}\n{tb}"
1358
+ raise RuntimeError(error_message) from e
1359
+
1281
1360
 
1282
1361
  def find_connector_object(project_path) -> Connector:
1283
1362
  """Finds the connector object in the given project path.
@@ -1301,8 +1380,8 @@ def find_connector_object(project_path) -> Connector:
1301
1380
  if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
1302
1381
  return obj_attr
1303
1382
 
1304
- print(
1305
- "SEVERE: The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector")
1383
+ print_library_log(
1384
+ "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
1306
1385
  sys.exit(1)
1307
1386
 
1308
1387
 
@@ -1369,6 +1448,8 @@ def main():
1369
1448
  """The main entry point for the script.
1370
1449
  Parses command line arguments and passes them to connector object methods
1371
1450
  """
1451
+ global EXECUTED_VIA_CLI
1452
+ EXECUTED_VIA_CLI = True
1372
1453
 
1373
1454
  parser = argparse.ArgumentParser(allow_abbrev=False)
1374
1455
 
@@ -1387,7 +1468,7 @@ def main():
1387
1468
  args = parser.parse_args()
1388
1469
 
1389
1470
  if args.command.lower() == "version":
1390
- print("fivetran_connector_sdk " + __version__)
1471
+ print_library_log("fivetran_connector_sdk " + __version__)
1391
1472
  return
1392
1473
 
1393
1474
  connector_object = find_connector_object(args.project_path)
@@ -1404,7 +1485,7 @@ def main():
1404
1485
 
1405
1486
  if args.command.lower() == "deploy":
1406
1487
  if args.state:
1407
- print("WARNING: 'state' parameter is not used for 'deploy' command")
1488
+ print_library_log("'state' parameter is not used for 'deploy' command", Logging.Level.WARNING)
1408
1489
  connector_object.deploy(args.project_path, args.force, ft_deploy_key, ft_group, ft_connection, configuration)
1409
1490
 
1410
1491
  elif args.command.lower() == "debug":
@@ -1428,6 +1509,9 @@ def validate_and_load_configuration(args, configuration):
1428
1509
  "Configuration must be provided as a JSON file. Please check your input. Reference: "
1429
1510
  "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
1430
1511
  else:
1512
+ json_filepath = os.path.join(args.project_path, "configuration.json")
1513
+ if os.path.exists(json_filepath):
1514
+ print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
1431
1515
  configuration = {}
1432
1516
  return configuration
1433
1517
 
@@ -1450,14 +1534,14 @@ def reset_local_file_directory(args):
1450
1534
  confirm = input(
1451
1535
  "This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
1452
1536
  if confirm.lower() != "y":
1453
- print("INFO: Reset canceled")
1537
+ print_library_log("Reset canceled")
1454
1538
  else:
1455
1539
  try:
1456
1540
  if os.path.exists(files_path) and os.path.isdir(files_path):
1457
1541
  shutil.rmtree(files_path)
1458
- print("INFO: Reset Successful")
1542
+ print_library_log("Reset Successful")
1459
1543
  except Exception as e:
1460
- print("ERROR: Reset Failed")
1544
+ print_library_log("Reset Failed", Logging.Level.SEVERE)
1461
1545
  raise e
1462
1546
 
1463
1547
 
@@ -1,19 +1,20 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: fivetran_connector_sdk
3
- Version: 0.11.14.1
3
+ Version: 0.12.9.2
4
4
  Summary: Build custom connectors on Fivetran platform
5
5
  Author-email: Fivetran <developers@fivetran.com>
6
6
  Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
7
7
  Project-URL: Github, https://github.com/fivetran/fivetran_connector_sdk
8
+ Project-URL: Changelog, https://fivetran.com/docs/connectors/connector-sdk/changelog
8
9
  Classifier: Programming Language :: Python :: 3
9
10
  Classifier: License :: OSI Approved :: MIT License
10
11
  Classifier: Operating System :: OS Independent
11
12
  Requires-Python: >=3.9
12
13
  Description-Content-Type: text/markdown
13
- Requires-Dist: grpcio ==1.60.1
14
- Requires-Dist: grpcio-tools ==1.60.1
15
- Requires-Dist: requests ==2.32.3
16
- Requires-Dist: pipreqs ==0.5.0
14
+ Requires-Dist: grpcio==1.60.1
15
+ Requires-Dist: grpcio-tools==1.60.1
16
+ Requires-Dist: requests==2.32.3
17
+ Requires-Dist: pipreqs==0.5.0
17
18
 
18
19
  # **fivetran-connector-sdk**
19
20
  [![Downloads](https://static.pepy.tech/badge/fivetran-connector-sdk)](https://pepy.tech/project/fivetran-connector-sdk)
@@ -1,4 +1,4 @@
1
- fivetran_connector_sdk/__init__.py,sha256=eEtA9kXpAxIJDncDEWWGDwg0SYT2jAHXG2wN5heX4rA,58725
1
+ fivetran_connector_sdk/__init__.py,sha256=2k64NC7j9Q8HeF4y7_NBeyO2fqtexxgPQeuHfEAIslQ,62965
2
2
  fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
3
  fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
4
4
  fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
6
6
  fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
7
7
  fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
8
8
  fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
9
- fivetran_connector_sdk-0.11.14.1.dist-info/METADATA,sha256=qfyFHEuw7Pe4k9frNRdyq7I1sKABLOgHIWxBcaRHHqI,2858
10
- fivetran_connector_sdk-0.11.14.1.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
11
- fivetran_connector_sdk-0.11.14.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
12
- fivetran_connector_sdk-0.11.14.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
13
- fivetran_connector_sdk-0.11.14.1.dist-info/RECORD,,
9
+ fivetran_connector_sdk-0.12.9.2.dist-info/METADATA,sha256=3yH28B6Qmc0TIL1tqPwmltqetrH3ewwBxMMfeJI35n0,2938
10
+ fivetran_connector_sdk-0.12.9.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
11
+ fivetran_connector_sdk-0.12.9.2.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
12
+ fivetran_connector_sdk-0.12.9.2.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
13
+ fivetran_connector_sdk-0.12.9.2.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.5.0)
2
+ Generator: setuptools (75.6.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5