fivetran-connector-sdk 0.8.19.1__py3-none-any.whl → 0.8.21.1__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -23,7 +23,7 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-__version__ = "0.8.19.1"
+__version__ = "0.8.21.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
@@ -129,9 +129,8 @@ class Operations:
             global TABLES
             for field in data.keys():
                 columns[field] = common_pb2.Column(
-                    name=field, type=common_pb2.DataType.UNSPECIFIED, primary_key=True)
+                    name=field, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
             new_table = common_pb2.Table(name=table, columns=columns.values())
-            TABLES[table] = new_table
 
             responses.append(connector_sdk_pb2.UpdateResponse(
                 operation=connector_sdk_pb2.Operation(
@@ -510,7 +509,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
             unused_deps = list(requirements.keys() - tmp_requirements.keys())
             if unused_deps:
-                print("INFO: The following dependencies are not used in connector.py:")
+                print("INFO: The following dependencies are not needed, "
+                      "they are not used or already installed. Please remove them from requirements.txt:")
                 print(*unused_deps)
         else:
             if os.path.exists("requirements.txt"):
@@ -532,7 +532,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         """
         if not deploy_key: print("SEVERE: The Fivetran API key is missing. Please provide a valid Fivetran API key to create the connector."); os._exit(1)
         if not connection: print("SEVERE: The connection name is missing. Please provide a valid connection name to create the connector."); os._exit(1)
-        _check_dict(configuration)
+        _check_dict(configuration, True)
 
         secrets_list = []
         if configuration:
@@ -774,31 +774,48 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Returns:
             tuple[str, str]: A tuple containing the group ID and group name.
         """
-        resp = rq.get("https://api.fivetran.com/v1/groups",
-                      headers={"Authorization": f"Basic {deploy_key}"})
+        groups_url = "https://api.fivetran.com/v1/groups"
+
+        params = {"limit": 500}
+        headers = {"Authorization": f"Basic {deploy_key}"}
+        resp = rq.get(groups_url, headers=headers, params=params)
 
         if not resp.ok:
-            print(f"SEVERE: Unable to fetch list of destination names, status code = {resp.status_code}")
+            print(
+                f"SEVERE: Unable to fetch list of destination names, status code = {resp.status_code}")
             os._exit(1)
 
-        # TODO: Do we need to implement pagination?
-        groups = resp.json()['data']['items']
+        data = resp.json().get("data", {})
+        groups = data.get("items")
+
         if not groups:
             print("SEVERE: No destinations defined in the account")
             os._exit(1)
 
-        if len(groups) == 1:
-            return groups[0]['id'], groups[0]['name']
-        else:
-            if not group:
-                print("SEVERE: Destination name is required when there are multiple destinations in the account")
+        if not group:
+            if len(groups) == 1:
+                return groups[0]['id'], groups[0]['name']
+            else:
+                print(
+                    "SEVERE: Destination name is required when there are multiple destinations in the account")
                 os._exit(1)
-
-            for grp in groups:
-                if grp['name'] == group:
-                    return grp['id'], grp['name']
-
-        print(f"SEVERE: Specified destination was not found in the account: {group}")
+        else:
+            while True:
+                for grp in groups:
+                    if grp['name'] == group:
+                        return grp['id'], grp['name']
+
+                next_cursor = data.get("next_cursor")
+                if not next_cursor:
+                    break
+
+                params = {"cursor": next_cursor, "limit": 500}
+                resp = rq.get(groups_url, headers=headers, params=params)
+                data = resp.json().get("data", {})
+                groups = data.get("items", [])
+
+        print(
+            f"SEVERE: The specified destination '{group}' was not found in your account.")
         os._exit(1)
 
     # Call this method to run the connector in production
@@ -1056,13 +1073,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 table = common_pb2.Table(name=table_name)
                 columns = {}
 
-                if "primary_key" not in entry:
-                    ValueError("Table requires at least one primary key: " + table_name)
-
-                for pkey_name in entry["primary_key"]:
-                    column = columns[pkey_name] if pkey_name in columns else common_pb2.Column(name=pkey_name)
-                    column.primary_key = True
-                    columns[pkey_name] = column
+                if "primary_key" in entry:
+                    for pkey_name in entry["primary_key"]:
+                        column = columns[pkey_name] if pkey_name in columns else common_pb2.Column(name=pkey_name)
+                        column.primary_key = True
+                        columns[pkey_name] = column
 
                 if "columns" in entry:
                     for name, type in entry["columns"].items():
@@ -1110,7 +1125,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                         else:
                             raise ValueError("Unrecognized column type: ", str(type))
 
-                        if name in entry["primary_key"]:
+                        if "primary_key" in entry and name in entry["primary_key"]:
                             column.primary_key = True
 
                         columns[name] = column
@@ -1179,7 +1194,7 @@ def main():
     parser = argparse.ArgumentParser(allow_abbrev=False)
 
     # Positional
-    parser.add_argument("command", help="debug|deploy")
+    parser.add_argument("command", help="debug|deploy|reset")
     parser.add_argument("project_path", nargs='?', default=os.getcwd(), help="Path to connector project directory")
 
     # Optional (Not all of these are valid with every mutually exclusive option below)
@@ -1232,6 +1247,20 @@ def main():
         port = 50051 if not args.port else args.port
         connector_object.debug(args.project_path, port, configuration, state)
 
+    elif args.command.lower() == "reset":
+        files_path = os.path.join(args.project_path, OUTPUT_FILES_DIR)
+        confirm = input("This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
+        if confirm.lower() != "y":
+            print("INFO: Reset canceled")
+        else:
+            try:
+                if os.path.exists(files_path) and os.path.isdir(files_path):
+                    shutil.rmtree(files_path)
+                print("INFO: Reset Successful")
+            except Exception as e:
+                print("ERROR: Reset Failed")
+                raise e
+
     else:
         raise NotImplementedError("Invalid command: ", args.command)
 

fivetran_connector_sdk-0.8.21.1.dist-info/METADATA

@@ -0,0 +1,57 @@
+Metadata-Version: 2.1
+Name: fivetran_connector_sdk
+Version: 0.8.21.1
+Summary: Build custom connectors on Fivetran platform
+Author-email: Fivetran <developers@fivetran.com>
+Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
+Project-URL: Github, https://github.com/fivetran/fivetran_connector_sdk
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: grpcio ==1.60.1
+Requires-Dist: grpcio-tools ==1.60.1
+Requires-Dist: requests ==2.31.0
+Requires-Dist: get-pypi-latest-version ==0.0.12
+Requires-Dist: pipreqs ==0.5.0
+
+# **fivetran-connector-sdk**
+The *fivetran-connector-sdk* is a Python module that enables you to write custom data connectors and deploy them as an extension of [Fivetran](https://www.fivetran.com/). Fivetran automatically manages running the connectors on your scheduled frequency and manages the required compute resources.
+
+The Connector SDK service is the best fit for the following use cases:
+- Fivetran doesn't have a connector for your source and is unlikely to support it soon.
+- You are using private APIs, custom applications, unsupported file formats, or those that require pre-processing.
+- You have sensitive data that needs filtering or anonymizing before entering the destination.
+- You don't want to introduce a 3rd party into your pipeline (e.g., to host a custom function).
+
+To learn more, see our [Connector SDK documentation](https://fivetran.com/docs/connectors/connector-sdk)
+
+## **Install**
+
+    pip install fivetran-connector-sdk
+
+## **Requirements**
+- Python ≥3.9 and ≤3.12
+- Operating System:
+    - Windows 10 or later
+    - MacOS 13 (Ventura) or later
+
+## **Getting started**
+See [Quickstart guide](https://fivetran.com/docs/connectors/connector-sdk/quickstart-guide) to get started.
+
+## **Usage**
+Import required classes from fivetran_connector_sdk to begin writing your custom data connector
+
+```python
+  from fivetran_connector_sdk import Connector
+  from fivetran_connector_sdk import Logging as log
+  from fivetran_connector_sdk import Operations as op
+```
+See our [Technical Reference](https://fivetran.com/docs/connectors/connector-sdk/technical-reference#update) and [Best Practices](https://fivetran.com/docs/connectors/connector-sdk/best-practices) documentation for more details.
+
+You can also refer to our [existing examples](https://github.com/fivetran/fivetran_connector_sdk) to better understand Fivetran's Connector SDK and start creating custom data connectors.
+
+## **Maintenance**
+This package is actively maintained by Fivetran Developers. Please reach out to our [Support team](https://support.fivetran.com/hc/en-us) for any inquiries.
+

{fivetran_connector_sdk-0.8.19.1.dist-info → fivetran_connector_sdk-0.8.21.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=DRdHyxqKeBGsvwYcqe2aCmM3AlVCiqVjbLQvRkfFVs0,49915
+fivetran_connector_sdk/__init__.py,sha256=1zgYuvadYRzaGcp0kG_icP55qS0lD_gRQhj3LzbI3X8,51084
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-0.8.19.1.dist-info/METADATA,sha256=fgkWljNC3yuszXoF9mo6HGx2mw8KvKLAhf6G5uMuxHs,708
-fivetran_connector_sdk-0.8.19.1.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-fivetran_connector_sdk-0.8.19.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-0.8.19.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-0.8.19.1.dist-info/RECORD,,
+fivetran_connector_sdk-0.8.21.1.dist-info/METADATA,sha256=jbJsixFXfw57UlFrRvPBGiiu6x4k4-Mx1wCsRaqzgU0,2796
+fivetran_connector_sdk-0.8.21.1.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
+fivetran_connector_sdk-0.8.21.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-0.8.21.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-0.8.21.1.dist-info/RECORD,,

{fivetran_connector_sdk-0.8.19.1.dist-info → fivetran_connector_sdk-0.8.21.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (73.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fivetran_connector_sdk-0.8.19.1.dist-info/METADATA

@@ -1,18 +0,0 @@
-Metadata-Version: 2.1
-Name: fivetran_connector_sdk
-Version: 0.8.19.1
-Summary: Build custom connectors on Fivetran platform
-Author-email: Fivetran <developers@fivetran.com>
-Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
-Project-URL: Github, https://github.com/fivetran/fivetran_connector_sdk
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-Requires-Dist: grpcio ==1.60.1
-Requires-Dist: grpcio-tools ==1.60.1
-Requires-Dist: requests ==2.31.0
-Requires-Dist: get-pypi-latest-version ==0.0.12
-Requires-Dist: pipreqs ==0.5.0
-