bizon 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

bizon/connectors/sources/cycle/src/source.py

@@ -0,0 +1,133 @@
+from typing import Any, List, Tuple
+
+from pydantic import Field
+from requests.auth import AuthBase
+
+from bizon.source.auth.builder import AuthBuilder
+from bizon.source.auth.config import AuthType
+from bizon.source.config import SourceConfig
+from bizon.source.models import SourceIteration, SourceRecord
+from bizon.source.source import AbstractSource
+
+
+class CycleSourceConfig(SourceConfig):
+    slug: str = Field(..., description="Slug of the Cycle account")
+
+
+class CycleSource(AbstractSource):
+    def __init__(self, config: CycleSourceConfig):
+        super().__init__(config)
+        self.config: CycleSourceConfig = config
+        self.url_graphql = "https://api.product.cycle.app/graphql"
+
+    def get_authenticator(self) -> AuthBase:
+        if self.config.authentication.type.value == AuthType.API_KEY:
+            return AuthBuilder.token(params=self.config.authentication.params)
+
+    @staticmethod
+    def streams() -> List[str]:
+        return ["customers"]
+
+    @staticmethod
+    def get_config_class() -> SourceConfig:
+        return CycleSourceConfig
+
+    def check_connection(self) -> Tuple[bool | Any | None]:
+        return True, None
+
+    def get_total_records_count(self) -> int | None:
+        return None
+
+    def run_graphql_query(self, query: str, variables: dict) -> dict:
+        """Run a graphql query and return the response"""
+
+        payload = {"query": query, "variables": variables}
+
+        response = self.session.post(self.url_graphql, json=payload)
+
+        data = response.json()
+        return data
+
+    def _get_pagination_str(self, pagination: dict) -> str:
+        if not pagination:
+            pagination_str = """
+            size: 100
+            """
+        else:
+            pagination_str = """
+            size: 100
+            where: {
+                cursor: "PAGINATION_CURSOR"
+                direction: AFTER
+            }
+            """.replace(
+                "PAGINATION_CURSOR", pagination.get("endCursor")
+            )
+
+        return pagination_str
+
+    def get_customers(self, pagination: dict) -> SourceIteration:
+        """Return all customers for the given slug"""
+
+        pagination_str = self._get_pagination_str(pagination=pagination)
+
+        query = """
+        query Customers($slug: DefaultString!) {
+            getProductBySlug(slug: $slug) {
+                customers(pagination: {
+                    PAGINATION_STRING
+                }) {
+                    edges {
+                        cursor
+                        node {
+                            id
+                            email
+                            name
+                            company {
+                                domain
+                                id
+                                name
+                            }
+                        }
+                    }
+                    pageInfo {
+                        hasPreviousPage
+                        hasNextPage
+                        startCursor
+                        endCursor
+                    }
+                }
+            }
+        }
+        """.replace(
+            "PAGINATION_STRING", pagination_str
+        )
+
+        variables = {"slug": self.config.slug}
+
+        data = self.run_graphql_query(query, variables)
+
+        # Parse edges from response
+        edges = data.get("data", {}).get("getProductBySlug", {}).get("customers", {}).get("edges", [])
+
+        records = []
+        for customer in edges:
+            customer_data = customer.get("node", {})
+            records.append(
+                SourceRecord(
+                    id=customer_data["id"],
+                    data=customer_data,
+                )
+            )
+
+        # Get pagination info from response
+        pagination_info = data.get("data", {}).get("getProductBySlug", {}).get("customers", {}).get("pageInfo", {})
+        next_pagination = pagination_info if pagination_info.get("hasNextPage") else {}
+
+        return SourceIteration(records=records, next_pagination=next_pagination)
+
+    def get(self, pagination: dict = None) -> SourceIteration:
+        if self.config.stream == "customers":
+            return self.get_customers(pagination)
+
+        raise NotImplementedError(f"Stream {self.config.stream} not implemented for Cycle")

bizon/{sources/periscope/tests/periscope_pipeline_dashboard.py → connectors/sources/cycle/tests/cycle_customers.py}

@@ -3,7 +3,7 @@ import os
 from bizon.cli.utils import parse_from_yaml
 from bizon.engine.engine import RunnerFactory
 
-config = parse_from_yaml(os.path.abspath("bizon/sources/periscope/config/periscope_dashboards.yml"))
+config = parse_from_yaml(os.path.abspath("bizon/connectors/sources/cycle/config/cycle.yml"))
 
 runner = RunnerFactory.create_from_config_dict(config=config)
 runner.run()

bizon/{sources → connectors/sources}/dummy/config/dummy.example.yml

@@ -1,8 +1,8 @@
 name: dummy to logger
 
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   authentication:
     type: api_key
     params:

bizon/{sources → connectors/sources}/dummy/src/fake_api.py

@@ -1,5 +1,10 @@
+import time
+
+
 # Function emulating an API call to a source endpoint
-def fake_api_call(url: str, cursor: str = None) -> dict:
+def fake_api_call(url: str, cursor: str = None, sleep: int = None) -> dict:
+    if sleep:
+        time.sleep(sleep)
     if url == "https://api.dummy.com/v1/creatures":
         return fake_api_call_creatures(cursor)
 

bizon/{sources → connectors/sources}/dummy/src/source.py

@@ -1,4 +1,5 @@
-from typing import List, Literal, Tuple, Type, Union
+import random
+from typing import List, Literal, Tuple, Union
 
 from pydantic import Field
 from requests.auth import AuthBase
@@ -7,7 +8,7 @@ from bizon.source.auth.authenticators.oauth import Oauth2AuthParams
 from bizon.source.auth.authenticators.token import TokenAuthParams
 from bizon.source.auth.builder import AuthBuilder
 from bizon.source.auth.config import AuthConfig, AuthType
-from bizon.source.config import SourceConfig
+from bizon.source.config import SourceConfig, SourceSyncModes
 from bizon.source.models import SourceIteration, SourceRecord
 from bizon.source.source import AbstractSource
 
@@ -23,12 +24,14 @@ class DummyAuthConfig(AuthConfig):
 
 class DummySourceConfig(SourceConfig):
     authentication: DummyAuthConfig
+    sleep: int = Field(0, description="Sleep time in seconds between API calls")
 
 
 class DummySource(AbstractSource):
 
     def __init__(self, config: DummySourceConfig):
         super().__init__(config)
+        self.config = config
 
     @staticmethod
     def streams() -> List[str]:
@@ -40,7 +43,7 @@ class DummySource(AbstractSource):
 
     @property
     def url_entity(self) -> str:
-        return f"https://api.dummy.com/v1/{self.config.stream_name}"
+        return f"https://api.dummy.com/v1/{self.config.stream}"
 
     def get_authenticator(self) -> AuthBase:
 
@@ -71,11 +74,11 @@ class DummySource(AbstractSource):
 
         # If no pagination data is passed, we want to reach first page
         if not pagination:
-            response = fake_api_call(url=self.url_entity)
+            response = fake_api_call(url=self.url_entity, sleep=self.config.sleep)
 
         # If we have pagination data we pass it to the API
         else:
-            response = fake_api_call(url=self.url_entity, cursor=pagination.get("cursor"))
+            response = fake_api_call(url=self.url_entity, cursor=pagination.get("cursor"), sleep=self.config.sleep)
 
         # Now we process the response to:
         # - allow bizon to process the records and write them to destination
@@ -87,6 +90,15 @@ class DummySource(AbstractSource):
 
         next_pagination = {"cursor": next_cursor} if next_cursor else {}
 
+        destination_id = None
+
+        # If we are in streaming mode, we need to get the destination id from the stream name
+        if self.config.sync_mode == SourceSyncModes.STREAM:
+            if next_pagination.get("cursor") == "final-cursor":
+                destination_id = "routed"
+            else:
+                destination_id = self.config.stream
+
         if records:
             return SourceIteration(
                 next_pagination=next_pagination,
@@ -94,6 +106,7 @@ class DummySource(AbstractSource):
                     SourceRecord(
                         id=record["id"],
                         data=record,
+                        destination_id=destination_id,
                     )
                     for record in records
                 ],

bizon/{sources → connectors/sources}/dummy/tests/dummy_pipeline.py

@@ -6,8 +6,8 @@ config_yaml = """
 name: dummy to logger
 
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   authentication:
     type: api_key
     params:

bizon/{sources → connectors/sources}/dummy/tests/dummy_pipeline_bigquery_backend.py

@@ -4,8 +4,8 @@ from bizon.engine.engine import RunnerFactory
 
 config_yaml = """
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   force_ignore_checkpoint: true
   authentication:
     type: api_key

bizon/{sources → connectors/sources}/dummy/tests/dummy_pipeline_kafka.py

@@ -4,8 +4,8 @@ from bizon.engine.engine import RunnerFactory
 
 config_yaml = """
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   authentication:
     type: api_key
     params:

bizon/{sources → connectors/sources}/dummy/tests/dummy_pipeline_rabbitmq.py

@@ -4,8 +4,8 @@ from bizon.engine.engine import RunnerFactory
 
 config_yaml = """
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   authentication:
     type: api_key
     params:

bizon/connectors/sources/dummy/tests/dummy_pipeline_unnest.py

@@ -0,0 +1,29 @@
+from yaml import safe_load
+
+from bizon.engine.engine import RunnerFactory
+
+config_yaml = """
+name: test_job
+
+source:
+  name: dummy
+  stream: creatures
+  authentication:
+    type: api_key
+    params:
+      token: dummy_key
+
+destination:
+  name: logger
+  config:
+    dummy: dummy
+
+transforms:
+  - label: failure_transform
+    python: |
+      data['cookies'] = data['key_that_does_not_exist'].upper()
+"""
+
+config = safe_load(config_yaml)
+runner = RunnerFactory.create_from_config_dict(config=config)
+runner.run()

bizon/{sources → connectors/sources}/dummy/tests/dummy_pipeline_write_data_bigquery.py

@@ -4,8 +4,8 @@ from bizon.engine.engine import RunnerFactory
 
 config_yaml = """
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   authentication:
     type: api_key
     params:

bizon/{sources → connectors/sources}/dummy/tests/dummy_pipeline_write_data_bigquery_through_kafka.py

@@ -4,8 +4,8 @@ from bizon.engine.engine import RunnerFactory
 
 config_yaml = """
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   authentication:
     type: api_key
     params:

bizon/{sources → connectors/sources}/gsheets/config/default_auth.example.yml

@@ -1,6 +1,8 @@
+name: gsheets to logger example
+
 source:
-  source_name: gsheets
-  stream_name: worksheet
+  name: gsheets
+  stream: worksheet
   spreadsheet_url: <MY_SPREADSHEET_URL>
   worksheet_name: Sheet1
 

bizon/{sources → connectors/sources}/gsheets/config/service_account.example.yml

@@ -1,6 +1,8 @@
+name: gsheets to logger example
+
 source:
-  source_name: gsheets
-  stream_name: worksheet
+  name: gsheets
+  stream: worksheet
   spreadsheet_url: <MY_SPREADSHEET_URL>
   worksheet_name: Sheet1
   service_account_key: >-

bizon/{sources → connectors/sources}/hubspot/config/api_key.example.yml

@@ -1,6 +1,8 @@
+name: hubspot contacts to logger
+
 source:
-  source_name: hubspot
-  stream_name: contacts
+  name: hubspot
+  stream: contacts
   properties:
     strategy: all
   authentication:

bizon/{sources → connectors/sources}/hubspot/config/oauth.example.yml

@@ -1,6 +1,8 @@
+name: hubspot contacts to bigquery
+
 source:
-  source_name: hubspot
-  stream_name: contacts
+  name: hubspot
+  stream: contacts
   properties:
     strategy: all
   authentication:

bizon/{sources → connectors/sources}/hubspot/src/hubspot_objects.py

@@ -40,7 +40,7 @@ class HubSpotObjectsSource(HubSpotBaseSource):
     def __init__(self, config: HubSpotSourceConfig):
         super().__init__(config)
         self.config: HubSpotSourceConfig = config
-        self.object = self.config.stream_name
+        self.object = self.config.stream
         self.selected_properties = []  # Initialize properties to empty list
 
         # If we are initializing the pipeline, we retrieve the selected properties from HubSpot

bizon/{sources → connectors/sources}/kafka/config/kafka.example.yml

@@ -1,15 +1,13 @@
 name: demo kafka to bigquery
 
 source:
-  source_name: kafka
-  stream_name: topic
+  name: kafka
+  stream: topic
 
   sync_mode: full_refresh
 
   topic: my-topic
 
-  nb_bytes_schema_id: 8
-
   batch_size: 1000
   consumer_timeout: 10
   bootstrap_servers: <bootstrap-severs>:9092
@@ -47,4 +45,4 @@ destination:
 #       syncCursorInDBEvery: 100
 
 #   runner:
-#     log_level: INFO
+#     log_level: INFO

bizon/connectors/sources/kafka/config/kafka_debezium.example.yml

@@ -0,0 +1,110 @@
+name: Kafka debezium messages to bigquery streaming
+
+source:
+  name: kafka
+  stream: topic
+
+  sync_mode: full_refresh
+
+  force_ignore_checkpoint: true
+
+  topic: <TOPIC_NAME>
+
+  batch_size: 1000
+  consumer_timeout: 10
+  bootstrap_servers: <BOOTSTRAP_SERVERS>
+  group_id: <GROUP_ID>
+
+  authentication:
+    type: basic
+
+    schema_registry_url: <SCHEMA_REGISTRY_URL>
+    schema_registry_username: <SCHEMA_REGISTRY_USERNAME>
+    schema_registry_password: <SCHEMA_REGISTRY_PASSWORD>
+
+    params:
+      username: <USERNAME>
+      password: <PASSWORD>
+
+destination:
+  name: bigquery_streaming
+
+  config:
+    buffer_size: 50
+    bq_max_rows_per_request: 10000
+    buffer_flush_timeout: 30
+
+    table_id: <TABLE_ID>
+    dataset_id: <DATASET_ID>
+    dataset_location: US
+    project_id: <PROJECT_ID>
+
+    unnest: true
+
+    time_partitioning:
+      # Mandatory if unnested
+      field: __event_timestamp
+
+    record_schema:
+      - name: account_id
+        type: INTEGER
+        mode: REQUIRED
+
+      - name: team_id
+        type: INTEGER
+        mode: REQUIRED
+
+      - name: user_id
+        type: INTEGER
+        mode: REQUIRED
+
+      - name: __deleted
+        type: BOOLEAN
+        mode: NULLABLE
+
+      - name: __cluster
+        type: STRING
+        mode: NULLABLE
+
+      - name: __kafka_partition
+        type: INTEGER
+        mode: NULLABLE
+
+      - name: __kafka_offset
+        type: INTEGER
+        mode: NULLABLE
+
+      - name: __event_timestamp
+        type: TIMESTAMP
+        mode: NULLABLE
+
+transforms:
+- label: debezium
+  python: |
+    from datetime import datetime
+
+    cluster = data['value']['source']['name'].replace('_', '-')
+    partition = data['partition']
+    offset = data['offset']
+
+    kafka_timestamp = datetime.utcfromtimestamp(data['value']['source']['ts_ms'] / 1000).strftime('%Y-%m-%d %H:%M:%S.%f')
+
+    deleted = False
+
+    if data['value']['op'] == 'd':
+      data = data['value']['before']
+      deleted = True
+    else:
+      data = data['value']['after']
+
+    data['__deleted'] = deleted
+    data['__cluster'] = cluster
+    data['__kafka_partition'] = partition
+    data['__kafka_offset'] = offset
+    data['__event_timestamp'] = kafka_timestamp
+
+engine:
+  queue:
+    type: python_queue
+    config:
+      max_nb_messages: 1000000

bizon/connectors/sources/kafka/src/callback.py

@@ -0,0 +1,18 @@
+from typing import List
+
+from bizon.source.callback import AbstractSourceCallback
+from bizon.source.models import SourceIteration
+
+from .config import KafkaSourceConfig
+
+
+class KafkaSourceCallback(AbstractSourceCallback):
+    def __init__(self, config: KafkaSourceConfig):
+        super().__init__(config)
+
+    def on_iterations_written(self, iterations: List[SourceIteration]):
+        """Commit the offsets of the iterations"""
+
+        # TODO: Implement the callback
+
+        pass

bizon/connectors/sources/kafka/src/config.py

@@ -0,0 +1,69 @@
+from enum import Enum
+from typing import Any, List, Literal, Mapping
+
+from pydantic import BaseModel, Field
+
+from bizon.source.auth.config import AuthConfig, AuthType
+from bizon.source.config import SourceConfig
+
+
+class SchemaRegistryType(str, Enum):
+    APICURIO = "apicurio"
+
+
+class MessageEncoding(str, Enum):
+    UTF_8 = "utf-8"
+    AVRO = "avro"
+
+
+class KafkaAuthConfig(AuthConfig):
+
+    type: Literal[AuthType.BASIC] = AuthType.BASIC  # username and password authentication
+
+    # Schema registry authentication
+    schema_registry_type: SchemaRegistryType = Field(
+        default=SchemaRegistryType.APICURIO, description="Schema registry type"
+    )
+
+    schema_registry_url: str = Field(default="", description="Schema registry URL with the format ")
+    schema_registry_username: str = Field(default="", description="Schema registry username")
+    schema_registry_password: str = Field(default="", description="Schema registry password")
+
+
+def default_kafka_consumer_config():
+    return {
+        "auto.offset.reset": "earliest",
+        "enable.auto.commit": False,  # Turn off auto-commit for manual offset handling
+        "session.timeout.ms": 45000,
+        "security.protocol": "SASL_SSL",
+    }
+
+
+class TopicConfig(BaseModel):
+    name: str = Field(..., description="Kafka topic name")
+    destination_id: str = Field(..., description="Destination id")
+
+
+class KafkaSourceConfig(SourceConfig):
+
+    # Mandatory Kafka configuration
+    topics: List[TopicConfig] = Field(..., description="Kafka topic, comma separated")
+    bootstrap_servers: str = Field(..., description="Kafka bootstrap servers")
+    group_id: str = Field(default="bizon", description="Kafka group id")
+
+    skip_message_empty_value: bool = Field(
+        default=True, description="Skip messages with empty value (tombstone messages)"
+    )
+
+    # Kafka consumer configuration
+    batch_size: int = Field(100, description="Kafka batch size, number of messages to fetch at once.")
+    consumer_timeout: int = Field(10, description="Kafka consumer timeout in seconds, before returning batch.")
+
+    consumer_config: Mapping[str, Any] = Field(
+        default_factory=default_kafka_consumer_config,
+        description="Kafka consumer configuration, as described in the confluent-kafka-python documentation",
+    )
+
+    message_encoding: str = Field(default=MessageEncoding.AVRO, description="Encoding to use to decode the message")
+
+    authentication: KafkaAuthConfig = Field(..., description="Authentication configuration")