bizon 0.0.1__tar.gz

bizon-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 data-collective
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bizon-0.0.1/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.1
+Name: bizon
+Version: 0.0.1
+Summary: Extract and load your data reliably from API Clients with native fault-tolerant and checkpointing mechanism.
+Author: Antoine Balliet
+Author-email: antoine.balliet@gmail.com
+Requires-Python: >=3.9,<3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: backoff (>=2.2.1,<3.0.0)
+Requires-Dist: dpath (>=2.2.0,<3.0.0)
+Requires-Dist: faker (>=26.0.0,<27.0.0)
+Requires-Dist: google-cloud-bigquery (>=3.25.0,<4.0.0)
+Requires-Dist: google-cloud-storage (>=2.17.0,<3.0.0)
+Requires-Dist: kafka-python (>=2.0.2,<3.0.0)
+Requires-Dist: loguru (>=0.7.2,<0.8.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: pendulum (>=3.0.0,<4.0.0)
+Requires-Dist: pyarrow (>=16.1.0,<17.0.0)
+Requires-Dist: pydantic (>=2.8.2,<3.0.0)
+Requires-Dist: pydantic-extra-types (>=2.9.0,<3.0.0)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: requests (>=2.28.2,<3.0.0)
+Requires-Dist: sqlalchemy (>=2.0.32,<3.0.0)
+Requires-Dist: sqlalchemy-bigquery (>=1.11.0,<2.0.0)
+Description-Content-Type: text/markdown
+
+# bizon ⚡️
+Extract and load your largest data streams with a framework you can trust for billion records.
+
+## Features
+- **Natively fault-tolerant**: Bizon uses a checkpointing mechanism to keep track of the progress and recover from the last checkpoint.
+- **Queue system agnostic**: Bizon is agnostic of the queuing system, you can use any queuing system like Python Queue, Kafka or Redpanda. Thanks to the `bizon.queue.Queue` interface, adapters can be written for any queuing system.
+- **Pipeline metrics**: Bizon provides exhaustive pipeline metrics and implement OpenTelemetry for tracing. You can monitor:
+    - ETAs for completion
+    - Number of records processed
+    - Completion percentage
+    - Latency Source <> Destination
+- **Lightweight & lean**: Bizon is lightweight, minimal codebase and only uses few dependencies:
+    - `requests` for HTTP requests
+    - `pyyaml` for configuration
+    - `sqlalchemy` for database / warehouse connections
+    - `pyarrow` for Parquet file format
+
+## Installation
+```bash
+pip install bizon
+```
+
+## Usage
+```python
+from yaml import safe_load
+from bizon.engine.runner import RunnerFactory
+
+yaml_config = """
+source:
+  source_name: dummy
+  stream_name: creatures
+  authentication:
+    type: api_key
+    params:
+      token: dummy_key
+
+destination:
+  name: logger
+  config:
+    dummy: dummy
+"""
+
+config = safe_load(yaml_config)
+runner = RunnerFactory.create_from_config_dict(config=config)
+runner.run()
+```
+
+## Backend configuration
+
+Backend is the interface used by Bizon to store its state. It can be configured in the `backend` section of the configuration file. The following backends are supported:
+- `sqlite`: In-memory SQLite database, useful for testing and development.
+- `biguquery`: Google BigQuery backend, perfect for light setup & production.
+- `postgres`: PostgreSQL backend, for production use and frequent cursor updates.
+
+## Queue configuration
+
+Queue is the interface used by Bizon to exchange data between `Source` and `Destination`. It can be configured in the `queue` section of the configuration file. The following queues are supported:
+- `python_queue`: Python Queue, useful for testing and development.
+- `kafka`: Apache Kafka, for production use and high throughput.
+
+## Start syncing your data 🚀
+
+### Quick setup without any dependencies ✌️
+
+Queue configuration can be set to `python_queue` and backend configuration to `sqlite`.
+This will allow you to test the pipeline without any external dependencies.
+
+
+### Local Kafka setup
+
+To test the pipeline with Kafka, you can use `docker compose` to setup Kafka or Redpanda locally.
+
+**Kafka**
+```bash
+docker compose --file ./scripts/kafka-compose.yml up
+```
+
+In your YAML configuration, set the `queue` configuration to Kafka under `engine`:
+```yaml
+engine:
+  queue:
+    type: kafka
+    config:
+      bootstrap_servers: localhost:9092
+```
+
+**Redpanda**
+```bash
+docker compose --file ./scripts/redpanda-compose.yml up
+```
+
+In your YAML configuration, set the `queue` configuration to Kafka under `engine`:
+
+```yaml
+engine:
+  queue:
+    type: kafka
+    config:
+      bootstrap_servers: localhost:19092
+```
+

bizon-0.0.1/README.md

@@ -0,0 +1,100 @@
+# bizon ⚡️
+Extract and load your largest data streams with a framework you can trust for billion records.
+
+## Features
+- **Natively fault-tolerant**: Bizon uses a checkpointing mechanism to keep track of the progress and recover from the last checkpoint.
+- **Queue system agnostic**: Bizon is agnostic of the queuing system, you can use any queuing system like Python Queue, Kafka or Redpanda. Thanks to the `bizon.queue.Queue` interface, adapters can be written for any queuing system.
+- **Pipeline metrics**: Bizon provides exhaustive pipeline metrics and implement OpenTelemetry for tracing. You can monitor:
+    - ETAs for completion
+    - Number of records processed
+    - Completion percentage
+    - Latency Source <> Destination
+- **Lightweight & lean**: Bizon is lightweight, minimal codebase and only uses few dependencies:
+    - `requests` for HTTP requests
+    - `pyyaml` for configuration
+    - `sqlalchemy` for database / warehouse connections
+    - `pyarrow` for Parquet file format
+
+## Installation
+```bash
+pip install bizon
+```
+
+## Usage
+```python
+from yaml import safe_load
+from bizon.engine.runner import RunnerFactory
+
+yaml_config = """
+source:
+  source_name: dummy
+  stream_name: creatures
+  authentication:
+    type: api_key
+    params:
+      token: dummy_key
+
+destination:
+  name: logger
+  config:
+    dummy: dummy
+"""
+
+config = safe_load(yaml_config)
+runner = RunnerFactory.create_from_config_dict(config=config)
+runner.run()
+```
+
+## Backend configuration
+
+Backend is the interface used by Bizon to store its state. It can be configured in the `backend` section of the configuration file. The following backends are supported:
+- `sqlite`: In-memory SQLite database, useful for testing and development.
+- `biguquery`: Google BigQuery backend, perfect for light setup & production.
+- `postgres`: PostgreSQL backend, for production use and frequent cursor updates.
+
+## Queue configuration
+
+Queue is the interface used by Bizon to exchange data between `Source` and `Destination`. It can be configured in the `queue` section of the configuration file. The following queues are supported:
+- `python_queue`: Python Queue, useful for testing and development.
+- `kafka`: Apache Kafka, for production use and high throughput.
+
+## Start syncing your data 🚀
+
+### Quick setup without any dependencies ✌️
+
+Queue configuration can be set to `python_queue` and backend configuration to `sqlite`.
+This will allow you to test the pipeline without any external dependencies.
+
+
+### Local Kafka setup
+
+To test the pipeline with Kafka, you can use `docker compose` to setup Kafka or Redpanda locally.
+
+**Kafka**
+```bash
+docker compose --file ./scripts/kafka-compose.yml up
+```
+
+In your YAML configuration, set the `queue` configuration to Kafka under `engine`:
+```yaml
+engine:
+  queue:
+    type: kafka
+    config:
+      bootstrap_servers: localhost:9092
+```
+
+**Redpanda**
+```bash
+docker compose --file ./scripts/redpanda-compose.yml up
+```
+
+In your YAML configuration, set the `queue` configuration to Kafka under `engine`:
+
+```yaml
+engine:
+  queue:
+    type: kafka
+    config:
+      bootstrap_servers: localhost:19092
+```

bizon-0.0.1/bizon/cli/parser.py

@@ -0,0 +1,7 @@
+import yaml
+
+
+def parse_from_yaml(path_to_yaml) -> dict:
+    with open(path_to_yaml) as f:
+        config = yaml.safe_load(f)
+    return config

bizon-0.0.1/bizon/common/errors/backoff.py

@@ -0,0 +1,14 @@
+import requests
+
+
+class BaseBackoffException(requests.exceptions.HTTPError):
+    def __init__(self, request: requests.PreparedRequest, response: requests.Response, error_message: str = ""):
+        error_message = (
+            error_message
+            or f"Request URL: {request.url}, Response Code: {response.status_code}, Response Text: {response.text}"
+        )
+        super().__init__(error_message, request=request, response=response)
+
+
+class DefaultBackoffException(BaseBackoffException):
+    pass

bizon-0.0.1/bizon/common/errors/errors.py

@@ -0,0 +1,18 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class FailureType(str, Enum):
+    SYSTEM_ERROR = "SYSTEM_ERROR"
+    CONFIG_ERROR = "CONFIG_ERROR"
+
+
+class ErrorTraceMessage(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    message: str = Field(..., description="A user-friendly message that indicates the cause of the error")
+    internal_message: Optional[str] = Field(None, description="The internal error that caused the failure")
+    stack_trace: Optional[str] = Field(None, description="The full stack trace of the error")
+    failure_type: Optional[FailureType] = Field(None, description="The type of error")

bizon-0.0.1/bizon/common/models.py

@@ -0,0 +1,40 @@
+import json
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+from uuid import uuid4
+
+from pydantic import BaseModel, ConfigDict, Field
+from pytz import UTC
+
+from bizon.destinations.config import DestinationConfig
+from bizon.engine.config import EngineConfig
+from bizon.source.config import SourceConfig
+
+
+class JobStatus(str, Enum):
+    NOT_STARTED = "not_started"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILED = "failed"
+
+
+class CursorStatus(str, Enum):
+    NOT_STARTED = "not_started"
+    SUCCESS = "success"
+    FAILED = "failed"
+
+
+class BizonConfig(BaseModel):
+
+    # Forbid extra keys in the model
+    model_config = ConfigDict(extra="forbid")
+
+    source: SourceConfig
+
+    destination: DestinationConfig
+
+    engine: Optional[EngineConfig] = Field(
+        description="Engine configuration",
+        default=EngineConfig(),
+    )

bizon-0.0.1/bizon/destinations/bigquery/config/bigquery.example.yml

@@ -0,0 +1,37 @@
+source:
+  name: hubspot
+  stream_name: contacts
+  properties:
+    strategy: all
+  authentication:
+    type: api_key
+    api_key: <MY_API_KEY>
+
+destination:
+  # Authentication: If empty it will be infered.
+  # Must have the bigquery.jobUser
+  # Must have the bigquery.dataEditor and storage.objectUser on the supplied dataset and bucket
+  name: bigquery
+  config:
+    dataset_id: bizon_test
+    dataset_location: US
+    project_id: my-gcp-project-id
+    gcs_buffer_bucket: bizon-buffer
+    gcs_buffer_format: parquet
+    service_account_key: >-
+      {
+        "type": "service_account",
+        "project_id": "my-gcp-project-id",
+        "private_key_id": "",
+        "private_key": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n",
+        "client_email": "a-bizon-service-account@my-gcp-project-id.iam.gserviceaccount.com",
+        "client_id": "",
+        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+        "token_uri": "https://oauth2.googleapis.com/token",
+        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+        "client_x509_cert_url": "",
+        "universe_domain": "googleapis.com"
+      }
+
+pipeline:
+  log_level: DEBUG

bizon-0.0.1/bizon/destinations/bigquery/src/config.py

@@ -0,0 +1,22 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class GCSBufferFormat(str, Enum):
+    PARQUET = "parquet"
+    CSV = "csv"
+
+
+class BigQueryConfig(BaseModel):
+    dataset_id: str
+    dataset_location: Optional[str] = "US"
+    project_id: str
+    gcs_buffer_bucket: str
+    gcs_buffer_format: Optional[GCSBufferFormat] = GCSBufferFormat.PARQUET
+
+    service_account_key: Optional[str] = Field(
+        description="Service Accouner Key JSON string. If empty it will be infered",
+        default="",
+    )

bizon-0.0.1/bizon/destinations/bigquery/src/destination.py

@@ -0,0 +1,114 @@
+import io
+import os
+import tempfile
+from typing import List
+from uuid import uuid4
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+from google.api_core.exceptions import NotFound
+from google.cloud import bigquery, storage
+from google.cloud.bigquery import DatasetReference
+from loguru import logger
+
+from bizon.destinations.destination import AbstractDestination
+from bizon.destinations.models import DestinationRecord
+from bizon.source.models import SourceRecord
+
+from .config import BigQueryConfig
+
+
+class BigQueryDestination(AbstractDestination):
+    def __init__(self, source_name: str, stream_name: str, config: BigQueryConfig):
+        super().__init__(source_name, stream_name, config)
+
+        if bool(config.service_account_key):
+            with tempfile.NamedTemporaryFile(delete=False) as temp:
+                temp.write(config.service_account_key.encode())
+                temp_file_path = temp.name
+            os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = temp_file_path
+
+        self.project_id = config.project_id
+        self.bq_client = bigquery.Client(project=self.project_id)
+        self.gcs_client = storage.Client(project=self.project_id)
+        self.buffer_bucket_name = config.gcs_buffer_bucket
+        self.buffer_bucket = self.gcs_client.bucket(config.gcs_buffer_bucket)
+        self.buffer_format = config.gcs_buffer_format
+        self.dataset_id = config.dataset_id
+        self.dataset_location = config.dataset_location
+
+    def convert_and_upload_to_buffer(self, source_records: List[SourceRecord]):
+
+        bizon_records = [
+            DestinationRecord.from_source_record(source_record).model_dump() for source_record in source_records
+        ]
+
+        logger.info(bizon_records[0])
+
+        df = pd.DataFrame(bizon_records)
+
+        # Convert DataFrame to Parquet in-memory
+        if self.buffer_format == "parquet":
+            table = pa.Table.from_pandas(df)
+            buffer = io.BytesIO()
+            pq.write_table(table, buffer)
+            buffer.seek(0)
+
+            # Upload the Parquet file to GCS
+            file_name = f"{self.source_name}/{self.stream_name}/{str(uuid4())}.parquet"
+            blob = self.buffer_bucket.blob(file_name)
+            blob.upload_from_file(buffer, content_type="application/octet-stream")
+            return file_name
+
+    def check_connection(self) -> bool:
+        dataset_ref = DatasetReference(self.project_id, self.dataset_id)
+
+        try:
+            self.bq_client.get_dataset(dataset_ref)
+        except NotFound:
+            dataset = bigquery.Dataset(dataset_ref)
+            dataset.location = self.dataset_location
+            dataset = self.bq_client.create_dataset(dataset)
+        return True
+
+    def cleanup(self, gcs_file: str):
+        blob = self.buffer_bucket.blob(gcs_file)
+        blob.delete()
+
+    # TO DO: Add backoff to common exceptions => looks like most are hanlded by the client
+    # https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout
+    # https://cloud.google.com/python/docs/reference/bigquery/latest/google.cloud.bigquery.dbapi.DataError
+    def load_to_bigquery(self, gcs_file: str):
+        job_config = bigquery.LoadJobConfig(
+            source_format=bigquery.SourceFormat.PARQUET,
+            write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
+            schema=[
+                bigquery.SchemaField("bizon_id", "STRING", mode="REQUIRED"),
+                bigquery.SchemaField("bizon_loaded_at", "STRING", mode="REQUIRED"),
+                bigquery.SchemaField("source_record_id", "STRING", mode="REQUIRED"),
+                bigquery.SchemaField("source_data", "STRING", mode="NULLABLE"),
+            ],
+        )
+
+        table_id = f"{self.project_id}.{self.dataset_id}.{self.source_name}_{self.stream_name}"
+
+        load_job = self.bq_client.load_table_from_uri(
+            f"gs://{self.buffer_bucket_name}/{gcs_file}", table_id, job_config=job_config
+        )
+        load_job.result()
+
+    def write_records(self, source_records: List[SourceRecord]):
+
+        # Here we can check if these IDs are already present in BigQuery
+        # Using SourceRecord.id values
+
+        gs_file_name = self.convert_and_upload_to_buffer(source_records=source_records)
+
+        try:
+            self.load_to_bigquery(gs_file_name)
+            self.cleanup(gs_file_name)
+        except Exception as e:
+            self.cleanup(gs_file_name)
+            raise e
+        return True

bizon-0.0.1/bizon/destinations/bigquery/tests/test_bigquery_client.py

@@ -0,0 +1,40 @@
+import logging
+import os
+import random
+from random import randint
+
+from faker import Faker
+
+from bizon.cli.parser import parse_from_yaml
+from bizon.destinations.bigquery.src.config import BigQueryConfig
+from bizon.destinations.bigquery.src.destination import BigQueryDestination
+from bizon.source.config import SourceConfig
+
+logger = logging.getLogger(__name__)
+
+fake = Faker("en_US")
+
+raw_config = parse_from_yaml(os.path.abspath("bizon/destinations/bigquery/config/bigquery.test.yml"))
+
+
+def test_config_models():
+    assert bool(SourceConfig.model_validate(raw_config["source"])) is True
+    logger.info("source validated...")
+
+    assert bool(BigQueryConfig.model_validate(raw_config["destination"])) is True
+    logger.info("destination validated...")
+
+
+# def test_load_records_to_bigquery():
+#     bigquery_config = BigQueryConfig.model_validate(raw_config["destination"])
+#     source_config = SourceConfig.model_validate(raw_config["source"])
+
+#     fake_records = [
+#         {"foo": randint(0, 100), "bar": {"baz": fake.name(), "poo": float(random.randrange(155, 389)) / 100}}
+#         for _ in range(100)
+#     ]
+#     client = BigQueryClient(config=bigquery_config.config, source_config=source_config)
+
+#     success = client.load_records_to_bigquery(json_records=fake_records)
+
+#     assert success is True

bizon-0.0.1/bizon/destinations/config.py

@@ -0,0 +1,17 @@
+from enum import Enum
+from typing import Union
+
+from pydantic import BaseModel, Field
+
+from .bigquery.src.config import BigQueryConfig
+from .logger.src.config import LoggerDestinationConfig
+
+
+class DestinationTypes(str, Enum):
+    BIGQUERY = "bigquery"
+    LOGGER = "logger"
+
+
+class DestinationConfig(BaseModel):
+    name: DestinationTypes = Field(..., description="Name of the destination")
+    config: Union[BigQueryConfig, LoggerDestinationConfig] = Field(..., description="Configuration for the destination")

bizon-0.0.1/bizon/destinations/destination.py

@@ -0,0 +1,49 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from bizon.source.models import SourceRecord
+
+from .config import DestinationConfig, DestinationTypes
+
+
+class AbstractDestination(ABC):
+
+    def __init__(self, source_name: str, stream_name: str, config: DestinationConfig):
+        self.source_name = source_name
+        self.stream_name = stream_name
+        self.config = config
+
+    @property
+    def name(self) -> str:
+        return self.config.name
+
+    @abstractmethod
+    def check_connection(self) -> bool:
+        pass
+
+    @abstractmethod
+    def write_records(self, source_records: List[SourceRecord]):
+        pass
+
+
+class DestinationFactory:
+    @staticmethod
+    def get_destination(source_name: str, stream_name: str, destination_config_dict: dict) -> AbstractDestination:
+
+        if destination_config_dict.get("name") == DestinationTypes.LOGGER:
+            from .logger.src.config import LoggerDestinationConfig
+            from .logger.src.destination import LoggerDestination
+
+            config = LoggerDestinationConfig.model_validate(obj=destination_config_dict.get("config"))
+            return LoggerDestination(source_name=source_name, stream_name=stream_name, config=config)
+
+        if destination_config_dict.get("name") == DestinationTypes.BIGQUERY:
+            from .bigquery.src.config import BigQueryConfig
+            from .bigquery.src.destination import BigQueryDestination
+
+            config = BigQueryConfig.model_validate(obj=destination_config_dict.get("config"))
+            return BigQueryDestination(source_name=source_name, stream_name=stream_name, config=config)
+
+        raise ValueError(
+            f"Destination {destination_config_dict.get('name')}" f"with params {destination_config_dict} not found"
+        )

bizon-0.0.1/bizon/destinations/logger/src/config.py

@@ -0,0 +1,7 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class LoggerDestinationConfig(BaseModel):
+    dummy: Optional[str] = "bizon"

bizon-0.0.1/bizon/destinations/logger/src/destination.py

@@ -0,0 +1,21 @@
+from typing import List
+
+from loguru import logger
+
+from bizon.destinations.destination import AbstractDestination
+from bizon.source.models import SourceRecord
+
+from .config import LoggerDestinationConfig
+
+
+class LoggerDestination(AbstractDestination):
+
+    def __init__(self, source_name: str, stream_name: str, config: LoggerDestinationConfig):
+        super().__init__(source_name, stream_name, config)
+
+    def check_connection(self) -> bool:
+        return True
+
+    def write_records(self, source_records: List[SourceRecord]):
+        for record in source_records:
+            logger.info(record.data)

bizon-0.0.1/bizon/destinations/models.py

@@ -0,0 +1,24 @@
+import json
+from datetime import datetime
+from uuid import uuid4
+
+from pydantic import BaseModel, Field
+from pytz import UTC
+
+from bizon.source.models import SourceRecord
+
+
+class DestinationRecord(BaseModel):
+    bizon_id: str = Field(..., description="Bizon unique identifier of the record")
+    bizon_loaded_at: str = Field(..., description="Datetime when the record was loaded")
+    source_record_id: str = Field(..., description="Source record id")
+    source_data: str = Field(..., description="Source record JSON string data")
+
+    @classmethod
+    def from_source_record(cls, source_record: SourceRecord) -> "DestinationRecord":
+        return cls(
+            bizon_id=uuid4().hex,
+            bizon_loaded_at=datetime.now(tz=UTC).isoformat(),
+            source_record_id=source_record.id,
+            source_data=json.dumps(source_record.data),
+        )