bizon 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

bizon/source/callback.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from bizon.source.config import SourceConfig
+from bizon.source.models import SourceIteration
+
+
+class AbstractSourceCallback(ABC):
+    @abstractmethod
+    def __init__(self, config: SourceConfig):
+        pass
+
+    @abstractmethod
+    def on_iterations_written(self, iterations: List[SourceIteration]):
+        pass
+
+
+class NoOpSourceCallback(AbstractSourceCallback):
+    def __init__(self, config: SourceConfig):
+        pass
+
+    def on_iterations_written(self, iterations: List[SourceIteration]):
+        """No-op callback"""
+        pass

bizon/source/config.py

@@ -21,7 +21,7 @@ class SourceSyncModes(str, Enum):
     INCREMENTAL = "incremental"
 
     # Stream mode
-    # - get single RUNNING streamJob  syncs data from the last destination cursor using last pagination, a single RUNNING StreamJob will be created and considered as always running.
+    # - get single RUNNING streamJob syncs data from the last successfuly committed offset in the source system
     STREAM = "stream"
 
 
@@ -32,8 +32,8 @@ class APIConfig(BaseModel):
 class SourceConfig(BaseModel, ABC):
 
     # Connector identifier to use, match a unique connector code
-    source_name: str = Field(..., description="Name of the source to sync")
-    stream_name: str = Field(..., description="Name of the stream to sync")
+    name: str = Field(..., description="Name of the source to sync")
+    stream: str = Field(..., description="Name of the stream to sync")
 
     source_file_path: Optional[str] = Field(
         default=None, description="Path to the source file, if not provided will look into bizon internal sources"

bizon/source/cursor.py

@@ -106,7 +106,7 @@ class Cursor:
             self.job_status = JobStatus.SUCCEEDED
 
             if self.total_records is not None and self.rows_fetched != self.total_records:
-                logger.warning(
+                logger.info(
                     (
                         f"Source: {self.source_full_name} - Iteration {self.iteration} - "
                         f"Total records fetched: {self.rows_fetched} "

bizon/source/discover.py

@@ -57,8 +57,9 @@ def find_inherited_classes(file_path):
 def get_python_import_path(relative_path: str) -> str:
     """Transform a relative path to a python import path"""
     python_path = relative_path.replace("/", ".").replace(".py", "")
+
     # Find the index where the desired substring starts
-    start_index = python_path.find("bizon.sources.")
+    start_index = python_path.find("bizon.connectors.sources")
 
     # Extract the substring from that point onwards
     if start_index != -1:
@@ -69,13 +70,13 @@ def get_python_import_path(relative_path: str) -> str:
 def find_all_source_paths(source_dir_name: str = "src") -> Mapping[str, str]:
     """Find all source code paths in the sources directory
     Return a dict mapping source_name to source_code_path
-    Like {'gsheets': '/path/to/bizon/sources/gsheets/src'}
+    Like {'gsheets': '/path/to/bizon/connectors/sources/gsheets/src'}
     """
 
     discovered_sources_paths = {}
 
     # Leading to the bizon internal sources directory
-    base_dir = os.path.join(BIZON_ABSOLUTE_PATH, "sources")
+    base_dir = os.path.join(BIZON_ABSOLUTE_PATH, "connectors", "sources")
 
     for source_name in os.listdir(base_dir):
 

bizon/source/models.py

@@ -1,6 +1,5 @@
-from dataclasses import dataclass
 from datetime import datetime
-from typing import List, Union
+from typing import List, Optional, Union
 
 import polars as pl
 from pydantic import BaseModel, Field, field_validator
@@ -12,6 +11,7 @@ source_record_schema = pl.Schema(
         ("id", str),
         ("data", str),  # JSON payload dumped as string
         ("timestamp", pl.Datetime(time_unit="us", time_zone="UTC")),
+        ("destination_id", str),
     ]
 )
 
@@ -28,6 +28,8 @@ class SourceRecord(BaseModel):
         description="Timestamp of the record as defined by the source. Default is the time of extraction",
     )
 
+    destination_id: Optional[str] = Field(None, description="Destination id")
+
     @field_validator("id", mode="before")
     def coerce_int_to_str(value: Union[int, str]) -> str:
         # Coerce int to str in case Source return id as int

bizon/source/source.py

@@ -1,11 +1,11 @@
 from abc import ABC, abstractmethod
-from datetime import datetime
 from typing import Any, List, Optional, Tuple, Type, Union
 
 from requests.auth import AuthBase
 
+from .callback import AbstractSourceCallback, NoOpSourceCallback
 from .config import SourceConfig
-from .models import SourceIncrementalState, SourceIteration
+from .models import SourceIncrementalState, SourceIteration, SourceRecord
 from .session import Session
 
 
@@ -36,6 +36,10 @@ class AbstractSource(ABC):
         """Return the config class for the source"""
         pass
 
+    def get_source_callback_instance(self) -> AbstractSourceCallback:
+        """Return an instance of the source callback"""
+        return NoOpSourceCallback(config=self.config)
+
     @abstractmethod
     def get_authenticator(self) -> Union[AuthBase, None]:
         """Return an authenticator for the source, it will be set in the session
@@ -96,3 +100,7 @@ class AbstractSource(ABC):
     def get_session(self) -> Session:
         """Return a new session"""
         return Session()
+
+    def commit(self):
+        """Commit the records to the source"""
+        pass

bizon/transform/config.py

@@ -0,0 +1,8 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class TransformModel(BaseModel):
+    label: str = Field(..., description="Label of the transform")
+    python: str = Field(..., description="Python code for the transform")

bizon/transform/transform.py

@@ -0,0 +1,48 @@
+import json
+import textwrap
+
+import polars as pl
+from loguru import logger
+
+from .config import TransformModel
+
+
+class Transform:
+
+    def __init__(self, transforms: list[TransformModel]):
+        self.transforms = transforms
+
+    def apply_transforms(self, df_source_records: pl.DataFrame) -> pl.DataFrame:
+        """Apply transformation on df_source_records"""
+
+        # Process the transformations
+        for transform in self.transforms:
+
+            logger.debug(f"Applying transform {transform.label}")
+
+            # Create a function to be executed in the desired context
+            def my_transform(data: str) -> str:
+
+                data = json.loads(data)
+
+                # Start writing here
+                local_vars = {"data": data}
+
+                # Normalize the indentation of the Python code
+                normalized_python = textwrap.dedent(transform.python)
+
+                exec(normalized_python, {}, local_vars)
+
+                # Stop writing here
+                return json.dumps(local_vars["data"])
+
+            transformed_source_records = []
+
+            for row in df_source_records["data"].to_list():
+                transformed_source_records.append(my_transform(row))
+
+            df_source_records = df_source_records.with_columns(
+                pl.Series("data", transformed_source_records, dtype=pl.String).alias("data")
+            )
+
+        return df_source_records

{bizon-0.1.0.dist-info → bizon-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: bizon
-Version: 0.1.0
+Version: 0.1.2
 Summary: Extract and load your data reliably from API Clients with native fault-tolerant and checkpointing mechanism.
 Author: Antoine Balliet
 Author-email: antoine.balliet@gmail.com
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: bigquery
+Provides-Extra: datadog
 Provides-Extra: gsheets
 Provides-Extra: kafka
 Provides-Extra: postgres
@@ -19,6 +20,8 @@ Requires-Dist: avro (>=1.12.0,<2.0.0) ; extra == "kafka"
 Requires-Dist: backoff (>=2.2.1,<3.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: confluent-kafka (>=2.6.0,<3.0.0) ; extra == "kafka"
+Requires-Dist: datadog (>=0.50.2,<0.51.0) ; extra == "datadog"
+Requires-Dist: ddtrace (>=3.10.0,<4.0.0) ; extra == "datadog"
 Requires-Dist: dpath (>=2.2.0,<3.0.0)
 Requires-Dist: fastavro (>=1.9.7,<2.0.0) ; extra == "kafka"
 Requires-Dist: google-cloud-bigquery (>=3.25.0,<4.0.0) ; extra == "bigquery"
@@ -27,6 +30,7 @@ Requires-Dist: google-cloud-storage (>=2.17.0,<3.0.0)
 Requires-Dist: gspread (>=6.1.2,<7.0.0) ; extra == "gsheets"
 Requires-Dist: kafka-python (>=2.0.2,<3.0.0) ; extra == "kafka"
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
+Requires-Dist: orjson (>=3.10.16,<4.0.0)
 Requires-Dist: pendulum (>=3.0.0,<4.0.0)
 Requires-Dist: pika (>=1.3.2,<2.0.0) ; extra == "rabbitmq"
 Requires-Dist: polars (>=1.16.0,<2.0.0)
@@ -39,8 +43,10 @@ Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: pytz (>=2024.2,<2025.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: requests (>=2.28.2,<3.0.0)
+Requires-Dist: simplejson (>=3.20.1,<4.0.0)
 Requires-Dist: sqlalchemy (>=2.0.32,<3.0.0)
 Requires-Dist: sqlalchemy-bigquery (>=1.11.0,<2.0.0) ; extra == "bigquery"
+Requires-Dist: tenacity (>=9.0.0,<10.0.0)
 Description-Content-Type: text/markdown
 
 # bizon ⚡️
@@ -48,13 +54,17 @@ Extract and load your largest data streams with a framework you can trust for bi
 
 ## Features
 - **Natively fault-tolerant**: Bizon uses a checkpointing mechanism to keep track of the progress and recover from the last checkpoint.
+
 - **High throughput**: Bizon is designed to handle high throughput and can process billions of records.
+
 - **Queue system agnostic**: Bizon is agnostic of the queuing system, you can use any queuing system among Python Queue, RabbitMQ, Kafka or Redpanda. Thanks to the `bizon.engine.queue.Queue` interface, adapters can be written for any queuing system.
-- **Pipeline metrics**: Bizon provides exhaustive pipeline metrics and implement OpenTelemetry for tracing. You can monitor:
+
+- **Pipeline metrics**: Bizon provides exhaustive pipeline metrics and implement Datadog & OpenTelemetry for tracing. You can monitor:
     - ETAs for completion
     - Number of records processed
     - Completion percentage
     - Latency Source <> Destination
+
 - **Lightweight & lean**: Bizon is lightweight, minimal codebase and only uses few dependencies:
     - `requests` for HTTP requests
     - `pyyaml` for configuration
@@ -83,8 +93,8 @@ Create a file named `config.yml` in your working directory with the following co
 name: demo-creatures-pipeline
 
 source:
-  source_name: dummy
-  stream_name: creatures
+  name: dummy
+  stream: creatures
   authentication:
     type: api_key
     params:
@@ -105,7 +115,7 @@ bizon run config.yml
 
 Backend is the interface used by Bizon to store its state. It can be configured in the `backend` section of the configuration file. The following backends are supported:
 - `sqlite`: In-memory SQLite database, useful for testing and development.
-- `biguquery`: Google BigQuery backend, perfect for light setup & production.
+- `bigquery`: Google BigQuery backend, perfect for light setup & production.
 - `postgres`: PostgreSQL backend, for production use and frequent cursor updates.
 
 ## Queue configuration
@@ -115,6 +125,13 @@ Queue is the interface used by Bizon to exchange data between `Source` and `Dest
 - `rabbitmq`: RabbitMQ, for production use and high throughput.
 - `kafka`: Apache Kafka, for production use and high throughput and strong persistence.
 
+## Runner configuration
+
+Runner is the interface used by Bizon to run the pipeline. It can be configured in the `runner` section of the configuration file. The following runners are supported:
+- `thread` (asynchronous)
+- `process` (asynchronous)
+- `stream` (synchronous)
+
 ## Start syncing your data 🚀
 
 ### Quick setup without any dependencies ✌️

bizon-0.1.2.dist-info/RECORD

@@ -0,0 +1,123 @@
+bizon/__main__.py,sha256=6GV4zEg2wC8UBnmESrr71ZmpWo4cNWlrYE3PQuKwFHA,69
+bizon/alerting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bizon/alerting/alerts.py,sha256=BQJaFL9cv7gH4fopB1MCX8BFqgjZMdfDIfBqj3e1pcs,676
+bizon/alerting/models.py,sha256=kWTeoT7dDC6UrkybU1sRAIXNAE0Wwipf8W7dEEhMZM0,553
+bizon/alerting/slack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bizon/alerting/slack/config.py,sha256=D_CAtzXu-O41OVoyrlYRlBxnGN9rNbnohYQ9CmBYt_E,84
+bizon/alerting/slack/handler.py,sha256=0m6IUSkxqDMlpDWslkImQX74ScD8wIx3YtrhfpYNGUA,1620
+bizon/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bizon/cli/main.py,sha256=pQnphPLznllBVzifyIrn3es0U9E1VGSzeEv_COzq9FI,3364
+bizon/cli/utils.py,sha256=aZ47YjFfifHkW95bAVzWfEQD3ZnxGSMT32bkRLmc5-c,953
+bizon/common/errors/backoff.py,sha256=z7RkQt1Npdh0sfD3hBDaiWQKe4iqS6ewvT1Q4Fds5aU,508
+bizon/common/errors/errors.py,sha256=mrYx1uE2kOuR2pEaB7ztK1l2m0E4V-_-hxq-DuILerY,682
+bizon/common/models.py,sha256=yJbYhdzsMWwp2qWXO1cjT3nUhLtqMKM9yLEecb5NFTg,2710
+bizon/connectors/destinations/bigquery/config/bigquery.example.yml,sha256=sy5-Piew00BlcjX5CFayFVrUq9G_vFYWXDmpWi9beTY,1263
+bizon/connectors/destinations/bigquery/src/config.py,sha256=erTcw4M3ki5uvB2UJD5yxI80T7KVZj57_e70TB8ddLY,4001
+bizon/connectors/destinations/bigquery/src/destination.py,sha256=5LtIH2i3tNivYiRj8RBXjRmeGJNIaYQJOIMNuXw42Ds,8969
+bizon/connectors/destinations/bigquery_streaming/src/config.py,sha256=LdBKEqHPaGll8PW6c6q_lH7PJvsGdtv2BCrtB-TukTA,1898
+bizon/connectors/destinations/bigquery_streaming/src/destination.py,sha256=aqMRDcSeX1hGO3VjyGs6lvd9BKbtwtbpdGSLI9xliko,16000
+bizon/connectors/destinations/bigquery_streaming_v2/src/config.py,sha256=cdHST5Vx1VQbLsIVsPkoEtOJKmbA35XjsKzj6fZ5DHw,1907
+bizon/connectors/destinations/bigquery_streaming_v2/src/destination.py,sha256=UgUne8dNsWIfhB7bXiPVZMlq5jx7eeAshc0sPglULzc,18638
+bizon/connectors/destinations/bigquery_streaming_v2/src/proto_utils.py,sha256=aWYVzMPMTgsdDapYniu8h6Tf2Pty4fDisT_33d9yEJ4,3692
+bizon/connectors/destinations/file/src/config.py,sha256=_SwVmFouv9HBCyBVR5B4CwSrfeOy8RFMcekvseTpD7Y,595
+bizon/connectors/destinations/file/src/destination.py,sha256=9jXPm6nRrdQAD0udmOJtTt5Aab_tUuuDg7mHhngBgaA,1832
+bizon/connectors/destinations/logger/src/config.py,sha256=IIB9DiGHTMJ0kJ7pkNojoyywiFwQlmVFdYL627nJ4rA,458
+bizon/connectors/destinations/logger/src/destination.py,sha256=OU2eGcVXGzCNTu8qYxokKt3OmDHG5-hsPRIW2PqfZrM,1233
+bizon/connectors/sources/cycle/config/cycle.example.yml,sha256=UDiqOa-8ZsykmNT625kxq9tyXOj_gKe9CFwg9r_8SYk,230
+bizon/connectors/sources/cycle/src/source.py,sha256=NCYiAnQZOCwwyMPoRhv3LFlG-UlMBCR3iPapuk3AwAA,4265
+bizon/connectors/sources/cycle/tests/cycle_customers.py,sha256=A48S20LxIC0A74JLoFn4NTHNTgBWV_5stTFtF1Gfk2c,271
+bizon/connectors/sources/dummy/config/dummy.example.yml,sha256=Wvn8v644u1aKvRRsPzPPML4kYmWk0ZhMGrCPFoSbVZQ,331
+bizon/connectors/sources/dummy/src/fake_api.py,sha256=5EEETp9INOmDJ57FNXm6wnsRtofQmR1xYp8hUKjn0Es,3067
+bizon/connectors/sources/dummy/src/source.py,sha256=QPjrfbFmxZpVplHsc773657cuBRc2LnMuVOwcua1hc0,4095
+bizon/connectors/sources/dummy/tests/dummy_pipeline.py,sha256=V9EKvugFXm3aNQVip-kMzlmvjlW6sZNGldxW1WdRa-E,397
+bizon/connectors/sources/dummy/tests/dummy_pipeline_bigquery_backend.py,sha256=SO_x_IH9iY1e09NHJlS1-CON0SI21fxGFqljSz4EmM4,542
+bizon/connectors/sources/dummy/tests/dummy_pipeline_kafka.py,sha256=zKhE7dhd6C4E_7VMepR8nSsGnesTJWXwlak-Dj01GD0,473
+bizon/connectors/sources/dummy/tests/dummy_pipeline_rabbitmq.py,sha256=A6KmSEOlWMK0UuIZhZxWO8y-z8h19rtWEV8DuhKX8Xs,686
+bizon/connectors/sources/dummy/tests/dummy_pipeline_unnest.py,sha256=64ZhjXUyZHzaIZTL_DO425yN6SlpB1vxzfhziE_pSEw,510
+bizon/connectors/sources/dummy/tests/dummy_pipeline_write_data_bigquery.py,sha256=oTV9HmOzjs3B2WwTPzDlIT_VkoYuXxy1BtSkeTGkQqw,686
+bizon/connectors/sources/dummy/tests/dummy_pipeline_write_data_bigquery_through_kafka.py,sha256=PFUhDuFw1Q1AMNMsnXPQxoqHIWf_wHEL1hLQodYlLcQ,596
+bizon/connectors/sources/gsheets/config/default_auth.example.yml,sha256=KOBp6MfO4uJwpwEYW0tJ4X5ctVwwdur9poJB4Ohba6s,348
+bizon/connectors/sources/gsheets/config/service_account.example.yml,sha256=XxVUnk9gGWc3lDb8CnzTHjTu8xz4Asyr5tXzY6qLvPg,1081
+bizon/connectors/sources/gsheets/src/source.py,sha256=2N0_TmuCdRyieVAiTsG3niov4l3KeibCpbDNNyRVbvQ,5660
+bizon/connectors/sources/gsheets/tests/gsheets_pipeline.py,sha256=lNSM3kZTd4W_-ajGIO3mdp8qGdEbnmWqsMm5pRiS0cw,181
+bizon/connectors/sources/hubspot/config/api_key.example.yml,sha256=VDTRloE5caqAdGdXgvsJZ6nQT46JHzX_YboxeGbpP18,389
+bizon/connectors/sources/hubspot/config/oauth.example.yml,sha256=YqBtj1IxIsdM9E85_4eVWl6mPiHsQNoQn41EzCqORy0,499
+bizon/connectors/sources/hubspot/src/hubspot_base.py,sha256=F9_jEZnQTCRazETJ2_HeDKv9NNxG11umYmJ0FhUmTu4,3425
+bizon/connectors/sources/hubspot/src/hubspot_objects.py,sha256=rlZl-pUb9IGq5Hbfa15JwrQZOr4uCp7q_rrQus7u3OI,6289
+bizon/connectors/sources/hubspot/src/models/hs_object.py,sha256=-Y20H3-nenJyySMlvM4TPttPz4O8qm3ArKP_I8pxsuo,1235
+bizon/connectors/sources/hubspot/tests/hubspot_pipeline.py,sha256=e6dCF5_MHMySkeiF6kKrSAuCa_48J22-ZeSCZSjrfUI,216
+bizon/connectors/sources/kafka/config/kafka.example.yml,sha256=taIj3QUL3jynQCpO-YlDtt6nGvQp8hOzGkS0_RJFUQU,933
+bizon/connectors/sources/kafka/config/kafka_debezium.example.yml,sha256=lqFbNbSAFRh1Q8a9xQ_7nhhvVsyVIZM4amGEvCqHXQE,2254
+bizon/connectors/sources/kafka/src/callback.py,sha256=NgP9PLquHbVagz6E9VJK5Vx-kK8K2l80MhoeenbhOXY,484
+bizon/connectors/sources/kafka/src/config.py,sha256=jyvpEkT2PUXXcyZgPK2T-YmwPpqeQBntY3gU5Zk5cKU,2510
+bizon/connectors/sources/kafka/src/decode.py,sha256=5Xr1_u1KEqpQsJR9cQ9Y-UTFd40lNNg48tAoQFSj7WQ,2822
+bizon/connectors/sources/kafka/src/source.py,sha256=uL1kQFLlrGXf-ebG8Cqx8mljVPnVsujOsQzDjxyZ9bM,15069
+bizon/connectors/sources/kafka/tests/kafka_pipeline.py,sha256=9LaCqXJIEx2ye3dkWq0YK_bPX7d4fCX_OcDOJCk34WE,206
+bizon/connectors/sources/periscope/config/periscope_charts.example.yml,sha256=9OgFDB7vguiNz2F2fmRqDNV8S_ddO9ncN5hgW9MhME4,350
+bizon/connectors/sources/periscope/config/periscope_dashboards.example.yml,sha256=mQey_04cfCzNpm-tYICDRUmPOFTZGSaTYI2OpK604d0,364
+bizon/connectors/sources/periscope/src/source.py,sha256=N4auF-PwME7SjsqR0ear3DO1WSRAIOaJCnOmsUFCYzE,12354
+bizon/connectors/sources/periscope/tests/periscope_pipeline_charts.py,sha256=mU0JtfhS1KmWsS3iovGhGxK7iPVWiYzjBM_QfRL3ZQI,275
+bizon/connectors/sources/periscope/tests/periscope_pipeline_dashboard.py,sha256=VmkixgCMLTqNPXYXPYqcK7ncHeeR96h68IXPcuC70iA,290
+bizon/connectors/sources/pokeapi/config/pokeapi_pokemon_to_json.example.yml,sha256=H3FpBsyft5qSvuyurh-hhaASpq7GnehNeVAOOn31Txk,283
+bizon/connectors/sources/pokeapi/config/pokeapi_pokemon_to_logger.example.yml,sha256=wuJUob6QF1jwm5pnfNcFfXeO01HCcu-TEoZCeuYl_lo,134
+bizon/connectors/sources/pokeapi/src/source.py,sha256=gmR58dk7mNOqimJTdJKVT1doSpEaKIM5mY1hfWGKJvk,2538
+bizon/destination/buffer.py,sha256=77sns18cjmo7_ejk9jjDF4uCT1mlaTCHqn82c2K2BIs,3156
+bizon/destination/config.py,sha256=YX_rxh7U2-gh6NEPFByD9VBaDG0eMkv3yOgR8Vzi8g4,3034
+bizon/destination/destination.py,sha256=Tx76KpOmLTzYx-HkNnVFVOL57Eh-GoqZwvKd6l_LjAo,14621
+bizon/destination/models.py,sha256=_LUnkbbD_9XauYrNTthh9VmbYwWsVgPHF90FX6vmSjg,1278
+bizon/engine/backend/adapters/sqlalchemy/backend.py,sha256=dXp4kszGuMDeJMEmf6MsVCC02Je1X9meAfmh92GQTfI,15454
+bizon/engine/backend/adapters/sqlalchemy/config.py,sha256=K-FpE_-VHnTSAQOduouhXFVy43EkrKbeZLqr9_OfeMw,1846
+bizon/engine/backend/backend.py,sha256=4J6uLKVh0hIPkOYO5XEg4ia8mPlOS13C3hSrIJ171VA,5849
+bizon/engine/backend/config.py,sha256=GhRzPWRGMaO-UJJRXkaqN_nlkFOCW6UOovwZLHLXrA8,900
+bizon/engine/backend/models.py,sha256=ECImDNji7u9eSkkjiw8sYq80l17vDF7MplTFvCpMgqA,5101
+bizon/engine/config.py,sha256=cKgI1IfzDncoxG3FsKUz-Aa3fU41ucQPaafjjhKeU90,2039
+bizon/engine/engine.py,sha256=l5S5I_7W-K4hQZsbxzZoDxk85A91tY4kR1lG1bcRddk,1706
+bizon/engine/pipeline/consumer.py,sha256=mgND9fD76T81Zu9P2Y4FgL9ZYoRs9E_R5aFZsdW6oO4,3208
+bizon/engine/pipeline/models.py,sha256=qOra2MJGN6-PuouKpKuZRjutnQmzom0mgWDFZ16LcM8,405
+bizon/engine/pipeline/producer.py,sha256=uARKW771TPLItDHSh0N7HqaQL2LqekSlaZMik0pB0PA,10059
+bizon/engine/queue/adapters/kafka/config.py,sha256=o7GAb_ls9N0nQV04B6Y4XjLo-Q57x28r63gjFG9LvVg,1091
+bizon/engine/queue/adapters/kafka/consumer.py,sha256=JPpvp3u5NXGvKRo60-ihRI4DlZxGJB1qH5Qw91XT8qc,2576
+bizon/engine/queue/adapters/kafka/queue.py,sha256=9XVuIH3MSEBLbNpnYwAugk6vfGz7IiuVKVdjeGZjLME,2001
+bizon/engine/queue/adapters/python_queue/config.py,sha256=_pyiIm1_PUjBo3WhKTATQcT2gazk-iHv0SdzNoGTeow,961
+bizon/engine/queue/adapters/python_queue/consumer.py,sha256=3Omq15fpU1nS1EZn8J9T_TdsHIPqxUmC75k8ZTFfG_8,1818
+bizon/engine/queue/adapters/python_queue/queue.py,sha256=XQ57hZCDFrdeF03pnTfb1Yx2MWhPHdJHC-UGV0y1vkg,2728
+bizon/engine/queue/adapters/rabbitmq/config.py,sha256=9N_7WREvNjJgcNTC3Y2kHII-iId2MZa3ssHHks6PyAs,987
+bizon/engine/queue/adapters/rabbitmq/consumer.py,sha256=EYhpPMb0c7vFX6M4ZBSQjUvkWjxW9B58dGpyfZkxcoc,2038
+bizon/engine/queue/adapters/rabbitmq/queue.py,sha256=pg_XUtXAd14Yi4H3k-yLXF3JM5yc-1WBlfFiEEfutfg,1745
+bizon/engine/queue/config.py,sha256=tTxo4OgSoDItMyWYIHLufFcOFqBGbIyPN5ZDNi1jZhI,1269
+bizon/engine/queue/queue.py,sha256=Y9uj31d-ZgW2f0F02iccp_o-m-RoMm_jR61NkLdMQ2M,3461
+bizon/engine/runner/adapters/process.py,sha256=0szaMKew7hbLcKR3k1pvwpDDynU7lcowpOJf48C8i3Y,3051
+bizon/engine/runner/adapters/streaming.py,sha256=fLl5JvW6__GD0yBsvraqr7XKRyHEmjO3bsjbcoCk_tE,5213
+bizon/engine/runner/adapters/thread.py,sha256=3BOGJlVFFWSA_NcK4NmwzB5FMFe6fqp0Ev_shukksPc,3792
+bizon/engine/runner/config.py,sha256=CK3rsudAo1cDhr2-fTmKcozWrmTsFmGzScKPl4cndWQ,2456
+bizon/engine/runner/runner.py,sha256=3AObfFotMkqaRPjBxJXmWRE-h0onMQBi3EpFU0JsKy8,11069
+bizon/monitoring/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bizon/monitoring/config.py,sha256=wRZRfW4ejjFDSG6swWj7kIULlaNYUPdbIKblQC9lzsk,1112
+bizon/monitoring/datadog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bizon/monitoring/datadog/monitor.py,sha256=NMtnIYq6n3I-QYmMJ6r8w63SO5DJ3wiDxXkcwqXEswY,6120
+bizon/monitoring/monitor.py,sha256=aJ3JEgD-f-HydGFAmprwL8YCA0WumDeXpm2P-uqp4wM,2627
+bizon/monitoring/noop/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bizon/monitoring/noop/monitor.py,sha256=9fALRj6LSxvNcqUTp6aKj5fLJqbpUMqu_xbP-T9Dbso,1079
+bizon/source/auth/authenticators/abstract_oauth.py,sha256=gJ40Sbrt0lnHfLupzkzOvUmse3X0Fp2XRHHqjqnVXdI,5274
+bizon/source/auth/authenticators/abstract_token.py,sha256=GMEQNyqkPYHmpY_FygChfokLLbohsz2r0WxttAQjsbw,899
+bizon/source/auth/authenticators/basic.py,sha256=78xtyw9fR1Nvr3DnKWT2qzyvJnBOT84evzZV53UhKrc,1290
+bizon/source/auth/authenticators/cookies.py,sha256=XSHthvRDRIxTJTgOLRzgQUhpQ3wdJW41Oy-gagTG2bU,802
+bizon/source/auth/authenticators/oauth.py,sha256=5P8y1kz2P8gIIMCCVr_cTNelSA4c34sxcnNNdOAwf0Q,3365
+bizon/source/auth/authenticators/token.py,sha256=P6SKRAarAEv28YiWp8hQLSKAV7twNlyNTGRr9sxlx58,956
+bizon/source/auth/builder.py,sha256=hc4zBNj31LZc-QqgIyx1VQEYTm9Xv81vY5pJiwQroJo,860
+bizon/source/auth/config.py,sha256=2jjcBLP95XsCkfKxdUei4X2yHI2WX92lJb8D8Txw86g,750
+bizon/source/callback.py,sha256=lfTwU_bzJwR0q5sbiKoK8uedQ-dhfHzoYkPVqm8b_Ho,602
+bizon/source/config.py,sha256=ELr9XL_WM-qJDuFWO01SlNEbyb3WRJcnTavu-So87Ns,2225
+bizon/source/cursor.py,sha256=ZQGXYvTOoJA77w3hbsocjpjT-QOYeqBki032Fc1FVB8,4239
+bizon/source/discover.py,sha256=4qPtHN_LFO4tVrq9mXwhoVDmaDP7R8I1zrksi3ksSEo,11156
+bizon/source/models.py,sha256=swyyS3bE6HtctX8_EBPwIVhChu1DQ7SFcPEIQ0eZAkw,1706
+bizon/source/session.py,sha256=z4dZlKC_PD8w_utTuAqs1vsfGuRkxHh5WQZhVKamNd0,1979
+bizon/source/source.py,sha256=j-aWCMd7Su8tVdWo143LU-n14ebNkgXlxnYMhMT6pHI,3703
+bizon/transform/config.py,sha256=DFfzUCprOy5mv0G5CDBKeZFXRtegIuRQ5fBdnye9czE,242
+bizon/transform/transform.py,sha256=IjluHZ2DY4uuuA-sn1x_1AqcavltglRbGbK4PJCr7Dk,1417
+bizon/utils.py,sha256=HXaPiyxpWKoy3XN5vSYOve1ezlFeOYin3aFqTjcabUQ,81
+bizon-0.1.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+bizon-0.1.2.dist-info/METADATA,sha256=Vr0kr2kAhglxQdDx9Re5LJoDZ8OVxL4RFUWcyLbdicw,6283
+bizon-0.1.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+bizon-0.1.2.dist-info/entry_points.txt,sha256=wtCd-6JswSY8lPWYSvOf7ASX1zfKgmgXtgg5XQS5274,44
+bizon-0.1.2.dist-info/RECORD,,

{bizon-0.1.0.dist-info → bizon-0.1.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any

bizon/destinations/bigquery/src/config.py

@@ -1,51 +0,0 @@
-from enum import Enum
-from typing import Literal, Optional
-
-from pydantic import BaseModel, Field
-
-from bizon.destinations.config import (
-    AbstractDestinationConfig,
-    AbstractDestinationDetailsConfig,
-    DestinationTypes,
-)
-
-
-class GCSBufferFormat(str, Enum):
-    PARQUET = "parquet"
-    CSV = "csv"
-
-
-class TimePartitioning(str, Enum):
-    DAY = "DAY"
-    HOUR = "HOUR"
-    MONTH = "MONTH"
-    YEAR = "YEAR"
-
-
-class BigQueryAuthentication(BaseModel):
-    service_account_key: str = Field(
-        description="Service Account Key JSON string. If empty it will be infered",
-        default="",
-    )
-
-
-class BigQueryConfigDetails(AbstractDestinationDetailsConfig):
-    project_id: str
-    dataset_id: str
-    dataset_location: Optional[str] = "US"
-    table_id: Optional[str] = Field(
-        default=None, description="Table ID, if not provided it will be inferred from source name"
-    )
-    gcs_buffer_bucket: str
-    gcs_buffer_format: Optional[GCSBufferFormat] = GCSBufferFormat.PARQUET
-
-    time_partitioning: Optional[TimePartitioning] = Field(
-        default=TimePartitioning.DAY, description="BigQuery Time partitioning type"
-    )
-    authentication: Optional[BigQueryAuthentication] = None
-
-
-class BigQueryConfig(AbstractDestinationConfig):
-    name: Literal[DestinationTypes.BIGQUERY]
-    buffer_size: Optional[int] = 2000
-    config: BigQueryConfigDetails

bizon/destinations/bigquery_streaming/src/config.py

@@ -1,43 +0,0 @@
-from enum import Enum
-from typing import Literal, Optional
-
-from pydantic import BaseModel, Field, field_validator
-
-from bizon.destinations.config import (
-    AbstractDestinationConfig,
-    AbstractDestinationDetailsConfig,
-    DestinationTypes,
-)
-
-
-class TimePartitioning(str, Enum):
-    DAY = "DAY"
-    HOUR = "HOUR"
-    MONTH = "MONTH"
-    YEAR = "YEAR"
-
-
-class BigQueryAuthentication(BaseModel):
-    service_account_key: str = Field(
-        description="Service Account Key JSON string. If empty it will be infered",
-        default="",
-    )
-
-
-class BigQueryStreamingConfigDetails(AbstractDestinationDetailsConfig):
-    project_id: str
-    dataset_id: str
-    dataset_location: Optional[str] = "US"
-    table_id: Optional[str] = Field(
-        default=None, description="Table ID, if not provided it will be inferred from source name"
-    )
-    time_partitioning: Optional[TimePartitioning] = Field(
-        default=TimePartitioning.DAY, description="BigQuery Time partitioning type"
-    )
-    authentication: Optional[BigQueryAuthentication] = None
-    bq_max_rows_per_request: Optional[int] = Field(30000, description="Max rows per buffer streaming request.")
-
-
-class BigQueryStreamingConfig(AbstractDestinationConfig):
-    name: Literal[DestinationTypes.BIGQUERY_STREAMING]
-    config: BigQueryStreamingConfigDetails