bizon 0.0.9__py3-none-any.whl → 0.0.10__py3-none-any.whl

bizon/destinations/destination.py

@@ -85,11 +85,16 @@ class AbstractDestination(ABC):
             pagination=self.buffer.pagination,
         )
 
+        logger.info(
+            f"Writing in destination from source iteration {self.buffer.from_iteration} to {self.buffer.to_iteration}"
+        )
+
         success, error_msg = self.write_records(destination_records=self.buffer.records)
 
         if success:
             # We wrote records to destination so we keep it
             destination_iteration.records_written = len(self.buffer.records)
+            logger.info(f"Successfully wrote {destination_iteration.records_written} records to destination")
 
         else:
             # We failed to write records to destination so we keep the error message
@@ -160,7 +165,7 @@ class AbstractDestination(ABC):
         logger.debug(f"Buffer free space {self.buffer.buffer_free_space_pct}%")
         logger.debug(f"Buffer current size {self.buffer.current_size} bytes")
         logger.info(
-            f"Buffer ripeness {self.buffer.ripeness / 60} min. Max ripeness {self.buffer.buffer_flush_timeout / 60} min."  # noqa
+            f"Buffer ripeness {round(self.buffer.ripeness / 60, 2)} min. Max ripeness {round(self.buffer.buffer_flush_timeout / 60, 2)} min."  # noqa
         )
 
         # Write buffer to destination if buffer is ripe and create a new buffer for the new iteration

bizon/engine/pipeline/producer.py

@@ -104,6 +104,8 @@ class Producer:
 
         while not cursor.is_finished:
 
+            timestamp_start_iteration = datetime.now(tz=UTC)
+
             # Handle the case where last cursor already reach max_iterations
             terminate = self.handle_max_iterations(cursor)
             if terminate:
@@ -178,6 +180,15 @@ class Producer:
                 return_value = PipelineReturnStatus.SOURCE_ERROR
                 break
 
+            # Items in queue
+            items_in_queue = f"{self.queue.get_size()} items in queue." if self.queue.get_size() else ""
+
+            logger.info(
+                (
+                    f"Iteration {cursor.iteration} finished in {datetime.now(tz=UTC) - timestamp_start_iteration}. {items_in_queue}"
+                )
+            )
+
         logger.info("Terminating destination ...")
 
         try:

bizon/engine/queue/adapters/kafka/queue.py

@@ -1,4 +1,5 @@
 import json
+from typing import Union
 
 from kafka import KafkaProducer
 from loguru import logger
@@ -36,6 +37,9 @@ class KafkaQueue(AbstractQueue):
     def on_error(e):
         logger.error(f"Error sending message: {e}")
 
+    def get_size(self) -> Union[int, None]:
+        return None
+
     def put_queue_message(self, queue_message: QueueMessage):
         future = self.producer.send(
             topic=self.config.queue.topic,

bizon/engine/queue/adapters/python_queue/queue.py

@@ -1,6 +1,7 @@
 import random
 import time
 from multiprocessing import Queue
+from typing import Union
 
 from loguru import logger
 
@@ -52,6 +53,11 @@ class PythonQueue(AbstractQueue):
             time.sleep(random.random())
             return self.get()
 
+    def get_size(self) -> Union[int, None]:
+        if hasattr(self.queue, "qsize"):
+            return self.queue.qsize()
+        return None
+
     def terminate(self, iteration: int) -> bool:
         self.put(source_records=[], iteration=iteration, signal=QUEUE_TERMINATION)
         logger.info("Sent termination signal to destination.")

bizon/engine/queue/adapters/rabbitmq/queue.py

@@ -1,3 +1,5 @@
+from typing import Union
+
 import pika
 from loguru import logger
 
@@ -31,6 +33,9 @@ class RabbitMQ(AbstractQueue):
             body=queue_message.model_dump_json(),
         )
 
+    def get_size(self) -> Union[int, None]:
+        return None
+
     def get(self) -> QueueMessage:
         raise NotImplementedError(
             "RabbitMQ does not support getting messages from the queue, directly use callback in consumer."

bizon/engine/queue/queue.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from pydantic import BaseModel
 from pytz import UTC
@@ -45,6 +45,11 @@ class AbstractQueue(ABC):
         """Get a QueueMessage object from the queue system"""
         pass
 
+    @abstractmethod
+    def get_size(self) -> Union[int, None]:
+        """If queue is compatible, return size of the queue"""
+        pass
+
     @abstractmethod
     def terminate(self, iteration: int) -> bool:
         """Send a termination signal in the queue system"""

bizon/source/discover.py

@@ -143,7 +143,7 @@ def parse_streams_from_filepath(source_name: str, filepath: str, skip_unavailabl
 
         # Transform the relative path to a python import path and import the module
         python_import_path = get_python_import_path(relative_path)
-        logger.info(f"Importing {python_import_path}")
+        logger.debug(f"Importing {python_import_path}")
 
         try:
             source_module = importlib.import_module(python_import_path, package="sources")

bizon/sources/kafka/src/source.py

@@ -98,7 +98,7 @@ class KafkaSource(AbstractSource):
         }
 
         # Consumer instance
-        self.consumer = Consumer(self.kafka_consumer_conf)
+        self.consumer = Consumer(self.kafka_consumer_conf, logger=logger)
 
     @staticmethod
     def streams() -> List[str]:
@@ -202,7 +202,7 @@ class KafkaSource(AbstractSource):
         source_timestamp = datetime.now(tz=timezone.utc)
 
         # Set consumer offset params
-        consumer = Consumer(self.kafka_consumer_conf)
+        consumer = Consumer(self.kafka_consumer_conf, logger=logger)
         consumer.assign([TopicPartition(self.config.topic, partition, topic_offsets.get_partition_offset(partition))])
         consumer.seek(TopicPartition(self.config.topic, partition, topic_offsets.get_partition_offset(partition)))
 

{bizon-0.0.9.dist-info → bizon-0.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bizon
-Version: 0.0.9
+Version: 0.0.10
 Summary: Extract and load your data reliably from API Clients with native fault-tolerant and checkpointing mechanism.
 Author: Antoine Balliet
 Author-email: antoine.balliet@gmail.com

{bizon-0.0.9.dist-info → bizon-0.0.10.dist-info}/RECORD

@@ -10,7 +10,7 @@ bizon/destinations/bigquery/src/config.py,sha256=QlD-FdBJ8Q6nKPrOf5q28lHnyFE8khT
 bizon/destinations/bigquery/src/destination.py,sha256=tPxE0IpHbR4zDkW5HaiHkgeDRDY2AibIPzY9iftZ2Uc,11079
 bizon/destinations/buffer.py,sha256=bFYkaoge-3AyKfGolqsuB3PWWtdPt65Fllrz-3X_uMI,2594
 bizon/destinations/config.py,sha256=jD4nkG-sg7mzJMFKLErQBkJu7ri0PMbCRVU3xIvFT7E,1686
-bizon/destinations/destination.py,sha256=VAyGPmowNimvK_joZj-6ESk2ezGxDZHnKCIpKRA-Vus,10995
+bizon/destinations/destination.py,sha256=qY2VIr_vEi7p589WPN6YeSRQcgehuDJtyLrvPDIC71Q,11276
 bizon/destinations/file/src/config.py,sha256=C4BBIKzBH5343iLGR3aCubAGjPo0b2LegsCLjb77uFA,513
 bizon/destinations/file/src/destination.py,sha256=1VCrVdtzAzwSKgYq0JUOc3r2cM7314dV-eIoAFhM_64,1003
 bizon/destinations/logger/src/config.py,sha256=AWY3R9q3ZjD3uQ_KBq8VcW60deKSIHe3qtgCKjdywKk,433
@@ -25,18 +25,18 @@ bizon/engine/config.py,sha256=cKgI1IfzDncoxG3FsKUz-Aa3fU41ucQPaafjjhKeU90,2039
 bizon/engine/engine.py,sha256=bdQksSQfxkeAHbbe52_MbqTJieOURjlMGYtkCCaDtuc,990
 bizon/engine/pipeline/consumer.py,sha256=HU3G2_h5ZUM217mnKSktdvib2nRc9r8OzvqWodRdFk0,424
 bizon/engine/pipeline/models.py,sha256=kfr_kqkJMEVlWX35rJiYMCuEBCrNhsx9R0a19E39i14,216
-bizon/engine/pipeline/producer.py,sha256=5Duhf_KHTtufV2j-KiZVwl5EEDKMpM4I8dH-MEdad7w,7924
+bizon/engine/pipeline/producer.py,sha256=k0dzSa6_7PiTJF0UtX0BAfvSTS0h7XFVZ3JN_3_ZvQQ,8330
 bizon/engine/queue/adapters/kafka/config.py,sha256=o7GAb_ls9N0nQV04B6Y4XjLo-Q57x28r63gjFG9LvVg,1091
 bizon/engine/queue/adapters/kafka/consumer.py,sha256=mh25mTjO7w6CGwJDWtxHVocwZi6DbTIVncm81rmhKrw,2576
-bizon/engine/queue/adapters/kafka/queue.py,sha256=fqROnv3HHcMvq4Aq9Is-jyiRWOQWC4zxNtzVpKFnKJs,1875
+bizon/engine/queue/adapters/kafka/queue.py,sha256=IS6akN7F81lkAajQdgqSqlqAg3r8uXbw6SdByDgvdMM,1965
 bizon/engine/queue/adapters/python_queue/config.py,sha256=D_CAuWJtdMQmQcm9gq9YBrkeFHAxZKRc7kIISliyp_4,847
 bizon/engine/queue/adapters/python_queue/consumer.py,sha256=yEoDF6QEmr9gjNGxXRqypdIHIJ50lQh_fFDhDXk6_g8,1566
-bizon/engine/queue/adapters/python_queue/queue.py,sha256=m4bfp0qD25lCxcok8yBXyjGGQjhQDqkm69uslutWLbc,2090
+bizon/engine/queue/adapters/python_queue/queue.py,sha256=VVc5A7qU2wgWEeeG6UOmgkmoIiwZ7GZGjSiBThloFzk,2259
 bizon/engine/queue/adapters/rabbitmq/config.py,sha256=9N_7WREvNjJgcNTC3Y2kHII-iId2MZa3ssHHks6PyAs,987
 bizon/engine/queue/adapters/rabbitmq/consumer.py,sha256=cN6K8wSBIQUSuRD7VsNltS6ElZ32PW92ZXiugzIDPJU,2019
-bizon/engine/queue/adapters/rabbitmq/queue.py,sha256=UPkrvrPizWNtwqSey714SB6kr70XWNJmqGQvGsgK5sY,1618
+bizon/engine/queue/adapters/rabbitmq/queue.py,sha256=gaTCIY_mCfWt8LCjfEymZuIiwqPkYQoVvaOacRYgLJo,1709
 bizon/engine/queue/config.py,sha256=PN9Je_Q9Sxo-3fI8lI6rZPQ9oeWatnD4rzUTWj3NhnA,792
-bizon/engine/queue/queue.py,sha256=Q3NlfS8DtKfVP2Y_bApVxjcNokCjtA4i7GH8dHorZk4,2778
+bizon/engine/queue/queue.py,sha256=_pOhmDZs79V7XgjthrKsFxZvae_4_cvol97jcZ-YR3g,2926
 bizon/engine/runner/adapters/process.py,sha256=idyknLADcmhCS4614WtyO-FqaYChV243gvjzPWvk0KE,2525
 bizon/engine/runner/adapters/thread.py,sha256=HUIJm5xg_yKdD2JOndvrod6x3qQ3uC6uakfc4m3XMso,2609
 bizon/engine/runner/config.py,sha256=QPgfy6YnS-EW8nhpTg1aRHshbGz5QTrQ5R3pDmLkIE0,1272
@@ -51,7 +51,7 @@ bizon/source/auth/builder.py,sha256=hc4zBNj31LZc-QqgIyx1VQEYTm9Xv81vY5pJiwQroJo,
 bizon/source/auth/config.py,sha256=2jjcBLP95XsCkfKxdUei4X2yHI2WX92lJb8D8Txw86g,750
 bizon/source/config.py,sha256=DPwJsBfU48yMvCw-pQCEha4X-IUjvmnQzjTwgsaCxAA,2307
 bizon/source/cursor.py,sha256=TSgWe1T9b4x7EEsbk22hwTWwVXCk5vdrs9eaHNhrevo,3983
-bizon/source/discover.py,sha256=ylv26jA02xOyWp5hZACCMpcqzZ4P3C5hvejBL7YX2cA,11119
+bizon/source/discover.py,sha256=C0_SnFxeHpz4VernxAfu2gbnQuoqv0cWX9z5J3WlCKw,11120
 bizon/source/models.py,sha256=iVp0H4muOWGst1W5DuxEVtHIY6lewOV8zDZUqvPTcBk,1337
 bizon/source/session.py,sha256=z4dZlKC_PD8w_utTuAqs1vsfGuRkxHh5WQZhVKamNd0,1979
 bizon/source/source.py,sha256=NhxMU1yXgi7mL64RyeymOYNqRk6fad9v_S8lhvXYUI0,3390
@@ -76,7 +76,7 @@ bizon/sources/hubspot/src/hubspot_objects.py,sha256=EmABx9XD8q6g4Uc5mHLv5YYl5KcI
 bizon/sources/hubspot/src/models/hs_object.py,sha256=-Y20H3-nenJyySMlvM4TPttPz4O8qm3ArKP_I8pxsuo,1235
 bizon/sources/hubspot/tests/hubspot_pipeline.py,sha256=e6dCF5_MHMySkeiF6kKrSAuCa_48J22-ZeSCZSjrfUI,216
 bizon/sources/kafka/config/kafka.example.yml,sha256=ZyHBmSWZ_5WQaBr9WzD05PuE6vi3hhYgHh2VZ-IU-Iw,755
-bizon/sources/kafka/src/source.py,sha256=28Cn_m8DOzsUdgbq0sUm36I7hB0TWRF6xEzg7TcrPrc,11343
+bizon/sources/kafka/src/source.py,sha256=0_LJKJBAy8LeW0AV0T6wGecwP-1Uid-ri9dDyU9KxKQ,11373
 bizon/sources/kafka/tests/kafka_pipeline.py,sha256=DrMHq96ZDiQ2lWmxEf_aX7HmBg_qNOsSFGTuGmuhly8,252
 bizon/sources/periscope/config/periscope_charts.example.yml,sha256=rpFDAWeU5oZ3UOiX0sSAgd1X5lv6t-s3iqiDPnRqutU,477
 bizon/sources/periscope/config/periscope_dashboards.example.yml,sha256=sN2iGGqCQCvrMXcwxNGq_dR7-KZ1KtYdXmNYKXlfEpg,481
@@ -84,8 +84,8 @@ bizon/sources/periscope/src/source.py,sha256=AZM-HDDjdTWj8akeeofQ_-G8YlnNHEKi2mj
 bizon/sources/periscope/tests/periscope_pipeline_charts.py,sha256=mU0JtfhS1KmWsS3iovGhGxK7iPVWiYzjBM_QfRL3ZQI,275
 bizon/sources/periscope/tests/periscope_pipeline_dashboard.py,sha256=vZKN7UfH-lQIWrnfjPqQFjZm28UIw2m9OSg4yS-Wckk,279
 bizon/utils.py,sha256=HXaPiyxpWKoy3XN5vSYOve1ezlFeOYin3aFqTjcabUQ,81
-bizon-0.0.9.dist-info/LICENSE,sha256=AW7SjYVT2bBnXOxgDxqy_e_JF8jDCFlMCaPCF11wFDI,1072
-bizon-0.0.9.dist-info/METADATA,sha256=POEtr3jEzvy8ogs2WvJ0rnlcpqFXcnZjLwQebLWxNnw,5646
-bizon-0.0.9.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-bizon-0.0.9.dist-info/entry_points.txt,sha256=wtCd-6JswSY8lPWYSvOf7ASX1zfKgmgXtgg5XQS5274,44
-bizon-0.0.9.dist-info/RECORD,,
+bizon-0.0.10.dist-info/LICENSE,sha256=AW7SjYVT2bBnXOxgDxqy_e_JF8jDCFlMCaPCF11wFDI,1072
+bizon-0.0.10.dist-info/METADATA,sha256=uG50AwUmj9iTn541sPTxfdXHpZ57NHX_rTo9cG-28jw,5647
+bizon-0.0.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+bizon-0.0.10.dist-info/entry_points.txt,sha256=wtCd-6JswSY8lPWYSvOf7ASX1zfKgmgXtgg5XQS5274,44
+bizon-0.0.10.dist-info/RECORD,,