bizon 0.0.4.dev3__tar.gz → 0.0.4.dev4__tar.gz

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bizon
-Version: 0.0.4.dev3
+Version: 0.0.4.dev4
 Summary: Extract and load your data reliably from API Clients with native fault-tolerant and checkpointing mechanism.
 Author: Antoine Balliet
 Author-email: antoine.balliet@gmail.com

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/cli/main.py

@@ -1,12 +1,13 @@
 import click
 
 from bizon.engine.engine import RunnerFactory
+from bizon.engine.runner.config import LoggerLevel
 from bizon.source.discover import discover_all_sources
 
 from .utils import (
     parse_from_yaml,
     set_custom_source_path_in_config,
-    set_debug_mode,
+    set_log_level,
     set_runner_in_config,
 )
 
@@ -88,25 +89,29 @@ def destination():
     help="Runner type to use. Thread or Process.",
 )
 @click.option(
-    "--debug",
+    "--log-level",
     required=False,
-    is_flag=True,
+    type=click.Choice([level.name for level in LoggerLevel]),
     show_default=True,
-    default=False,
-    help="Enable debug mode.",
+    help="Log level to use.",
 )
-def run(filename: str, custom_source: str, runner: str, debug, help="Run a bizon pipeline from a YAML file."):
+def run(
+    filename: str,
+    custom_source: str,
+    runner: str,
+    log_level: LoggerLevel,
+    help="Run a bizon pipeline from a YAML file.",
+):
     """Run a bizon pipeline from a YAML file."""
-    ctx = click.get_current_context()
 
     # Parse config from YAML file as a dictionary
     config = parse_from_yaml(filename)
 
     # Set debug mode
-    set_debug_mode(debug)
+    set_log_level(config=config, level=log_level)
 
     # Override source_file_path param in config
-    set_custom_source_path_in_config(config=config, custom_source=ctx.get_parameter_source("custom-source"))
+    set_custom_source_path_in_config(config=config, custom_source=custom_source)
 
     # Override runner param in config
     set_runner_in_config(config=config, runner=runner)

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/cli/utils.py

@@ -1,5 +1,3 @@
-import os
-
 import yaml
 
 
@@ -9,10 +7,17 @@ def parse_from_yaml(path_to_yaml) -> dict:
     return config
 
 
-def set_debug_mode(debug: bool):
+# TODO: Refacto
+def set_log_level(config: dict, level: str):
     # Set Log Level to DEBUG
-    if debug:
-        os.environ["LOGURU_LEVEL"] = "DEBUG"
+    if level:
+        if "engine" not in config:
+            config["engine"] = {}
+
+        if "runner" not in config["engine"]:
+            config["engine"]["runner"] = {"log_level": level}
+
+        config["engine"]["runner"]["log_level"] = level
 
 
 def set_custom_source_path_in_config(config: dict, custom_source: str):
@@ -20,6 +25,7 @@ def set_custom_source_path_in_config(config: dict, custom_source: str):
         config["source"]["source_file_path"] = custom_source
 
 
+# TODO: Refacto
 def set_runner_in_config(config: dict, runner: str):
     if runner:
         if "engine" not in config:

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/destinations/destination.py

@@ -83,29 +83,24 @@ class AbstractDestination(ABC):
             self.delete_table()
 
     def buffer_flush_handler(self, session=None) -> DestinationIteration:
-        nb_records_to_write = len(self.buffer.records)
+        # TODO: Add retry strategy
+
         # Initialize destination iteration
         destination_iteration = DestinationIteration(
             success=False,
             records_written=0,
             pagination=self.buffer.pagination,
         )
+
         success, error_msg = self.write_records(destination_records=self.buffer.records)
+
         if success:
-            destination_iteration.records_written = nb_records_to_write
-            self.backend.update_stream_job_status(
-                job_id=self.sync_metadata.job_id,
-                job_status=JobStatus.SUCCEEDED,
-                session=session,
-            )
+            # We wrote records to destination so we keep it
+            destination_iteration.records_written = len(self.buffer.records)
 
         else:
-            self.backend.update_stream_job_status(
-                job_id=self.sync_metadata.job_id,
-                job_status=JobStatus.FAILED,
-                error_message=error_msg,
-                session=session,
-            )
+            # We failed to write records to destination so we keep the error message
+            destination_iteration.error_message = error_msg
 
         destination_iteration.success = success
         destination_iteration.from_source_iteration = self.buffer.from_iteration
@@ -115,6 +110,8 @@ class AbstractDestination(ABC):
         # Update destination cursor
         self.create_cursors(destination_iteration=destination_iteration)
 
+        return destination_iteration
+
     def write_or_buffer_records(
         self,
         destination_records: List[DestinationRecord],
@@ -129,7 +126,16 @@ class AbstractDestination(ABC):
         if last_iteration:
             logger.debug("Writing last iteration records to destination")
             assert len(destination_records) == 0, "Last iteration should not have any records"
-            self.buffer_flush_handler(session=session)
+            destination_iteration = self.buffer_flush_handler(session=session)
+
+            if destination_iteration.success:
+                # Update job status to success
+                self.backend.update_stream_job_status(
+                    job_id=self.sync_metadata.job_id,
+                    job_status=JobStatus.SUCCEEDED,
+                    session=session,
+                )
+
             self.buffer.flush()
             return DestinationBufferStatus.RECORDS_WRITTEN
 
@@ -140,7 +146,7 @@ class AbstractDestination(ABC):
 
         # Write records to destination if buffer size is 0
         if self.buffer.buffer_size == 0:
-            logger.debug("Writing last iteration records to destination")
+            logger.info("Writing last iteration records to destination")
             self.buffer.add_source_iteration_records_to_buffer(
                 iteration=iteration, records=destination_records, pagination=pagination
             )
@@ -148,15 +154,15 @@ class AbstractDestination(ABC):
             self.buffer.flush()
             return DestinationBufferStatus.RECORDS_WRITTEN
 
-        logger.info(f"Buffer free space {self.buffer.buffer_free_space_pct}%")
-        logger.info(f"Buffer current size {self.buffer.current_size} bytes")
+        logger.debug(f"Buffer free space {self.buffer.buffer_free_space_pct}%")
+        logger.debug(f"Buffer current size {self.buffer.current_size} bytes")
         logger.info(
             f"Buffer ripeness {self.buffer.ripeness / 60} min. Max ripeness {self.buffer.buffer_flush_timeout / 60} min."  # noqa
         )
 
         # Write buffer to destination if buffer is ripe and create a new buffer for the new iteration
         if self.buffer.is_ripe:
-            logger.debug(
+            logger.info(
                 f"Buffer is ripe (buffering for longer than buffer_flush_timeout: {self.buffer.buffer_flush_timeout} seconds), writing buffer to destination"  # noqa
             )
             self.buffer_flush_handler(session=session)

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/engine/backend/adapters/sqlalchemy/backend.py

@@ -252,7 +252,7 @@ class SQLAlchemyBackend(AbstractBackend):
         if job:
             return job
 
-        logger.debug(f"No running job found for source={source_name} stream={stream_name}")
+        logger.info(f"No running job found for source={source_name} stream={stream_name}")
         return None
 
     #### SOURCE CURSOR ####

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/engine/backend/models.py

@@ -18,7 +18,6 @@ def generate_uuid():
 class JobStatus(str, Enum):
     STARTED = "started"
     RUNNING = "running"
-    SUCCESS = "success"
     FAILED = "failed"
     CANCELED = "cancelled"
     SUCCEEDED = "succeeded"

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/engine/pipeline/producer.py

@@ -1,11 +1,12 @@
 import ast
+import os
 from datetime import datetime
 
 from loguru import logger
 from pytz import UTC
 
 from bizon.engine.backend.backend import AbstractBackend
-from bizon.engine.backend.models import CursorStatus, DestinationCursor
+from bizon.engine.backend.models import CursorStatus
 from bizon.engine.queue.queue import AbstractQueue
 from bizon.source.cursor import Cursor
 from bizon.source.source import AbstractSource

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/engine/runner/config.py

@@ -9,6 +9,15 @@ class RunnerTypes(str, Enum):
     PROCESS = "process"
 
 
+class LoggerLevel(str, Enum):
+    TRACE = "TRACE"
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
 class RunnerFuturesConfig(BaseModel):
     max_workers: Optional[int] = Field(
         description="Number of workers to use for the runner",
@@ -36,7 +45,7 @@ class RunnerConfig(BaseModel):
         default=RunnerFuturesConfig(),
     )
 
-    log_level: str = Field(
+    log_level: LoggerLevel = Field(
         description="Logging level",
-        default="INFO",
+        default=LoggerLevel.INFO,
     )

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/engine/runner/runner.py

@@ -1,3 +1,5 @@
+import os
+import sys
 from abc import ABC, abstractmethod
 
 from loguru import logger
@@ -23,6 +25,11 @@ class AbstractRunner(ABC):
         self.config: dict = config
         self.bizon_config = BizonConfig.model_validate(obj=config)
 
+        # Set log level
+        logger.info(f"Setting log level to {self.bizon_config.engine.runner.log_level.name}")
+        logger.remove()
+        logger.add(sys.stderr, level=self.bizon_config.engine.runner.log_level)
+
     @property
     def is_running(self) -> bool:
         """Return True if the pipeline is running"""
@@ -103,7 +110,7 @@ class AbstractRunner(ABC):
                 logger.info(f"Job {job.id} canceled. Creating a new one...")
             # Otherwise we return the existing job
             else:
-                logger.debug(f"Found an existing job: {job.id}")
+                logger.info(f"Found an existing job: {job.id}")
                 return job
 
         # If no job is running, we create a new one:
@@ -119,6 +126,8 @@ class AbstractRunner(ABC):
             job_status=JobStatus.STARTED,
         )
 
+        logger.info(f"Created a new job: {job.id}")
+
         return job
 
     @staticmethod

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/bizon/sources/periscope/src/source.py

@@ -166,7 +166,8 @@ class PeriscopeSource(AbstractSource):
         return dashboard_charts
 
     def get_charts(self, pagination: dict = None) -> SourceIteration:
-        MAX_CONCURRENT_REQUESTS = 10
+
+        BATCH_SIZE = 10
 
         if not pagination:
             dashboard_ids = self.get_dashboard_ids()
@@ -177,14 +178,15 @@ class PeriscopeSource(AbstractSource):
 
             pagination = {"dashboard_ids": dashboard_ids}
 
-        with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS) as executor:
-            result = list(executor.map(self._get_charts, pagination["dashboard_ids"][:10]))
+        with ThreadPoolExecutor(max_workers=BATCH_SIZE) as executor:
+            result = list(executor.map(self._get_charts, pagination["dashboard_ids"][:BATCH_SIZE]))
 
         # Remove the first 10 dashboard ids if there are more
-        if len(pagination["dashboard_ids"]) > 10:
-            pagination["dashboard_ids"] = pagination["dashboard_ids"][10:]
+        if len(pagination["dashboard_ids"]) > BATCH_SIZE:
+            pagination["dashboard_ids"] = pagination["dashboard_ids"][BATCH_SIZE:]
         else:
-            pagination["dashboard_ids"] = []
+            # We end the pagination
+            pagination = {}
 
         records = list(itertools.chain.from_iterable(result))
 

{bizon-0.0.4.dev3 → bizon-0.0.4.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bizon"
-version = "0.0.4.dev3"
+version = "0.0.4.dev4"
 description = "Extract and load your data reliably from API Clients with native fault-tolerant and checkpointing mechanism."
 authors = ["Antoine Balliet <antoine.balliet@gmail.com>", "Anas El Mhamdi <anas.elmhamdi@gmail.com>"]
 readme = "README.md"