bizon 0.0.14__py3-none-any.whl → 0.1.1__py3-none-any.whl

bizon/connectors/sources/pokeapi/src/source.py

@@ -0,0 +1,79 @@
+from enum import Enum
+from typing import Any, List, Tuple
+
+from requests.auth import AuthBase
+
+from bizon.source.config import SourceConfig
+from bizon.source.models import SourceIteration, SourceRecord
+from bizon.source.source import AbstractSource
+
+BASE_URL = "https://pokeapi.co/api/v2"
+
+
+# Define the streams that the source supports
+class PokeAPIStreams(str, Enum):
+    POKEMON = "pokemon"
+    BERRY = "berry"
+    ITEM = "item"
+
+
+# Define the config class for the source
+class PokeAPISourceConfig(SourceConfig):
+    stream: PokeAPIStreams
+
+
+class PeriscopeSource(AbstractSource):
+
+    def __init__(self, config: PokeAPISourceConfig):
+        super().__init__(config)
+        self.config: PokeAPISourceConfig = config
+
+    @property
+    def url_entity(self) -> str:
+        return f"{BASE_URL}/{self.config.stream}"
+
+    @staticmethod
+    def streams() -> List[str]:
+        return [item.value for item in PokeAPIStreams]
+
+    @staticmethod
+    def get_config_class() -> AbstractSource:
+        return PokeAPISourceConfig
+
+    def check_connection(self) -> Tuple[bool | Any | None]:
+        # Make a request to the base URL to check if the connection is successful
+        _ = self.session.get(self.url_entity)
+        return True, None
+
+    def get_authenticator(self) -> AuthBase:
+        # We return None because we don't need any authentication
+        return None
+
+    def get_total_records_count(self) -> int | None:
+        # Return the total number of records in the stream
+        response = self.session.get(self.url_entity)
+        return response.json().get("count")
+
+    def get_entity_list(self, pagination: dict = None) -> SourceIteration:
+        # If pagination is provided, use the next URL to get the next set of records
+        url = pagination.get("next") if pagination else self.url_entity
+        response = self.session.get(url)
+
+        data = response.json()
+
+        return SourceIteration(
+            next_pagination={"next": data.get("next")} if data.get("next") else {},
+            records=[
+                SourceRecord(
+                    id=record["name"],
+                    data=record,
+                )
+                for record in data["results"]
+            ],
+        )
+
+    def get(self, pagination: dict = None) -> SourceIteration:
+        if self.config.stream in [PokeAPIStreams.POKEMON, PokeAPIStreams.BERRY, PokeAPIStreams.ITEM]:
+            return self.get_entity_list(pagination)
+
+        raise NotImplementedError(f"Stream {self.config.stream} not implemented for PokeAPI source")

bizon/{destinations → destination}/buffer.py

@@ -18,6 +18,11 @@ class DestinationBuffer:
         self.pagination = {}
         self.modified_at: List[datetime] = [datetime.now(tz=UTC)]
 
+    @property
+    def is_empty(self) -> bool:
+        """Check if buffer is empty"""
+        return self.df_destination_records.height == 0
+
     @property
     def current_size(self) -> int:
         """Return buffer size"""

bizon/destination/config.py

@@ -0,0 +1,74 @@
+from abc import ABC
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class DestinationTypes(str, Enum):
+    BIGQUERY = "bigquery"
+    BIGQUERY_STREAMING = "bigquery_streaming"
+    BIGQUERY_STREAMING_V2 = "bigquery_streaming_v2"
+    LOGGER = "logger"
+    FILE = "file"
+
+
+class DestinationColumn(BaseModel, ABC):
+    name: str = Field(..., description="Name of the column")
+    type: str = Field(..., description="Type of the column")
+    description: Optional[str] = Field(None, description="Description of the column")
+
+
+class RecordSchemaConfig(BaseModel):
+    # Forbid extra keys in the model
+    model_config = ConfigDict(extra="forbid")
+
+    destination_id: str = Field(..., description="Destination ID")
+    record_schema: list[DestinationColumn] = Field(..., description="Record schema")
+
+
+class AbstractDestinationDetailsConfig(BaseModel):
+
+    # Forbid extra keys in the model
+    model_config = ConfigDict(extra="forbid")
+
+    buffer_size: int = Field(
+        default=50,
+        description="Buffer size in Mb for the destination. Set to 0 to disable and write directly to the destination.",
+    )
+
+    buffer_flush_timeout: int = Field(
+        default=600,
+        description="Maximum time in seconds for buffering after which the records will be written to the destination. Set to 0 to deactivate the timeout buffer check.",  # noqa
+    )
+
+    record_schemas: Optional[list[RecordSchemaConfig]] = Field(
+        default=None, description="Schemas for the records. Required if unnest is set to true."
+    )
+
+    unnest: bool = Field(
+        default=False,
+        description="Unnest the data before writing to the destination. Schema should be provided in the model_config.",
+    )
+
+    authentication: Optional[BaseModel] = Field(
+        description="Authentication configuration for the destination, if needed", default=None
+    )
+
+    destination_id: Optional[str] = Field(
+        description="Destination ID, identifier to use to store the records in the destination", default=None
+    )
+
+    @field_validator("unnest", mode="before")
+    def validate_record_schema_if_unnest(cls, value, values):
+        if bool(value) and not values.data.get("record_schemas", []):
+            raise ValueError("At least one `record_schemas` must be provided if `unnest` is set to True.")
+        return value
+
+
+class AbstractDestinationConfig(BaseModel):
+    # Forbid extra keys in the model
+    model_config = ConfigDict(extra="forbid")
+
+    name: DestinationTypes = Field(..., description="Name of the destination")
+    config: AbstractDestinationDetailsConfig = Field(..., description="Configuration for the destination")

bizon/{destinations → destination}/destination.py

@@ -10,6 +10,7 @@ from pydantic import BaseModel, Field
 from bizon.common.models import SyncMetadata
 from bizon.engine.backend.backend import AbstractBackend
 from bizon.engine.backend.models import JobStatus
+from bizon.source.callback import AbstractSourceCallback
 from bizon.source.config import SourceSyncModes
 
 from .buffer import DestinationBuffer
@@ -43,13 +44,40 @@ class DestinationIteration(BaseModel):
 
 class AbstractDestination(ABC):
 
-    def __init__(self, sync_metadata: SyncMetadata, config: AbstractDestinationDetailsConfig, backend: AbstractBackend):
+    def __init__(
+        self,
+        sync_metadata: SyncMetadata,
+        config: AbstractDestinationDetailsConfig,
+        backend: AbstractBackend,
+        source_callback: AbstractSourceCallback,
+    ):
         self.sync_metadata = sync_metadata
         self.config = config
         self.backend = backend
         self.buffer = DestinationBuffer(
             buffer_size=self.config.buffer_size, buffer_flush_timeout=self.config.buffer_flush_timeout
         )
+        self.source_callback = source_callback
+        self.destination_id = config.destination_id
+
+        self._record_schemas = None
+        self._clustering_keys = None
+
+    @property
+    def record_schemas(self):
+        if self._record_schemas is None and self.config.record_schemas:
+            self._record_schemas = {
+                schema.destination_id: schema.record_schema for schema in self.config.record_schemas
+            }
+        return self._record_schemas
+
+    @property
+    def clustering_keys(self):
+        if self._clustering_keys is None and self.config.record_schemas:
+            self._clustering_keys = {
+                schema.destination_id: schema.clustering_keys for schema in self.config.record_schemas
+            }
+        return self._clustering_keys
 
     @abstractmethod
     def check_connection(self) -> bool:
@@ -75,7 +103,7 @@ class AbstractDestination(ABC):
         )
 
         logger.info(
-            f"Writing in destination from source iteration {self.buffer.from_iteration} to {self.buffer.to_iteration}"
+            f"Writing in destination {self.destination_id} from source iteration {self.buffer.from_iteration} to {self.buffer.to_iteration}"
         )
 
         success, error_msg = self.write_records(df_destination_records=self.buffer.df_destination_records)
@@ -83,7 +111,9 @@ class AbstractDestination(ABC):
         if success:
             # We wrote records to destination so we keep it
             destination_iteration.records_written = self.buffer.df_destination_records.height
-            logger.info(f"Successfully wrote {destination_iteration.records_written} records to destination")
+            logger.info(
+                f"Successfully wrote {destination_iteration.records_written} records to destination {self.destination_id}"
+            )
 
         else:
             # We failed to write records to destination so we keep the error message
@@ -112,8 +142,8 @@ class AbstractDestination(ABC):
         # Last iteration, write all records to destination
         if last_iteration:
 
-            if self.buffer.df_destination_records.height == 0 and self.config.buffer_size == 0:
-                logger.warning("No records to write to destination, already written, buffer is empty.")
+            if self.buffer.df_destination_records.height == 0 and self.buffer.is_empty:
+                logger.info("No records to write to destination, already written, buffer is empty.")
                 return DestinationBufferStatus.RECORDS_WRITTEN
 
             logger.debug("Writing last iteration records to destination")
@@ -143,12 +173,12 @@ class AbstractDestination(ABC):
 
         # Don't write empty records to destination
         if df_destination_records.height == 0 and not last_iteration:
-            logger.warning("No records to write to destination. Check source and queue provider.")
+            logger.info("No records to write to destination. Check source and queue provider.")
             return DestinationBufferStatus.NO_RECORDS
 
         # Write records to destination if buffer size is 0 or streaming
         if self.buffer.buffer_size == 0:
-            logger.info("Writing records to destination.")
+            logger.info(f"Writing records to destination {self.destination_id}.")
             self.buffer.add_source_iteration_records_to_buffer(
                 iteration=iteration, df_destination_records=df_destination_records, pagination=pagination
             )
@@ -245,26 +275,52 @@ class DestinationFactory:
         sync_metadata: SyncMetadata,
         config: AbstractDestinationConfig,
         backend: AbstractBackend,
+        source_callback: AbstractSourceCallback,
     ) -> AbstractDestination:
 
         if config.name == DestinationTypes.LOGGER:
-            from .logger.src.destination import LoggerDestination
+            from bizon.connectors.destinations.logger.src.destination import (
+                LoggerDestination,
+            )
 
-            return LoggerDestination(sync_metadata=sync_metadata, config=config.config, backend=backend)
+            return LoggerDestination(
+                sync_metadata=sync_metadata, config=config.config, backend=backend, source_callback=source_callback
+            )
 
         elif config.name == DestinationTypes.BIGQUERY:
-            from .bigquery.src.destination import BigQueryDestination
+            from bizon.connectors.destinations.bigquery.src.destination import (
+                BigQueryDestination,
+            )
 
-            return BigQueryDestination(sync_metadata=sync_metadata, config=config.config, backend=backend)
+            return BigQueryDestination(
+                sync_metadata=sync_metadata, config=config.config, backend=backend, source_callback=source_callback
+            )
 
         elif config.name == DestinationTypes.BIGQUERY_STREAMING:
-            from .bigquery_streaming.src.destination import BigQueryStreamingDestination
+            from bizon.connectors.destinations.bigquery_streaming.src.destination import (
+                BigQueryStreamingDestination,
+            )
+
+            return BigQueryStreamingDestination(
+                sync_metadata=sync_metadata, config=config.config, backend=backend, source_callback=source_callback
+            )
+
+        elif config.name == DestinationTypes.BIGQUERY_STREAMING_V2:
+            from bizon.connectors.destinations.bigquery_streaming_v2.src.destination import (
+                BigQueryStreamingV2Destination,
+            )
 
-            return BigQueryStreamingDestination(sync_metadata=sync_metadata, config=config.config, backend=backend)
+            return BigQueryStreamingV2Destination(
+                sync_metadata=sync_metadata, config=config.config, backend=backend, source_callback=source_callback
+            )
 
         elif config.name == DestinationTypes.FILE:
-            from .file.src.destination import FileDestination
+            from bizon.connectors.destinations.file.src.destination import (
+                FileDestination,
+            )
 
-            return FileDestination(sync_metadata=sync_metadata, config=config.config, backend=backend)
+            return FileDestination(
+                sync_metadata=sync_metadata, config=config.config, backend=backend, source_callback=source_callback
+            )
 
         raise ValueError(f"Destination {config.name}" f"with params {config} not found")

bizon/engine/backend/adapters/sqlalchemy/backend.py

@@ -1,6 +1,6 @@
 import json
 from datetime import datetime
-from typing import Generator, Optional, Union
+from typing import Optional, Union
 
 from loguru import logger
 from pytz import UTC
@@ -39,9 +39,9 @@ class SQLAlchemyBackend(AbstractBackend):
         ] = config
 
         self.kwargs = kwargs
+        self.session = self.get_session()
 
-    @property
-    def session(self) -> Generator[Session, None, None]:
+    def get_session(self) -> Session:
         """yields a SQLAlchemy connection"""
         engine = self.get_engine()
         session_ = scoped_session(
@@ -51,20 +51,17 @@ class SQLAlchemyBackend(AbstractBackend):
             )
         )
 
-        yield session_
-
-        session_.close()
-
-        # For SQLite, we need to dispose the engine after each operation to prevent database lock
-        self.handle_dispose_sqlite(engine)
+        return session_
 
     def _get_engine_bigquery(self) -> Engine:
+        # If service account key is provided, use it
         if hasattr(self.config, "service_account_key") and self.config.service_account_key:
             return create_engine(
                 f"bigquery://{self.config.database}/{self.config.schema_name}",
                 echo=self.config.echoEngine,
                 credentials_info=self.config.service_account_key,
             )
+        # Otherwise we will rely on the default Google Authentication mechanism (e.g. GOOGLE_APPLICATION_CREDENTIALS)
         return create_engine(
             f"bigquery://{self.config.database}/{self.config.schema_name}", echo=self.config.echoEngine
         )
@@ -78,7 +75,9 @@ class SQLAlchemyBackend(AbstractBackend):
     def _get_engine_sqlite(self) -> Engine:
         return create_engine(
             f"sqlite:///{self.config.database}.sqlite3",
-            connect_args={"check_same_thread": False},
+            connect_args={"check_same_thread": False, "timeout": 30},
+            pool_size=5,  # Adjust based on expected concurrency
+            pool_pre_ping=True,  # Ensures connections are alive
         )
 
     def _get_engine(self) -> Engine:
@@ -120,12 +119,6 @@ class SQLAlchemyBackend(AbstractBackend):
                     f"Schema or dataset {self.config.schema_name} does not exist in the database, you need to create it first."
                 )
 
-        self.handle_dispose_sqlite(engine)
-
-    def handle_dispose_sqlite(self, engine: Engine):
-        if self.type == BackendTypes.SQLITE:
-            engine.dispose()
-
     def get_engine(self) -> Engine:
         """Return the SQLAlchemy engine"""
         if not self._engine:
@@ -138,12 +131,10 @@ class SQLAlchemyBackend(AbstractBackend):
     def create_all_tables(self):
         engine = self.get_engine()
         Base.metadata.create_all(engine)
-        self.handle_dispose_sqlite(engine)
 
     def drop_all_tables(self):
         engine = self.get_engine()
         Base.metadata.drop_all(engine)
-        self.handle_dispose_sqlite(engine)
 
     def check_prerequisites(self) -> bool:
         """Check if the database contains the necessary tables, return True if entities are present
@@ -171,20 +162,20 @@ class SQLAlchemyBackend(AbstractBackend):
             all_entities_exist = False
             logger.info(f"Table {TABLE_DESTINATION_CURSOR} does not exist in the database, we will create it")
 
-        self.handle_dispose_sqlite(engine)
-
         return all_entities_exist
 
     def _add_and_commit(self, obj, session: Optional[Session] = None):
         """Add the object to the session and commit it, return its ID"""
-        session = session or next(self.session)
+        session = session or self.session
         session.add(obj)
         session.commit()
         return obj
 
     def _execute(self, select: Select, session: Optional[Session] = None) -> Result:
-        session = session or next(self.session)
-        return session.execute(select)
+        session = session or self.session
+        result = session.execute(select)
+        session.commit()
+        return result
 
     #### STREAM JOB ####
 

bizon/engine/engine.py

@@ -1,4 +1,4 @@
-from loguru import logger
+from os import getenv
 
 from bizon.cli.utils import parse_from_yaml
 from bizon.common.models import BizonConfig
@@ -7,10 +7,24 @@ from .config import RunnerTypes
 from .runner.runner import AbstractRunner
 
 
+def replace_env_variables_in_config(config: dict) -> dict:
+    """Replace templated secrets with actual values from environment variables"""
+    for key, value in config.items():
+        if isinstance(value, dict):
+            config[key] = replace_env_variables_in_config(value)
+        elif isinstance(value, str):
+            if value.startswith("BIZON_ENV_"):
+                config[key] = getenv(value)
+    return config
+
+
 class RunnerFactory:
     @staticmethod
     def create_from_config_dict(config: dict) -> AbstractRunner:
 
+        # Replace env variables in config
+        config = replace_env_variables_in_config(config=config)
+
         bizon_config = BizonConfig.model_validate(obj=config)
 
         if bizon_config.engine.runner.type == RunnerTypes.THREAD:
@@ -23,6 +37,11 @@ class RunnerFactory:
 
             return ProcessRunner(config=config)
 
+        if bizon_config.engine.runner.type == RunnerTypes.STREAM:
+            from .runner.adapters.streaming import StreamingRunner
+
+            return StreamingRunner(config=config)
+
         raise ValueError(f"Runner type {bizon_config.engine.runner.type} is not supported")
 
     @staticmethod

bizon/engine/pipeline/consumer.py

@@ -1,15 +1,83 @@
+import multiprocessing
+import multiprocessing.synchronize
+import threading
+import traceback
 from abc import ABC, abstractmethod
-from enum import Enum
+from typing import Union
 
-from bizon.destinations.destination import AbstractDestination
-from bizon.engine.queue.config import AbstractQueueConfig
+from loguru import logger
+
+from bizon.destination.destination import AbstractDestination
+from bizon.engine.pipeline.models import PipelineReturnStatus
+from bizon.engine.queue.config import (
+    QUEUE_TERMINATION,
+    AbstractQueueConfig,
+    QueueMessage,
+)
+from bizon.monitoring.monitor import AbstractMonitor
+from bizon.transform.transform import Transform
 
 
 class AbstractQueueConsumer(ABC):
-    def __init__(self, config: AbstractQueueConfig, destination: AbstractDestination):
+    def __init__(
+        self,
+        config: AbstractQueueConfig,
+        destination: AbstractDestination,
+        transform: Transform,
+        monitor: AbstractMonitor,
+    ):
         self.config = config
         self.destination = destination
+        self.transform = transform
+        self.monitor = monitor
 
     @abstractmethod
-    def run(self):
+    def run(self, stop_event: Union[multiprocessing.synchronize.Event, threading.Event]) -> PipelineReturnStatus:
         pass
+
+    def process_queue_message(self, queue_message: QueueMessage) -> PipelineReturnStatus:
+
+        # Apply the transformation
+        try:
+            df_source_records = self.transform.apply_transforms(df_source_records=queue_message.df_source_records)
+        except Exception as e:
+            logger.error(f"Error applying transformation: {e}")
+            logger.error(traceback.format_exc())
+            self.monitor.track_pipeline_status(PipelineReturnStatus.TRANSFORM_ERROR)
+            return PipelineReturnStatus.TRANSFORM_ERROR
+
+        # Handle last iteration
+        try:
+            if queue_message.signal == QUEUE_TERMINATION:
+                logger.info("Received termination signal, waiting for destination to close gracefully ...")
+                self.destination.write_records_and_update_cursor(
+                    df_source_records=df_source_records,
+                    iteration=queue_message.iteration,
+                    extracted_at=queue_message.extracted_at,
+                    pagination=queue_message.pagination,
+                    last_iteration=True,
+                )
+                self.monitor.track_pipeline_status(PipelineReturnStatus.SUCCESS)
+                return PipelineReturnStatus.SUCCESS
+
+        except Exception as e:
+            logger.error(f"Error writing records to destination: {e}")
+            self.monitor.track_pipeline_status(PipelineReturnStatus.DESTINATION_ERROR)
+            return PipelineReturnStatus.DESTINATION_ERROR
+
+        # Write the records to the destination
+        try:
+            self.destination.write_records_and_update_cursor(
+                df_source_records=df_source_records,
+                iteration=queue_message.iteration,
+                extracted_at=queue_message.extracted_at,
+                pagination=queue_message.pagination,
+            )
+            return PipelineReturnStatus.RUNNING
+
+        except Exception as e:
+            logger.error(f"Error writing records to destination: {e}")
+            self.monitor.track_pipeline_status(PipelineReturnStatus.DESTINATION_ERROR)
+            return PipelineReturnStatus.DESTINATION_ERROR
+
+        raise RuntimeError("Should not reach this point")

bizon/engine/pipeline/models.py

@@ -1,10 +1,15 @@
 from enum import Enum
 
 
-class PipelineReturnStatus(Enum):
+class PipelineReturnStatus(str, Enum):
     """Producer error types"""
 
-    SUCCESS = "success"
+    BACKEND_ERROR = "backend_error"
+    DESTINATION_ERROR = "destination_error"
+    KILLED_BY_RUNNER = "killed_by_runner"
     QUEUE_ERROR = "queue_error"
+    RUNNING = "running"
     SOURCE_ERROR = "source_error"
-    BACKEND_ERROR = "backend_error"
+    SUCCESS = "success"
+    TRANSFORM_ERROR = "transform_error"
+    STREAM_ERROR = "stream_error"

bizon/engine/pipeline/producer.py

@@ -1,8 +1,11 @@
 import ast
+import multiprocessing
+import multiprocessing.synchronize
+import threading
 import traceback
 from datetime import datetime
 from time import sleep
-from typing import Tuple
+from typing import Tuple, Union
 
 from loguru import logger
 from pytz import UTC
@@ -28,7 +31,7 @@ class Producer:
 
     @property
     def name(self) -> str:
-        return f"producer-{self.source.config.source_name}-{self.source.config.stream_name}"
+        return f"producer-{self.source.config.name}-{self.source.config.stream}"
 
     def get_or_create_cursor(self, job_id: str, session=None) -> Cursor:
         """Get or create a cursor for the current stream, return the cursor"""
@@ -47,8 +50,8 @@ class Producer:
 
             # Initialize the recovery from the DestinationCursor
             cursor = Cursor.from_db(
-                source_name=self.source.config.source_name,
-                stream_name=self.source.config.stream_name,
+                source_name=self.source.config.name,
+                stream_name=self.source.config.stream,
                 job_id=job_id,
                 total_records=job.total_records_to_fetch,
                 iteration=cursor_from_db.to_source_iteration + 1,
@@ -60,8 +63,8 @@ class Producer:
             total_records = self.source.get_total_records_count()
             # Initialize the cursor
             cursor = Cursor(
-                source_name=self.source.config.source_name,
-                stream_name=self.source.config.stream_name,
+                source_name=self.source.config.name,
+                stream_name=self.source.config.stream,
                 job_id=job_id,
                 total_records=total_records,
             )
@@ -99,7 +102,9 @@ class Producer:
 
         return False, queue_size, approximate_nb_records_in_queue
 
-    def run(self, job_id: int):
+    def run(
+        self, job_id: int, stop_event: Union[multiprocessing.synchronize.Event, threading.Event]
+    ) -> PipelineReturnStatus:
 
         return_value: PipelineReturnStatus = PipelineReturnStatus.SUCCESS
 
@@ -128,6 +133,10 @@ class Producer:
 
         while not cursor.is_finished:
 
+            if stop_event.is_set():
+                logger.info("Stop event is set, terminating producer ...")
+                return PipelineReturnStatus.KILLED_BY_RUNNER
+
             timestamp_start_iteration = datetime.now(tz=UTC)
 
             # Handle the case where last cursor already reach max_iterations
@@ -142,8 +151,8 @@ class Producer:
                     self.backend.create_source_cursor(
                         job_id=job_id,
                         name=self.bizon_config.name,
-                        source_name=self.source.config.source_name,
-                        stream_name=self.source.config.stream_name,
+                        source_name=self.source.config.name,
+                        stream_name=self.source.config.stream,
                         iteration=cursor.iteration,
                         rows_fetched=cursor.rows_fetched,
                         next_pagination=cursor.pagination,

bizon/engine/queue/adapters/kafka/consumer.py

@@ -3,9 +3,9 @@ import json
 from kafka import KafkaConsumer
 from loguru import logger
 
-from bizon.destinations.destination import AbstractDestination
+from bizon.destination.destination import AbstractDestination
 from bizon.engine.pipeline.consumer import AbstractQueueConsumer
-from bizon.engine.queue.queue import QUEUE_TERMINATION, QueueMessage
+from bizon.engine.queue.config import QUEUE_TERMINATION, QueueMessage
 
 from .config import KafkaConfigDetails
 

bizon/engine/queue/adapters/kafka/queue.py

@@ -4,8 +4,9 @@ from typing import Union
 from kafka import KafkaProducer
 from loguru import logger
 
-from bizon.destinations.destination import AbstractDestination
-from bizon.engine.queue.queue import QUEUE_TERMINATION, AbstractQueue, QueueMessage
+from bizon.destination.destination import AbstractDestination
+from bizon.engine.queue.config import QUEUE_TERMINATION, QueueMessage
+from bizon.engine.queue.queue import AbstractQueue
 
 from .config import KafkaConfigDetails
 from .consumer import KafkaConsumer_