core-cdc 1.0.0__tar.gz

core-cdc-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Alejandro Cora González.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

core-cdc-1.0.0/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.1
+Name: core-cdc
+Version: 1.0.0
+Summary: This project/library contains the core mechanism and resources for Change Data Capture services...
+Author-email: Alejandro Cora González <alek.cora.glez@gmail.com>
+Maintainer: Alejandro Cora González
+License: MIT
+Project-URL: Homepage, https://gitlab.com/bytecode-solutions/core/core-cdc
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Utilities
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: core-mixins
+Requires-Dist: core-tests
+Requires-Dist: core-aws
+Requires-Dist: core-db
+Requires-Dist: mysql-replication==0.31
+Requires-Dist: pymongo==4.6.0
+Provides-Extra: test
+
+# core-cdc (CDC a.k.a Change Data Capture)
+_______________________________________________________________________________
+
+It provides the core mechanism and required resources to 
+implement "Change Data Capture" services...
+
+## Execution Environment
+
+### Install libraries
+```commandline
+pip install --upgrade pip 
+pip install virtualenv
+```
+
+### Create the Python Virtual Environment.
+```commandline
+virtualenv --python=python3.11 .venv
+```
+
+### Activate the Virtual Environment.
+```commandline
+source .venv/bin/activate
+```
+
+### Install required libraries.
+```commandline
+pip install .
+```
+
+### Check tests and coverage...
+```commandline
+python manager.py run-test
+python manager.py run-coverage
+```

core-cdc-1.0.0/README.md

@@ -0,0 +1,34 @@
+# core-cdc (CDC a.k.a Change Data Capture)
+_______________________________________________________________________________
+
+It provides the core mechanism and required resources to 
+implement "Change Data Capture" services...
+
+## Execution Environment
+
+### Install libraries
+```commandline
+pip install --upgrade pip 
+pip install virtualenv
+```
+
+### Create the Python Virtual Environment.
+```commandline
+virtualenv --python=python3.11 .venv
+```
+
+### Activate the Virtual Environment.
+```commandline
+source .venv/bin/activate
+```
+
+### Install required libraries.
+```commandline
+pip install .
+```
+
+### Check tests and coverage...
+```commandline
+python manager.py run-test
+python manager.py run-coverage
+```

core-cdc-1.0.0/core_cdc/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import targets

core-cdc-1.0.0/core_cdc/base.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import json
+from datetime import date, datetime
+from enum import StrEnum
+from typing import Dict, Tuple, Optional
+
+
+class EventType(StrEnum):
+    GLOBAL = "GLOBAL"        # Event related to the processor itself.
+    DDL_STATEMENT = "QUERY"  # Event for DDL statement (like create table, etc.).
+    INSERT = "INSERT"        # Event for DML INSERT operation.
+    UPDATE = "UPDATE"        # Event for DML UPDATE operation.
+    DELETE = "DELETE"        # Event for DML DELETE operation.
+
+
+class Record:
+    """
+    It provides a wrapper or common object useful for integration
+    across services that needs to handle data replication
+    via Change Data Capture producers and consumers...
+    """
+
+    def __init__(
+            self, event_timestamp: int, event_type: EventType, record: Dict, service: str,
+            source: str, position: int, primary_key: Tuple | str, schema_name: str, table_name: str,
+            global_id: Optional[str], transaction_id: Optional[str]) -> None:
+
+        """
+        It creates a new record with all the attributes required for streaming and
+        replication using a Data Change Capture service...
+
+        :param: global_id: Unique identifier. (like: Global Transaction Identifier (gtid) in MySQL engine).
+        :param: transaction_id: Identifier for the transaction. (like: xid in MySQL engine).
+
+        :param event_timestamp: Timestamp when the record was generated.
+        :param event_type: It specifies the operation that generated the event.
+        :param record: The record (like a record/row in a database).
+
+        :param service: The service from where the record came.
+        :param source: The source from where the record was retrieved (like binlog file name).
+        :param position: Record position in the source.
+
+        :param: primary_key: Attributes to use to identify a record as unique.
+        :param: schema_name: Schema from where the record was retrieved.
+        :param: table: Table from where the record was retrieved.
+        """
+
+        self.global_id = global_id
+        self.transaction_id = transaction_id
+
+        self.event_timestamp = event_timestamp
+        self.event_type = event_type
+        self.record = record
+
+        self.service = service
+        self.source = source
+        self.position = position
+
+        self.primary_key = primary_key
+        self.schema_name = schema_name
+        self.table_name = table_name
+
+    def __str__(self):
+        return json.dumps(self.to_json())
+
+    def to_json(self):
+        """
+        It returns the JSON version of the record required to be streamed...
+
+        :return: A dictionary that follows the below structure:
+            {
+                "global_id": "",
+                "transaction_id": "",
+                "event_timestamp": 1653685384,
+                "event_type": "INSERT | UPDATE | DELETE",
+                "service": "service-name",
+                "source": "binlog.000001 | FileName | Something",
+                "position": 8077,
+                "primary_key": "(str | tuple)",
+                "schema_name": "schema_name",
+                "table_name": "table_name",
+                "record": {
+                    ...
+                    // This is an example...
+                    "id": "000-1",
+                    "category": "Marketing",
+                    "price": 100.0,
+                    ...
+                }
+            }
+        """
+
+        return {
+            "global_id": self.global_id,
+            "transaction_id": self.transaction_id,
+            "event_timestamp": self.event_timestamp,
+            "event_type": self.event_type.value,
+            "service": self.service,
+            "source": self.source,
+            "position": self.position,
+            "primary_key": self.primary_key,
+            "schema_name": self.schema_name,
+            "table_name": self.table_name,
+            "record": {
+                key: value.isoformat() if type(value) in (datetime, date) else value
+                for key, value in self.record.items()
+            }
+        }

core-cdc-1.0.0/core_cdc/processors/base.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+from abc import abstractmethod
+from logging import Logger
+from typing import Any, Dict, List, Iterator, Iterable, Type, Self
+
+from core_mixins.interfaces.factory import IFactory
+
+from core_cdc.base import EventType, Record
+from core_cdc.targets.base import Target
+
+
+class IProcessor(IFactory):
+    """ Interface for all "Change Data Capture" implementations """
+
+    _impls: Dict[str, Type[Self]] = {}
+
+    def __init__(
+            self, service: str, targets: List[Target], logger: Logger,
+            events_to_stream: Iterable[EventType] = (EventType.INSERT, EventType.UPDATE, EventType.DELETE),
+            add_event_timestamp: bool = False) -> None:
+
+        """
+        :param service: The name of the service that is doing the CDC process.
+        :param events_to_stream: By default, insert, update and, delete operations will be streamed.
+        :param logger: Logger used.
+
+        :param add_event_timestamp:
+            If True, the column event_timestamp will be added when a table is created. The column
+            could be useful for UPSERT or MERGE operations.
+        """
+
+        self.service = service
+        self.targets = targets
+        self.events_to_stream = events_to_stream
+        self.add_event_timestamp = add_event_timestamp
+        self.logger = logger
+
+    def execute(self):
+        self.logger.info("Reading events from the stream...")
+
+        for event in self.get_events():
+            event_type = self.get_event_type(event)
+
+            if event_type == EventType.DDL_STATEMENT:
+                for target in self.targets:
+                    if target.execute_ddl:
+                        try:
+                            query = target.get_ddl_query(event)
+                            if query:
+                                target.execute(query)
+                                self.logger.info(f"The below query was executed in: {target.register_name()}.")
+                                self.logger.info(query)
+
+                            if self.add_event_timestamp:
+                                if query.lower().count("create table"):
+                                    schema, table = Target.get_schema_table_from_query(query)
+
+                                    target.execute(
+                                        target.get_add_column_ddl(
+                                            schema=schema, table=table,
+                                            column="event_timestamp",
+                                            type_="bigint"
+                                        )
+                                    )
+
+                        except Exception as error:
+                            self.logger.error(f"[{target.register_name()}] -- {error}")
+
+            elif event_type in self.events_to_stream:
+                for target in self.targets:
+                    try:
+                        records = self.process_dml_event(event)
+                        target.save(records)
+
+                        self.logger.info({
+                            "target": target.register_name(),
+                            "number_of_records": len(records),
+                            "event_type": event_type,
+                            "schema": records[0].schema_name,
+                            "table": records[0].table_name
+                        })
+
+                    except Exception as error:
+                        self.logger.error(f"[{target.register_name()}] -- {error}")
+
+            else:
+                try:
+                    self.process_event(event)
+
+                except Exception as error:
+                    self.logger.error(f"Error: {error}")
+
+    @abstractmethod
+    def get_events(self) -> Iterator[Any]:
+        """ It returns an iterator with the events to process """
+
+    @abstractmethod
+    def get_event_type(self, event: Any) -> EventType:
+        """ It returns the event type """
+
+    @abstractmethod
+    def process_dml_event(self, event: Any, **kwargs) -> List[Record]:
+        """ It processes the event and return the records to stream """
+
+    def process_event(self, event: Any):
+        """
+        It should be implemented if another event (apart from DDL
+        and DML) must be processed...
+        """

core-cdc-1.0.0/core_cdc/processors/mongo_stream.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any, List, Iterator, Dict
+
+from pymongo.change_stream import ChangeStream
+
+from core_cdc.base import Record, EventType
+from .base import IProcessor
+
+
+class MongoDbStreamProcessor(IProcessor):
+    """
+    It processes the events from the MongoDB Stream.
+
+    A change stream is a real-time stream of database changes that flows from your
+    database to your application. With change streams, your applications can react—in real
+    time—to data changes in a single collection, a database, or even an entire
+    deployment. For apps that rely on notifications of changing data, change
+    streams are critical.
+
+    More information:
+    https://www.mongodb.com/basics/change-streams
+    """
+
+    def __init__(self, stream: ChangeStream, save_full_event: bool = True, **kwargs):
+        """
+        :param stream: DatabaseChangeStream object.
+        :param save_full_event: If True, all the event will be streamed, otherwise only fullDocument.
+
+        To create a stream you can use:
+            * db.collection.watch()
+            * db.watch()
+
+        Example:
+            pipeline = [{'$match': {'operationType': 'insert'}}, {'$match': {'operationType': 'replace'}}]
+            MongoDbStreamProcessor(
+                stream = MongoClient(...)["database"].<collection>.watch(pipeline)
+            )
+
+        More information...
+            * https://www.mongodb.com/basics/change-streams
+            * https://www.mongodb.com/docs/manual/changeStreams/#open-a-change-stream
+            * https://www.mongodb.com/docs/manual/reference/method/db.watch/#db.watch--
+        """
+
+        super(MongoDbStreamProcessor, self).__init__(**kwargs)
+        self.save_full_event = save_full_event
+        self.stream = stream
+
+    def get_events(self) -> Iterator[Any]:
+        for event in self.stream:
+            self.logger.info(
+                f"Received event: {event.get('operationType')} "
+                f"for document: {event.get('documentKey', {}).get('_id')}."
+            )
+
+            event["clusterTime"] = event["clusterTime"].time if event.get("clusterTime") else None
+            yield event
+
+            # To store the token to resume execution...
+            self.save_resume_token(event["_id"])
+
+    def get_event_type(self, event: Dict) -> EventType:
+        opt_type = event.get("operationType")
+        if opt_type == "insert":
+            return EventType.INSERT
+
+        elif opt_type in ("replace", "update"):
+            return EventType.UPDATE
+
+        elif opt_type == "delete":
+            return EventType.DELETE
+
+        return EventType.GLOBAL
+
+    def process_dml_event(self, event: Any, **kwargs) -> List[Record]:
+        metadata = {
+            "global_id": None,
+            "transaction_id": None,
+            "event_timestamp": event["clusterTime"],
+            "event_type": self.get_event_type(event),
+            "service": self.service,
+            "source": None,
+            "position": 0,
+            "primary_key": "_id",
+            "schema_name": event["ns"]["db"],
+            "table_name": event["ns"]["coll"]
+        }
+
+        event["documentKey"]["_id"] = str(event["documentKey"]["_id"])
+        if event.get("fullDocument"):
+            event["fullDocument"]["_id"] = str(event["fullDocument"]["_id"])
+
+        return [
+            Record(
+                record=event if self.save_full_event else event.get("fullDocument", {}),
+                **metadata
+            )
+        ]
+
+    def save_resume_token(self, token):
+        """
+        It stores the token that can be used to resume the
+        process in a certain point...
+        """

core-cdc-1.0.0/core_cdc/processors/mssql.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+from core_cdc.processors.base import IProcessor
+
+
+class MsSqlProcessor(IProcessor):
+    """
+    Change Data Capture (CDC) is a feature provided by Microsoft SQL Server to capture
+    and track changes made to tables at the database level. It is designed to efficiently
+    identify and record changes to data, making it a powerful tool for applications
+    that require real-time or near-real-time data integration, synchronization,
+    and auditing. Here are some key points
+    about SQL Server Change Data Capture:
+
+    1. **Purpose:**
+       - **Data Integration:** CDC facilitates the integration of data between different systems and databases by capturing changes at the source.
+       - **Synchronization:** It allows keeping multiple databases in sync by identifying and applying changes made to one database to another.
+       - **Auditing:** CDC provides a reliable and efficient way to audit changes to data, tracking who made the change and when.
+
+    2. **How it Works:**
+       - CDC works by enabling change tracking on selected tables. When enabled, SQL Server maintains change tables to store the details of changes made to the tracked tables.
+       - The change tables store information about INSERTs, UPDATEs, and DELETEs, including the columns affected and the corresponding values before and after the change.
+
+    3. **Configuration:**
+       - To use CDC, you need to enable it at the database level and then enable it for specific tables.
+       - Enabling CDC involves creating the required system tables and stored procedures for change tracking.
+       - Once enabled, SQL Server automatically populates change tables with the details of changes made to the tracked tables.
+
+    4. **Querying Changes:**
+       - After CDC is enabled, you can query the change tables to retrieve information about changes made to tracked tables.
+       - The `cdc.fn_cdc_get_all_changes_<capture_instance>` function is commonly used to retrieve changes for a specific capture instance.
+
+    5. **Retention Period:**
+       - CDC allows you to specify a retention period for change data. After this period, older change data is automatically purged.
+
+    6. **Integration with ETL and Replication:**
+       - CDC is often used in conjunction with Extract, Transform, Load (ETL) processes to capture changes at the source and propagate them to a data warehouse or another database.
+       - It is also integrated with SQL Server Replication to capture and replicate changes to other SQL Server instances.
+
+    7. **Security Considerations:**
+       - CDC respects SQL Server security settings, ensuring that only authorized users can access the change data.
+
+    Here's a basic example of enabling CDC for a table:
+
+    ```sql
+    -- Enable CDC at the database level
+    EXEC sys.sp_cdc_enable_db;
+
+    -- Enable CDC for a specific table
+    EXEC sys.sp_cdc_enable_table
+      @source_schema = 'dbo',
+      @source_name   = 'YourTableName',
+      @role_name     = 'cdc_admin';
+    ```
+
+    Once CDC is enabled, you can query the change tables to retrieve
+    information about the changes made to the tracked table. Keep in mind that
+    using CDC introduces some overhead, and you should carefully consider the
+    impact on performance and storage when enabling it. It's a powerful
+    feature for scenarios requiring change tracking, but it may not be
+    necessary for every application.
+    """

core-cdc-1.0.0/core_cdc/processors/mysql_binlog.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any, Iterator, List
+
+from pymysqlreplication import BinLogStreamReader
+from pymysqlreplication.event import GtidEvent
+from pymysqlreplication.event import QueryEvent
+from pymysqlreplication.event import RotateEvent
+from pymysqlreplication.event import XidEvent
+from pymysqlreplication.row_event import DeleteRowsEvent
+from pymysqlreplication.row_event import UpdateRowsEvent
+from pymysqlreplication.row_event import WriteRowsEvent
+
+from core_cdc.base import Record, EventType
+from .base import IProcessor
+
+
+class MySqlBinlogProcessor(IProcessor):
+    """
+    It processes the events from the BinLog files.
+
+    The binary log contains “events” that describe database changes such as table creation
+    operations or changes to table data. It also contains events for statements that potentially
+    could have made changes (for example, a DELETE which matched no rows), unless row-based
+    logging is used. The binary log also contains information about how long each
+    statement took that updated data.
+
+    More information:
+    https://dev.mysql.com/doc/refman/8.0/en/binary-log.html
+    """
+
+    def __init__(self, stream: BinLogStreamReader, **kwargs) -> None:
+        """
+        https://python-mysql-replication.readthedocs.io/en/stable/binlogstream.html
+        :param stream: BinLogStreamReader object.
+        """
+
+        super(MySqlBinlogProcessor, self).__init__(**kwargs)
+        self.stream = stream
+
+        # To keep the tracking of the processed elements...
+        self.log_file = None
+        self.log_pos = None
+        self.gtid = None
+        self.xid = None
+
+    def get_events(self) -> Iterator[Any]:
+        for event in self.stream:
+            self.logger.info(f"Received event: {event.__class__.__name__}.")
+            self.log_file, self.log_pos = self.stream.log_file, self.stream.log_pos
+            self.logger.info(f"File: {self.log_file}, Position: {self.log_pos}.")
+
+            yield event
+            self._update_log_pos(self.log_pos)
+
+    def get_event_type(self, event: Any) -> EventType:
+        if isinstance(event, QueryEvent):
+            return EventType.DDL_STATEMENT
+
+        elif isinstance(event, WriteRowsEvent):
+            return EventType.INSERT
+
+        elif isinstance(event, UpdateRowsEvent):
+            return EventType.UPDATE
+
+        elif isinstance(event, DeleteRowsEvent):
+            return EventType.DELETE
+
+        return EventType.GLOBAL
+
+    def process_dml_event(self, event: Any, **kwargs) -> List[Record]:
+        metadata = {
+            "global_id": self.gtid,
+            "transaction_id": self.xid,
+            "event_timestamp": event.timestamp,
+            "event_type": "",
+            "service": self.service,
+            "source": self.log_file,
+            "position": self.log_pos,
+            "primary_key": event.primary_key,
+            "schema_name": event.schema,
+            "table_name": event.table
+        }
+
+        if isinstance(event, WriteRowsEvent):
+            metadata["event_type"] = EventType.INSERT
+
+        if isinstance(event, DeleteRowsEvent):
+            metadata["event_type"] = EventType.DELETE
+
+        if isinstance(event, UpdateRowsEvent):
+            metadata["event_type"] = EventType.UPDATE
+
+            return [
+                Record(record=row.get("after_values", {}), **metadata)
+                for row in event.rows
+            ]
+
+        return [
+            Record(record=row.get("values", {}), **metadata)
+            for row in event.rows
+        ]
+
+    def process_event(self, event: Any, **kwargs):
+        if isinstance(event, GtidEvent):
+            self.gtid = event.gtid
+
+        elif isinstance(event, XidEvent):
+            self.xid = event.xid
+
+        elif isinstance(event, RotateEvent):
+            self.logger.info(f"NEXT FILE: {event.next_binlog}. POSITION: {event.position}.")
+            self._update_log_file(event.next_binlog)
+            self._update_log_pos(event.position)
+
+    def _update_log_file(self, log_file_name: str):
+        """ It updates the log_file name """
+
+    def _update_log_pos(self, position: int):
+        """ It updates the log_file position """

core-cdc-1.0.0/core_cdc/processors/oracle.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+
+from core_cdc.processors.base import IProcessor
+
+
+class OracleProcessor(IProcessor):
+    """
+    Oracle Database does not provide a dedicated Python library similar to
+    the SQL Server Change Data Capture (CDC) feature. However, you can implement change
+    data capture in Oracle using a combination of Oracle features and Oracle Database
+    connectors for Python.
+
+    Here's a general approach:
+
+    1. **Oracle Flashback Technology:**
+       - Oracle offers Flashback Technology, including Flashback Query and Flashback Version Query, which allows you to query data as it appeared at a previous point in time.
+       - You can leverage these features to implement a form of change data capture by periodically querying the database for changes within a specified time range.
+
+    2. **Database Triggers:**
+       - Oracle supports triggers, which are procedures that are automatically executed in response to specific events on a particular table or view.
+       - You can create triggers to capture information about changes (e.g., inserts, updates, deletes) into separate audit or change tables.
+
+    3. **LogMiner:**
+       - Oracle LogMiner is a utility that allows you to mine redo log files and obtain information about changes made to the database.
+       - You can use Oracle's LogMiner to track changes and update a change tracking table.
+
+    4. **Oracle Database Connectors for Python:**
+       - You can use Python libraries such as `cx_Oracle` to connect to Oracle Database and execute queries or procedures that implement your change data capture logic.
+       - Install `cx_Oracle` using: `pip install cx_Oracle`.
+
+    Here's a simplified example using Python and `cx_Oracle`:
+
+    ```python
+    import cx_Oracle
+
+    # Connect to Oracle
+    connection = cx_Oracle.connect("username/password@localhost/orcl")
+
+    # Create a cursor
+    cursor = connection.cursor()
+
+    # Execute a query using Flashback Query
+    query = "SELECT * FROM your_table AS OF TIMESTAMP (SYSTIMESTAMP - INTERVAL '1' DAY)"
+    cursor.execute(query)
+
+    # Fetch the results
+    for row in cursor.fetchall():
+        print(row)
+
+    # Close the cursor and connection
+    cursor.close()
+    connection.close()
+    ```
+
+    Remember that Oracle Database may have specific requirements and considerations
+    for implementing change data capture based on your use case. You might also want to
+    explore Oracle GoldenGate, a comprehensive solution for real-time data integration
+    and replication that goes beyond basic change tracking.
+    """

core-cdc-1.0.0/core_cdc/targets/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from .kinesis import KinesisDataStreamTarget
+from .mysql import MySQLTarget
+from .sns import SnsTarget
+from .sqs import SqsTarget

core-cdc-1.0.0/core_cdc/targets/base.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from logging import Logger
+from re import compile
+from typing import Dict, List, Tuple, Type, Self
+
+from core_mixins.interfaces.factory import IFactory
+from pymysqlreplication.event import QueryEvent
+
+from core_cdc.base import Record
+
+
+class Target(IFactory):
+    """
+    This is the base class for the specific implementations of the targets the data will be sent
+    or replicated. A target could be a database, queue, topic, data warehouse, etc...
+    """
+
+    # Registered targets...
+    _impls: Dict[str, Type[Self]] = {}
+
+    def __init__(
+            self, logger: Logger, execute_ddl: bool = False, send_data: bool = False,
+            **kwargs) -> None:
+
+        self.execute_ddl = execute_ddl
+        self.send_data = send_data
+        self.logger = logger
+        self.client = None
+
+        for attr in kwargs:
+            setattr(self, attr, kwargs[attr])
+
+    def init_client(self, **kwargs) -> None:
+        """ The target's implementations must implement this method """
+
+    def connect(self):
+        if self.client:
+            self.client.connect()
+
+    def get_ddl_query(self, event: QueryEvent) -> str | None:
+        """
+        Each engine could use a different query for DDL operations...
+        :return: The query or None if not supported.
+        """
+
+        # sql_statement = sub(r"/\*.*?\*/", "", event.query.lower()).strip()
+        sql_statement = event.query.lower()
+
+        if sql_statement.count("create schema") or sql_statement.count("create database"):
+            return self.get_create_schema_statement(event)
+
+        elif sql_statement.count("drop schema") or sql_statement.count("drop database"):
+            return self.get_drop_schema_statement(event)
+
+        elif sql_statement.count("create table"):
+            return self.get_create_table_statement(event)
+
+        elif sql_statement.count("alter table"):
+            return self.get_alter_table_statement(event)
+
+        elif sql_statement.count("drop table"):
+            return self.get_drop_table_statement(event)
+
+    @staticmethod
+    def get_add_column_ddl(schema: str, table: str, column: str, type_: str) -> str:
+        """ Returns the DDL to add a new column """
+
+        return f"ALTER TABLE `{schema}`.`{table}` ADD COLUMN `{column}` {type_};"
+
+    @staticmethod
+    def get_create_schema_statement(event: QueryEvent) -> str:
+        return event.query
+
+    @staticmethod
+    def get_drop_schema_statement(event: QueryEvent) -> str:
+        return event.query
+
+    @staticmethod
+    def get_create_table_statement(event: QueryEvent) -> str:
+        return event.query
+
+    @staticmethod
+    def get_alter_table_statement(event: QueryEvent) -> str:
+        return event.query
+
+    @staticmethod
+    def get_schema_table_from_query(query: str) -> Tuple[str, str]:
+        """ Returns schema, table from query using Regex """
+
+        # TODO if the query does not contains "`" this will not work...
+        data = compile("`[a-z_]+`.`[a-z_]+`").search(query).group(0)
+        schema, table = data.replace("`", "").split(".")
+        return schema, table
+
+    @staticmethod
+    def get_drop_table_statement(event: QueryEvent) -> str:
+        return event.query
+
+    def execute(self, query: str):
+        if self.client:
+            self.client.execute(query)
+
+    def save(self, records: List[Record], **kwargs):
+        if self.send_data:
+            self._save(records, **kwargs)
+            self.logger.info(f"{len(records)} records were sent to: {self.register_name()}!")
+
+    @abstractmethod
+    def _save(self, records: List[Record], **kwargs):
+        """ Specific implementation to store the data into the Engine """
+
+    def close(self):
+        """ Implement it if is required to release or close resources """

core-cdc-1.0.0/core_cdc/targets/kinesis.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+from typing import List
+
+from core_aws.services.kinesis.client import KinesisClient
+
+from core_cdc.base import Record
+from .base import Target
+
+
+class KinesisDataStreamTarget(Target):
+    """ Send data to a Kinesis Data Stream """
+
+    def __init__(self, aws_region: str, stream_name: str, partition_key: str, **kwargs):
+        super(KinesisDataStreamTarget, self).__init__(**kwargs)
+
+        self.aws_region = aws_region
+        self.client = KinesisClient(region=self.aws_region)
+        self.partition_key = partition_key
+        self.stream_name = stream_name
+
+    def _save(self, records: List[Record], **kwargs):
+        return self.client.send_records(
+            records=[x.to_json() for x in records],
+            stream_name=self.stream_name,
+            partition_key=self.partition_key,
+            **kwargs)

core-cdc-1.0.0/core_cdc/targets/mysql.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+from typing import List
+
+from core_db.engines.mysql import MySQLClient
+
+from core_cdc.base import Record
+from .base import Target
+
+
+class MySQLTarget(Target):
+    """
+    This is the processor for synchronization with another MySQL instance. This capability
+    is provided because even if the replication process is done outside this service
+    is required to apply the DDL statements to keep the structure up to date...
+    """
+
+    def init_client(self, **kwargs) -> None:
+        """
+        Expecting at least: host, user, password, database
+        More information: https://pymysql.readthedocs.io/en/latest/user/index.html
+        """
+
+        self.client = MySQLClient(**kwargs)
+
+    def _save(self, records: List[Record], **kwargs):
+        """
+        We are not implementing the mechanism to persist the data directly
+        into the database because we believe decoupling (by sending the
+        data to SQS queue or SNS topic) is a better solution. But you
+        could modify the behavior or inherit the class...
+
+        Possible solutions:
+          * self.client.get_insert_dml(...)
+          * self.client.get_merge_dml(...)
+        """

core-cdc-1.0.0/core_cdc/targets/snowflake.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+
+import re
+from typing import List
+
+from core_db.engines.snowflake import SnowflakeClient
+from pymysqlreplication.event import QueryEvent
+
+from core_cdc.base import Record
+from .base import Target
+
+
+class SnowflakeTarget(Target):
+    """ Processor for synchronization with Snowflake """
+
+    def init_client(self, **kwargs) -> None:
+        """
+        Expecting: host, user, password, account, database, warehouse, schema, role.
+        https://docs.snowflake.com/en/user-guide/python-connector-example.html
+        """
+
+        self.client = SnowflakeClient(**kwargs)
+
+    def _save(self, records: List[Record], **kwargs):
+        """
+        We are not implementing the mechanism to persist the data directly
+        into the data warehouse because we believe decoupling (by sending the
+        data to SQS queue or SNS topic) is a better solution. But you
+        could modify the behavior or inherit the class...
+        """
+
+    @staticmethod
+    def get_create_schema_statement(event: QueryEvent):
+        return f"CREATE IF NOT EXIST SCHEMA {event.schema.decode()};"
+
+    def get_create_table_statement(self, event: QueryEvent):
+        return self.transform_create_table_query(event.query.lower())
+
+    @staticmethod
+    def transform_create_table_query(query: str) -> str:
+        """ Transform a MySQL query to a Snowflake one """
+
+        mapper = [
+            (r"`", ""),
+            (r"create table", "create table if not exists"),
+            (r"unique index \w+ \(+\w+ \w+\) \w+", ""),
+
+            (r"not null", ""),
+            (r"null", ""),
+
+            # Not necessary because it's a replication...
+            (r" generated always as[.]*(.*?)\)", ""),
+            (r"auto_increment", ""),
+            (r"unsigned", ""),
+            (r"zerofill", ""),
+
+            # Updating types...
+            (r" blob", " binary"),
+            (r" varbinary[.]*(.*?)\)", " binary"),
+            (r" year[.]*(.*?)\)", " varchar(4)"),
+            (r" json", " object"),
+            (r" enum[.]*(.*?)\)", " varchar"),
+            (r" set[.]*(.*?)\)", " varchar"),
+
+            # Removing unnecessary blank spaces...
+            (r"[ ]{2,}", ""),
+
+            # Removing \n...
+            (r"[\n]*,", ","),
+
+            # Removing unnecessary commas...
+            (r",,", ","),
+            (r",[\n]*\)", ")")
+        ]
+
+        for pattern, replacement in mapper:
+            regex = re.compile(pattern, re.MULTILINE)
+            query = re.sub(regex, replacement, query)
+
+        return query

core-cdc-1.0.0/core_cdc/targets/sns.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+from typing import List
+
+from core_aws.services.sns.client import SnsClient
+from core_aws.services.sns.client import SnsMessage
+
+from core_cdc.base import Record
+from core_cdc.targets.base import Target
+
+
+class SnsTarget(Target):
+    """ To send data to a SQS queue """
+
+    def __init__(self, aws_region: str, topic_arn: str, batch_size: int = 10, **kwargs):
+        super(SnsTarget, self).__init__(**kwargs)
+
+        self.aws_region = aws_region
+        self.client = SnsClient(region=aws_region, batch_size=batch_size)
+        self.topic_arn = topic_arn
+        self.execute_ddl = False
+
+    def _save(self, records: List[Record], **kwargs):
+        self.client.publish_batch(
+            topic_arn=self.topic_arn,
+            messages=[
+                SnsMessage(Id=f"{rec.table_name}-{x}", Message=rec.to_json())
+                for x, rec in enumerate(records)
+            ]
+        )

core-cdc-1.0.0/core_cdc/targets/sqs.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+import json
+from typing import List
+from uuid import uuid4
+
+from core_aws.services.sqs.client import SqsClient
+from core_mixins.utils import get_batches
+
+from core_cdc.base import Record
+from core_cdc.targets.base import Target
+
+
+class SqsTarget(Target):
+    """ To send data to a SQS queue """
+
+    def __init__(self, aws_region: str, queue_name: str, **kwargs):
+        super(SqsTarget, self).__init__(**kwargs)
+
+        self.aws_region = aws_region
+        self.execute_ddl = False
+        self.queue_url = None
+
+        self.queue_name = queue_name
+        self.client = SqsClient(region=aws_region)
+        self.queue_url = None
+
+    def init_client(self, **kwargs) -> None:
+        self.queue_url = self.client.get_queue_by_name(self.queue_name).url
+
+    def _save(self, records: List[Record], **kwargs):
+        for batch in get_batches([x.to_json() for x in records], 10):
+            self.client.send_message_batch(
+                queue_url=self.queue_url,
+                entries=[
+                    {
+                        "Id": str(uuid4()),
+                        "MessageBody": json.dumps(entry),
+                        "DelaySeconds": 0,
+                        # "MessageDeduplicationId": "",
+                        # "MessageGroupId": ""
+                    } for entry in batch
+                ]
+            )

core-cdc-1.0.0/core_cdc.egg-info/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.1
+Name: core-cdc
+Version: 1.0.0
+Summary: This project/library contains the core mechanism and resources for Change Data Capture services...
+Author-email: Alejandro Cora González <alek.cora.glez@gmail.com>
+Maintainer: Alejandro Cora González
+License: MIT
+Project-URL: Homepage, https://gitlab.com/bytecode-solutions/core/core-cdc
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Utilities
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: core-mixins
+Requires-Dist: core-tests
+Requires-Dist: core-aws
+Requires-Dist: core-db
+Requires-Dist: mysql-replication==0.31
+Requires-Dist: pymongo==4.6.0
+Provides-Extra: test
+
+# core-cdc (CDC a.k.a Change Data Capture)
+_______________________________________________________________________________
+
+It provides the core mechanism and required resources to 
+implement "Change Data Capture" services...
+
+## Execution Environment
+
+### Install libraries
+```commandline
+pip install --upgrade pip 
+pip install virtualenv
+```
+
+### Create the Python Virtual Environment.
+```commandline
+virtualenv --python=python3.11 .venv
+```
+
+### Activate the Virtual Environment.
+```commandline
+source .venv/bin/activate
+```
+
+### Install required libraries.
+```commandline
+pip install .
+```
+
+### Check tests and coverage...
+```commandline
+python manager.py run-test
+python manager.py run-coverage
+```

core-cdc-1.0.0/core_cdc.egg-info/SOURCES.txt

@@ -0,0 +1,24 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+core_cdc/__init__.py
+core_cdc/base.py
+core_cdc.egg-info/PKG-INFO
+core_cdc.egg-info/SOURCES.txt
+core_cdc.egg-info/dependency_links.txt
+core_cdc.egg-info/requires.txt
+core_cdc.egg-info/top_level.txt
+core_cdc/processors/__init__.py
+core_cdc/processors/base.py
+core_cdc/processors/mongo_stream.py
+core_cdc/processors/mssql.py
+core_cdc/processors/mysql_binlog.py
+core_cdc/processors/oracle.py
+core_cdc/targets/__init__.py
+core_cdc/targets/base.py
+core_cdc/targets/kinesis.py
+core_cdc/targets/mysql.py
+core_cdc/targets/snowflake.py
+core_cdc/targets/sns.py
+core_cdc/targets/sqs.py

core-cdc-1.0.0/core_cdc.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

core-cdc-1.0.0/core_cdc.egg-info/requires.txt

@@ -0,0 +1,8 @@
+core-mixins
+core-tests
+core-aws
+core-db
+mysql-replication==0.31
+pymongo==4.6.0
+
+[test]

core-cdc-1.0.0/core_cdc.egg-info/top_level.txt

@@ -0,0 +1 @@
+core_cdc

core-cdc-1.0.0/pyproject.toml

@@ -0,0 +1,50 @@
+# Other project metadata fields as specified in:
+# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/
+# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html
+
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "core-cdc"
+description = "This project/library contains the core mechanism and resources for Change Data Capture services..."
+version = "1.0.0"
+
+authors = [
+    {name = "Alejandro Cora González", email = "alek.cora.glez@gmail.com"}
+]
+
+maintainers = [
+    {name = "Alejandro Cora González"}
+]
+
+requires-python = ">=3.9"
+license = {text = "MIT"}
+readme = "README.md"
+
+classifiers = [
+    # Classifiers -> https://pypi.org/classifiers/
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.11",
+    "Intended Audience :: Developers",
+    "Topic :: Utilities"
+]
+
+dependencies = [
+    "core-mixins",
+    "core-tests",
+    "core-aws",
+    "core-db",
+    "mysql-replication==0.31",
+    "pymongo==4.6.0"
+]
+
+[project.urls]
+Homepage = "https://gitlab.com/bytecode-solutions/core/core-cdc"
+
+[project.optional-dependencies]
+test = []

core-cdc-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

core-cdc-1.0.0/setup.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from setuptools import setup
+
+
+setup()