py-auto-migrate 0.3.5__py3-none-any.whl

py_auto_migrate/__init__.py

@@ -0,0 +1,3 @@
+"""
+Py-Auto-Migrate: A tool for migrating data between MongoDB and MySQL databases.
+"""

py_auto_migrate/base_models/base.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+
+class BaseModel(ABC):
+    
+    def __init__(self, uri):
+        self.uri = uri
+    
+    @abstractmethod
+    def _connect(self):
+        pass
+    
+    @abstractmethod
+    def get_tables(self):
+        pass
+    
+    @abstractmethod
+    def read_table(self, table_name):
+        pass

py_auto_migrate/base_models/base_clickhouse.py

@@ -0,0 +1,88 @@
+import pandas as pd
+from clickhouse_driver import Client
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseClickHouse(BaseModel):
+    def __init__(self, clickhouse_uri):
+        super().__init__(clickhouse_uri)
+        self.user, self.password, self.host, self.port, self.db_name = self._parse_clickhouse_uri()
+    
+    def _parse_clickhouse_uri(self):
+        if self.uri.startswith("clickhouse://"):
+            uri = self.uri.replace("clickhouse://", "")
+        else:
+            uri = self.uri
+
+        if "@" in uri:
+            auth_part, host_db_part = uri.split("@")
+            if ":" in auth_part:
+                user, password = auth_part.split(":")
+            else:
+                user, password = auth_part, ""
+        else:
+            user, password = "default", "" 
+            host_db_part = uri
+            
+        if "/" in host_db_part:
+            host_port, db_name = host_db_part.split("/", 1)
+        else:
+            host_port, db_name = host_db_part, ""
+
+        if ":" in host_port:
+            host, port = host_port.split(":")
+            port = int(port)
+        else:
+            host, port = host_port, 9000
+
+        return user, password, host, port, db_name
+
+    def _connect(self):
+        try:
+            client = Client(
+                host=self.host,
+                port=self.port,
+                user=self.user,
+                password=self.password,
+                database=self.db_name
+            )
+            client.execute('SELECT 1')
+            return client
+        except Exception as e:
+            print(f"❌ Error connecting to ClickHouse: {e}")
+            return None
+
+    def get_tables(self):
+        client = self._connect()
+        if client is None:
+            return []
+            
+        query = "SELECT name FROM system.tables WHERE database = %s"
+        try:
+            tables_result = client.execute(query, [self.db_name])
+            tables = [r[0] for r in tables_result]
+            client.disconnect()
+            return tables
+        except Exception as e:
+            print(f"❌ Error fetching tables: {e}")
+            return []
+
+    def read_table(self, table_name):
+        client = self._connect()
+        if client is None:
+            return pd.DataFrame()
+
+        query = f'SELECT * FROM "{table_name}"'
+        try:
+            data = client.execute(query)
+            columns = [desc[0] for desc in client.description_of_result_set]
+            
+            client.disconnect()
+            
+            df = pd.DataFrame(data, columns=columns)
+            if df.empty:
+                print(f"❌ Table '{table_name}' is empty.")
+            return df.fillna(0)
+        except Exception as e:
+            print(f"❌ Error reading table {table_name}: {e}")
+            return pd.DataFrame()

py_auto_migrate/base_models/base_dynamodb.py

@@ -0,0 +1,67 @@
+import boto3
+import pandas as pd
+from botocore.exceptions import ClientError
+from urllib.parse import urlparse, parse_qs
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseDynamoDB(BaseModel):
+    def __init__(self, dynamo_uri):
+        super().__init__(dynamo_uri)
+        self._parse_uri()
+
+    def _parse_uri(self):
+        try:
+            parsed = urlparse(self.uri)
+            self.host = parsed.hostname
+            self.port = parsed.port 
+            self.aws_access_key = parsed.username or None
+            self.aws_secret_key = parsed.password or None
+            self.table_prefix = parsed.path.lstrip('/') or 'default'
+
+            qs = parse_qs(parsed.query)
+            self.region_name = qs.get('region', [ 'us-west-2' ])[0]
+
+        except Exception as e:
+            self.aws_access_key = None
+            self.aws_secret_key = None
+            self.host = 'localhost'
+            self.port = 9000
+            self.table_prefix = 'default'
+            self.region_name = 'us-west-2'
+
+    def _connect(self):
+        try:
+            endpoint = f"http://{self.host}:{self.port}" if self.host and self.port else None
+            client = boto3.resource(
+                'dynamodb',
+                region_name=self.region_name,
+                aws_access_key_id=self.aws_access_key,
+                aws_secret_access_key=self.aws_secret_key,
+                endpoint_url=endpoint
+            )
+            return client
+        except Exception as e:
+            return None
+
+    def get_tables(self):
+        conn = self._connect()
+        if not conn:
+            return []
+        try:
+            tables = [t.name for t in conn.tables.all() if t.name.startswith(self.table_prefix)]
+            return tables
+        except ClientError as e:
+            return []
+
+    def read_table(self, table_name):
+        conn = self._connect()
+        if not conn:
+            return pd.DataFrame()
+        try:
+            table = conn.Table(table_name)
+            response = table.scan()
+            data = response.get("Items", [])
+            return pd.DataFrame(data)
+        except ClientError as e:
+            return pd.DataFrame()

py_auto_migrate/base_models/base_elasticsearch.py

@@ -0,0 +1,52 @@
+import pandas as pd
+from elasticsearch import Elasticsearch
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseElasticsearch(BaseModel):
+    def __init__(self, es_uri):
+        super().__init__(es_uri)
+
+    def _connect(self):
+        try:
+            es = Elasticsearch(self.uri)
+            if not es.ping():
+                return None
+            return es
+        except Exception as e:
+            return None
+
+    def get_tables(self):
+        es = self._connect()
+        if es is None:
+            return []
+        try:
+            indices = es.indices.get_alias("*").keys()
+            return list(indices)
+        except Exception as e:
+            return []
+
+    def read_table(self, index_name):
+        es = self._connect()
+        if es is None:
+            return pd.DataFrame()
+
+        try:
+            query = {
+                "query": {"match_all": {}},
+                "size": 10000
+            }
+
+            result = es.search(index=index_name, body=query)
+
+            hits = result.get("hits", {}).get("hits", [])
+            if not hits:
+                return pd.DataFrame()
+
+            records = [hit["_source"] | {"_id": hit["_id"]} for hit in hits]
+
+            df = pd.DataFrame(records)
+            return df
+
+        except Exception as e:
+            return pd.DataFrame()

py_auto_migrate/base_models/base_mariadb.py

@@ -0,0 +1,40 @@
+import pandas as pd
+import pymysql
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseMariaDB(BaseModel):
+    def __init__(self, maria_uri):
+        super().__init__(maria_uri)
+
+    def _parse_maria_uri(self, maria_uri=None):
+        if maria_uri is None:
+            maria_uri = self.uri
+        maria_uri = maria_uri.replace("mariadb://", "").replace("mysql://", "")
+        user_pass, host_db = maria_uri.split("@")
+        user, password = user_pass.split(":")
+        host_port, db_name = host_db.split("/")
+        if ":" in host_port:
+            host, port = host_port.split(":")
+            port = int(port)
+        else:
+            host, port = host_port, 3306
+        return host, port, user, password, db_name
+
+    def _connect(self):
+        host, port, user, password, db_name = self._parse_maria_uri()
+        return pymysql.connect(host=host, port=port, user=user, password=password, database=db_name)
+
+    def get_tables(self):
+        conn = self._connect()
+        cur = conn.cursor()
+        cur.execute("SHOW TABLES")
+        tables = [r[0] for r in cur.fetchall()]
+        conn.close()
+        return tables
+
+    def read_table(self, table_name):
+        conn = self._connect()
+        df = pd.read_sql(f"SELECT * FROM `{table_name}`", conn)
+        conn.close()
+        return df.fillna(0)

py_auto_migrate/base_models/base_mongodb.py

@@ -0,0 +1,70 @@
+import pandas as pd
+from pymongo import MongoClient
+from urllib.parse import urlparse
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseMongoDB(BaseModel):
+    def __init__(self, mongo_uri):
+        super().__init__(mongo_uri)
+
+    def _connect(self):
+        try:
+            parsed = urlparse(self.uri)
+            db_name = parsed.path.lstrip('/')
+
+            if not db_name:
+                db_name = 'admin'
+
+            auth_source = 'admin'
+
+            if parsed.query:
+                query_params = dict(param.split('=')
+                                    for param in parsed.query.split('&') if '=' in param)
+                auth_source = query_params.get('authSource', 'admin')
+
+            client = MongoClient(
+                host=parsed.hostname,
+                port=parsed.port or 27017,
+                username=parsed.username,
+                password=parsed.password,
+                authSource=auth_source,
+                serverSelectionTimeoutMS=5000
+            )
+
+            client.admin.command('ping')
+            return client[db_name]
+
+        except Exception as e:
+            print(f"❌ MongoDB Connection Error: {e}")
+            return None
+
+    def get_tables(self):
+        db = self._connect()
+        if db is None:
+            return []
+        try:
+            return db.list_collection_names()
+        except Exception as e:
+            print(f"❌ Error getting collections: {e}")
+            return []
+
+    def read_table(self, collection_name):
+        db = self._connect()
+        if db is None:
+            return pd.DataFrame()
+
+        try:
+            data = list(db[collection_name].find())
+            if not data:
+                print(f"❌ Collection '{collection_name}' is empty.")
+                return pd.DataFrame()
+
+            df = pd.DataFrame(data)
+            if "_id" in df.columns:
+                df = df.drop(columns="_id")
+
+            return df.fillna(0)
+        except Exception as e:
+            print(f"❌ Error reading collection '{collection_name}': {e}")
+            return pd.DataFrame()

py_auto_migrate/base_models/base_mssql.py

@@ -0,0 +1,61 @@
+import pyodbc
+import pandas as pd
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseMSSQL(BaseModel):
+    def __init__(self, mssql_uri):
+        super().__init__(mssql_uri)
+
+    def _parse_mssql_uri(self, uri=None):
+        if uri is None:
+            uri = self.uri
+        uri = uri.replace("mssql://", "")
+        if uri.startswith("@") or "@" not in uri:
+            if uri.startswith("@"):
+                uri = uri[1:]
+            host_port, db_name = uri.split("/", 1)
+            if ":" in host_port:
+                host, port = host_port.split(":")
+            else:
+                host, port = host_port, "1433"
+            return {"auth": "windows", "host": host, "port": port, "database": db_name}
+        else:
+            user_pass, host_db = uri.split("@", 1)
+            user, password = user_pass.split(":", 1)
+            host_port, db_name = host_db.split("/", 1)
+            if ":" in host_port:
+                host, port = host_port.split(":")
+            else:
+                host, port = host_port, "1433"
+            return {
+                "auth": "sql",
+                "host": host,
+                "port": port,
+                "database": db_name,
+                "user": user,
+                "password": password
+            }
+
+    def _connect(self):
+        cfg = self._parse_mssql_uri()
+        if cfg["auth"] == "windows":
+            conn_str = f"DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={cfg['host']},{cfg['port']};DATABASE={cfg['database']};Trusted_Connection=yes;"
+        else:
+            conn_str = f"DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={cfg['host']},{cfg['port']};DATABASE={cfg['database']};UID={cfg['user']};PWD={cfg['password']};"
+        return pyodbc.connect(conn_str)
+
+    def get_tables(self):
+        conn = self._connect()
+        cur = conn.cursor()
+        cur.execute(
+            "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'")
+        tables = [r[0] for r in cur.fetchall()]
+        conn.close()
+        return tables
+
+    def read_table(self, table_name):
+        conn = self._connect()
+        df = pd.read_sql(f"SELECT * FROM [{table_name}]", conn)
+        conn.close()
+        return df.fillna(0)

py_auto_migrate/base_models/base_mysql.py

@@ -0,0 +1,51 @@
+from mysqlSaver import Connection
+import pandas as pd
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseMySQL(BaseModel):
+    def __init__(self, mysql_uri):
+        super().__init__(mysql_uri)
+    
+    def _parse_mysql_uri(self, uri=None):
+        if uri is None:
+            uri = self.uri
+        uri = uri.replace("mysql://", "")
+        user_pass, host_db = uri.split("@")
+        user, password = user_pass.split(":")
+        host_port, db_name = host_db.split("/")
+        
+        if ":" in host_port:
+            host, port = host_port.split(":")
+            port = int(port)
+        else:
+            host, port = host_port, 3306
+            
+        return host, port, user, password, db_name
+
+    def _connect(self, db_name=None):
+        host, port, user, password, uri_db = self._parse_mysql_uri()
+        if db_name is None:
+            db_name = uri_db
+        return Connection.connect(host, port, user, password, db_name)
+
+    def get_tables(self):
+        conn = self._connect()
+        cursor = conn.cursor()
+        cursor.execute("SHOW TABLES")
+        tables = [row[0] for row in cursor.fetchall()]
+        conn.close()
+        return tables
+
+    def read_table(self, table_name):
+        conn = self._connect()
+        cursor = conn.cursor()
+        cursor.execute(f"SELECT * FROM `{table_name}`")
+        data = cursor.fetchall()
+        columns = [desc[0] for desc in cursor.description]
+        conn.close()
+        
+        df = pd.DataFrame(data, columns=columns)
+        if df.empty:
+            print(f"❌ Table '{table_name}' is empty.")
+        return df.fillna(0)

py_auto_migrate/base_models/base_oracle.py

@@ -0,0 +1,36 @@
+import oracledb
+import pandas as pd
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseOracle(BaseModel):
+    def __init__(self, oracle_uri):
+        super().__init__(oracle_uri)
+
+    def _parse_oracle_uri(self):
+        uri = self.uri.replace("oracle://", "")
+        user_pass, host_db = uri.split("@")
+        user, password = user_pass.split(":")
+        host_port, db_name = host_db.split("/")
+        if ":" in host_port:
+            host, port = host_port.split(":")
+        else:
+            host, port = host_port, "1521"
+        return user, password, host, port, db_name
+
+    def _connect(self):
+        user, password, host, port, db_name = self._parse_oracle_uri()
+        dsn = f"{host}:{port}/{db_name}"
+        return oracledb.connect(user=user, password=password, dsn=dsn)
+
+    def get_tables(self):
+        with self._connect() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT table_name FROM user_tables")
+            tables = [r[0] for r in cursor.fetchall()]
+        return tables
+
+    def read_table(self, table_name):
+        with self._connect() as conn:
+            df = pd.read_sql(f'SELECT * FROM "{table_name}"', conn)
+        return df.fillna(0)

py_auto_migrate/base_models/base_postgressql.py

@@ -0,0 +1,43 @@
+import psycopg2
+import pandas as pd
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BasePostgresSQL(BaseModel):
+    def __init__(self, pg_uri):
+        super().__init__(pg_uri)
+
+    def _connect(self):
+        user_pass, host_db = self.uri.replace(
+            "postgresql://", "").split("@")
+        user, password = user_pass.split(":")
+        host_port, db_name = host_db.split("/")
+        if ":" in host_port:
+            host, port = host_port.split(":")
+            port = int(port)
+        else:
+            host, port = host_port, 5432
+        try:
+            return psycopg2.connect(host=host, port=port, user=user, password=password, dbname=db_name)
+        except Exception as e:
+            return None
+
+    def get_tables(self):
+        conn = self._connect()
+        if conn is None:
+            return []
+        cursor = conn.cursor()
+        cursor.execute(
+            "SELECT table_name FROM information_schema.tables WHERE table_schema='public'")
+        tables = [row[0] for row in cursor.fetchall()]
+        cursor.close()
+        conn.close()
+        return tables
+
+    def read_table(self, table_name):
+        conn = self._connect()
+        if conn is None:
+            return pd.DataFrame()
+        df = pd.read_sql(f'SELECT * FROM "{table_name}"', conn)
+        conn.close()
+        return df.fillna(0)

py_auto_migrate/base_models/base_redis.py

@@ -0,0 +1,55 @@
+import redis
+import pandas as pd
+import json
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseRedis(BaseModel):
+    def __init__(self, redis_uri):
+        super().__init__(redis_uri)
+
+    def _connect(self):
+        try:
+            conn = redis.from_url(self.uri)
+            conn.ping()
+            return conn
+        except Exception as e:
+            return None
+
+    def get_tables(self):
+        conn = self._connect()
+        if conn is None:
+            return []
+        try:
+            keys = conn.keys('*')
+            return [key.decode('utf-8') if isinstance(key, bytes) else key for key in keys]
+        except Exception as e:
+            return []
+
+    def read_table(self, table_name):
+        conn = self._connect()
+        if conn is None:
+            return pd.DataFrame()
+        try:
+            value = conn.get(table_name)
+            if value is None:
+                return pd.DataFrame()
+
+            if isinstance(value, bytes):
+                value = value.decode('utf-8')
+            
+            data = json.loads(value)
+
+            if isinstance(data, dict) and all(str(k).isdigit() for k in data.keys()):
+                data = list(data.values())
+
+            if isinstance(data, list):
+                df = pd.DataFrame(data)
+            elif isinstance(data, dict):
+                df = pd.DataFrame([data])
+            else:
+                return pd.DataFrame()
+
+            return df.fillna(0)
+        except Exception as e:
+            return pd.DataFrame()

py_auto_migrate/base_models/base_sqlite.py

@@ -0,0 +1,26 @@
+import sqlite3
+import pandas as pd
+from py_auto_migrate.base_models.base import BaseModel
+
+
+class BaseSQLite(BaseModel):
+    def __init__(self, sqlite_path):
+        super().__init__(sqlite_path)
+
+    def _connect(self):
+        return sqlite3.connect(self.uri)
+
+    def get_tables(self):
+        conn = self._connect()
+        cursor = conn.cursor()
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        tables = [row[0] for row in cursor.fetchall()]
+        cursor.close()
+        conn.close()
+        return tables
+
+    def read_table(self, table_name):
+        conn = self._connect()
+        df = pd.read_sql(f'SELECT * FROM "{table_name}"', conn)
+        conn.close()
+        return df.fillna(0)