mkpipe-extractor-postgres 0.1.0__tar.gz

mkpipe_extractor_postgres-0.1.0/MANIFEST.in

@@ -0,0 +1 @@
+include mkpipe_extractor_postgres/jars/*

mkpipe_extractor_postgres-0.1.0/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.1
+Name: mkpipe-extractor-postgres
+Version: 0.1.0
+Summary: PostgreSQL extractor for mkpipe.
+Author: Metin Karakus
+Author-email: metin_karakus@yahoo.com
+License: Apache License 2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.8

mkpipe_extractor_postgres-0.1.0/README.md

@@ -0,0 +1,66 @@
+# MkPipe
+
+**MkPipe** is a modular, open-source ETL (Extract, Transform, Load) tool that allows you to integrate various data sources and sinks easily. It is designed to be extensible with a plugin-based architecture that supports extractors, transformers, and loaders.
+
+## Features
+
+- Extract data from multiple sources (e.g., PostgreSQL, MongoDB).
+- Transform data using custom Python logic and Apache Spark.
+- Load data into various sinks (e.g., ClickHouse, PostgreSQL, Parquet).
+- Plugin-based architecture that supports future extensions.
+- Cloud-native architecture, can be deployed on Kubernetes and other environments.
+
+## Installation
+
+You can install the core package and extractors using pip:
+
+### Install the core package:
+```bash
+pip install mkpipe-core
+```
+
+### Install the Postgres extractor:
+```bash
+pip install mkpipe-extractor-postgres
+```
+
+### Install additional extractors or loaders as needed:
+You can find or contribute new extractors and loaders in the future.
+
+## Usage
+
+To run the ETL process, use the following command:
+
+```py
+from mkpipe_core.plugins.registry import EXTRACTORS
+
+def test_postgres_extractor():
+    postgres_extractor = EXTRACTORS.get("postgres")
+    if not postgres_extractor:
+        print("Postgres extractor not found!")
+        return
+    instance = postgres_extractor()
+    instance.extract()
+
+if __name__ == "__main__":
+    test_postgres_extractor()
+
+```
+
+Where `elt.yaml` is your configuration file that specifies the extractors, transformers, and loaders.
+
+## Documentation
+
+For more detailed documentation, please visit the [GitHub repository](https://github.com/m-karakus/mkpipe).
+
+## License
+
+This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.
+
+## Support Me
+
+If you enjoy this project, consider supporting me:
+
+<iframe src="https://github.com/sponsors/m-karakus/button" title="Sponsor m-karakus" height="32" width="114" style="border: 0; border-radius: 6px;"></iframe>
+
+[Support on GitHub](https://github.com/sponsors/m-karakus?frequency=one-time&sponsor=m-karakus)

mkpipe_extractor_postgres-0.1.0/mkpipe_extractor_postgres/extractor.py

@@ -0,0 +1,4 @@
+
+class PostgresExtractor:
+    def extract(self):
+        print("Extracting data from PostgreSQL...")

mkpipe_extractor_postgres-0.1.0/mkpipe_extractor_postgres/jar_paths.py

@@ -0,0 +1,18 @@
+import pkg_resources
+import os
+
+def get_jar_path(filename):
+    package_name = "mkpipe-core"  # Adjust according to where the jars are located
+    try:
+        jar_path = pkg_resources.resource_filename(package_name, f'jars/{filename}')
+        if os.path.exists(jar_path):
+            return jar_path
+        else:
+            print(f"JAR file {filename} not found in package")
+    except Exception as e:
+        print(f"Error accessing JAR files: {e}")
+    return None
+
+jar_path = get_jar_path("com.postgresql_driver.jar")
+if jar_path:
+    print(f"Found JAR at {jar_path}")

mkpipe_extractor_postgres-0.1.0/mkpipe_extractor_postgres.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.1
+Name: mkpipe-extractor-postgres
+Version: 0.1.0
+Summary: PostgreSQL extractor for mkpipe.
+Author: Metin Karakus
+Author-email: metin_karakus@yahoo.com
+License: Apache License 2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.8

mkpipe_extractor_postgres-0.1.0/mkpipe_extractor_postgres.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+MANIFEST.in
+README.md
+setup.py
+mkpipe_extractor_postgres/__init__.py
+mkpipe_extractor_postgres/extractor.py
+mkpipe_extractor_postgres/jar_paths.py
+mkpipe_extractor_postgres.egg-info/PKG-INFO
+mkpipe_extractor_postgres.egg-info/SOURCES.txt
+mkpipe_extractor_postgres.egg-info/dependency_links.txt
+mkpipe_extractor_postgres.egg-info/entry_points.txt
+mkpipe_extractor_postgres.egg-info/top_level.txt

mkpipe_extractor_postgres-0.1.0/mkpipe_extractor_postgres.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

mkpipe_extractor_postgres-0.1.0/mkpipe_extractor_postgres.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[mkpipe.extractors]
+postgres = mkpipe_extractor_postgres.extractor:PostgresExtractor

mkpipe_extractor_postgres-0.1.0/mkpipe_extractor_postgres.egg-info/top_level.txt

@@ -0,0 +1 @@
+mkpipe_extractor_postgres

mkpipe_extractor_postgres-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mkpipe_extractor_postgres-0.1.0/setup.py

@@ -0,0 +1,24 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="mkpipe-extractor-postgres",
+    version="0.1.0",
+    license="Apache License 2.0",
+    packages=find_packages(),
+    install_requires=[],
+    include_package_data=True,
+    entry_points={
+        "mkpipe.extractors": [
+            "postgres = mkpipe_extractor_postgres.extractor:PostgresExtractor",
+        ],
+    },
+    description="PostgreSQL extractor for mkpipe.",
+    author="Metin Karakus",
+    author_email="metin_karakus@yahoo.com",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache Software License",
+    ],
+    python_requires=">=3.8",
+)
+