ingestflow-sdk 1.0.9__tar.gz

ingestflow_sdk-1.0.9/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: ingestflow-sdk
+Version: 1.0.9
+Summary: A Python SDK for data ingestion and tracking
+Author: InduPrakash
+Requires-Python: >=3.10
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: psycopg2-binary>=2.9.9
+Requires-Dist: python-dotenv>=1.0.1
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

ingestflow_sdk-1.0.9/ingestflow/__init__.py

@@ -0,0 +1,23 @@
+from ingestflow.database.connection import DatabaseConnection
+from ingestflow.trackers.metadata_manager import MetadataManager
+from ingestflow.logging.logger import Logger
+from ingestflow.trackers.file_tracker import FileTracker
+from ingestflow.readers.csv_reader import CsvReader
+from ingestflow.readers.json_reader import JsonReader
+from ingestflow.utils.run_id_generator import RunIdGenerator
+from ingestflow.trackers.record_tracker import RecordTracker
+from ingestflow.writers.postgres_writer import PostgresWriter
+from ingestflow.validators.schema_validator import SchemaValidator
+
+__all__ = [
+    "DatabaseConnection",
+    "MetadataManager",
+    "FileTracker",
+    "CsvReader",
+    "JsonReader",
+    "RunIdGenerator",
+    "Logger",
+    "RecordTracker",
+    "PostgresWriter",
+    "SchemaValidator",
+]

ingestflow_sdk-1.0.9/ingestflow_sdk.egg-info/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: ingestflow-sdk
+Version: 1.0.9
+Summary: A Python SDK for data ingestion and tracking
+Author: InduPrakash
+Requires-Python: >=3.10
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: psycopg2-binary>=2.9.9
+Requires-Dist: python-dotenv>=1.0.1
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

ingestflow_sdk-1.0.9/ingestflow_sdk.egg-info/SOURCES.txt

@@ -0,0 +1,22 @@
+pyproject.toml
+setup.py
+ingestflow/__init__.py
+ingestflow_sdk.egg-info/PKG-INFO
+ingestflow_sdk.egg-info/SOURCES.txt
+ingestflow_sdk.egg-info/dependency_links.txt
+ingestflow_sdk.egg-info/requires.txt
+ingestflow_sdk.egg-info/top_level.txt
+tests/test_csv_reader.py
+tests/test_data_validator.py
+tests/test_database_connection.py
+tests/test_duplicate_file.py
+tests/test_file_tracker.py
+tests/test_incremental_load.py
+tests/test_json_reader.py
+tests/test_logger.py
+tests/test_merge_load.py
+tests/test_metadata_manager.py
+tests/test_postgres_writer.py
+tests/test_reject_file.py
+tests/test_run_id_generator.py
+tests/test_schema_validator.py

ingestflow_sdk-1.0.9/ingestflow_sdk.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ingestflow_sdk-1.0.9/ingestflow_sdk.egg-info/requires.txt

@@ -0,0 +1,3 @@
+pandas>=2.2.0
+psycopg2-binary>=2.9.9
+python-dotenv>=1.0.1

ingestflow_sdk-1.0.9/ingestflow_sdk.egg-info/top_level.txt

@@ -0,0 +1 @@
+ingestflow

ingestflow_sdk-1.0.9/pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=69.0",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

ingestflow_sdk-1.0.9/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ingestflow_sdk-1.0.9/setup.py

@@ -0,0 +1,24 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="ingestflow-sdk",
+    #same name should be given in PyPi account
+    version="1.0.9",
+    description="A Python SDK for data ingestion and tracking",  
+    author="InduPrakash",  
+    packages=find_packages(),       
+    
+    # Minimal Python version
+    python_requires=">=3.10",
+    
+    # Runtime dependencies
+    install_requires=[
+        "pandas>=2.2.0",
+        "psycopg2-binary>=2.9.9",
+        "python-dotenv>=1.0.1"
+    ],
+
+    # If you want to include non-python files (README, LICENSE, etc.)
+    include_package_data=True,
+    
+)

ingestflow_sdk-1.0.9/tests/test_csv_reader.py

@@ -0,0 +1,11 @@
+from ingestflow.readers.csv_reader import CsvReader
+
+def test_csv_reader():
+    df = CsvReader.read(
+        "tests/source_file/employees.csv"
+    )
+    print(df)
+
+if __name__ == "__main__":
+
+    test_csv_reader()

ingestflow_sdk-1.0.9/tests/test_data_validator.py

@@ -0,0 +1,34 @@
+import pandas as pd
+from ingestflow.validators.data_validator import DataValidator
+
+def test_data_validator():
+    df = pd.DataFrame(
+        {
+            "id": [1, None, 3, 4, None],
+            "name": [
+                "John",
+                None,
+                "Bob",
+                None,
+                "Alice"
+            ]
+        }
+    )
+    valid_df, invalid_df = (
+        DataValidator.validate_dataframe(df)
+    )
+    print("\nVALID DATA")
+    print(valid_df)
+    print("\nINVALID DATA")
+    print(invalid_df)
+
+    reject_file = (
+        DataValidator.generate_reject_file(
+            invalid_df,
+            "RUN_20260903_001"
+        )
+    )
+    print(reject_file)
+
+if __name__ == "__main__":
+    test_data_validator()

ingestflow_sdk-1.0.9/tests/test_database_connection.py

@@ -0,0 +1,10 @@
+from ingestflow.database.connection import DatabaseConnection
+
+
+def test_database_connection():
+    db = DatabaseConnection()
+    db.test_connection()
+
+if __name__ == "__main__":
+
+    test_database_connection()

ingestflow_sdk-1.0.9/tests/test_duplicate_file.py

@@ -0,0 +1,35 @@
+# from ingestflow.trackers.file_tracker import FileTracker
+
+# tracker = FileTracker()
+# file_hash = (
+#     tracker.get_file_hash(
+#         "tests/source_file/employees.csv"
+#     )
+# )
+# is_duplicate = (
+#     tracker.is_duplicate_file(
+#         file_hash
+#     )
+# )
+# print(
+#     f"Duplicate File : "
+#     f"{is_duplicate}"
+# )
+
+from ingestflow.trackers.file_tracker import FileTracker
+from ingestflow.trackers.metadata_manager import MetadataManager
+
+
+def test_duplicate_file():
+    metadata = MetadataManager()
+    metadata.create_metadata_tables()
+    tracker = FileTracker()
+    file_hash = tracker.get_file_hash(
+        "tests/source_file/employees.csv"
+    )
+
+    result = tracker.is_duplicate_file(
+        file_hash
+    )
+
+    assert result in [True, False]

ingestflow_sdk-1.0.9/tests/test_file_tracker.py

@@ -0,0 +1,13 @@
+from ingestflow.trackers.file_tracker import FileTracker
+
+def test_file_hash():
+    tracker = FileTracker()
+    file_hash = tracker.generate_file_hash(
+        "tests/source_file/employees.csv"
+    )
+    print(
+        f"File Hash : {file_hash}"
+    )
+
+if __name__ == "__main__":
+    test_file_hash()

ingestflow_sdk-1.0.9/tests/test_incremental_load.py

@@ -0,0 +1,54 @@
+# import pandas as pd
+
+# from ingestflow.trackers.record_tracker import RecordTracker
+
+
+# def test_incremental_load():
+
+#     source_df = pd.DataFrame(
+#         {
+#             "id": [1, 2, 3],
+#             "updated_date": [
+#                 "2026-01-01",
+#                 "2026-02-01",
+#                 "2026-03-01"
+#             ]
+#         }
+#     )
+
+#     tracker = RecordTracker()
+
+#     incremental_df = (
+#         tracker.get_incremental_records(
+#             source_df,
+#             "employees",
+#             "updated_date"
+#         )
+#     )
+
+#     assert incremental_df is not None
+
+import pandas as pd
+
+from ingestflow.trackers.record_tracker import RecordTracker
+
+
+def test_incremental_load():
+
+    source_df = pd.DataFrame(
+        {
+            "id": [1, 2, 3, 4, 5]
+        }
+    )
+
+    tracker = RecordTracker()
+
+    incremental_df = (
+        tracker.get_incremental_records(
+            source_df,
+            "employees",
+            "id"
+        )
+    )
+
+    assert incremental_df is not None

ingestflow_sdk-1.0.9/tests/test_json_reader.py

@@ -0,0 +1,15 @@
+from ingestflow.readers.json_reader import JsonReader
+
+
+def test_json_reader():
+
+    df = JsonReader.read(
+        "tests/source_file/employees.json"
+    )
+
+    print(df)
+
+
+if __name__ == "__main__":
+
+    test_json_reader()

ingestflow_sdk-1.0.9/tests/test_logger.py

@@ -0,0 +1,12 @@
+from ingestflow.logging.logger import Logger
+
+summary = {
+    "Run ID": "RUN_20260603_001",
+    "File Name": "employees.csv",
+    "Load Type": "merge",
+    "Inserted Records": 10,
+    "Updated Records": 2,
+    "Status": "SUCCESS"
+}
+
+Logger.print_summary(summary)

ingestflow_sdk-1.0.9/tests/test_merge_load.py

@@ -0,0 +1,40 @@
+import pandas as pd
+
+from ingestflow.trackers.record_tracker import RecordTracker
+
+
+def test_merge_load():
+
+    source_df = pd.DataFrame(
+        {
+            "id": [
+                1,
+                2,
+                3
+            ],
+            "name": [
+                "John Updated",
+                "Alice",
+                "Bob"
+            ]
+        }
+    )
+
+    tracker = RecordTracker()
+
+    insert_df, update_df = (
+        tracker.identify_merge_records(
+            source_df,
+            "employees",
+            ["id"]
+        )
+    )
+
+    print("\nINSERT RECORDS")
+    print(insert_df)
+
+    print("\nUPDATE RECORDS")
+    print(update_df)
+
+    assert insert_df is not None
+    assert update_df is not None

ingestflow_sdk-1.0.9/tests/test_metadata_manager.py

@@ -0,0 +1,13 @@
+from ingestflow.trackers.metadata_manager import MetadataManager
+
+
+def test_metadata_tables():
+    metadata = MetadataManager()
+    metadata.create_metadata_tables()
+    print(
+        "Metadata tables created successfully"
+    )
+
+if __name__ == "__main__":
+
+    test_metadata_tables()

ingestflow_sdk-1.0.9/tests/test_postgres_writer.py

@@ -0,0 +1,20 @@
+import pandas as pd
+from ingestflow.writers.postgres_writer import PostgresWriter
+
+def test_postgres_writer():
+    df = pd.DataFrame(
+        {
+            "id": [1, 2],
+            "name": ["John", "Alice"]
+        }
+    )
+    writer = PostgresWriter()
+    writer.create_table_if_not_exists(
+        "employees",
+        df
+    )
+    inserted = writer.insert_records(
+        "employees",
+        df
+    )
+    assert inserted >= 0

ingestflow_sdk-1.0.9/tests/test_reject_file.py

@@ -0,0 +1,23 @@
+import pandas as pd
+from ingestflow.validators.data_validator import DataValidator
+
+def test_reject_file():
+    invalid_df = pd.DataFrame(
+        {
+            "id": [2],
+            "name": ["Alice"],
+            "reject_reason": [
+                "Sample Reject"
+            ]
+        }
+    )
+    reject_file = (
+        DataValidator.generate_reject_file(
+            invalid_df,
+            "RUN_20260603_001"
+        )
+    )
+    print(reject_file)
+
+if __name__ == "__main__":
+    test_reject_file()

ingestflow_sdk-1.0.9/tests/test_run_id_generator.py

@@ -0,0 +1,18 @@
+from ingestflow.utils.run_id_generator import RunIdGenerator
+
+def test_run_id():
+
+    print(
+        RunIdGenerator.generate()
+    )
+
+    print(
+        RunIdGenerator.generate()
+    )
+
+    print(
+        RunIdGenerator.generate()
+    )
+
+if __name__ == "__main__":
+    test_run_id()

ingestflow_sdk-1.0.9/tests/test_schema_validator.py

@@ -0,0 +1,20 @@
+import pandas as pd
+from ingestflow.validators.schema_validator import SchemaValidator
+
+def test_schema_validator():
+    df = pd.DataFrame(
+        {
+            "id": [1, 2],
+            "name": ["John", "Alice"]
+        }
+    )
+    validator = SchemaValidator()
+    result = validator.validate_schema(
+        dataframe=df,
+        table_name="employees",
+        schema_mode="fail"
+    )
+    print(result)
+
+if __name__ == "__main__":
+    test_schema_validator()