data-processing-pipeline 0.1.0__tar.gz

data_processing_pipeline-0.1.0/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: data-processing-pipeline
+Version: 0.1.0
+Summary: Data processing pipeline
+Author: Mikolaj Kawaler

data_processing_pipeline-0.1.0/README.md

@@ -0,0 +1 @@
+# Data-Processing-Pipeline-Test

data_processing_pipeline-0.1.0/data_processing_pipeline.egg-info/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: data-processing-pipeline
+Version: 0.1.0
+Summary: Data processing pipeline
+Author: Mikolaj Kawaler

data_processing_pipeline-0.1.0/data_processing_pipeline.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+data_processing_pipeline.egg-info/PKG-INFO
+data_processing_pipeline.egg-info/SOURCES.txt
+data_processing_pipeline.egg-info/dependency_links.txt
+data_processing_pipeline.egg-info/top_level.txt
+pipeline/__init__.py
+pipeline/config.py
+pipeline/db.py
+pipeline/main.py
+pipeline/monitoring.py
+pipeline/processing.py
+tests/test_monitoring.py
+tests/test_processing.py

data_processing_pipeline-0.1.0/data_processing_pipeline.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data_processing_pipeline-0.1.0/data_processing_pipeline.egg-info/top_level.txt

@@ -0,0 +1 @@
+pipeline

data_processing_pipeline-0.1.0/pipeline/config.py

@@ -0,0 +1,8 @@
+import logging
+
+def setup_logging():
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
+    )
+    return logging.getLogger("data_pipeline")

data_processing_pipeline-0.1.0/pipeline/db.py

@@ -0,0 +1,39 @@
+import sqlite3
+from pathlib import Path
+
+DB_PATH = Path(__file__).parent / "example.db"
+
+
+def init_db():
+    conn = sqlite3.connect(DB_PATH)
+    cur = conn.cursor()
+
+    # Tabela zamówień
+    cur.execute(
+        """
+        CREATE TABLE IF NOT EXISTS orders (
+            id INTEGER PRIMARY KEY,
+            customer TEXT NOT NULL,
+            quantity INTEGER NOT NULL,
+            price_per_unit REAL NOT NULL
+        )
+        """
+    )
+
+    # Proste dane przykładowe
+    cur.execute("DELETE FROM orders")
+    cur.executemany(
+        "INSERT INTO orders (customer, quantity, price_per_unit) VALUES (?, ?, ?)",
+        [
+            ("Alice", 10, 5.0),
+            ("Bob", 3, 12.5),
+            ("Charlie", 0, 7.0),   # quantity 0 – przyda się do walidacji
+        ],
+    )
+
+    conn.commit()
+    conn.close()
+
+
+def get_connection():
+    return sqlite3.connect(DB_PATH)

data_processing_pipeline-0.1.0/pipeline/main.py

@@ -0,0 +1,180 @@
+from pathlib import Path
+import argparse
+import json
+from .config import setup_logging
+from .db import init_db, get_connection
+from .processing import (
+    load_orders,
+    validate_orders,
+    deduplicate_orders,
+    transform_orders,
+    save_to_csv,
+    save_to_json,
+    save_to_parquet,
+)
+from .monitoring import start_monitoring, end_monitoring
+
+OUTPUT_PATH = Path(__file__).parent / "orders_processed.csv"
+
+
+def run_pipeline(
+    output_path: str = None,
+    output_format: str = "csv",
+    dry_run: bool = False,
+    enable_dedup: bool = True,
+):
+    """Run the data processing pipeline."""
+    logger = setup_logging()
+    logger.info("Starting pipeline (dry_run=%s)", dry_run)
+
+    # Inicjalizacja bazy i danych
+    init_db()
+    conn = get_connection()
+
+    # Monitoring start
+    start_time = start_monitoring()
+
+    try:
+        # ETL
+        df_raw = load_orders(conn)
+        df_valid = validate_orders(df_raw)
+        
+        # Deduplikacja (opcjonalna)
+        if enable_dedup:
+            df_valid = deduplicate_orders(df_valid)
+        
+        df_transformed = transform_orders(df_valid)
+
+        # Zapis (jeśli nie dry-run)
+        if not dry_run:
+            output_file = output_path or str(OUTPUT_PATH)
+            
+            if output_format == "csv":
+                save_to_csv(df_transformed, output_file)
+            elif output_format == "json":
+                save_to_json(df_transformed, output_file)
+            elif output_format == "parquet":
+                save_to_parquet(df_transformed, output_file)
+            else:
+                logger.error("Unknown format: %s", output_format)
+                raise ValueError(f"Unsupported format: {output_format}")
+        else:
+            logger.info("[DRY RUN] Would save %d records to %s", len(df_transformed), output_path or OUTPUT_PATH)
+
+        # Monitoring end
+        metrics = end_monitoring(
+            start_time=start_time,
+            records_in=len(df_raw),
+            records_out=len(df_transformed),
+        )
+
+        logger.info(
+            "Pipeline finished in %.3f s (records in: %d, records out: %d)",
+            metrics.duration,
+            metrics.records_in,
+            metrics.records_out,
+        )
+        
+        return metrics
+
+    finally:
+        conn.close()
+
+
+def run_batch_pipeline(config_file: str, dry_run: bool = False):
+    """Run pipeline on multiple configs from JSON file."""
+    logger = setup_logging()
+    logger.info("Starting batch pipeline from config: %s", config_file)
+
+    with open(config_file, "r") as f:
+        configs = json.load(f)
+
+    results = []
+    for idx, config in enumerate(configs, 1):
+        logger.info("Processing batch %d/%d", idx, len(configs))
+        output_format = config.get("format", "csv")
+        output_path = config.get("output")
+        enable_dedup = config.get("deduplicate", True)
+        
+        try:
+            metrics = run_pipeline(
+                output_path=output_path,
+                output_format=output_format,
+                dry_run=dry_run,
+                enable_dedup=enable_dedup,
+            )
+            results.append({"status": "success", "config": config, "metrics": metrics.__dict__})
+        except Exception as e:
+            logger.error("Batch %d failed: %s", idx, str(e))
+            results.append({"status": "error", "config": config, "error": str(e)})
+
+    return results
+
+
+def main():
+    """CLI entry point for the pipeline."""
+    parser = argparse.ArgumentParser(
+        description="Data processing pipeline with advanced transformations"
+    )
+    
+    subparsers = parser.add_subparsers(dest="command", help="Command to run")
+    
+    # Single pipeline command
+    single_parser = subparsers.add_parser("run", help="Run single pipeline")
+    single_parser.add_argument(
+        "-o", "--output",
+        type=str,
+        default=None,
+        help="Output file path (default: pipeline/orders_processed.csv)",
+    )
+    single_parser.add_argument(
+        "-f", "--format",
+        type=str,
+        choices=["csv", "json", "parquet"],
+        default="csv",
+        help="Output format (default: csv)",
+    )
+    single_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Preview without saving",
+    )
+    single_parser.add_argument(
+        "--no-dedup",
+        action="store_true",
+        help="Disable deduplication",
+    )
+    
+    # Batch pipeline command
+    batch_parser = subparsers.add_parser("batch", help="Run batch pipeline from config")
+    batch_parser.add_argument(
+        "config",
+        type=str,
+        help="JSON config file with batch configurations",
+    )
+    batch_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Preview without saving",
+    )
+    
+    args = parser.parse_args()
+    
+    if args.command == "run" or args.command is None:
+        run_pipeline(
+            output_path=args.output if hasattr(args, 'output') else None,
+            output_format=args.format if hasattr(args, 'format') else "csv",
+            dry_run=args.dry_run if hasattr(args, 'dry_run') else False,
+            enable_dedup=not (args.no_dedup if hasattr(args, 'no_dedup') else False),
+        )
+    elif args.command == "batch":
+        run_batch_pipeline(
+            config_file=args.config,
+            dry_run=args.dry_run,
+        )
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

data_processing_pipeline-0.1.0/pipeline/monitoring.py

@@ -0,0 +1,27 @@
+import time
+from dataclasses import dataclass
+
+@dataclass
+class RunMetrics:
+    start_time: float
+    end_time: float
+    records_in: int
+    records_out: int
+
+    @property
+    def duration(self) -> float:
+        return self.end_time - self.start_time
+
+
+def start_monitoring():
+    return time.time()
+
+
+def end_monitoring(start_time: float, records_in: int, records_out: int) -> RunMetrics:
+    end_time = time.time()
+    return RunMetrics(
+        start_time=start_time,
+        end_time=end_time,
+        records_in=records_in,
+        records_out=records_out,
+    )

data_processing_pipeline-0.1.0/pipeline/processing.py

@@ -0,0 +1,144 @@
+import pandas as pd
+import logging
+from functools import wraps
+from time import sleep
+from typing import Callable
+
+logger = logging.getLogger("data_pipeline")
+
+
+def retry_on_error(max_retries: int = 3, delay: float = 1.0):
+    """Decorator to retry a function on exception."""
+    def decorator(func: Callable):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            for attempt in range(max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    if attempt < max_retries - 1:
+                        logger.warning(
+                            "Attempt %d/%d failed: %s. Retrying in %.1fs...",
+                            attempt + 1, max_retries, str(e), delay
+                        )
+                        sleep(delay)
+                    else:
+                        logger.error("All %d attempts failed for %s", max_retries, func.__name__)
+                        raise
+        return wrapper
+    return decorator
+
+
+def load_orders(conn) -> pd.DataFrame:
+    logger.info("Loading orders from database")
+    df = pd.read_sql_query("SELECT * FROM orders", conn)
+    logger.info("Loaded %d records", len(df))
+    return df
+
+
+def validate_orders(df: pd.DataFrame) -> pd.DataFrame:
+    logger.info("Validating orders")
+
+    # Sprawdzenie braków
+    missing = df.isna().sum().sum()
+    if missing > 0:
+        logger.warning("Found %d missing values", missing)
+
+    # Sprawdzenie typów (proste)
+    assert "quantity" in df.columns
+    assert "price_per_unit" in df.columns
+
+    # Zakresy – np. quantity > 0
+    invalid_quantity = (df["quantity"] <= 0).sum()
+    if invalid_quantity > 0:
+        logger.warning("Found %d orders with non-positive quantity", invalid_quantity)
+
+    # Filtrowanie niepoprawnych rekordów
+    df_valid = df[df["quantity"] > 0].copy()
+    logger.info("After validation: %d valid records", len(df_valid))
+    return df_valid
+
+
+def deduplicate_orders(df: pd.DataFrame) -> pd.DataFrame:
+    """Remove duplicate orders based on all columns."""
+    logger.info("Deduplicating orders")
+    initial_count = len(df)
+    
+    # Usuń dokładne duplikaty
+    df_dedup = df.drop_duplicates().copy()
+    
+    # Opcjonalnie: usuń duplikaty bazując na customer i quantity
+    # df_dedup = df.drop_duplicates(subset=['customer', 'quantity'], keep='first')
+    
+    removed = initial_count - len(df_dedup)
+    if removed > 0:
+        logger.warning("Removed %d duplicate records", removed)
+    
+    logger.info("After deduplication: %d records", len(df_dedup))
+    return df_dedup
+
+
+def transform_orders(df: pd.DataFrame) -> pd.DataFrame:
+    logger.info("Transforming orders")
+    
+    # Feature engineering: total_value
+    df["total_value"] = df["quantity"] * df["price_per_unit"]
+    
+    # Normalizacja – standaryzacja total_value
+    total_mean = df["total_value"].mean()
+    total_std = df["total_value"].std()
+    if total_std > 0:
+        df["total_value_norm"] = (df["total_value"] - total_mean) / total_std
+    else:
+        df["total_value_norm"] = 0
+    
+    # Zaawansowane transformacje
+    # 1. Percentile bucketing
+    df["value_percentile"] = pd.qcut(df["total_value"], q=4, labels=["Q1", "Q2", "Q3", "Q4"], duplicates="drop")
+    
+    # 2. Kwantyle dla ilości
+    df["quantity_category"] = pd.cut(df["quantity"], bins=[0, 5, 15, float('inf')], labels=["small", "medium", "large"], include_lowest=True)
+    
+    # 3. Average price normalizacja
+    if "price_per_unit" in df.columns:
+        price_mean = df["price_per_unit"].mean()
+        df["price_norm"] = df["price_per_unit"] / price_mean if price_mean > 0 else df["price_per_unit"]
+    
+    # 4. Log transform dla total_value (aby uniknąć log(0), dodajemy 1)
+    df["total_value_log"] = df["total_value"].apply(lambda x: __import__('math').log(x + 1) if x >= 0 else 0)
+    
+    # 5. Customer-level aggregations (jeśli mamy customer info)
+    if "customer" in df.columns:
+        customer_stats = df.groupby("customer").agg({
+            "total_value": ["count", "sum", "mean"]
+        }).reset_index()
+        customer_stats.columns = ["customer", "customer_order_count", "customer_total_spend", "customer_avg_spend"]
+        df = df.merge(customer_stats, on="customer", how="left")
+    
+    logger.info("Transformation complete with %d features", len(df.columns))
+    return df
+
+
+@retry_on_error(max_retries=3, delay=1.0)
+def save_to_csv(df: pd.DataFrame, path: str):
+    logger.info("Saving result to %s", path)
+    df.to_csv(path, index=False)
+    logger.info("Saved %d records", len(df))
+
+
+def save_to_json(df: pd.DataFrame, path: str):
+    """Save dataframe to JSON format."""
+    logger.info("Saving result to JSON: %s", path)
+    df.to_json(path, orient="records", indent=2)
+    logger.info("Saved %d records to JSON", len(df))
+
+
+def save_to_parquet(df: pd.DataFrame, path: str):
+    """Save dataframe to Parquet format."""
+    logger.info("Saving result to Parquet: %s", path)
+    try:
+        df.to_parquet(path, index=False)
+        logger.info("Saved %d records to Parquet", len(df))
+    except ImportError:
+        logger.error("Parquet format requires 'pyarrow' or 'fastparquet' library")
+        raise

data_processing_pipeline-0.1.0/pyproject.toml

@@ -0,0 +1,12 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "data-processing-pipeline"
+version = "0.1.0"
+description = "Data processing pipeline"
+authors = [
+    { name="Mikolaj Kawaler" }
+]
+dependencies = []

data_processing_pipeline-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

data_processing_pipeline-0.1.0/tests/test_monitoring.py

@@ -0,0 +1,10 @@
+from pipeline.monitoring import start_monitoring, end_monitoring
+
+
+def test_monitoring_metrics():
+    start = start_monitoring()
+    metrics = end_monitoring(start, records_in=10, records_out=8)
+
+    assert metrics.records_in == 10
+    assert metrics.records_out == 8
+    assert metrics.duration >= 0

data_processing_pipeline-0.1.0/tests/test_processing.py

@@ -0,0 +1,32 @@
+import pandas as pd
+from pipeline.processing import validate_orders, transform_orders
+
+
+def test_validate_orders_filters_non_positive_quantity():
+    df = pd.DataFrame(
+        {
+            "id": [1, 2],
+            "customer": ["A", "B"],
+            "quantity": [10, 0],
+            "price_per_unit": [5.0, 7.0],
+        }
+    )
+
+    df_valid = validate_orders(df)
+    assert len(df_valid) == 1
+    assert (df_valid["quantity"] > 0).all()
+
+
+def test_transform_orders_creates_features():
+    df = pd.DataFrame(
+        {
+            "id": [1, 2],
+            "customer": ["A", "B"],
+            "quantity": [10, 3],
+            "price_per_unit": [5.0, 12.5],
+        }
+    )
+
+    df_transformed = transform_orders(df)
+    assert "total_value" in df_transformed.columns
+    assert "total_value_norm" in df_transformed.columns