dlt-utils 0.1.0__py3-none-any.whl

dlt_utils/__init__.py

@@ -0,0 +1,32 @@
+"""
+dlt_utils: Shared utilities for dlt data pipelines with multi-company support.
+
+This package provides common utilities for building dlt pipelines that work with
+multiple companies/tenants, including:
+
+- PartitionedIncremental: State tracking per partition key
+- Date utilities: Generate (year, week) and (year, month) sequences
+- Schema utilities: Ensure database tables exist
+"""
+
+from .incremental import PartitionedIncremental
+from .dates import generate_year_weeks, generate_year_months
+from .schema import (
+    ensure_all_tables_exist,
+    ensure_tables_for_resources,
+    get_tables_for_resources,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Incremental
+    "PartitionedIncremental",
+    # Dates
+    "generate_year_weeks",
+    "generate_year_months",
+    # Schema
+    "ensure_all_tables_exist",
+    "ensure_tables_for_resources",
+    "get_tables_for_resources",
+]

dlt_utils/dates.py

@@ -0,0 +1,160 @@
+"""
+Utility functions for date-based partitioning in dlt data extraction.
+
+This module provides helper functions to generate sequences of (year, week) and
+(year, month) tuples for partitioning data extraction across time periods.
+These are used by resources that require parametric date-based queries.
+"""
+
+from datetime import date, datetime, timedelta
+from typing import List, Optional, Tuple
+
+
+def generate_year_weeks(
+    start_year: Optional[int] = None,
+    start_week: Optional[int] = 1,
+    end_year: Optional[int] = None,
+    end_week: Optional[int] = None,
+    years_back: int = 3,
+    weeks_forward: int = 52,
+) -> List[Tuple[int, int]]:
+    """
+    Generate a list of (year, week) tuples using ISO week numbers.
+
+    Uses ISO 8601 week numbers where weeks start on Monday and the first
+    week of the year contains at least 4 days in that year.
+
+    Args:
+        start_year: Start year. Defaults to `years_back` years ago.
+        start_week: Start ISO week number 1-53. Defaults to 1.
+        end_year: End year. Defaults to the year of calculated end date.
+        end_week: End ISO week number 1-53. Defaults to current + `weeks_forward`.
+        years_back: Years to go back for default start_year. Defaults to 3.
+        weeks_forward: Weeks to add for default end calculation. Defaults to 52.
+
+    Returns:
+        List of (year, week) tuples in chronological order.
+
+    Examples:
+        >>> # Specific range
+        >>> generate_year_weeks(2024, 1, 2024, 4)
+        [(2024, 1), (2024, 2), (2024, 3), (2024, 4)]
+
+        >>> # Default: 3 years back to 52 weeks forward
+        >>> weeks = generate_year_weeks()
+        >>> weeks[0]  # First week
+        (2022, 1)
+
+        >>> # Custom defaults
+        >>> generate_year_weeks(years_back=1, weeks_forward=4)
+        # From 1 year ago to 4 weeks from now
+    """
+    today = date.today()
+
+    # Default start_year: years_back years ago
+    if start_year is None:
+        start_year = today.year - years_back
+
+    # Default end: current date + weeks_forward
+    if end_year is None or end_week is None:
+        future_date = today + timedelta(weeks=weeks_forward)
+        iso_cal = future_date.isocalendar()
+        end_year = end_year or iso_cal.year
+        end_week = end_week or iso_cal.week
+
+    # Start from start_week of start_year
+    start_date = datetime.strptime(f"{start_year}-W{start_week:02d}-1", "%G-W%V-%u").date()
+    end_date = datetime.strptime(f"{end_year}-W{end_week:02d}-1", "%G-W%V-%u").date()
+
+    year_weeks = []
+    current = start_date
+    while current <= end_date:
+        iso_cal = current.isocalendar()
+        year_weeks.append((iso_cal.year, iso_cal.week))
+        current += timedelta(weeks=1)
+
+    return year_weeks
+
+
+def generate_year_months(
+    start_year: Optional[int] = None,
+    start_month: Optional[int] = 1,
+    end_year: Optional[int] = None,
+    end_month: Optional[int] = None,
+    years_back: int = 3,
+    months_forward: int = 1,
+) -> List[Tuple[int, int]]:
+    """
+    Generate a list of (year, month) tuples from start to end.
+
+    Args:
+        start_year: Start year. Defaults to `years_back` years ago.
+        start_month: Start month 1-12. Defaults to 1 (January).
+        end_year: End year. Defaults to the year of calculated end date.
+        end_month: End month 1-12. Defaults to current + `months_forward`.
+        years_back: Years to go back for default start_year. Defaults to 3.
+        months_forward: Months to add for default end calculation. Defaults to 1.
+
+    Returns:
+        List of (year, month) tuples in chronological order.
+
+    Examples:
+        >>> generate_year_months(2024, 10, 2025, 2)
+        [(2024, 10), (2024, 11), (2024, 12), (2025, 1), (2025, 2)]
+
+        >>> # Default: 3 years back to 1 month forward
+        >>> months = generate_year_months()
+        >>> months[0]
+        (2022, 1)
+    """
+    today = date.today()
+
+    # Default start_year: years_back years ago
+    if start_year is None:
+        start_year = today.year - years_back
+
+    # Default end: current date + months_forward
+    if end_year is None or end_month is None:
+        # Calculate future date by adding months
+        future_month = today.month + months_forward
+        future_year = today.year
+        while future_month > 12:
+            future_month -= 12
+            future_year += 1
+        end_year = end_year or future_year
+        end_month = end_month or future_month
+
+    year_months = []
+    current_year = start_year
+    current_month = start_month
+
+    while (current_year, current_month) <= (end_year, end_month):
+        year_months.append((current_year, current_month))
+        current_month += 1
+        if current_month > 12:
+            current_month = 1
+            current_year += 1
+
+    return year_months
+
+
+def get_current_iso_week() -> Tuple[int, int]:
+    """
+    Get the current ISO year and week number.
+
+    Returns:
+        Tuple of (year, week) for today.
+    """
+    iso_cal = date.today().isocalendar()
+    return (iso_cal.year, iso_cal.week)
+
+
+def get_current_year_month() -> Tuple[int, int]:
+    """
+    Get the current year and month.
+
+    Returns:
+        Tuple of (year, month) for today.
+    """
+    today = date.today()
+    return (today.year, today.month)

dlt_utils/incremental.py

@@ -0,0 +1,161 @@
+"""
+Incremental state tracking voor dlt resources met partitionering.
+
+Dit module biedt een PartitionedIncremental class die werkt als dlt.sources.incremental,
+maar state per partition key (bijv. company_id) bijhoudt in plaats van per resource.
+"""
+
+from typing import Any, Callable, Dict, List, TypeVar
+
+TCursorValue = TypeVar("TCursorValue")
+
+
+class PartitionedIncremental:
+    """
+    Incremental state tracking per partition key (bijv. company_id).
+
+    Werkt als dlt.sources.incremental, maar partitioneert state per key zodat
+    meerdere companies onafhankelijke cursors kunnen hebben binnen dezelfde resource.
+
+    Voorbeeld:
+        ```python
+        @dlt.resource
+        def sync_resource():
+            state = dlt.current.resource_state()
+            inc = PartitionedIncremental(
+                state=state,
+                state_key="sequences",
+                cursor_path="sequenceNumber",
+                initial_value=0,
+            )
+
+            for company_id in ["company_a", "company_b"]:
+                start_seq = inc.get_last_value(company_id)
+                for record in fetch_data(company_id, since=start_seq):
+                    inc.track(company_id, record["sequenceNumber"])
+                    yield record
+        ```
+
+    State structuur in dlt:
+        ```json
+        {
+            "sequences": {
+                "company_a": 12345,
+                "company_b": 67890
+            }
+        }
+        ```
+
+    Use cases:
+        - sequenceNumber tracking per company (Floriday sync endpoints)
+        - last_modified_at per company + period (Easyflex journaalposten)
+        - Any cursor that needs to be tracked per partition
+    """
+
+    def __init__(
+        self,
+        state: Dict[str, Any],
+        state_key: str,
+        cursor_path: str = None,
+        initial_value: TCursorValue = None,
+        last_value_func: Callable[[List[TCursorValue]], TCursorValue] = max,
+    ):
+        """
+        Initialiseer PartitionedIncremental.
+
+        Args:
+            state: Resource state dict van dlt.current.resource_state().
+            state_key: Sleutel in state dict waar partition values worden opgeslagen.
+            cursor_path: Optioneel pad naar cursor veld in records (voor track_record).
+            initial_value: Waarde voor partitions zonder bestaande state.
+            last_value_func: Functie om "laatste" waarde te bepalen (default: max).
+                             Gebruik min() voor aflopende cursors.
+        """
+        self._state = state
+        self._state_key = state_key
+        self._cursor_path = cursor_path
+        self._initial_value = initial_value
+        self._last_value_func = last_value_func
+
+        # Zorg dat state structuur bestaat
+        if state_key not in state:
+            state[state_key] = {}
+        self._partition_state = state[state_key]
+
+        # Track huidige waarden tijdens streaming (voor running max/min)
+        self._current_values: Dict[str, TCursorValue] = {}
+
+    def get_last_value(self, partition_key: str) -> TCursorValue:
+        """
+        Haal de laatst opgeslagen waarde op voor een partition.
+
+        Args:
+            partition_key: Identifier voor de partition (bijv. company_id).
+
+        Returns:
+            De opgeslagen waarde, of initial_value als er geen state is.
+        """
+        return self._partition_state.get(partition_key, self._initial_value)
+
+    def track(self, partition_key: str, cursor_value: TCursorValue) -> None:
+        """
+        Track een cursor waarde voor een partition.
+
+        Werkt running max/min bij en persisted naar state. Roep dit aan voor
+        elk record tijdens het streamen zodat de hoogste/laagste waarde
+        wordt onthouden.
+
+        Args:
+            partition_key: Identifier voor de partition (bijv. company_id).
+            cursor_value: De te tracken waarde (bijv. sequenceNumber).
+        """
+        current = self._current_values.get(partition_key)
+        if current is None:
+            current = self.get_last_value(partition_key)
+
+        if cursor_value is not None:
+            new_value = self._last_value_func(
+                [cursor_value, current] if current is not None else [cursor_value]
+            )
+            self._current_values[partition_key] = new_value
+            # Direct persisten naar state (dlt commit na resource completion)
+            self._partition_state[partition_key] = new_value
+
+    def track_record(self, partition_key: str, record: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Track cursor waarde uit een record via cursor_path.
+
+        Convenience methode voor gebruik in map functies.
+
+        Args:
+            partition_key: Identifier voor de partition.
+            record: Record dict met cursor veld.
+
+        Returns:
+            Het record ongewijzigd (voor chaining in pipelines).
+        """
+        if self._cursor_path:
+            cursor_value = record.get(self._cursor_path)
+            self.track(partition_key, cursor_value)
+        return record
+
+    def get_all_partitions(self) -> Dict[str, TCursorValue]:
+        """
+        Haal alle opgeslagen partition states op.
+
+        Returns:
+            Dict van partition_key -> cursor_value.
+        """
+        return dict(self._partition_state)
+
+    def reset_partition(self, partition_key: str) -> None:
+        """
+        Reset state voor een specifieke partition.
+
+        Args:
+            partition_key: Identifier voor de partition om te resetten.
+        """
+        if partition_key in self._partition_state:
+            del self._partition_state[partition_key]
+        if partition_key in self._current_values:
+            del self._current_values[partition_key]

dlt_utils/schema.py

@@ -0,0 +1,157 @@
+"""
+Schema utilities for dlt pipelines.
+
+Provides functions to ensure database tables exist before running pipelines,
+which is useful for scenarios where you need tables created without data load.
+"""
+
+import logging
+from typing import Any, Dict, Iterable, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dlt import Pipeline
+
+logger = logging.getLogger(__name__)
+
+
+def ensure_all_tables_exist(
+    pipeline: "Pipeline",
+    only_tables: Optional[Iterable[str]] = None,
+) -> Dict[str, Any]:
+    """
+    Ensure all tables from the schema exist in the database.
+
+    Uses dlt's own SQL generation and executes CREATE TABLE statements for
+    tables that don't exist yet.
+
+    IMPORTANT: This bypasses dlt's schema hash check, so tables are created
+    even if the schema appears "up to date".
+
+    Args:
+        pipeline: An initialized dlt pipeline with a loaded schema.
+        only_tables: Optional - only create these specific tables.
+
+    Returns:
+        Dict with info about created tables (schema_update).
+
+    Example:
+        ```python
+        pipeline = dlt.pipeline(
+            pipeline_name="my_pipeline",
+            destination="postgres",
+            dataset_name="my_dataset",
+        )
+
+        # Load schema from file or run once
+        pipeline.run(my_source().with_resources("__nothing__"))
+
+        # Now ensure all tables exist
+        ensure_all_tables_exist(pipeline)
+        ```
+    """
+    tables_to_check = (
+        list(only_tables) if only_tables else list(pipeline.default_schema.tables.keys())
+    )
+
+    with pipeline.destination_client() as client:
+        # Get existing tables from the database
+        # get_storage_tables returns (table_name, columns_dict) tuples
+        storage_tables = list(client.get_storage_tables(tables_to_check))
+
+        # Build CREATE/ALTER statements
+        sql_scripts, schema_update = client._build_schema_update_sql(storage_tables)
+
+        if sql_scripts:
+            logger.info(f"Executing {len(sql_scripts)} SQL statements to create/update tables")
+            for sql in sql_scripts:
+                logger.debug(f"SQL: {sql[:100]}...")
+
+            # Execute the SQL
+            client.sql_client.execute_many(sql_scripts)
+
+            logger.info(f"Created/updated {len(schema_update)} tables: {list(schema_update.keys())}")
+        else:
+            logger.info("All tables already exist and are up to date")
+
+        return schema_update or {}
+
+
+def get_tables_for_resources(
+    pipeline: "Pipeline",
+    resource_names: List[str],
+) -> List[str]:
+    """
+    Find all tables (root + children) for given resources.
+
+    This is useful when you need to know which tables will be created
+    for a set of resources, including nested/child tables.
+
+    Args:
+        pipeline: An initialized dlt pipeline with a loaded schema.
+        resource_names: List of resource names to find tables for.
+
+    Returns:
+        List of table names including child tables.
+
+    Example:
+        ```python
+        tables = get_tables_for_resources(pipeline, ["trade_items"])
+        # Returns: ["trade_items", "trade_items__photos", "trade_items__characteristics", ...]
+        ```
+    """
+    schema = pipeline.default_schema
+    tables = schema.tables
+
+    relevant_tables = set()
+
+    for resource_name in resource_names:
+        # Root table
+        if resource_name in tables:
+            relevant_tables.add(resource_name)
+
+        # Child tables (tables with parent relation or matching prefix)
+        for table_name, table_def in tables.items():
+            # Check prefix (common pattern for nested tables)
+            if table_name.startswith(f"{resource_name}__"):
+                relevant_tables.add(table_name)
+
+            # Or via parent chain
+            parent = table_def.get("parent")
+            checked = set()
+            while parent and parent not in checked:
+                checked.add(parent)
+                if parent == resource_name:
+                    relevant_tables.add(table_name)
+                    break
+                parent = tables.get(parent, {}).get("parent")
+
+    return list(relevant_tables)
+
+
+def ensure_tables_for_resources(
+    pipeline: "Pipeline",
+    resource_names: List[str],
+) -> Dict[str, Any]:
+    """
+    Ensure tables exist for specific resources (including child tables).
+
+    Combines get_tables_for_resources and ensure_all_tables_exist for
+    convenient resource-based table creation.
+
+    Args:
+        pipeline: An initialized dlt pipeline with a loaded schema.
+        resource_names: List of resource names to ensure tables for.
+
+    Returns:
+        Dict with info about created tables.
+
+    Example:
+        ```python
+        # Only create tables for trade_items and organizations
+        ensure_tables_for_resources(pipeline, ["trade_items", "organizations"])
+        ```
+    """
+    tables = get_tables_for_resources(pipeline, resource_names)
+    logger.info(f"Ensuring {len(tables)} tables exist for resources {resource_names}")
+
+    return ensure_all_tables_exist(pipeline, only_tables=tables)

dlt_utils-0.1.0.dist-info/METADATA

@@ -0,0 +1,181 @@
+Metadata-Version: 2.4
+Name: dlt_utils
+Version: 0.1.0
+Summary: Shared utilities for dlt data pipelines with multi-company support
+Author: Jay van den Bos
+License: MIT
+Project-URL: Homepage, https://pypi.org/project/dlt_utils/
+Project-URL: Repository, https://dev.azure.com/your-org/your-project/_git/dlt_utils
+Keywords: dlt,data,pipeline,etl,incremental
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+Requires-Dist: dlt>=1.19.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+
+# dlt_utils
+
+[![PyPI version](https://badge.fury.io/py/dlt_utils.svg)](https://badge.fury.io/py/dlt_utils)
+[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Shared utilities for [dlt](https://dlthub.com/) data pipelines with multi-company support.
+
+## Features
+
+- **PartitionedIncremental**: Incremental state tracking per partition key (e.g., company_id)
+- **Date utilities**: Generate (year, week) and (year, month) tuples for time-based partitioning
+- **Schema utilities**: Ensure tables exist in destination database
+
+## Installation
+
+```bash
+# From PyPI
+pip install dlt_utils
+
+# For development
+pip install -e ".[dev]"
+```
+
+## Usage
+
+### PartitionedIncremental
+
+Track incremental state per company (or any partition key):
+
+```python
+import dlt
+from dlt_utils import PartitionedIncremental
+
+@dlt.resource
+def sync_resource():
+    state = dlt.current.resource_state()
+    inc = PartitionedIncremental(
+        state=state,
+        state_key="sequences",
+        cursor_path="sequenceNumber",
+        initial_value=0,
+    )
+
+    for company_id in ["company_a", "company_b"]:
+        start_seq = inc.get_last_value(company_id)
+        for record in fetch_data(company_id, since=start_seq):
+            inc.track(company_id, record["sequenceNumber"])
+            yield record
+```
+
+### Date utilities
+
+Generate time periods for partitioned data extraction:
+
+```python
+from dlt_utils import generate_year_weeks, generate_year_months
+
+# Generate weeks from 2024 to now + 52 weeks
+weeks = generate_year_weeks(start_year=2024)
+# [(2024, 1), (2024, 2), ..., (2025, 52)]
+
+# Generate months from October 2024 to February 2025
+months = generate_year_months(2024, 10, 2025, 2)
+# [(2024, 10), (2024, 11), (2024, 12), (2025, 1), (2025, 2)]
+```
+
+### Schema utilities
+
+Ensure tables exist before running pipeline:
+
+```python
+from dlt_utils import ensure_all_tables_exist, ensure_tables_for_resources
+
+# Create all tables from schema
+ensure_all_tables_exist(pipeline)
+
+# Create only specific resource tables (including child tables)
+ensure_tables_for_resources(pipeline, ["trade_items", "organizations"])
+```
+
+## Development
+
+```bash
+# Install dev dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+
+# Run linter
+ruff check dlt_utils/
+```
+
+## CI/CD Pipeline
+
+De pipeline draait automatisch bij:
+- **Push naar `main`**: Voert tests uit
+- **Tag met `v*` prefix**: Voert tests uit én publiceert naar PyPI
+
+### Pipeline Workflow
+
+```
+┌─────────────────┐     ┌─────────────────┐     ┌─────────────────┐
+│   Push/Tag      │────▶│   Test Stage    │────▶│  Publish Stage  │
+│   naar repo     │     │   (altijd)      │     │  (alleen tags)  │
+└─────────────────┘     └─────────────────┘     └─────────────────┘
+                              │                        │
+                              ▼                        ▼
+                        - Install deps           - Build package
+                        - Run pytest             - Upload to PyPI
+                        - Publish results
+```
+
+### Nieuwe Versie Releasen
+
+#### Optie 1: Via Git CLI
+
+```bash
+# 1. Zorg dat alle changes gecommit zijn
+git add .
+git commit -m "Release v0.2.0"
+
+# 2. Maak een tag aan
+git tag v0.2.0
+
+# 3. Push commit én tag naar remote
+git push origin main
+git push origin v0.2.0
+```
+
+#### Optie 2: Via Azure DevOps
+
+1. Ga naar **Repos** → **Tags**
+2. Klik op **New tag**
+3. Vul in:
+   - **Name**: `v0.2.0` (moet beginnen met `v`)
+   - **Based on**: selecteer de commit of branch (bijv. `main`)
+   - **Description**: optioneel, bijv. "Added new feature X"
+4. Klik op **Create**
+
+De pipeline wordt automatisch getriggered en publiceert naar PyPI.
+
+### Versienummering
+
+Gebruik [Semantic Versioning](https://semver.org/):
+- `vMAJOR.MINOR.PATCH` (bijv. `v1.2.3`)
+- **MAJOR**: Breaking changes
+- **MINOR**: Nieuwe features (backwards compatible)
+- **PATCH**: Bugfixes
+
+> ⚠️ **Belangrijk**: Vergeet niet de versie in `pyproject.toml` bij te werken vóór het taggen!
+
+## License
+
+MIT

dlt_utils-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+dlt_utils/__init__.py,sha256=WDJm8BP6dgqBTuvm3Lj7igKoDP2lm5mJHX8ZZT05yV8,875
+dlt_utils/dates.py,sha256=HX27XaRE68tEVazKWmE8NFjOQutTcKjpBZ7ILU36qS0,5231
+dlt_utils/incremental.py,sha256=qzk5_JDP21BV3XSnJnelM1FxNXpU_VeYL0OePHslTqU,5677
+dlt_utils/schema.py,sha256=udCmTYGnRfTQWRfMFTBJetRf6lrEy-rbEx7sc0PrR4Y,5042
+dlt_utils-0.1.0.dist-info/METADATA,sha256=1teOYnnZDtwbYk3JMlDEta7atN4qPYF7gUAIBMKvQMg,5377
+dlt_utils-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dlt_utils-0.1.0.dist-info/top_level.txt,sha256=KQIiBkukfDFtKV6NmbNK651_k5Oj0jLk5dpU0ZO1xuQ,10
+dlt_utils-0.1.0.dist-info/RECORD,,

dlt_utils-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dlt_utils-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dlt_utils