ingestify 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

ingestify/__init__.py

@@ -7,5 +7,6 @@ except NameError:
 if not __INGESTIFY_SETUP__:
     from .infra import retrieve_http
     from .source_base import Source, DatasetResource
+    from .main import debug_source
 
-__version__ = "0.7.0"
+__version__ = "0.8.0"

ingestify/application/ingestion_engine.py

@@ -110,6 +110,9 @@ class IngestionEngine:
         else:
             do_load()
 
+    # Alias for load() - more intuitive name for running ingestion
+    run = load
+
     def list_datasets(self, as_count: bool = False):
         """Consider moving this to DataStore"""
         datasets = sorted(

ingestify/application/loader.py

@@ -307,7 +307,17 @@ class Loader:
             auto_ingest_config=auto_ingest_config,
             **selector_filters,
         )
-        if selector_filters and not selectors:
-            logger.warning(f"No data found matching {selector_filters}")
+        if (provider or source or dataset_type or selector_filters) and not selectors:
+            filters_applied = {
+                k: v
+                for k, v in {
+                    "provider": provider,
+                    "source": source,
+                    "dataset_type": dataset_type,
+                    **selector_filters,
+                }.items()
+                if v
+            }
+            logger.warning(f"No data found matching filters: {filters_applied}")
         else:
             self.run(selectors, dry_run=dry_run)

ingestify/domain/models/dataset/file.py

@@ -39,6 +39,12 @@ class DraftFile(BaseModel):
             stream = BytesIO(file_.read().encode("utf-8"))
         elif isinstance(file_, BytesIO):
             stream = file_
+        elif hasattr(file_, "read"):
+            data = file_.read()
+            if isinstance(data, bytes):
+                stream = BytesIO(data)
+            else:
+                stream = BytesIO(data.encode("utf-8"))
         else:
             raise Exception(f"Not possible to create DraftFile from {type(file_)}")
 

ingestify/main.py

@@ -279,3 +279,158 @@ def get_engine(
         ingestion_engine.add_ingestion_plan(ingestion_plan_)
 
     return ingestion_engine
+
+
+def get_dev_engine(
+    source: Source,
+    dataset_type: str,
+    data_spec_versions: dict,
+    ephemeral: bool = True,
+    configure_logging: bool = True,
+    dev_dir: Optional[str] = None,
+) -> IngestionEngine:
+    """
+    Quick development helper - creates an engine with minimal setup.
+
+    Args:
+        source: The source to test
+        dataset_type: Dataset type to ingest
+        data_spec_versions: Dict like {"hops": "v1"}
+        ephemeral: If True, uses temp dir that gets cleaned. If False, uses persistent /tmp storage.
+        configure_logging: If True, configures basic logging (default: True)
+        dev_dir: Optional custom directory for data storage (overrides ephemeral)
+
+    Returns:
+        IngestionEngine configured for development
+
+    Example:
+        >>> source = MySource(name="test", ...)
+        >>> engine = get_dev_engine(source, "hops", {"hops": "v1"})
+        >>> engine.run()
+        >>>
+        >>> # Access the datasets
+        >>> datasets = engine.store.get_dataset_collection()
+        >>> print(f"Ingested {len(datasets)} datasets")
+    """
+    import tempfile
+    from pathlib import Path
+
+    if configure_logging:
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        )
+
+    if dev_dir:
+        # Use provided directory
+        dev_dir = Path(dev_dir)
+    elif ephemeral:
+        # Use temp directory that will be cleaned up
+        import uuid
+
+        dev_dir = Path(tempfile.gettempdir()) / f"ingestify-dev-{uuid.uuid4().hex[:8]}"
+    else:
+        # Use persistent directory
+        dev_dir = Path(tempfile.gettempdir()) / "ingestify-dev"
+
+    dev_dir.mkdir(parents=True, exist_ok=True)
+    metadata_url = f"sqlite:///{dev_dir / 'database.db'}"
+    file_url = f"file://{dev_dir}"
+
+    logger.info(f"Dev mode: storing data in {dev_dir}")
+
+    engine = get_engine(
+        metadata_url=metadata_url,
+        file_url=file_url,
+        bucket="main",
+        disable_events=True,
+    )
+
+    data_spec_versions_obj = DataSpecVersionCollection.from_dict(data_spec_versions)
+
+    engine.add_ingestion_plan(
+        IngestionPlan(
+            source=source,
+            dataset_type=dataset_type,
+            selectors=[Selector.build({}, data_spec_versions=data_spec_versions_obj)],
+            fetch_policy=FetchPolicy(),
+            data_spec_versions=data_spec_versions_obj,
+        )
+    )
+
+    return engine
+
+
+def debug_source(
+    source: Source,
+    *,
+    dataset_type: str,
+    data_spec_versions: dict,
+    ephemeral: bool = True,
+    configure_logging: bool = True,
+    dev_dir: Optional[str] = None,
+    **kwargs,
+) -> IngestionEngine:
+    """
+    Debug helper - creates a dev engine, runs ingestion, and shows results.
+
+    This is a convenience wrapper around get_dev_engine() that does everything:
+    creates the engine, runs ingestion, and displays results.
+
+    Args:
+        source: The source to debug
+        dataset_type: Dataset type (e.g., "match")
+        data_spec_versions: Dict like {"match": "v1"} - explicit, no defaults!
+        ephemeral: If True, uses temp dir. If False, uses persistent /tmp storage.
+        configure_logging: If True, configures basic logging (default: True)
+        dev_dir: Optional custom directory for data storage (overrides ephemeral)
+        **kwargs: Selector arguments. For sources with discover_selectors(), these
+                  filter discovered selectors. Otherwise passed to find_datasets().
+
+    Returns:
+        IngestionEngine: The engine used for ingestion (for further inspection)
+
+    Example:
+        >>> # Simple source without discover_selectors
+        >>> source = StatsBombHOPSS3(name="test", s3_bucket="my-bucket", s3_prefix="HOPS")
+        >>> engine = debug_source(source, dataset_type="hops", data_spec_versions={"hops": "v1"})
+
+        >>> # Source with discover_selectors - discovers all competitions
+        >>> source = StatsBombMatchAPI(name="test", ...)
+        >>> engine = debug_source(
+        ...     source,
+        ...     dataset_type="match",
+        ...     data_spec_versions={"match": "v6"}
+        ... )
+
+        >>> # Filter discovered selectors
+        >>> engine = debug_source(
+        ...     source,
+        ...     dataset_type="match",
+        ...     data_spec_versions={"match": "v6"},
+        ...     competition_id=46  # Filters to specific competition
+        ... )
+    """
+    logger.info(f"Debug mode for source: {source.name}")
+
+    engine = get_dev_engine(
+        source=source,
+        dataset_type=dataset_type,
+        data_spec_versions=data_spec_versions,
+        ephemeral=ephemeral,
+        configure_logging=configure_logging,
+        dev_dir=dev_dir,
+    )
+
+    # Run ingestion
+    # Empty selector {} automatically triggers discover_selectors() if available
+    # kwargs filter discovered selectors or are passed to find_datasets()
+    engine.run(**kwargs)
+
+    # Show results
+    datasets = engine.store.get_dataset_collection()
+    logger.info("=" * 60)
+    logger.info(f"✓ Ingestion complete: {len(datasets)} dataset(s)")
+    logger.info("=" * 60)
+
+    return engine

{ingestify-0.7.0.dist-info → ingestify-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ingestify
-Version: 0.7.0
+Version: 0.8.0
 Summary: Data Ingestion Framework
 Author: Koen Vossen
 Author-email: info@koenvossen.nl
@@ -77,6 +77,43 @@ Ingestify fixes that by building **your own data lake** of untouched provider fi
 pip install ingestify            # or: pip install git+https://github.com/PySport/ingestify.git
 ```
 
+### Developing a new Source
+
+When developing a new `Source`, use the `debug_source()` helper for rapid iteration:
+
+```python
+from ingestify import Source, debug_source
+
+class MyCustomSource(Source):
+    provider = "my_provider"
+
+    def __init__(self, name: str, api_key: str):
+        super().__init__(name)
+        self.api_key = api_key
+
+    def find_datasets(self, dataset_type, data_spec_versions, **kwargs):
+        # Your source implementation
+        ...
+
+# Quick debug - runs full ingestion with temp storage
+if __name__ == "__main__":
+    source = MyCustomSource(name="test", api_key="...")
+
+    debug_source(
+        source,
+        dataset_type="match",
+        data_spec_versions={"events": "v1"},
+    )
+```
+
+The `debug_source()` helper:
+- ✅ Creates an ephemeral dev engine with temp storage
+- ✅ Configures logging automatically
+- ✅ Runs the full ingestion cycle
+- ✅ Shows storage location and results
+
+Perfect for testing your source before adding it to production config!
+
 ### Minimal `config.yaml`
 
 ```yaml
@@ -175,8 +212,16 @@ pip install kloppy
 ```
 
 ```python
+import logging, sys
+
 from ingestify.main import get_engine
 
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    stream=sys.stderr,
+)
+
 engine = get_engine(
     metadata_url="sqlite:///database_open_data/catalog.db",
     file_url="file://database_open_data/files/"
@@ -188,12 +233,13 @@ dataset_iter = engine.iter_datasets(
 
     provider="statsbomb",
     dataset_type="match",
-    competition_id=43,
-    season_id=281
+    competition_id=43,  # "FIFA World Cup"
+    #season_id=281
 )
 
 for dataset in dataset_iter:
     kloppy_dataset = engine.load_dataset_with_kloppy(dataset)
+    logging.info(f"Loaded {kloppy_dataset}")
 ```
 
 

{ingestify-0.7.0.dist-info → ingestify-0.8.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
-ingestify/__init__.py,sha256=IuO5KQRTWAjWxmJ6Knte5-Q2Ybq1BDDkM5UlZjYRl84,301
+ingestify/__init__.py,sha256=FeK7pau-iTc6ooJiPelblIhkrPLojVHKpTHXIrkdpq8,336
 ingestify/cmdline.py,sha256=Rs1_lSKSIJrcygH5fvtOGicOl_e0sZYW7deqp4_jGbY,6233
 ingestify/exceptions.py,sha256=izRzaLQmMy-4P8ZqGqVZyf4k6LFYOYqwYLuRaUH8BJw,187
-ingestify/main.py,sha256=YegWoI_xIgoz30BSS7N6Ew3SAXSr1-jPFMPqvFda3DI,9797
+ingestify/main.py,sha256=WjhcsT21F7dOibrg_S7wRiui6Ytj5ScsWqMCGuv9fs8,14938
 ingestify/server.py,sha256=OVrf_XtpAQIn88MzqQzShXgsA9_jbnqYvD8YPBjn3cs,2413
 ingestify/source_base.py,sha256=GXAFCoT11Zov9M2v-fqQr9gFCXbtVfEIEH32V7r2oE8,382
 ingestify/utils.py,sha256=tsoo-GgeSrwK161WCqW793BAm5bjvnGwI8yGgLTJ1lk,6486
 ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestify/application/dataset_store.py,sha256=GP6wGjVirefEn6hlqWIkOBqdELad9L_mmTpdHdzj18M,20353
-ingestify/application/ingestion_engine.py,sha256=cG4JgU667PcsHBngOWUk58KffDrHkJOMv8LrVjaRQ1o,11163
-ingestify/application/loader.py,sha256=xImeaOdj97iCpprZ0WxRbQJ4w6nS1cEmPba7TN1lE6I,13038
+ingestify/application/ingestion_engine.py,sha256=we16yiDS9QGOlAUiP1vidDycihjWK3B2jo64uqKmrXE,11246
+ingestify/application/loader.py,sha256=K99ZJuHMEJFO6CIlxoyHKGSQtXw63JgOYu3moUD6sR0,13400
 ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
 ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
 ingestify/domain/models/__init__.py,sha256=WuKS34uiR1EwyczKujBHYGupqseJP-U2P5IQS4kpsA8,838
@@ -25,7 +25,7 @@ ingestify/domain/models/dataset/dataset.py,sha256=OiP03nY0-m06y2GTrs_m-RiZE8Hwyp
 ingestify/domain/models/dataset/dataset_repository.py,sha256=bf3F_1cKw0CvUberD3FMROE8iowAmYefnD4L6aPB39k,989
 ingestify/domain/models/dataset/dataset_state.py,sha256=IaYG02WzgooGaM_AuwRhZgljs-9NhCF_LpBZXkl5ELY,324
 ingestify/domain/models/dataset/events.py,sha256=M8jrHWCm9iXapAy3xjvZZtiiOxXDnfefBixiMwkas24,786
-ingestify/domain/models/dataset/file.py,sha256=2wpBfluS8i_mCPQkdSu1x1Af3kc15bVBLAKXeogB4jA,4243
+ingestify/domain/models/dataset/file.py,sha256=cXDjSw19HRMCGFpVN4u1oejxE1V8SMQptfNVDVixj6o,4464
 ingestify/domain/models/dataset/file_collection.py,sha256=yaQmqFlmbajLCkU5QnjgqCvKzvVEZJrXVvinx5UGHcM,1193
 ingestify/domain/models/dataset/file_repository.py,sha256=9EQprch9isAH2pbK7e7tfOKl6ulip4Ij1kBCTbO_rTc,1721
 ingestify/domain/models/dataset/identifier.py,sha256=EJYsxt0OS_43Y989DZQq8U9NjwmtvnHGYGMe6-hOBlI,575
@@ -72,8 +72,8 @@ ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRX
 ingestify/infra/store/file/dummy_file_repository.py,sha256=azUq9c43Mz9-GWk9j0E97BaqyUKu-ZMrcuaIednLq5E,723
 ingestify/infra/store/file/local_file_repository.py,sha256=1hhLqds5LlppJq2QBB0oN0Q98j6aXreCtYQYz3Q1P8g,819
 ingestify/infra/store/file/s3_file_repository.py,sha256=tz_EZ_gun7W2qJMlI3j_R03iKBZlJSDcG7AUJ1JkdpE,1501
-ingestify-0.7.0.dist-info/METADATA,sha256=cNjbTVDpw0MTkPj-xoywVXzdrynoRAO5z7qyu1LftLg,6871
-ingestify-0.7.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-ingestify-0.7.0.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
-ingestify-0.7.0.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
-ingestify-0.7.0.dist-info/RECORD,,
+ingestify-0.8.0.dist-info/METADATA,sha256=rpC2ALX0e4Ii-XzhJWmRKfW7YoBgl6gEpP2cUGFlQp4,8089
+ingestify-0.8.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+ingestify-0.8.0.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.8.0.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.8.0.dist-info/RECORD,,