ingestify 0.6.4__py3-none-any.whl → 0.8.0__py3-none-any.whl

ingestify/application/loader.py

@@ -9,6 +9,9 @@ from ingestify.utils import TaskExecutor
 
 from .dataset_store import DatasetStore
 from ingestify.domain.models.ingestion.ingestion_plan import IngestionPlan
+from ingestify.domain.models.fetch_policy import FetchPolicy
+from ingestify.domain import DataSpecVersionCollection
+from ingestify.infra.source.statsbomb_github import StatsbombGithub
 from ..domain.models.ingestion.ingestion_job import IngestionJob
 from ..exceptions import ConfigurationError
 
@@ -21,6 +24,34 @@ else:
 logger = logging.getLogger(__name__)
 
 
+# Registry of open data sources that can be auto-instantiated
+OPEN_DATA_SOURCES = {
+    "statsbomb": StatsbombGithub,
+    # Add more open data sources here as they become available
+}
+
+
+def _create_open_data_plan(provider: str, dataset_type: str) -> Optional[IngestionPlan]:
+    """Create a temporary ingestion plan for open data sources."""
+    if provider not in OPEN_DATA_SOURCES:
+        return None
+
+    source_class = OPEN_DATA_SOURCES[provider]
+    source = source_class(name=f"open_data_{provider}")
+
+    # Create empty selector to trigger discover_selectors
+    data_spec_versions = DataSpecVersionCollection.from_dict({"default": {"v1"}})
+    empty_selector = Selector.build({}, data_spec_versions=data_spec_versions)
+
+    return IngestionPlan(
+        source=source,
+        fetch_policy=FetchPolicy(),
+        selectors=[empty_selector],
+        dataset_type=dataset_type,
+        data_spec_versions=data_spec_versions,
+    )
+
+
 class Loader:
     def __init__(self, store: DatasetStore):
         self.store = store
@@ -29,30 +60,67 @@ class Loader:
     def add_ingestion_plan(self, ingestion_plan: IngestionPlan):
         self.ingestion_plans.append(ingestion_plan)
 
-    def collect_and_run(
+    def collect(
         self,
-        dry_run: bool = False,
         provider: Optional[str] = None,
         source: Optional[str] = None,
+        dataset_type: Optional[str] = None,
+        auto_ingest_config: Optional[dict] = None,
+        **selector_filters,
     ):
+        """Collect and prepare selectors for execution."""
         ingestion_plans = []
         for ingestion_plan in self.ingestion_plans:
             if provider is not None:
                 if ingestion_plan.source.provider != provider:
-                    logger.info(
+                    logger.debug(
                         f"Skipping {ingestion_plan} because provider doesn't match '{provider}'"
                     )
                     continue
 
             if source is not None:
                 if ingestion_plan.source.name != source:
-                    logger.info(
+                    logger.debug(
                         f"Skipping {ingestion_plan} because source doesn't match '{source}'"
                     )
                     continue
 
+            if dataset_type is not None:
+                if ingestion_plan.dataset_type != dataset_type:
+                    logger.debug(
+                        f"Skipping {ingestion_plan} because dataset_type doesn't match '{dataset_type}'"
+                    )
+                    continue
+
+            # Note: Selector filtering is now done after all selectors are collected
+            # to allow discover_selectors to run for plans with empty selectors
+
             ingestion_plans.append(ingestion_plan)
 
+        # Check if we need to add open data plans
+        auto_ingest_config = auto_ingest_config or {}
+        if auto_ingest_config.get("use_open_data", False):
+            # Validate prerequisites for open data
+            if not provider:
+                raise ConfigurationError(
+                    "use_open_data requires 'provider' to be specified"
+                )
+            if not dataset_type:
+                raise ConfigurationError(
+                    "use_open_data requires 'dataset_type' to be specified"
+                )
+
+            # Only add open data plan if no matching configured plans found
+            if not ingestion_plans:
+                open_data_plan = _create_open_data_plan(provider, dataset_type)
+                if open_data_plan:
+                    logger.info(f"Auto-discovered open data source: {open_data_plan}")
+                    ingestion_plans.append(open_data_plan)
+                else:
+                    logger.warning(
+                        f"No open data source available for provider '{provider}'"
+                    )
+
         # First collect all selectors, before discovering datasets
         selectors = {}
         for ingestion_plan in ingestion_plans:
@@ -134,32 +202,45 @@ class Loader:
                 else:
                     selectors[key] = (ingestion_plan, selector)
 
-        """
-            Data is denormalized:
-            
-            It actually looks like:
-                - IngestionPlan #1
-                    - Selector 1.1
-                    - Selector 1.2
-                    - Selector 1.3
-                - IngestionPlan #2
-                    - Selector 2.1
-                    - Selector 2.2
-                    
-            We process this as:
-            - IngestionPlan #1, Selector 1.1
-            - IngestionPlan #1, Selector 1.2
-            - IngestionPlan #1, Selector 1.3
-            - IngestionPlan #2, Selector 2.1
-            - IngestionPlan #2, Selector 2.2 
-            
-            IngestionJobSummary holds the summary for an IngestionPlan and a single Selector
-        """
+        # Convert to list
+        collected_selectors = list(selectors.values())
+
+        # Apply selector filters if provided
+        if selector_filters:
+            filtered_selectors = []
+            for ingestion_plan, selector in collected_selectors:
+                if selector.matches(selector_filters):
+                    # Merge selector with user filters to make it more strict
+                    merged_attributes = {
+                        **selector.filtered_attributes,
+                        **selector_filters,
+                    }
+                    strict_selector = Selector.build(
+                        merged_attributes,
+                        data_spec_versions=selector.data_spec_versions,
+                    )
+
+                    # Check if selector was actually made more strict
+                    if len(strict_selector.filtered_attributes) > len(
+                        selector.filtered_attributes
+                    ):
+                        logger.debug(
+                            f"Made selector more strict: {selector} -> {strict_selector}"
+                        )
 
+                    filtered_selectors.append((ingestion_plan, strict_selector))
+                else:
+                    logger.debug(
+                        f"Filtering out selector {selector} because it doesn't match filters"
+                    )
+            collected_selectors = filtered_selectors
+
+        return collected_selectors
+
+    def run(self, selectors, dry_run: bool = False):
+        """Execute the collected selectors."""
         ingestion_job_prefix = str(uuid.uuid1())
-        for ingestion_job_idx, (ingestion_plan, selector) in enumerate(
-            selectors.values()
-        ):
+        for ingestion_job_idx, (ingestion_plan, selector) in enumerate(selectors):
             logger.info(
                 f"Discovering datasets from {ingestion_plan.source.__class__.__name__} using selector {selector}"
             )
@@ -186,3 +267,57 @@ class Loader:
                     self.store.save_ingestion_job_summary(ingestion_job_summary)
 
         logger.info("Done")
+
+    def collect_and_run(
+        self,
+        dry_run: bool = False,
+        provider: Optional[str] = None,
+        source: Optional[str] = None,
+        dataset_type: Optional[str] = None,
+        auto_ingest_config: Optional[dict] = None,
+        **selector_filters,
+    ):
+        """
+        Backward compatibility method - collect then run.
+
+        Data flow explanation:
+
+        IngestionPlans are structured hierarchically:
+            - IngestionPlan #1
+                - Selector 1.1
+                - Selector 1.2
+                - Selector 1.3
+            - IngestionPlan #2
+                - Selector 2.1
+                - Selector 2.2
+
+        But we process them as flat (plan, selector) pairs for execution:
+            - (IngestionPlan #1, Selector 1.1)
+            - (IngestionPlan #1, Selector 1.2)
+            - (IngestionPlan #1, Selector 1.3)
+            - (IngestionPlan #2, Selector 2.1)
+            - (IngestionPlan #2, Selector 2.2)
+
+        Each IngestionJobSummary tracks the execution of one (IngestionPlan, Selector) pair.
+        """
+        selectors = self.collect(
+            provider=provider,
+            source=source,
+            dataset_type=dataset_type,
+            auto_ingest_config=auto_ingest_config,
+            **selector_filters,
+        )
+        if (provider or source or dataset_type or selector_filters) and not selectors:
+            filters_applied = {
+                k: v
+                for k, v in {
+                    "provider": provider,
+                    "source": source,
+                    "dataset_type": dataset_type,
+                    **selector_filters,
+                }.items()
+                if v
+            }
+            logger.warning(f"No data found matching filters: {filters_applied}")
+        else:
+            self.run(selectors, dry_run=dry_run)

ingestify/cmdline.py

@@ -5,7 +5,6 @@ from pathlib import Path
 from typing import Optional
 
 import click
-import jinja2
 from dotenv import find_dotenv, load_dotenv
 
 from ingestify.exceptions import ConfigurationError
@@ -219,53 +218,6 @@ def delete_dataset(
     logger.info("Done")
 
 
-@cli.command()
-@click.option(
-    "--template",
-    "template",
-    required=True,
-    help="Template",
-    type=click.Choice(["wyscout", "statsbomb_github"]),
-)
-@click.argument("project_name")
-def init(template: str, project_name: str):
-    logger.warning(
-        "`ingestify init` is currently not supported. See https://github.com/PySport/ingestify/issues/11"
-    )
-    return
-
-    directory = Path(project_name)
-    if directory.exists():
-        logger.warning(f"Directory '{directory}' already exists")
-        return sys.exit(1)
-
-    if template == "wyscout":
-        template_dir = Path(__file__).parent / "static/templates/wyscout"
-    elif template == "statsbomb_github":
-        template_dir = Path(__file__).parent / "static/templates/statsbomb_github"
-    else:
-        raise Exception(f"Template {template} not found")
-
-    directory.mkdir(parents=True)
-
-    for file in template_dir.glob("*"):
-        filename = file.name
-        if file.is_file():
-            data = file.open("r").read()
-
-            if filename.endswith(".jinja2"):
-                raw_input = jinja2.Template(data)
-                data = raw_input.render(ingestify_version=__version__)
-                filename = filename.rstrip(".jinja2")
-
-            with open(directory / filename, "w") as fp:
-                fp.write(data)
-        elif file.is_dir():
-            (directory / filename).mkdir()
-
-    logger.info(f"Initialized project at `{directory}` with template `{template}`")
-
-
 #
 # @cli.command("list")
 # @click.option(

ingestify/domain/models/__init__.py

@@ -12,6 +12,7 @@ from .dataset import (
     Selector,
     Revision,
 )
+from .dataset.dataset_state import DatasetState
 from .sink import Sink
 from .source import Source
 from .task import Task, TaskSet
@@ -37,4 +38,5 @@ __all__ = [
     "Task",
     "Sink",
     "DataSpecVersionCollection",
+    "DatasetState",
 ]

ingestify/domain/models/dataset/collection.py

@@ -19,9 +19,6 @@ class DatasetCollection:
         }
         self.metadata = metadata
 
-    def loaded(self):
-        return self.metadata.count == len(self.datasets)
-
     def get(self, dataset_identifier: Identifier) -> Dataset:
         return self.datasets.get(dataset_identifier.key)
 
@@ -31,12 +28,6 @@ class DatasetCollection:
     def __iter__(self):
         return iter(self.datasets.values())
 
-    def get_dataset_by_id(self, dataset_id):
-        for dataset in self:
-            if dataset.dataset_id == dataset_id:
-                return dataset
-        return None
-
     def first(self):
         try:
             return next(iter(self.datasets.values()))

ingestify/domain/models/dataset/dataset_repository.py

@@ -3,6 +3,7 @@ from typing import Optional, List, Union
 
 from .collection import DatasetCollection
 from .dataset import Dataset
+from .dataset_state import DatasetState
 from .selector import Selector
 
 
@@ -16,6 +17,9 @@ class DatasetRepository(ABC):
         provider: Optional[str] = None,
         selector: Optional[Union[Selector, List[Selector]]] = None,
         metadata_only: bool = False,
+        dataset_state: Optional[List[DatasetState]] = None,
+        page: Optional[int] = None,
+        page_size: Optional[int] = None,
     ) -> DatasetCollection:
         pass
 

ingestify/domain/models/dataset/dataset_state.py

@@ -1,4 +1,9 @@
+import logging
 from enum import Enum
+from typing import Union, List, Optional, NewType
+
+
+logger = logging.getLogger(__name__)
 
 
 class DatasetState(str, Enum):

ingestify/domain/models/dataset/events.py

@@ -4,6 +4,7 @@ from pydantic import BaseModel
 
 from ingestify.domain.models.event.domain_event import DomainEvent
 from .dataset import Dataset
+from .selector import Selector
 
 
 class DatasetCreated(DomainEvent):
@@ -19,3 +20,15 @@ class RevisionAdded(DomainEvent):
 class MetadataUpdated(DomainEvent):
     dataset: Dataset
     event_type: ClassVar[str] = "metadata_updated"
+
+
+class SelectorSkipped(DomainEvent):
+    model_config = {"arbitrary_types_allowed": True}
+
+    selector: Selector
+    event_type: ClassVar[str] = "selector_skipped"
+
+
+class DatasetSkipped(DomainEvent):
+    dataset: Dataset
+    event_type: ClassVar[str] = "dataset_skipped"

ingestify/domain/models/dataset/file.py

@@ -39,6 +39,12 @@ class DraftFile(BaseModel):
             stream = BytesIO(file_.read().encode("utf-8"))
         elif isinstance(file_, BytesIO):
             stream = file_
+        elif hasattr(file_, "read"):
+            data = file_.read()
+            if isinstance(data, bytes):
+                stream = BytesIO(data)
+            else:
+                stream = BytesIO(data.encode("utf-8"))
         else:
             raise Exception(f"Not possible to create DraftFile from {type(file_)}")
 
@@ -135,4 +141,4 @@ class LoadedFile(BaseModel):
         return self.stream_
 
 
-__all__ = ["File", "DraftFile", "LoadedFile"]
+__all__ = ["File", "DraftFile", "LoadedFile", "NotModifiedFile"]

ingestify/domain/models/dataset/selector.py

@@ -38,10 +38,17 @@ class Selector(AttributeBag):
         except AttributeError:
             return None
 
+    @property
+    def name(self) -> Optional[str]:
+        try:
+            return self._name
+        except AttributeError:
+            return None
+
     @property
     def custom_attributes(self):
         return {
             k: v
             for k, v in self.items()
-            if k not in ("_matcher", "_data_spec_versions", "_last_modified")
+            if k not in ("_matcher", "_data_spec_versions", "_last_modified", "_name")
         }

ingestify/domain/models/event/event_bus.py

@@ -7,6 +7,14 @@ from .dispatcher import Dispatcher
 logger = logging.getLogger(__name__)
 
 
+class QueueForwarder:
+    def __init__(self, queue):
+        self.queue = queue
+
+    def dispatch(self, event):
+        self.queue.put(event)
+
+
 class EventBus:
     def __init__(self):
         self.dispatchers: list[Dispatcher] = []
@@ -14,8 +22,15 @@ class EventBus:
     def register(self, dispatcher: Dispatcher):
         self.dispatchers.append(dispatcher)
 
-    def dispatch(self, event):
+        def unregister():
+            self.dispatchers.remove(dispatcher)
+
+        return unregister
 
+    def register_queue(self, queue):
+        return self.register(QueueForwarder(queue))
+
+    def dispatch(self, event):
         for dispatcher in self.dispatchers:
             try:
                 dispatcher.dispatch(event)

ingestify/domain/models/ingestion/ingestion_job.py

@@ -5,6 +5,8 @@ import uuid
 from enum import Enum
 from typing import Optional, Iterator, Union
 
+from pydantic import ValidationError
+
 from ingestify import retrieve_http
 from ingestify.application.dataset_store import DatasetStore
 from ingestify.domain import Selector, Identifier, TaskSet, Dataset, DraftFile, Task
@@ -14,12 +16,13 @@ from ingestify.domain.models.ingestion.ingestion_job_summary import (
     IngestionJobSummary,
 )
 from ingestify.domain.models.ingestion.ingestion_plan import IngestionPlan
+from ingestify.domain.models.dataset.events import SelectorSkipped, DatasetSkipped
 from ingestify.domain.models.resources.dataset_resource import (
     FileResource,
     DatasetResource,
 )
 from ingestify.domain.models.task.task_summary import TaskSummary
-from ingestify.exceptions import SaveError
+from ingestify.exceptions import SaveError, IngestifyError
 from ingestify.utils import TaskExecutor, chunker
 
 logger = logging.getLogger(__name__)
@@ -241,6 +244,9 @@ class IngestionJob:
                     f"'{self.selector.last_modified}' < metadata last_modified "
                     f"'{dataset_collection_metadata.last_modified}'"
                 )
+                # Emit event for streaming datasets
+                store.dispatch(SelectorSkipped(selector=self.selector))
+
                 ingestion_job_summary.set_skipped()
                 yield ingestion_job_summary
                 return
@@ -260,6 +266,16 @@ class IngestionJob:
 
                 # We need to include the to_batches as that will start the generator
                 batches = to_batches(dataset_resources)
+        except ValidationError as e:
+            # Make sure to pass this to the highest level as this means the Source is wrong
+            if "Field required" in str(e):
+                raise IngestifyError("failed to run find_datasets") from e
+            else:
+                logger.exception("Failed to find datasets")
+
+                ingestion_job_summary.set_exception(e)
+                yield ingestion_job_summary
+                return
         except Exception as e:
             logger.exception("Failed to find datasets")
 
@@ -327,6 +343,8 @@ class IngestionJob:
                                 )
                             )
                         else:
+                            # Emit event for streaming datasets
+                            store.dispatch(DatasetSkipped(dataset=dataset))
                             skipped_tasks += 1
                     else:
                         if self.ingestion_plan.fetch_policy.should_fetch(
@@ -348,9 +366,10 @@ class IngestionJob:
                         f"using selector {self.selector} => {len(task_set)} tasks. {skipped_tasks} skipped."
                     )
                     logger.info(f"Running {len(task_set)} tasks")
-                    ingestion_job_summary.add_task_summaries(
-                        task_executor.run(run_task, task_set)
-                    )
+
+                    task_summaries = task_executor.run(run_task, task_set)
+
+                    ingestion_job_summary.add_task_summaries(task_summaries)
                 else:
                     logger.info(
                         f"Discovered {len(dataset_identifiers)} datasets from {self.ingestion_plan.source.__class__.__name__} "

ingestify/domain/models/resources/dataset_resource.py

@@ -1,4 +1,3 @@
-from dataclasses import dataclass
 from datetime import datetime
 from typing import Optional, Callable, Any, Protocol, TYPE_CHECKING  # noqa
 from pydantic import Field

ingestify/infra/source/statsbomb/base.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+import requests
+
+from ingestify import Source
+from ingestify.exceptions import ConfigurationError
+
+
+class StatsBombBaseAPI(Source):
+    provider = "statsbomb"
+    BASE_URL = "https://data.statsbombservices.com/api"
+
+    def __init__(self, name: str, username: str, password: str):
+        super().__init__(name)
+
+        self.username = username.strip()
+        self.password = password.strip()
+
+        if not self.username:
+            raise ConfigurationError(
+                f"Username of StatsBomb source named '{self.name}' cannot be empty"
+            )
+
+        if not self.password:
+            raise ConfigurationError(
+                f"Username of StatsBomb source named '{self.name}' cannot be empty"
+            )
+
+    def get_url(self, data_feed_key: str, data_spec_version: str, path: str):
+        return f"{self.BASE_URL}/{data_spec_version}/{data_feed_key}/{path}"
+
+    def get(self, data_spec_version: str, path: str):
+        url = f"{self.BASE_URL}/{data_spec_version}/{path}"
+        res = requests.get(url, auth=(self.username, self.password))
+        res.raise_for_status()
+        return res.json()