ingestify 0.1.0__py3-none-any.whl

ingestify/static/templates/wyscout/config.yaml.jinja2

@@ -0,0 +1,18 @@
+ingestify_version: {{ ingestify_version }}
+
+main:
+  dataset_url: sqlite:///database/catalog.db
+  file_url: file://database/files/
+  default_bucket: main
+
+sources:
+  wyscout:
+    type: ingestify.wyscout
+    configuration:
+      username: !ENV ${WYSCOUT_USERNAME}
+      password: !ENV ${WYSCOUT_PASSWORD}
+
+extract_jobs:
+  - source: wyscout
+    selectors:
+      - season_id: 188105

ingestify/static/templates/wyscout/database/README.md

@@ -0,0 +1 @@
+# This will contain the database

ingestify/static/templates/wyscout/query.py

@@ -0,0 +1,14 @@
+from ingestify.main import get_datastore
+
+
+def main():
+    store = get_datastore("config.yaml")
+    dataset_collection = store.get_dataset_collection()
+
+    for dataset in dataset_collection:
+        kloppy_dataset = store.load_with_kloppy(dataset)
+        print(f"Loaded dataset with {len(kloppy_dataset.records)} events")
+
+
+if __name__ == "__main__":
+    main()

ingestify/utils.py

@@ -0,0 +1,276 @@
+import abc
+import inspect
+import logging
+import os
+import time
+import re
+from multiprocessing import get_context, cpu_count, get_all_start_methods
+
+from datetime import datetime, timezone
+from string import Template
+from typing import Dict, Generic, Type, TypeVar, Tuple, Optional, Any
+
+import cloudpickle
+from typing_extensions import Self
+
+
+from itertools import islice
+
+
+logger = logging.getLogger(__name__)
+
+
+def chunker(it, size):
+    iterator = iter(it)
+    while chunk := list(islice(iterator, size)):
+        yield chunk
+
+
+def sanitize_exception_message(exception_message):
+    """
+    Sanitizes an exception message by removing any sensitive information such as passwords.
+    """
+    # Regular expression to identify potential sensitive information like URLs with passwords
+    sensitive_info_pattern = r":(\w+)@"
+
+    # Replace sensitive information with a placeholder
+    sanitized_message = re.sub(sensitive_info_pattern, ":******@", exception_message)
+
+    return sanitized_message
+
+
+class ComponentRegistry:
+    def __init__(self):
+        self.__registered_components = {}
+
+        class _Registered(abc.ABCMeta):
+            def __new__(mcs, cls_name, bases, class_dict):
+                class_dict["name"] = cls_name
+                component_cls = super(_Registered, mcs).__new__(
+                    mcs, cls_name, bases, class_dict
+                )
+                if not inspect.isabstract(component_cls):
+                    self.register_component(cls_name, component_cls)
+                else:
+                    if bases[0] != abc.ABC:
+                        raise Exception(
+                            f"Class '{cls_name}' seems to be an concrete class, but missing some abstract methods"
+                        )
+                return component_cls
+
+        self.__metaclass = _Registered
+
+    @property
+    def metaclass(self):
+        return self.__metaclass
+
+    def register_component(self, cls_name, component_cls):
+        self.__registered_components[cls_name] = component_cls
+
+    def get_component(self, cls_name: str):
+        return self.__registered_components[cls_name]
+
+    def get_supporting_component(self, **kwargs) -> str:
+        for cls_name, class_ in self.__registered_components.items():
+            if not hasattr(class_, "supports"):
+                raise Exception(
+                    f"Class '{cls_name}' does not implemented a 'supports' classmethod. "
+                    f"This is required when using 'get_supporting_component'."
+                )
+
+            if class_.supports(**kwargs):
+                return cls_name
+
+        kwargs_str = sanitize_exception_message(str(kwargs))
+        raise Exception(f"No supporting class found for {kwargs_str}")
+
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+class ComponentFactory(Generic[T]):
+    def __init__(self, registry: ComponentRegistry):
+        self.registry = registry
+
+    @classmethod
+    def build_factory(
+        cls, component_cls: Type[R], registry: ComponentRegistry
+    ) -> "ComponentFactory[R]":
+        return cls[component_cls](registry)
+
+    def build(self, cls_name, **kwargs) -> T:
+        component_cls = self.registry.get_component(cls_name)
+        try:
+            return component_cls.from_dict(**kwargs)
+        except AttributeError:
+            pass
+        try:
+            return component_cls(**kwargs)
+        except TypeError as e:
+            raise e
+            # raise TypeError(f"Could not initialize {cls_name}")
+
+    def build_if_supports(self, **kwargs) -> T:
+        cls_name = self.registry.get_supporting_component(**kwargs)
+        return self.build(cls_name, **kwargs)
+
+
+def key_from_dict(d: dict) -> str:
+    return "/".join([f"{k}={v}" for k, v in sorted(d.items()) if not k.startswith("_")])
+
+
+def utcnow() -> datetime:
+    return datetime.fromtimestamp(time.time(), timezone.utc)
+
+
+NOT_SET = object()
+
+
+class AttributeBag:
+    def __init__(self, attributes=NOT_SET, **kwargs):
+        if attributes is not NOT_SET:
+            self.attributes = attributes
+        else:
+            self.attributes = kwargs
+        self.key = key_from_dict(self.attributes)
+
+    def __getattr__(self, item):
+        if item in self.__dict__:
+            return self.__dict__[item]
+        if "attributes" in self.__dict__ and item in self.attributes:
+            return self.attributes[item]
+        raise AttributeError(f"{item} not found")
+
+    def items(self):
+        return self.attributes.items()
+
+    def format_string(self, string: str):
+        return Template(string).substitute(**self.attributes)
+
+    def matches(self, attributes: Dict) -> bool:
+        for k, v in self.attributes.items():
+            if attributes.get(k) != v:
+                return False
+        return True
+
+    @property
+    def filtered_attributes(self):
+        return {k: v for k, v in self.attributes.items() if not k.startswith("_")}
+
+    def __eq__(self, other):
+        if isinstance(other, AttributeBag):
+            return self.key == other.key
+
+    def __hash__(self):
+        return hash(self.key)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({', '.join([f'{k}={v}' for k, v in self.filtered_attributes.items()])})"
+
+    def __str__(self):
+        return "/".join([f"{k}={v}" for k, v in self.filtered_attributes.items()])
+
+    @classmethod
+    def create_from(cls, other: "AttributeBag", **kwargs):
+        _args = dict(**other.attributes)
+        _args.update(kwargs)
+
+        return cls(**_args)
+
+    def split(self, attribute_name: str) -> Tuple[Self, Optional[Any]]:
+        return self.attributes.get(attribute_name), self.__class__(
+            **{k: v for k, v in self.attributes.items() if k != attribute_name}
+        )
+
+
+def cloud_unpack_and_call(args):
+    f_pickled, org_args = args
+
+    f = cloudpickle.loads(f_pickled)
+    return f(org_args)
+
+
+def map_in_pool(func, iterable, processes=0):
+    # TODO: move to cmdline
+    if os.environ.get("INGESTIFY_RUN_EAGER") == "true":
+        return list(map(func, iterable))
+
+    if not processes:
+        processes = int(os.environ.get("INGESTIFY_CONCURRENCY", "0"))
+
+    if "fork" in get_all_start_methods():
+        ctx = get_context("fork")
+    else:
+        ctx = get_context("spawn")
+
+    wrapped_fn = cloudpickle.dumps(func)
+
+    with ctx.Pool(processes or cpu_count()) as pool:
+        return pool.map(
+            cloud_unpack_and_call, ((wrapped_fn, item) for item in iterable)
+        )
+
+
+class SyncPool:
+    def map(self, func, iterable):
+        return [func(item) for item in iterable]
+
+    def join(self):
+        return True
+
+    def close(self):
+        return True
+
+
+class DummyPool:
+    def map(self, func, iterable):
+        logger.info(f"DummyPool: not running {len(list(iterable))} tasks")
+        return None
+
+    def join(self):
+        return True
+
+    def close(self):
+        return True
+
+
+class TaskExecutor:
+    def __init__(self, processes=0, dry_run: bool = False):
+        if dry_run:
+            pool = DummyPool()
+        elif os.environ.get("INGESTIFY_RUN_EAGER") == "true":
+            pool = SyncPool()
+        else:
+            if not processes:
+                processes = int(os.environ.get("INGESTIFY_CONCURRENCY", "0"))
+
+            if "fork" in get_all_start_methods():
+                ctx = get_context("fork")
+            else:
+                ctx = get_context("spawn")
+
+            pool = ctx.Pool(processes or cpu_count())
+        self.pool = pool
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.join()
+
+    def run(self, func, iterable):
+        wrapped_fn = cloudpickle.dumps(func)
+        start_time = time.time()
+        res = self.pool.map(
+            cloud_unpack_and_call, ((wrapped_fn, item) for item in iterable)
+        )
+        if res:
+            took = time.time() - start_time
+            logger.info(
+                f"Finished {len(res)} tasks in {took:.1f} seconds. {(len(res)/took):.1f} tasks/sec"
+            )
+
+    def join(self):
+        self.pool.close()
+        self.pool.join()

ingestify-0.1.0.dist-info/METADATA

@@ -0,0 +1,265 @@
+Metadata-Version: 2.1
+Name: ingestify
+Version: 0.1.0
+Summary: Standardizing soccer tracking- and event data
+Author: Koen Vossen
+Author-email: info@koenvossen.nl
+License: AGPL
+Description-Content-Type: text/markdown
+Requires-Dist: requests<3,>=2.0.0
+Requires-Dist: SQLAlchemy
+Requires-Dist: dataclass-factory
+Requires-Dist: cloudpickle
+Requires-Dist: click
+Requires-Dist: jinja2
+Requires-Dist: python-dotenv
+Requires-Dist: pyaml-env
+Requires-Dist: boto3
+Requires-Dist: pytz
+Provides-Extra: test
+Requires-Dist: pytest<7,>=6.2.5; extra == "test"
+
+# Ingestify
+
+## Data Management Platform
+
+In general a data management platform contains:
+1. Ingestion of data (Extract from Source into Load into Data Lake)
+2. Transformation of data (Extract from Data Lake, Transform and Load into Data Warehouse)
+3. Utilization of data
+
+<img src="https://www.getdbt.com/ui/img/blog/what-exactly-is-dbt/1-BogoeTTK1OXFU1hPfUyCFw.png" />
+Source: https://www.getdbt.com/blog/what-exactly-is-dbt/
+
+TODO: Improve drawings and explain more
+
+## Ingestify
+
+Ingestify focus' on Ingestion of data. 
+
+### How does Ingestify work?
+
+1. A `Source` is asked for all available `Datasets` using the `discover_datasets` method
+2. All available `Datasets` are compared with what's already fetched, and if it's changed (using a `FetchPolicy`)
+3. A `TaskQueue` is filled with `Tasks` to fetch all missing or stale `Datasets`
+
+<img src="https://raw.githubusercontent.com/PySport/ingestify/refs/heads/main/docs/overview.svg" />
+
+- [Source](blob/main/ingestify/domain/models/source.py) is the main entrance from Ingestify to external sources. A Source must always define:
+  - `discover_datasets` - Creates a list of all available datasets on the Source
+  - `fetch_dataset_files` - Fetches a single dataset for a Source
+- [Dataset Store](blob/main/ingestify/application/dataset_store.py) manages the access to the Metadata storage and the file storage. It keeps track of versions, and knows how to load data.
+- [Loader](blob/main/ingestify/application/loader.py) organizes the fetching process. It does this by executing the following steps:
+  1. Ask `Source` for all available datasets for a selector
+  2. Ask `Dataset Store` for all available datasets for a selector
+  3. Determines missing `Datasets`
+  4. Create tasks for data retrieval and puts in `TaskQueue`
+  5. Use multiprocessing to execute all tasks
+
+## Get started
+
+### Install
+
+Make sure you have installed the latest version:
+```bash
+pip install git+https://github.com/PySport/ingestify.git
+
+# OR
+
+pip install git+ssh://git@github.com/PySport/ingestify.git
+```
+
+### Using a template
+
+Ingestify provides some templates to get started quickly. When using `ingestify init` a new project will be created and example files are copied.
+Currently, Ingestify offers a `statsbomb_github` and `wyscout` template. 
+
+#### Statsbomb Github
+
+This uses https://github.com/statsbomb/open-data as source and syncs some competitions.
+
+```
+bash# ingestify init --template statsbomb_github /tmp/ingestify-test
+
+2023-05-23 08:57:51,250 [INFO] ingestify.cmdline: Initialized project at `/tmp/ingestify-test` with template `statsbomb_github`
+```
+
+#### Wyscout
+
+This requires valid Wyscout credentials. The templates includes some security best practices like using a `.env` file for credentials which isn't part of version control. 
+
+```
+bash# ingestify init --template wyscout /tmp/ingestify-test
+
+2023-05-23 08:58:18,720 [INFO] ingestify.cmdline: Initialized project at `/tmp/ingestify-test` with template `wyscout`
+```
+
+### Running Ingestify
+
+To actually run Ingestify you first change the current directory to the project directory.
+
+Then run:
+```bash
+bash# ingestify run
+
+2023-05-23 08:59:07,066 [INFO] ingestify.main: Initializing sources
+2023-05-23 08:59:07,068 [INFO] ingestify.main: Initializing IngestionEngine
+2023-05-23 08:59:07,086 [INFO] ingestify.main: Determining tasks...
+2023-05-23 08:59:07,364 [INFO] ingestify.application.loader: Discovered 33 datasets from StatsbombGithub using selector competition_id=11/season_id=42 => 33 tasks. 0 skipped.
+2023-05-23 08:59:07,625 [INFO] ingestify.application.loader: Discovered 35 datasets from StatsbombGithub using selector competition_id=11/season_id=90 => 35 tasks. 0 skipped.
+2023-05-23 08:59:07,625 [INFO] ingestify.application.loader: Scheduled 68 tasks. With 10 processes
+2023-05-23 08:59:07,654 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303516)
+2023-05-23 08:59:07,654 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303731)
+2023-05-23 08:59:07,655 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303430)
+2023-05-23 08:59:07,655 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303504)
+2023-05-23 08:59:07,655 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303421)
+2023-05-23 08:59:07,655 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303400)
+2023-05-23 08:59:07,656 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303664)
+2023-05-23 08:59:07,656 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303680)
+2023-05-23 08:59:07,657 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303487)
+2023-05-23 08:59:07,658 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303615)
+2023-05-23 08:59:08,419 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303532)
+2023-05-23 08:59:08,421 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303682)
+2023-05-23 08:59:08,444 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303451)
+2023-05-23 08:59:08,462 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303596)
+2023-05-23 08:59:08,518 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303634)
+2023-05-23 08:59:08,528 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303479)
+2023-05-23 08:59:08,541 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303696)
+2023-05-23 08:59:08,638 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303725)
+2023-05-23 08:59:08,684 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303600)
+2023-05-23 08:59:08,962 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303493)
+2023-05-23 08:59:09,270 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303548)
+2023-05-23 08:59:09,276 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303674)
+2023-05-23 08:59:09,292 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303700)
+2023-05-23 08:59:09,332 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303666)
+2023-05-23 08:59:09,411 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303377)
+2023-05-23 08:59:09,462 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303517)
+2023-05-23 08:59:09,491 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303473)
+2023-05-23 08:59:09,511 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773631)
+2023-05-23 08:59:09,726 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773497)
+2023-05-23 08:59:09,757 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773593)
+2023-05-23 08:59:09,957 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303652)
+2023-05-23 08:59:09,999 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303715)
+2023-05-23 08:59:10,075 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303470)
+2023-05-23 08:59:10,103 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303707)
+2023-05-23 08:59:10,188 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773457)
+2023-05-23 08:59:10,248 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303524)
+2023-05-23 08:59:10,282 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773665)
+2023-05-23 08:59:10,411 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=42/match_id=303610)
+2023-05-23 08:59:10,563 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773466)
+2023-05-23 08:59:10,711 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773585)
+2023-05-23 08:59:10,768 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773672)
+2023-05-23 08:59:10,778 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773565)
+2023-05-23 08:59:10,867 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773660)
+2023-05-23 08:59:10,954 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773656)
+2023-05-23 08:59:10,974 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773586)
+2023-05-23 08:59:11,026 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773387)
+2023-05-23 08:59:11,136 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773369)
+2023-05-23 08:59:11,438 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773552)
+2023-05-23 08:59:11,515 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773597)
+2023-05-23 08:59:11,586 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773571)
+2023-05-23 08:59:11,610 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773587)
+2023-05-23 08:59:11,690 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773386)
+2023-05-23 08:59:11,727 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773377)
+2023-05-23 08:59:11,757 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773372)
+2023-05-23 08:59:11,899 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3764661)
+2023-05-23 08:59:11,901 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773695)
+2023-05-23 08:59:12,006 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773661)
+2023-05-23 08:59:12,186 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773474)
+2023-05-23 08:59:12,283 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773523)
+2023-05-23 08:59:12,339 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773403)
+2023-05-23 08:59:12,426 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773428)
+2023-05-23 08:59:12,582 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773415)
+2023-05-23 08:59:12,583 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773689)
+2023-05-23 08:59:12,705 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773526)
+2023-05-23 08:59:13,510 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773477)
+2023-05-23 08:59:13,538 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3764440)
+2023-05-23 08:59:13,592 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773625)
+2023-05-23 08:59:15,017 [INFO] ingestify.application.loader: Running task CreateDatasetTask(StatsbombGithub -> competition_id=11/season_id=90/match_id=3773547)
+2023-05-23 08:59:15,917 [INFO] ingestify.cmdline: Done
+```
+
+When we run it for the second time:
+```bash
+bash# ingestify run
+
+2023-05-23 08:59:48,001 [INFO] ingestify.main: Initializing sources
+2023-05-23 08:59:48,002 [INFO] ingestify.main: Initializing IngestionEngine
+2023-05-23 08:59:48,006 [INFO] ingestify.main: Determining tasks...
+2023-05-23 08:59:48,067 [INFO] ingestify.application.loader: Discovered 33 datasets from StatsbombGithub using selector competition_id=11/season_id=42 => 0 tasks. 33 skipped.
+2023-05-23 08:59:48,118 [INFO] ingestify.application.loader: Discovered 35 datasets from StatsbombGithub using selector competition_id=11/season_id=90 => 0 tasks. 35 skipped.
+2023-05-23 08:59:48,118 [INFO] ingestify.application.loader: Nothing to do.
+2023-05-23 08:59:48,119 [INFO] ingestify.cmdline: Done
+```
+
+## Using the data
+
+The project contains a `query.py` file with an example of how to use the data.
+
+```bash
+bash# python query.py
+
+Loaded dataset with 3702 events
+Loaded dataset with 3994 events
+Loaded dataset with 3831 events
+Loaded dataset with 3647 events
+Loaded dataset with 4062 events
+Loaded dataset with 4051 events
+
+.....
+
+```
+
+
+How to go from raw data to parquet files:
+
+```python
+from ingestify.main import get_datastore
+
+store = get_datastore("config.yaml")
+
+dataset_collection = store.get_dataset_collection(
+    provider="statsbomb", stage="raw"
+)
+
+# Store.map is using multiprocessing by default
+store.map(
+    lambda dataset: (
+        store
+        
+        # As it's related to https://github.com/PySport/kloppy the store can load files using kloppy
+        .load_with_kloppy(dataset)
+        
+        # Convert it into a polars dataframe using all columns in the original data and some more additional ones
+        .to_df(
+            "*", 
+            match_id=dataset.identifier.match_id,
+            competition_id=dataset.identifier.competition_id,
+            season_id=dataset.identifier.season_id, 
+            
+            engine="polars"
+        )
+        
+        # Write to parquet format
+        .write_parquet(
+            f"/tmp/files/blaat/{dataset.identifier.match_id}.parquet"
+        )
+    ),
+    dataset_collection,
+)
+
+# TODO: 
+#  - when a file is written in parquet format (on any other format) it should be added as such to the store.
+```
+
+
+## Future work
+
+Some future work include:
+- Workflow tools - Run custom workflows using with tools like [Airflow](https://airflow.apache.org/), [Dagster](https://docs.dagster.io/getting-started), [Prefect](https://www.prefect.io/), [DBT](https://www.getdbt.com/)
+- Execution engines - Run tasks on other execution engines like [AWS Lambda](https://aws.amazon.com/lambda/), [Dask](https://www.dask.org/)
+- Lineage - Keep track of lineage with tools like [SQLLineage](https://sqllineage.readthedocs.io/en/latest/index.html)
+- Data quality - Monitor data quality with tools like [Great Expectations](https://docs.greatexpectations.io/docs/tutorials/quickstart/)
+- Event Bus - Automatically publish events to external systems like [AWS Event Bridge](https://aws.amazon.com/eventbridge/), [Azure Event Grid](https://learn.microsoft.com/en-us/azure/event-grid/overview), [Google Cloud Pub/Sub](https://cloud.google.com/pubsub/docs/overview), [Kafka](https://kafka.apache.org/), [RabbitMQ](https://www.rabbitmq.com/)
+- Query Engines - Integrate with query engines to run SQL queries directly on the store using tools like [DuckDB](https://duckdb.org/), [DataBend](https://databend.rs/), [DataFusion](https://arrow.apache.org/datafusion/), [Polars](https://www.pola.rs/), [Spark](https://spark.apache.org/)
+- Streaming Data - Ingest streaming data

ingestify-0.1.0.dist-info/RECORD

@@ -0,0 +1,79 @@
+ingestify/__init__.py,sha256=DdhKleT3RggJUMj5Auq_ImGLmKm-3HHs5Yerx_VsH_w,301
+ingestify/cmdline.py,sha256=gLy79Cq3OnEyoEcI6koWIEbCwvgFZ1E8n3UU1sKS8FM,7143
+ingestify/exceptions.py,sha256=wMMuajl4AkQRfW60TLN7btJmQaH8-lUczXyW_2g9kOU,143
+ingestify/main.py,sha256=YjrAOiGzwurtoDyIf981DSJHHA6IT5q09k3QNzTKCC8,6814
+ingestify/server.py,sha256=OVrf_XtpAQIn88MzqQzShXgsA9_jbnqYvD8YPBjn3cs,2413
+ingestify/source_base.py,sha256=GXAFCoT11Zov9M2v-fqQr9gFCXbtVfEIEH32V7r2oE8,382
+ingestify/utils.py,sha256=eEHwulqNEb2YTRDrCMVxr6mWZYI6KOcNCAIWFTi74u0,8029
+ingestify/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/application/dataset_store.py,sha256=NAW-XSvp118Lr2hXZd3qtuQr6VkPdWCLksIwd5MSs30,11489
+ingestify/application/ingestion_engine.py,sha256=GYIhb8a9ePkEcNOBPdfu-YawiD7eRZMRlxCA-6g9DRA,2249
+ingestify/application/loader.py,sha256=d7iXmdHN_yhDkEc2MMZ_6BLMEdRz9ChpBMy4yCWvxQo,13317
+ingestify/application/secrets_manager.py,sha256=5qCbPfUvRGP1Xbq6xPkMfpgYl8uPzF_0NbiKeRY5zxU,1757
+ingestify/domain/__init__.py,sha256=M7_fVTJjQUx53P4UQUPhowRKPKsIIjx4JYZL1yjHKsM,46
+ingestify/domain/models/__init__.py,sha256=xHVQZP57ZQYUKwAtbccnDKX89_yTOvBKAtn4XDVbEbY,930
+ingestify/domain/models/data_spec_version_collection.py,sha256=qjEM6-gt-Uf5orQlv64P6NJCEdWiUPX2oTZv8cC-KVY,1203
+ingestify/domain/models/extract_job.py,sha256=yXrlF2Vt5hxB1Vo9CicpgyW5rjvJaEPfSiMzaAqhqB0,624
+ingestify/domain/models/fetch_policy.py,sha256=d7K1TzliNJXxqaqzqEOQWLhvgIvmmqhUQEliXvSUcTs,1405
+ingestify/domain/models/sink.py,sha256=AieqDQ76Vj7WGxCrl3-F93AKe-VBfoPHtMNH28GTQM4,384
+ingestify/domain/models/source.py,sha256=sB3aqr2LfjIbtw7ODJpHnPj3RUeo7gYmTU7MXvfaYg4,973
+ingestify/domain/models/dataset/__init__.py,sha256=kSn3XZo0o-D0WzMb2VDxhOXw9Rr9jvS-8fkHdOnrccU,748
+ingestify/domain/models/dataset/collection.py,sha256=E2utQ6oyaFFrfQFMiwP9J_I7Wm21z0sRvE4Zc3QEs20,1310
+ingestify/domain/models/dataset/collection_metadata.py,sha256=gI5cb9M0QRsheIr2jA71wOyWfI5lGx5ES2Qw7rbDIoA,371
+ingestify/domain/models/dataset/dataset.py,sha256=m0iVJPXd1KOAHbDg7fmY_7MCdrKQaILUekIWUfo5pXI,2893
+ingestify/domain/models/dataset/dataset_repository.py,sha256=eiloP5msmDau4WRHee8gA7pLoH_ca2JXAhPx9UecPIA,1185
+ingestify/domain/models/dataset/events.py,sha256=x4l_pdzBHbemE_722EyCYXzWy9t8IcTx5j-wNFxWs6o,708
+ingestify/domain/models/dataset/file.py,sha256=O-yJom9dr13PaHfmc_4crtSa9B1Q9iruHsnf-m01McU,3943
+ingestify/domain/models/dataset/file_collection.py,sha256=V5wh2aSc61UA4HWcHi9PvyQUIUvssDRkaPVe2YR6XwU,1140
+ingestify/domain/models/dataset/file_repository.py,sha256=lxf3Dh8e-_67dRspMZHT1DZ79IWW_vlvb3z8lKjypj4,1514
+ingestify/domain/models/dataset/identifier.py,sha256=EJYsxt0OS_43Y989DZQq8U9NjwmtvnHGYGMe6-hOBlI,575
+ingestify/domain/models/dataset/revision.py,sha256=fiHnd_mad0iYmNCGswKImUHpauhIf2gW_ukztDFVP48,781
+ingestify/domain/models/dataset/selector.py,sha256=kEGpU8pIyjZ0zwE9n2uo_NY5xrNanWiTTgapyMAUEsw,1039
+ingestify/domain/models/event/__init__.py,sha256=OdPTpE9bj5QqdGmrYqRTLPX1f-LR9GWJYlGMPPEsuL8,138
+ingestify/domain/models/event/_old_event.py,sha256=RktgCAj9SMdtqkAc_bOwoghEb2Z6m4r5_xWXin9wqx4,472
+ingestify/domain/models/event/dispatcher.py,sha256=5WnyUJ7Qzr612btAtl1dMG9JBXDPcsBLyLmW6H7Q1zk,154
+ingestify/domain/models/event/domain_event.py,sha256=a5nNNwDWSAqou8aSBGIEA6aQOHTOxYyMEUXB91fYUIM,187
+ingestify/domain/models/event/event_bus.py,sha256=iseourbCwdUg-ODM5bM_u6cageJmceWLstOxiP3-2qU,576
+ingestify/domain/models/event/publisher.py,sha256=TOAawYYiPQCLR2Gm17LumMEzeapMDYcAYeklLFmwqAY,620
+ingestify/domain/models/event/subscriber.py,sha256=tP1ZFSvpJWKUITnATYekRxJzepz85UY7egBTMiP-dwg,1039
+ingestify/domain/models/resources/__init__.py,sha256=ZuY9DPRfwk-aLB3Lj6DYP_NqMkcQfcYjZp4VejTtcbU,46
+ingestify/domain/models/resources/dataset_resource.py,sha256=g0tu9QZQEdAGR-dRXQPL3ddcbEEGI__pvkDJGoscUTE,3027
+ingestify/domain/models/task/__init__.py,sha256=BdlyIPvE07Xax_IzLgO9DUw0wsz9OZutxnxdDNyRlys,79
+ingestify/domain/models/task/set.py,sha256=04txDYgS5rotXofD9TqChKdW0VZIYshrkfPIpXtlhW4,430
+ingestify/domain/models/task/task.py,sha256=R6tEZub-N_Wjl4VjwlPySdFb3L9D7nH4St2CcDzFoKA,107
+ingestify/domain/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/domain/services/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/domain/services/transformers/kloppy_to_pandas.py,sha256=NcN6nTBGVn9gz-_hWZJTMcduS1Gg7EM4X95Cqxi1QIM,809
+ingestify/infra/__init__.py,sha256=V0hpLzPVTcOHRVh0gguF6FT30YIgEOUd5v87xUHkfZ4,88
+ingestify/infra/fetch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/infra/fetch/http.py,sha256=gm7x0dACp3sTY1FMlbv8zRoQLZuZgtXmBg3HbhQ0syI,3086
+ingestify/infra/serialization/__init__.py,sha256=LwfmRoO4qykZkJZXxVPSKpwoVIkg9qzXa7Egut9JjL4,1772
+ingestify/infra/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/infra/sink/postgresql.py,sha256=SxuM3LntfYcpCriUpqJhMvgAf0s9cohXf6WkxSEDYDY,1816
+ingestify/infra/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/infra/source/statsbomb_github.py,sha256=CuHZoJn6fU8ZKQl4f1-gyaVYsmxL6R33n0cbOx1jQmI,2895
+ingestify/infra/source/wyscout.py,sha256=DxCzdkzYpVRHTfV9GpF8pe3FzwIk-WHYUlea6nOUGxE,5626
+ingestify/infra/store/__init__.py,sha256=3dA6NWfB6FS5SFdQiSlJ0ZghBfnUAUuGIP5Vr4rkCqk,43
+ingestify/infra/store/dataset/__init__.py,sha256=8oVJFiA-IKccrEpiYxAmSc65dfpNut7PYx8PUhylmdU,113
+ingestify/infra/store/dataset/local_dataset_repository.py,sha256=UMgSe1M9u_629V4WyuTJ-QegZJiDczzMo7vkNbNleqA,2064
+ingestify/infra/store/dataset/sqlalchemy/__init__.py,sha256=Z5JHWGO_hwT6rO-ecMOOAmOKjFFJi449KZvJTQgt6vQ,52
+ingestify/infra/store/dataset/sqlalchemy/mapping.py,sha256=Q7Od3zBnoZgxE5aThdZE93waWeKVut9dstrCnEYb9nc,3981
+ingestify/infra/store/dataset/sqlalchemy/repository.py,sha256=ynoIVMVD0_w9aa2hFKkcLxRKzJDoET_SNfGHXPIoN40,7067
+ingestify/infra/store/file/__init__.py,sha256=DuEekZa2pmDuRCFiulbgoGotN0wGv3OrRXSvokY0PhY,104
+ingestify/infra/store/file/local_file_repository.py,sha256=0oIzjjKO5U_7gPXhsBJFUqQBarQTFQS499ZK7HNxMxo,893
+ingestify/infra/store/file/s3_file_repository.py,sha256=txDviBrY9EHn3soqLFvTrjSPkyh548RxUgx4T83j0QY,1331
+ingestify/static/templates/statsbomb_github/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/static/templates/statsbomb_github/config.yaml.jinja2,sha256=_gAuAipfBL3ddLacyS1IBP5JluvPS2vmrb8GGaFtcUM,386
+ingestify/static/templates/statsbomb_github/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
+ingestify/static/templates/statsbomb_github/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
+ingestify/static/templates/wyscout/.env,sha256=o2kfuDC_seZNIqDscPf2Ww5TGiJmLh_DMOUNykGvs8Q,141
+ingestify/static/templates/wyscout/.gitignore,sha256=db0A2IjIeZf5fLLwXKD-bLmC4pETofxm848bljymnNs,13
+ingestify/static/templates/wyscout/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ingestify/static/templates/wyscout/config.yaml.jinja2,sha256=0zQXuvJVwd0oL2OJsPMZ8sOvRbdfRbieSGLQ44ezmYc,379
+ingestify/static/templates/wyscout/query.py,sha256=wjAOMoKvhX-BzCRqEm1SJp6YAcF8Fsq7ddrOaOpAeOk,364
+ingestify/static/templates/wyscout/database/README.md,sha256=7IuzjKo7Pqkx5wkmOETRZDljVOslqfA3ALuHMONq5dg,32
+ingestify-0.1.0.dist-info/METADATA,sha256=ryin_4RwMcyvqa4l6nESqysjlutcffHHWwoBvcwU784,18822
+ingestify-0.1.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+ingestify-0.1.0.dist-info/entry_points.txt,sha256=czYYXeX2ul4zdeB6bKlz3HaUF7zyVVcj9E_sRNDisI0,53
+ingestify-0.1.0.dist-info/top_level.txt,sha256=Lwnjgns4KequS7KiicXhh6mLUvcdfjzLyPI4qf_s4A0,10
+ingestify-0.1.0.dist-info/RECORD,,