ingestify 0.1.0__py3-none-any.whl

ingestify/infra/store/dataset/sqlalchemy/repository.py

@@ -0,0 +1,239 @@
+import json
+import uuid
+from typing import Optional, Union, List
+
+from sqlalchemy import create_engine, func, text, tuple_
+from sqlalchemy.engine import make_url
+from sqlalchemy.exc import NoSuchModuleError
+from sqlalchemy.orm import Session, joinedload
+
+from ingestify.domain import File
+from ingestify.domain.models import (
+    Dataset,
+    DatasetCollection,
+    DatasetRepository,
+    Identifier,
+    Selector,
+)
+from ingestify.domain.models.dataset.collection_metadata import (
+    DatasetCollectionMetadata,
+)
+
+from .mapping import dataset_table, metadata
+
+
+def parse_value(v):
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+def json_serializer(o):
+    return json.dumps(o)
+
+
+def json_deserializer(o):
+    o = json.loads(o)
+    # THIS BREAKS WHEN USING OTHER JSON COLUMNS!!
+    o = Identifier(**o)
+    return o
+
+
+# @compiles(DateTime, "mysql")
+# def compile_datetime_mysql(type_, compiler, **kw):
+#     return "DATETIME(6)"
+
+
+def isfloat(x):
+    try:
+        a = float(x)
+    except (TypeError, ValueError):
+        return False
+    else:
+        return True
+
+
+def isint(x):
+    try:
+        a = float(x)
+        b = int(a)
+    except (TypeError, ValueError):
+        return False
+    else:
+        return a == b
+
+
+class SqlAlchemyDatasetRepository(DatasetRepository):
+    @staticmethod
+    def fix_url(url: str) -> str:
+        if url.startswith("postgres://"):
+            url = url.replace("postgres://", "postgresql://")
+        return url
+
+    @classmethod
+    def supports(cls, url: str) -> bool:
+        url = cls.fix_url(url)
+
+        _url = make_url(url)
+        try:
+            _url.get_dialect()
+        except NoSuchModuleError:
+            return False
+        return True
+
+    def _init_engine(self):
+        self.engine = create_engine(
+            self.url,
+            # Use the default isolation level, don't need SERIALIZABLE
+            # isolation_level="SERIALIZABLE",
+            json_serializer=json_serializer,
+            json_deserializer=json_deserializer,
+        )
+        self.session = Session(bind=self.engine)
+
+    def __init__(self, url: str):
+        url = self.fix_url(url)
+
+        self.url = url
+        self._init_engine()
+
+        metadata.create_all(self.engine)
+
+    def __getstate__(self):
+        return {"url": self.url}
+
+    def __setstate__(self, state):
+        self.url = state["url"]
+        self._init_engine()
+
+    def __del__(self):
+        self.session.close()
+        self.engine.dispose()
+
+    def _filter_query(
+        self,
+        query,
+        bucket: str,
+        dataset_type: Optional[str] = None,
+        provider: Optional[str] = None,
+        dataset_id: Optional[Union[str, List[str]]] = None,
+        selector: Optional[Union[Selector, List[Selector]]] = None,
+    ):
+        query = query.filter(Dataset.bucket == bucket)
+        if dataset_type:
+            query = query.filter(Dataset.dataset_type == dataset_type)
+        if provider:
+            query = query.filter(Dataset.provider == provider)
+        if dataset_id is not None:
+            if isinstance(dataset_id, list):
+                if len(dataset_id) == 0:
+                    # When an empty list is explicitly passed, make sure we
+                    # return an empty DatasetCollection
+                    return DatasetCollection()
+
+                query = query.filter(Dataset.dataset_id.in_(dataset_id))
+            else:
+                query = query.filter(Dataset.dataset_id == dataset_id)
+
+        dialect = self.session.bind.dialect.name
+
+        if not isinstance(selector, list):
+            where, selector = selector.split("where")
+        else:
+            where = None
+
+        if selector:
+            if isinstance(selector, list):
+                selectors = selector
+            else:
+                selectors = [selector]
+
+            if not selectors:
+                raise ValueError("Selectors must contain at least one item")
+
+            keys = list(selectors[0].filtered_attributes.keys())
+
+            columns = []
+            first_selector = selectors[0].filtered_attributes
+
+            # Create a query like this:
+            #  SELECT * FROM dataset WHERE (column1, column2, column3) IN ((1, 2, 3), (4, 5, 6), (7, 8, 9))
+            for k in keys:
+                if dialect == "postgresql":
+                    column = dataset_table.c.identifier[k]
+
+                    # Take the value from the first selector to determine the type.
+                    # TODO: check all selectors to determine the type
+                    v = first_selector[k]
+                    if isint(v):
+                        column = column.as_integer()
+                    elif isfloat(v):
+                        column = column.as_float()
+                    else:
+                        column = column.as_string()
+                else:
+                    column = func.json_extract(Dataset.identifier, f"$.{k}")
+                columns.append(column)
+
+            values = []
+            for selector in selectors:
+                filtered_attributes = selector.filtered_attributes
+                values.append(tuple([filtered_attributes[k] for k in keys]))
+
+            query = query.filter(tuple_(*columns).in_(values))
+
+        if where:
+            query = query.filter(text(where))
+        return query
+
+    def get_dataset_collection(
+        self,
+        bucket: str,
+        dataset_type: Optional[str] = None,
+        provider: Optional[str] = None,
+        dataset_id: Optional[Union[str, List[str]]] = None,
+        selector: Optional[Union[Selector, List[Selector]]] = None,
+        metadata_only: bool = False,
+    ) -> DatasetCollection:
+        def apply_query_filter(query):
+            return self._filter_query(
+                query,
+                bucket=bucket,
+                dataset_type=dataset_type,
+                provider=provider,
+                dataset_id=dataset_id,
+                selector=selector,
+            )
+
+        if not metadata_only:
+            dataset_query = apply_query_filter(
+                self.session.query(Dataset).options(joinedload(Dataset.revisions))
+            )
+            datasets = list(dataset_query)
+        else:
+            datasets = []
+
+        metadata_result_row = apply_query_filter(
+            self.session.query(
+                func.min(File.modified_at).label("first_modified_at"),
+                func.max(File.modified_at).label("last_modified_at"),
+                func.count().label("row_count"),
+            ).join(Dataset, Dataset.dataset_id == File.dataset_id)
+        ).first()
+        dataset_collection_metadata = DatasetCollectionMetadata(*metadata_result_row)
+
+        return DatasetCollection(dataset_collection_metadata, datasets)
+
+    def save(self, bucket: str, dataset: Dataset):
+        # Just make sure
+        dataset.bucket = bucket
+        self.session.add(dataset)
+        self.session.commit()
+
+    def destroy(self, dataset: Dataset):
+        self.session.delete(dataset)
+        self.session.commit()
+
+    def next_identity(self):
+        return str(uuid.uuid4())

ingestify/infra/store/file/__init__.py

@@ -0,0 +1,2 @@
+from .local_file_repository import LocalFileRepository
+from .s3_file_repository import S3FileRepository

ingestify/infra/store/file/local_file_repository.py

@@ -0,0 +1,32 @@
+import os
+import shutil
+from pathlib import Path
+from typing import IO, AnyStr, BinaryIO
+
+from ingestify.domain.models import Dataset, FileRepository
+
+
+class LocalFileRepository(FileRepository):
+    @classmethod
+    def supports(cls, url: str) -> bool:
+        return url.startswith("file://")
+
+    def save_content(
+        self,
+        bucket: str,
+        dataset: Dataset,
+        revision_id: int,
+        filename: str,
+        stream: BinaryIO,
+    ) -> Path:
+        path = self.get_path(bucket, dataset, revision_id, filename)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(path, "wb") as fp:
+            shutil.copyfileobj(stream, fp)
+        return path
+
+    def load_content(
+        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
+    ) -> BinaryIO:
+        return open(self.get_path(bucket, dataset, revision_id, filename), "rb")

ingestify/infra/store/file/s3_file_repository.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+from typing import BinaryIO
+
+import boto3 as boto3
+
+from ingestify.domain import Dataset
+from ingestify.domain.models import FileRepository
+
+
+class S3FileRepository(FileRepository):
+    def __init__(self, url):
+        super().__init__(url)
+
+        self._s3 = None
+
+    @property
+    def s3(self):
+        if not self._s3:
+            self._s3 = boto3.resource("s3")
+        return self._s3
+
+    def __getstate__(self):
+        return {"base_dir": self.base_dir, "_s3": None}
+
+    def save_content(
+        self,
+        bucket: str,
+        dataset: Dataset,
+        revision_id: int,
+        filename: str,
+        stream: BinaryIO,
+    ) -> Path:
+        key = self.get_path(bucket, dataset, revision_id, filename)
+        s3_bucket = Path(key.parts[0])
+
+        self.s3.Object(str(s3_bucket), str(key.relative_to(s3_bucket))).put(Body=stream)
+        return key
+
+    def load_content(
+        self, bucket: str, dataset: Dataset, revision_id: int, filename: str
+    ) -> BinaryIO:
+        key = self.get_path(bucket, dataset, revision_id, filename)
+        s3_bucket = Path(key.parts[0])
+        return self.s3.Object(str(s3_bucket), str(key.relative_to(s3_bucket))).get()[
+            "Body"
+        ]
+
+    @classmethod
+    def supports(cls, url: str) -> bool:
+        return url.startswith("s3://")

ingestify/main.py

@@ -0,0 +1,205 @@
+import importlib
+import logging
+import os
+import sys
+from itertools import product
+from typing import Optional, Type
+
+from pyaml_env import parse_config
+
+from ingestify import Source
+from ingestify.application.dataset_store import DatasetStore
+from ingestify.application.ingestion_engine import IngestionEngine
+from ingestify.application.secrets_manager import SecretsManager
+from ingestify.domain import Selector
+from ingestify.domain.models import (
+    dataset_repository_factory,
+    file_repository_factory,
+)
+from ingestify.domain.models.data_spec_version_collection import (
+    DataSpecVersionCollection,
+)
+from ingestify.domain.models.event import EventBus, Publisher, Subscriber
+
+from ingestify.domain.models.extract_job import ExtractJob
+from ingestify.domain.models.fetch_policy import FetchPolicy
+from ingestify.exceptions import ConfigurationError
+
+logger = logging.getLogger(__name__)
+
+secrets_manager = SecretsManager()
+
+
+def _product_selectors(selector_args):
+    if not selector_args:
+        # Empty selector passed. This is a special case when
+        # a Source doesn't have discover_selectors but also doesn't require
+        # selectors
+        yield dict()
+        return
+
+    if isinstance(selector_args, str):
+        if selector_args == "*":
+            yield lambda dict_selector: True
+        else:
+            yield lambda dict_selector: eval(selector_args, {}, dict_selector)
+        return
+
+    selector_args_ = {
+        k: v if isinstance(v, list) else [v] for k, v in selector_args.items()
+    }
+    keys, values = zip(*selector_args_.items())
+    for bundle in product(*values):
+        yield dict(zip(keys, bundle))
+
+
+def import_cls(name):
+    components = name.split(".")
+    mod = importlib.import_module(".".join(components[:-1]))
+    return getattr(mod, components[-1])
+
+
+def get_dataset_store_by_urls(
+    dataset_url: str, file_url: str, bucket: str
+) -> DatasetStore:
+    """
+    Initialize a DatasetStore by a DatasetRepository and a FileRepository
+    """
+    if not bucket:
+        raise Exception("Bucket is not specified")
+
+    file_repository = file_repository_factory.build_if_supports(url=file_url)
+
+    if secrets_manager.supports(dataset_url):
+        dataset_url = secrets_manager.load_as_db_url(dataset_url)
+
+    if dataset_url.startswith("postgres://"):
+        dataset_url = dataset_url.replace("postgress://", "postgress+")
+
+    dataset_repository = dataset_repository_factory.build_if_supports(url=dataset_url)
+    return DatasetStore(
+        dataset_repository=dataset_repository,
+        file_repository=file_repository,
+        bucket=bucket,
+    )
+
+
+def get_datastore(config_file, bucket: Optional[str] = None) -> DatasetStore:
+    config = parse_config(config_file, default_value="")
+
+    return get_dataset_store_by_urls(
+        dataset_url=config["main"]["dataset_url"],
+        file_url=config["main"]["file_url"],
+        bucket=bucket or config["main"].get("default_bucket"),
+    )
+
+
+def get_remote_datastore(url: str, bucket: str, **kwargs) -> DatasetStore:
+    return get_dataset_store_by_urls(dataset_url=url, file_url=url, bucket=bucket)
+
+
+def get_source_cls(key: str) -> Type[Source]:
+    if key.startswith("ingestify."):
+        _, type_ = key.split(".")
+        if type_ == "wyscout":
+            from ingestify.infra.source.wyscout import Wyscout
+
+            return Wyscout
+
+        elif type_ == "statsbomb_github":
+            from ingestify.infra.source.statsbomb_github import StatsbombGithub
+
+            return StatsbombGithub
+        else:
+            raise Exception(f"Unknown source type 'ingestify.{type_}'")
+    else:
+        return import_cls(key)
+
+
+def build_source(name, source_args):
+    source_cls = get_source_cls(source_args["type"])
+    raw_configuration = source_args.get("configuration", {})
+    configuration = {}
+    if isinstance(raw_configuration, list):
+        # This normally means the data needs to be loaded from somewhere else
+        for item in raw_configuration:
+            if isinstance(item, dict):
+                configuration.update(item)
+            elif secrets_manager.supports(item):
+                item = secrets_manager.load_as_dict(item)
+                configuration.update(item)
+            else:
+                raise ConfigurationError(
+                    f"Don't know how to use source configuration '{item}'"
+                )
+    elif isinstance(raw_configuration, str):
+        configuration = secrets_manager.load_as_dict(raw_configuration)
+    else:
+        configuration = raw_configuration
+
+    return source_cls(name=name, **configuration)
+
+
+def get_event_subscriber_cls(key: str) -> Type[Subscriber]:
+    return import_cls(key)
+
+
+def get_engine(config_file, bucket: Optional[str] = None) -> IngestionEngine:
+    config = parse_config(config_file, default_value="")
+
+    logger.info("Initializing sources")
+    sources = {}
+    sys.path.append(os.path.dirname(config_file))
+    for name, source_args in config["sources"].items():
+        sources[name] = build_source(name=name, source_args=source_args)
+
+    logger.info("Initializing IngestionEngine")
+    store = get_dataset_store_by_urls(
+        dataset_url=config["main"]["dataset_url"],
+        file_url=config["main"]["file_url"],
+        bucket=bucket or config["main"].get("default_bucket"),
+    )
+
+    # Setup an EventBus and wire some more components
+    event_bus = EventBus()
+    publisher = Publisher()
+    for subscriber in config.get("event_subscribers", []):
+        cls = get_event_subscriber_cls(subscriber["type"])
+        publisher.add_subscriber(cls(store))
+    event_bus.register(publisher)
+    store.set_event_bus(event_bus)
+
+    ingestion_engine = IngestionEngine(
+        store=store,
+    )
+
+    logger.info("Determining tasks...")
+
+    fetch_policy = FetchPolicy()
+
+    for job in config["extract_jobs"]:
+        data_spec_versions = DataSpecVersionCollection.from_dict(
+            job.get("data_spec_versions", {"default": {"v1"}})
+        )
+
+        if "selectors" in job:
+            selectors = [
+                Selector.build(selector, data_spec_versions=data_spec_versions)
+                for selector_args in job["selectors"]
+                for selector in _product_selectors(selector_args)
+            ]
+        else:
+            # Add a single empty selector. This won't match anything
+            # but makes it easier later one where we loop over selectors.
+            selectors = [Selector.build({}, data_spec_versions=data_spec_versions)]
+
+        import_job = ExtractJob(
+            source=sources[job["source"]],
+            dataset_type=job["dataset_type"],
+            selectors=selectors,
+            fetch_policy=fetch_policy,
+            data_spec_versions=data_spec_versions,
+        )
+        ingestion_engine.add_extract_job(import_job)
+
+    return ingestion_engine

ingestify/server.py

@@ -0,0 +1,78 @@
+import flask
+from flask import Response, request
+from flask_restful import Resource, Api
+
+
+# Dataset
+from ingestify.infra.serialization import serialize
+from ingestify.main import get_datastore
+
+
+def create_server(config_file: str):
+    app = flask.Flask(__name__)
+    api = Api(app, prefix="/api")
+
+    # @api.representation('application/json')
+    # def output_json(data, code, headers=None):
+    #     resp = make_response(json.dumps(data, cls=DecimalEncoder), code)
+    #     resp.headers.extend(headers or {})
+    #     return resp
+
+    datastore_cache = {}
+
+    def get_datastore_by_bucket(bucket: str):
+        try:
+            return datastore_cache[bucket]
+        except KeyError:
+            datastore_cache[bucket] = get_datastore(config_file, bucket=bucket)
+            return datastore_cache[bucket]
+
+    class DatasetResource(Resource):
+        def patch(self, bucket: str, dataset_id: str):
+            # TODO: Filter out dataset from body
+            return "OK"
+
+        def delete(self, bucket: str, dataset_id: str):
+            pass
+
+    class DatasetListResource(Resource):
+        def get(self, bucket: str):
+            return serialize(get_datastore_by_bucket(bucket).get_dataset_collection())
+
+    class FileResource(Resource):
+        def get(self, bucket, dataset_id: str, version: int, filename: str):
+            return Response(
+                get_datastore_by_bucket(bucket)
+                .load_content(dataset_id, version, filename)
+                .read()
+            )
+
+        def put(self, bucket, dataset_id: str, version: int, filename: str):
+            return Response(
+                get_datastore_by_bucket(bucket).save_content(
+                    dataset_id, version, filename, request.stream
+                )
+            )
+
+    api.add_resource(
+        DatasetListResource, "/buckets/<string:bucket>/datasets", methods=["GET"]
+    )
+    api.add_resource(
+        DatasetResource,
+        "/buckets/<string:bucket>/datasets/<string:dataset_id>",
+        methods=["PATCH", "DELETE"],
+    )
+    api.add_resource(
+        FileResource,
+        "/buckets/<string:bucket>/"
+        "datasets/<string:dataset_id>/files/"
+        "<string:version>/<string:filename>",
+        methods=["GET", "PUT"],
+    )
+
+    return app
+
+
+if __name__ == "__main__":
+    app = create_server(config_file="../examples/statsbomb/config_local.yaml")
+    app.run(host="0.0.0.0", port=8080)

ingestify/source_base.py

@@ -0,0 +1,23 @@
+from ingestify.application.dataset_store import DatasetStore
+from ingestify.domain.models import (
+    Dataset,
+    DatasetResource,
+    DraftFile,
+    File,
+    Identifier,
+    Selector,
+    Source,
+    Revision,
+)
+
+__all__ = [
+    "Selector",
+    "Identifier",
+    "Source",
+    "DatasetStore",
+    "Dataset",
+    "DatasetResource",
+    "Revision",
+    "File",
+    "DraftFile",
+]

ingestify/static/templates/statsbomb_github/config.yaml.jinja2

@@ -0,0 +1,19 @@
+ingestify_version: {{ ingestify_version }}
+
+main:
+  dataset_url: sqlite:///database/catalog.db
+  file_url: file://database/files/
+  default_bucket: main
+
+sources:
+  statsbomb:
+    type: ingestify.statsbomb_github
+
+extract_jobs:
+  - source: statsbomb
+    selectors:
+      - competition_id: 11
+        season_id: [42, 90]
+
+    # passing an empty selector means: fetch everything
+    #  -

ingestify/static/templates/statsbomb_github/database/README.md

@@ -0,0 +1 @@
+# This will contain the database

ingestify/static/templates/statsbomb_github/query.py

@@ -0,0 +1,14 @@
+from ingestify.main import get_datastore
+
+
+def main():
+    store = get_datastore("config.yaml")
+    dataset_collection = store.get_dataset_collection()
+
+    for dataset in dataset_collection:
+        kloppy_dataset = store.load_with_kloppy(dataset)
+        print(f"Loaded dataset with {len(kloppy_dataset.records)} events")
+
+
+if __name__ == "__main__":
+    main()

ingestify/static/templates/wyscout/.env

@@ -0,0 +1,5 @@
+# Template .env file from Ingestify
+# You should not add this file to a version control system like git
+
+WYSCOUT_USERNAME=
+WYSCOUT_PASSWORD=

ingestify/static/templates/wyscout/.gitignore

@@ -0,0 +1,2 @@
+.env
+database