ingestr 0.7.4__py3-none-any.whl → 0.7.6__py3-none-any.whl

ingestr/main.py

@@ -1,6 +1,7 @@
 import hashlib
 from datetime import datetime
 from enum import Enum
+import tempfile
 from typing import Optional
 
 import dlt
@@ -236,6 +237,13 @@ def ingest(
             envvar="SCHEMA_NAMING",
         ),
     ] = SchemaNaming.default,  # type: ignore
+    pipelines_dir: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The path to store dlt-related pipeline metadata. By default, ingestr will create a temporary directory and delete it after the execution is done in order to make retries stateless.",
+            envvar="PIPELINES_DIR",
+        ),
+    ] = None,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -280,13 +288,18 @@ def ingest(
         if progress == Progress.log:
             progressInstance = LogCollector(dump_system_stats=False)
 
+        is_pipelines_dir_temp = False
+        if pipelines_dir is None:
+            pipelines_dir = tempfile.mkdtemp()
+            is_pipelines_dir_temp = True
+
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),
             destination=destination.dlt_dest(
                 uri=dest_uri,
             ),
             progress=progressInstance,
-            pipelines_dir="pipeline_data",
+            pipelines_dir=pipelines_dir,
             refresh="drop_resources" if full_refresh else None,
         )
 
@@ -362,6 +375,8 @@ def ingest(
         if incremental_strategy != IncrementalStrategy.none:
             write_disposition = incremental_strategy.value
 
+        start_time = datetime.now()
+
         run_info: LoadInfo = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(
@@ -389,11 +404,17 @@ def ingest(
 
         destination.post_load()
 
+        end_time = datetime.now()
         elapsedHuman = ""
         if run_info.started_at:
-            elapsed = run_info.finished_at - run_info.started_at
+            elapsed = end_time - start_time
             elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
 
+        # remove the pipelines_dir folder if it was created by ingestr
+        if is_pipelines_dir_temp:
+            import shutil
+            shutil.rmtree(pipelines_dir)
+
         print(
             f"[bold green]Successfully finished loading data from '{factory.source_scheme}' to '{factory.destination_scheme}' {elapsedHuman} [/bold green]"
         )

ingestr/src/chess/__init__.py

@@ -0,0 +1,166 @@
+"""A source loading player profiles and games from chess.com api"""
+
+from typing import Any, Callable, Dict, Iterator, List, Sequence
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers import requests
+
+from .helpers import get_path_with_retry, get_url_with_retry, validate_month_string
+from .settings import UNOFFICIAL_CHESS_API_URL
+
+
+@dlt.source(name="chess")
+def source(
+    players: List[str], start_month: str = None, end_month: str = None
+) -> Sequence[DltResource]:
+    """
+    A dlt source for the chess.com api. It groups several resources (in this case chess.com API endpoints) containing
+    various types of data: user profiles or chess match results
+    Args:
+        players (List[str]): A list of the player usernames for which to get the data.
+        start_month (str, optional): Filters out all the matches happening before `start_month`. Defaults to None.
+        end_month (str, optional): Filters out all the matches happening after `end_month`. Defaults to None.
+    Returns:
+        Sequence[DltResource]: A sequence of resources that can be selected from including players_profiles,
+        players_archives, players_games, players_online_status
+    """
+    return (
+        players_profiles(players),
+        players_archives(players),
+        players_games(players, start_month=start_month, end_month=end_month),
+        players_online_status(players),
+    )
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "last_online": {"data_type": "timestamp"},
+        "joined": {"data_type": "timestamp"},
+    },
+)
+def players_profiles(players: List[str]) -> Iterator[TDataItem]:
+    """
+    Yields player profiles for a list of player usernames.
+    Args:
+        players (List[str]): List of player usernames to retrieve profiles for.
+    Yields:
+        Iterator[TDataItem]: An iterator over player profiles data.
+    """
+
+    # get archives in parallel by decorating the http request with defer
+    @dlt.defer
+    def _get_profile(username: str) -> TDataItem:
+        return get_path_with_retry(f"player/{username}")
+
+    for username in players:
+        yield _get_profile(username)
+
+
+@dlt.resource(write_disposition="replace", selected=False)
+def players_archives(players: List[str]) -> Iterator[List[TDataItem]]:
+    """
+    Yields url to game archives for specified players.
+    Args:
+        players (List[str]): List of player usernames to retrieve archives for.
+    Yields:
+        Iterator[List[TDataItem]]: An iterator over list of player archive data.
+    """
+    for username in players:
+        data = get_path_with_retry(f"player/{username}/games/archives")
+        yield data.get("archives", [])
+
+
+@dlt.resource(
+    write_disposition="append", columns={"end_time": {"data_type": "timestamp"}}
+)
+def players_games(
+    players: List[str], start_month: str = None, end_month: str = None
+) -> Iterator[Callable[[], List[TDataItem]]]:
+    """
+    Yields `players` games that happened between `start_month` and `end_month`.
+    Args:
+        players (List[str]): List of player usernames to retrieve games for.
+        start_month (str, optional): The starting month in the format "YYYY/MM". Defaults to None.
+        end_month (str, optional): The ending month in the format "YYYY/MM". Defaults to None.
+    Yields:
+        Iterator[Callable[[], List[TDataItem]]]: An iterator over callables that return a list of games for each player.
+    """  # do a simple validation to prevent common mistakes in month format
+    validate_month_string(start_month)
+    validate_month_string(end_month)
+
+    # get a list of already checked archives
+    # from your point of view, the state is python dictionary that will have the same content the next time this function is called
+    checked_archives = dlt.current.resource_state().setdefault("archives", [])
+    # get player archives, note that you can call the resource like any other function and just iterate it like a list
+    archives = players_archives(players)
+
+    # get archives in parallel by decorating the http request with defer
+    @dlt.defer
+    def _get_archive(url: str) -> List[TDataItem]:
+        try:
+            games = get_url_with_retry(url).get("games", [])
+            return games  # type: ignore
+        except requests.HTTPError as http_err:
+            # sometimes archives are not available and the error seems to be permanent
+            if http_err.response.status_code == 404:
+                return []
+            raise
+
+    # enumerate the archives
+    for url in archives:
+        # the `url` format is https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM}
+        if start_month and url[-7:] < start_month:
+            continue
+        if end_month and url[-7:] > end_month:
+            continue
+        # do not download archive again
+        if url in checked_archives:
+            continue
+        checked_archives.append(url)
+        # get the filtered archive
+        yield _get_archive(url)
+
+
+@dlt.resource(write_disposition="append")
+def players_online_status(players: List[str]) -> Iterator[TDataItem]:
+    """
+    Returns current online status for a list of players.
+    Args:
+        players (List[str]): List of player usernames to check online status for.
+    Yields:
+        Iterator[TDataItem]: An iterator over the online status of each player.
+    """
+    # we'll use unofficial endpoint to get online status, the official seems to be removed
+    for player in players:
+        status = get_url_with_retry(f"{UNOFFICIAL_CHESS_API_URL}user/popup/{player}")
+        # return just relevant selection
+        yield {
+            "username": player,
+            "onlineStatus": status["onlineStatus"],
+            "lastLoginDate": status["lastLoginDate"],
+            "check_time": pendulum.now(),  # dlt can deal with native python dates
+        }
+
+
+@dlt.source
+def chess_dlt_config_example(
+    secret_str: str = dlt.secrets.value,
+    secret_dict: Dict[str, Any] = dlt.secrets.value,
+    config_int: int = dlt.config.value,
+) -> DltResource:
+    """
+    An example of a source that uses dlt to provide secrets and config values.
+    Args:
+        secret_str (str, optional): Secret string provided by dlt.secrets.value. Defaults to dlt.secrets.value.
+        secret_dict (Dict[str, Any], optional): Secret dictionary provided by dlt.secrets.value. Defaults to dlt.secrets.value.
+        config_int (int, optional): Config integer provided by dlt.config.value. Defaults to dlt.config.value.
+    Returns:
+        DltResource: Returns a resource yielding the configured values.
+    """
+
+    # returns a resource yielding the configured values - it is just a test
+    return dlt.resource([secret_str, secret_dict, config_int], name="config_values")

ingestr/src/chess/helpers.py

@@ -0,0 +1,21 @@
+"""Chess source helpers"""
+
+from dlt.common.typing import StrAny
+from dlt.sources.helpers import requests
+
+from .settings import OFFICIAL_CHESS_API_URL
+
+
+def get_url_with_retry(url: str) -> StrAny:
+    r = requests.get(url)
+    return r.json()  # type: ignore
+
+
+def get_path_with_retry(path: str) -> StrAny:
+    return get_url_with_retry(f"{OFFICIAL_CHESS_API_URL}{path}")
+
+
+def validate_month_string(string: str) -> None:
+    """Validates that the string is in YYYY/MM format"""
+    if string and string[4] != "/":
+        raise ValueError(string)

ingestr/src/chess/settings.py

@@ -0,0 +1,4 @@
+"""Chess source settings and constants"""
+
+OFFICIAL_CHESS_API_URL = "https://api.chess.com/pub/"
+UNOFFICIAL_CHESS_API_URL = "https://www.chess.com/callback/"

ingestr/src/factory.py

@@ -15,13 +15,16 @@ from ingestr.src.destinations import (
     SynapseDestination,
 )
 from ingestr.src.sources import (
+    ChessSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HubspotSource,
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
     ShopifySource,
     SqlSource,
+    StripeAnalyticsSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -102,6 +105,12 @@ class SourceDestinationFactory:
             return ShopifySource()
         elif self.source_scheme == "gorgias":
             return GorgiasSource()
+        elif self.source_scheme == "chess":
+            return ChessSource()
+        elif self.source_scheme == "stripe":
+            return StripeAnalyticsSource()
+        elif self.source_scheme == "hubspot":
+            return HubspotSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
 

ingestr/src/hubspot/__init__.py

@@ -0,0 +1,281 @@
+"""
+This is a module that provides a DLT source to retrieve data from multiple endpoints of the HubSpot API using a specified API key. The retrieved data is returned as a tuple of Dlt resources, one for each endpoint.
+
+The source retrieves data from the following endpoints:
+- CRM Companies
+- CRM Contacts
+- CRM Deals
+- CRM Tickets
+- CRM Products
+- CRM Quotes
+- Web Analytics Events
+
+For each endpoint, a resource and transformer function are defined to retrieve data and transform it to a common format.
+The resource functions yield the raw data retrieved from the API, while the transformer functions are used to retrieve
+additional information from the Web Analytics Events endpoint.
+
+The source also supports enabling Web Analytics Events for each endpoint by setting the corresponding enable flag to True.
+
+Example:
+To retrieve data from all endpoints, use the following code:
+
+python
+
+>>> resources = hubspot(api_key="your_api_key")
+"""
+
+from typing import Any, Dict, Iterator, List, Literal, Sequence
+from urllib.parse import quote
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.typing import TDataItems
+from dlt.sources import DltResource
+
+from .helpers import _get_property_names, fetch_data, fetch_property_history
+from .settings import (
+    ALL,
+    CRM_OBJECT_ENDPOINTS,
+    DEFAULT_COMPANY_PROPS,
+    DEFAULT_CONTACT_PROPS,
+    DEFAULT_DEAL_PROPS,
+    DEFAULT_PRODUCT_PROPS,
+    DEFAULT_QUOTE_PROPS,
+    DEFAULT_TICKET_PROPS,
+    OBJECT_TYPE_PLURAL,
+    STARTDATE,
+    WEB_ANALYTICS_EVENTS_ENDPOINT,
+)
+
+THubspotObjectType = Literal["company", "contact", "deal", "ticket", "product", "quote"]
+
+
+@dlt.source(name="hubspot")
+def hubspot(
+    api_key: str = dlt.secrets.value,
+    include_history: bool = False,
+    include_custom_props: bool = True,
+) -> Sequence[DltResource]:
+    """
+    A DLT source that retrieves data from the HubSpot API using the
+    specified API key.
+
+    This function retrieves data for several HubSpot API endpoints,
+    including companies, contacts, deals, tickets, products and web
+    analytics events. It returns a tuple of Dlt resources, one for
+    each endpoint.
+
+    Args:
+        api_key (Optional[str]):
+            The API key used to authenticate with the HubSpot API. Defaults
+            to dlt.secrets.value.
+        include_history (Optional[bool]):
+            Whether to load history of property changes along with entities.
+            The history entries are loaded to separate tables.
+
+    Returns:
+        Sequence[DltResource]: Dlt resources, one for each HubSpot API endpoint.
+
+    Notes:
+        This function uses the `fetch_data` function to retrieve data from the
+        HubSpot CRM API. The API key is passed to `fetch_data` as the
+        `api_key` argument.
+    """
+
+    @dlt.resource(name="companies", write_disposition="replace")
+    def companies(
+        api_key: str = api_key,
+        include_history: bool = include_history,
+        props: Sequence[str] = DEFAULT_COMPANY_PROPS,
+        include_custom_props: bool = include_custom_props,
+    ) -> Iterator[TDataItems]:
+        """Hubspot companies resource"""
+        yield from crm_objects(
+            "company",
+            api_key,
+            include_history=include_history,
+            props=props,
+            include_custom_props=include_custom_props,
+        )
+
+    @dlt.resource(name="contacts", write_disposition="replace")
+    def contacts(
+        api_key: str = api_key,
+        include_history: bool = include_history,
+        props: Sequence[str] = DEFAULT_CONTACT_PROPS,
+        include_custom_props: bool = include_custom_props,
+    ) -> Iterator[TDataItems]:
+        """Hubspot contacts resource"""
+        yield from crm_objects(
+            "contact",
+            api_key,
+            include_history,
+            props,
+            include_custom_props,
+        )
+
+    @dlt.resource(name="deals", write_disposition="replace")
+    def deals(
+        api_key: str = api_key,
+        include_history: bool = include_history,
+        props: Sequence[str] = DEFAULT_DEAL_PROPS,
+        include_custom_props: bool = include_custom_props,
+    ) -> Iterator[TDataItems]:
+        """Hubspot deals resource"""
+        yield from crm_objects(
+            "deal",
+            api_key,
+            include_history,
+            props,
+            include_custom_props,
+        )
+
+    @dlt.resource(name="tickets", write_disposition="replace")
+    def tickets(
+        api_key: str = api_key,
+        include_history: bool = include_history,
+        props: Sequence[str] = DEFAULT_TICKET_PROPS,
+        include_custom_props: bool = include_custom_props,
+    ) -> Iterator[TDataItems]:
+        """Hubspot tickets resource"""
+        yield from crm_objects(
+            "ticket",
+            api_key,
+            include_history,
+            props,
+            include_custom_props,
+        )
+
+    @dlt.resource(name="products", write_disposition="replace")
+    def products(
+        api_key: str = api_key,
+        include_history: bool = include_history,
+        props: Sequence[str] = DEFAULT_PRODUCT_PROPS,
+        include_custom_props: bool = include_custom_props,
+    ) -> Iterator[TDataItems]:
+        """Hubspot products resource"""
+        yield from crm_objects(
+            "product",
+            api_key,
+            include_history,
+            props,
+            include_custom_props,
+        )
+
+    @dlt.resource(name="quotes", write_disposition="replace")
+    def quotes(
+        api_key: str = api_key,
+        include_history: bool = include_history,
+        props: Sequence[str] = DEFAULT_QUOTE_PROPS,
+        include_custom_props: bool = include_custom_props,
+    ) -> Iterator[TDataItems]:
+        """Hubspot quotes resource"""
+        yield from crm_objects(
+            "quote",
+            api_key,
+            include_history,
+            props,
+            include_custom_props,
+        )
+
+    return companies, contacts, deals, tickets, products, quotes
+
+
+def crm_objects(
+    object_type: str,
+    api_key: str = dlt.secrets.value,
+    include_history: bool = False,
+    props: Sequence[str] = None,
+    include_custom_props: bool = True,
+) -> Iterator[TDataItems]:
+    """Building blocks for CRM resources."""
+    if props == ALL:
+        props = list(_get_property_names(api_key, object_type))
+
+    if include_custom_props:
+        all_props = _get_property_names(api_key, object_type)
+        custom_props = [prop for prop in all_props if not prop.startswith("hs_")]
+        props = props + custom_props  # type: ignore
+
+    props = ",".join(sorted(list(set(props))))
+
+    if len(props) > 2000:
+        raise ValueError(
+            "Your request to Hubspot is too long to process. "
+            "Maximum allowed query length is 2000 symbols, while "
+            f"your list of properties `{props[:200]}`... is {len(props)} "
+            "symbols long. Use the `props` argument of the resource to "
+            "set the list of properties to extract from the endpoint."
+        )
+
+    params = {"properties": props, "limit": 100}
+
+    yield from fetch_data(CRM_OBJECT_ENDPOINTS[object_type], api_key, params=params)
+    if include_history:
+        # Get history separately, as requesting both all properties and history together
+        # is likely to hit hubspot's URL length limit
+        for history_entries in fetch_property_history(
+            CRM_OBJECT_ENDPOINTS[object_type],
+            api_key,
+            props,
+        ):
+            yield dlt.mark.with_table_name(
+                history_entries,
+                OBJECT_TYPE_PLURAL[object_type] + "_property_history",
+            )
+
+
+@dlt.resource
+def hubspot_events_for_objects(
+    object_type: THubspotObjectType,
+    object_ids: List[str],
+    api_key: str = dlt.secrets.value,
+    start_date: pendulum.DateTime = STARTDATE,
+) -> DltResource:
+    """
+    A standalone DLT resources that retrieves web analytics events from the HubSpot API for a particular object type and list of object ids.
+
+    Args:
+        object_type(THubspotObjectType, required): One of the hubspot object types see definition of THubspotObjectType literal
+        object_ids: (List[THubspotObjectType], required): List of object ids to track events
+        api_key (str, optional): The API key used to authenticate with the HubSpot API. Defaults to dlt.secrets.value.
+        start_date (datetime, optional): The initial date time from which start getting events, default to STARTDATE
+
+    Returns:
+        incremental dlt resource to track events for objects from the list
+    """
+
+    end_date = pendulum.now().isoformat()
+    name = object_type + "_events"
+
+    def get_web_analytics_events(
+        occurred_at: dlt.sources.incremental[str],
+    ) -> Iterator[List[Dict[str, Any]]]:
+        """
+        A helper function that retrieves web analytics events for a given object type from the HubSpot API.
+
+        Args:
+            object_type (str): The type of object for which to retrieve web analytics events.
+
+        Yields:
+            dict: A dictionary representing a web analytics event.
+        """
+        for object_id in object_ids:
+            yield from fetch_data(
+                WEB_ANALYTICS_EVENTS_ENDPOINT.format(
+                    objectType=object_type,
+                    objectId=object_id,
+                    occurredAfter=quote(occurred_at.last_value),
+                    occurredBefore=quote(end_date),
+                ),
+                api_key=api_key,
+            )
+
+    return dlt.resource(
+        get_web_analytics_events,
+        name=name,
+        primary_key="id",
+        write_disposition="append",
+        selected=True,
+        table_name=lambda e: name + "_" + str(e["eventType"]),
+    )(dlt.sources.incremental("occurredAt", initial_value=start_date.isoformat()))

ingestr/src/hubspot/helpers.py

@@ -0,0 +1,188 @@
+"""Hubspot source helpers"""
+
+import urllib.parse
+from typing import Any, Dict, Iterator, List, Optional
+
+from dlt.sources.helpers import requests
+
+from .settings import OBJECT_TYPE_PLURAL
+
+BASE_URL = "https://api.hubapi.com/"
+
+
+def get_url(endpoint: str) -> str:
+    """Get absolute hubspot endpoint URL"""
+    return urllib.parse.urljoin(BASE_URL, endpoint)
+
+
+def _get_headers(api_key: str) -> Dict[str, str]:
+    """
+    Return a dictionary of HTTP headers to use for API requests, including the specified API key.
+
+    Args:
+        api_key (str): The API key to use for authentication, as a string.
+
+    Returns:
+        dict: A dictionary of HTTP headers to include in API requests, with the `Authorization` header
+            set to the specified API key in the format `Bearer {api_key}`.
+
+    """
+    # Construct the dictionary of HTTP headers to use for API requests
+    return dict(authorization=f"Bearer {api_key}")
+
+
+def extract_property_history(objects: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]:
+    for item in objects:
+        history = item.get("propertiesWithHistory")
+        if not history:
+            return
+        # Yield a flat list of property history entries
+        for key, changes in history.items():
+            if not changes:
+                continue
+            for entry in changes:
+                yield {"object_id": item["id"], "property_name": key, **entry}
+
+
+def fetch_property_history(
+    endpoint: str,
+    api_key: str,
+    props: str,
+    params: Optional[Dict[str, Any]] = None,
+) -> Iterator[List[Dict[str, Any]]]:
+    """Fetch property history from the given CRM endpoint.
+
+    Args:
+        endpoint: The endpoint to fetch data from, as a string.
+        api_key: The API key to use for authentication, as a string.
+        props: A comma separated list of properties to retrieve the history for
+        params: Optional dict of query params to include in the request
+
+    Yields:
+         List of property history entries (dicts)
+    """
+    # Construct the URL and headers for the API request
+    url = get_url(endpoint)
+    headers = _get_headers(api_key)
+
+    params = dict(params or {})
+    params["propertiesWithHistory"] = props
+    params["limit"] = 50
+    # Make the API request
+    r = requests.get(url, headers=headers, params=params)
+    # Parse the API response and yield the properties of each result
+
+    # Parse the response JSON data
+    _data = r.json()
+    while _data is not None:
+        if "results" in _data:
+            yield list(extract_property_history(_data["results"]))
+
+        # Follow pagination links if they exist
+        _next = _data.get("paging", {}).get("next", None)
+        if _next:
+            next_url = _next["link"]
+            # Get the next page response
+            r = requests.get(next_url, headers=headers)
+            _data = r.json()
+        else:
+            _data = None
+
+
+def fetch_data(
+    endpoint: str, api_key: str, params: Optional[Dict[str, Any]] = None
+) -> Iterator[List[Dict[str, Any]]]:
+    """
+    Fetch data from HUBSPOT endpoint using a specified API key and yield the properties of each result.
+    For paginated endpoint this function yields item from all pages.
+
+    Args:
+        endpoint (str): The endpoint to fetch data from, as a string.
+        api_key (str): The API key to use for authentication, as a string.
+        params: Optional dict of query params to include in the request
+
+    Yields:
+        A List of CRM object dicts
+
+    Raises:
+        requests.exceptions.HTTPError: If the API returns an HTTP error status code.
+
+    Notes:
+        This function uses the `requests` library to make a GET request to the specified endpoint, with
+        the API key included in the headers. If the API returns a non-successful HTTP status code (e.g.
+        404 Not Found), a `requests.exceptions.HTTPError` exception will be raised.
+
+        The `endpoint` argument should be a relative URL, which will be appended to the base URL for the
+        API. The `params` argument is used to pass additional query parameters to the request
+
+        This function also includes a retry decorator that will automatically retry the API call up to
+        3 times with a 5-second delay between retries, using an exponential backoff strategy.
+    """
+    # Construct the URL and headers for the API request
+    url = get_url(endpoint)
+    headers = _get_headers(api_key)
+
+    # Make the API request
+    r = requests.get(url, headers=headers, params=params)
+    # Parse the API response and yield the properties of each result
+    # Parse the response JSON data
+    _data = r.json()
+    # Yield the properties of each result in the API response
+    while _data is not None:
+        if "results" in _data:
+            _objects: List[Dict[str, Any]] = []
+            for _result in _data["results"]:
+                _obj = _result.get("properties", _result)
+                if "id" not in _obj and "id" in _result:
+                    # Move id from properties to top level
+                    _obj["id"] = _result["id"]
+                if "associations" in _result:
+                    for association in _result["associations"]:
+                        __values = [
+                            {
+                                "value": _obj["hs_object_id"],
+                                f"{association}_id": __r["id"],
+                            }
+                            for __r in _result["associations"][association]["results"]
+                        ]
+
+                        # remove duplicates from list of dicts
+                        __values = [
+                            dict(t) for t in {tuple(d.items()) for d in __values}
+                        ]
+
+                        _obj[association] = __values
+                _objects.append(_obj)
+            yield _objects
+
+        # Follow pagination links if they exist
+        _next = _data.get("paging", {}).get("next", None)
+        if _next:
+            next_url = _next["link"]
+            # Get the next page response
+            r = requests.get(next_url, headers=headers)
+            _data = r.json()
+        else:
+            _data = None
+
+
+def _get_property_names(api_key: str, object_type: str) -> List[str]:
+    """
+    Retrieve property names for a given entity from the HubSpot API.
+
+    Args:
+        entity: The entity name for which to retrieve property names.
+
+    Returns:
+        A list of property names.
+
+    Raises:
+        Exception: If an error occurs during the API request.
+    """
+    properties = []
+    endpoint = f"/crm/v3/properties/{OBJECT_TYPE_PLURAL[object_type]}"
+
+    for page in fetch_data(endpoint, api_key):
+        properties.extend([prop["name"] for prop in page])
+
+    return properties

ingestr/src/hubspot/settings.py

@@ -0,0 +1,99 @@
+"""Hubspot source settings and constants"""
+
+from dlt.common import pendulum
+
+STARTDATE = pendulum.datetime(year=2000, month=1, day=1)
+
+CRM_CONTACTS_ENDPOINT = (
+    "/crm/v3/objects/contacts?associations=deals,products,tickets,quotes"
+)
+CRM_COMPANIES_ENDPOINT = (
+    "/crm/v3/objects/companies?associations=contacts,deals,products,tickets,quotes"
+)
+CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
+CRM_PRODUCTS_ENDPOINT = "/crm/v3/objects/products"
+CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
+CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
+
+CRM_OBJECT_ENDPOINTS = {
+    "contact": CRM_CONTACTS_ENDPOINT,
+    "company": CRM_COMPANIES_ENDPOINT,
+    "deal": CRM_DEALS_ENDPOINT,
+    "product": CRM_PRODUCTS_ENDPOINT,
+    "ticket": CRM_TICKETS_ENDPOINT,
+    "quote": CRM_QUOTES_ENDPOINT,
+}
+
+WEB_ANALYTICS_EVENTS_ENDPOINT = "/events/v3/events?objectType={objectType}&objectId={objectId}&occurredAfter={occurredAfter}&occurredBefore={occurredBefore}&sort=-occurredAt"
+
+OBJECT_TYPE_SINGULAR = {
+    "companies": "company",
+    "contacts": "contact",
+    "deals": "deal",
+    "tickets": "ticket",
+    "products": "product",
+    "quotes": "quote",
+}
+
+OBJECT_TYPE_PLURAL = {v: k for k, v in OBJECT_TYPE_SINGULAR.items()}
+
+DEFAULT_DEAL_PROPS = [
+    "amount",
+    "closedate",
+    "createdate",
+    "dealname",
+    "dealstage",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "pipeline",
+]
+
+DEFAULT_COMPANY_PROPS = [
+    "createdate",
+    "domain",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "name",
+]
+
+DEFAULT_CONTACT_PROPS = [
+    "createdate",
+    "email",
+    "firstname",
+    "hs_object_id",
+    "lastmodifieddate",
+    "lastname",
+]
+
+DEFAULT_TICKET_PROPS = [
+    "createdate",
+    "content",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "hs_pipeline",
+    "hs_pipeline_stage",
+    "hs_ticket_category",
+    "hs_ticket_priority",
+    "subject",
+]
+
+DEFAULT_PRODUCT_PROPS = [
+    "createdate",
+    "description",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "name",
+    "price",
+]
+
+DEFAULT_QUOTE_PROPS = [
+    "hs_createdate",
+    "hs_expiration_date",
+    "hs_lastmodifieddate",
+    "hs_object_id",
+    "hs_public_url_key",
+    "hs_status",
+    "hs_title",
+]
+
+ALL = ("ALL",)

ingestr/src/sources.py

@@ -1,17 +1,21 @@
 import base64
 import csv
 import json
+from datetime import date
 from typing import Any, Callable, Optional
 from urllib.parse import parse_qs, urlparse
 
 import dlt
 
+from ingestr.src.chess import source
 from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.gorgias import gorgias_source
+from ingestr.src.hubspot import hubspot
 from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
 from ingestr.src.sql_database import sql_table
+from ingestr.src.stripe_analytics import stripe_source
 from ingestr.src.table_definition import table_string_to_dataclass
 
 
@@ -295,3 +299,135 @@ class GoogleSheetsSource:
             range_names=[table_fields.dataset],
             get_named_ranges=False,
         )
+
+
+class ChessSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    # chess://?players=john,peter
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Chess takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        source_fields = urlparse(uri)
+        source_params = parse_qs(source_fields.query)
+        list_players = None
+        if "players" in source_params:
+            list_players = source_params["players"][0].split(",")
+        else:
+            list_players = [
+                "MagnusCarlsen",
+                "HikaruNakamura",
+                "ArjunErigaisi",
+                "IanNepomniachtchi",
+            ]
+
+        date_args = {}
+        start_date = kwargs.get("interval_start")
+        end_date = kwargs.get("interval_end")
+        if start_date and end_date:
+            if isinstance(start_date, date) and isinstance(end_date, date):
+                date_args["start_month"] = start_date.strftime("%Y/%m")
+                date_args["end_month"] = end_date.strftime("%Y/%m")
+
+        table_mapping = {
+            "profiles": "players_profiles",
+            "games": "players_games",
+            "archives": "players_archives",
+        }
+
+        if table not in table_mapping:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Chess source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        return source(players=list_players, **date_args).with_resources(table_mapping[table])
+
+
+class StripeAnalyticsSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Stripe takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        api_key = None
+        source_field = urlparse(uri)
+        source_params = parse_qs(source_field.query)
+        api_key = source_params.get("api_key")
+
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Stripe")
+
+        endpoint = None
+        table = str.capitalize(table)
+
+        if table in [
+            "Subscription",
+            "Account",
+            "Coupon",
+            "Customer",
+            "Product",
+            "Price",
+            "BalanceTransaction",
+            "Invoice",
+            "Event",
+        ]:
+            endpoint = table
+        else:
+            raise ValueError(
+                f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        date_args = {}
+        if kwargs.get("interval_start"):
+            date_args["start_date"] = kwargs.get("interval_start")
+
+        if kwargs.get("interval_end"):
+            date_args["end_date"] = kwargs.get("interval_end")
+
+        return stripe_source(
+            endpoints=[
+                endpoint,
+            ],
+            stripe_secret_key=api_key[0],
+            **date_args,
+        ).with_resources(endpoint)
+
+
+class HubspotSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    # hubspot://?api_key=<api_key>
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Hubspot takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        api_key = None
+        source_parts = urlparse(uri)
+        source_parmas = parse_qs(source_parts.query)
+        api_key = source_parmas.get("api_key")
+
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Hubspot")
+
+        endpoint = None
+        if table in ["contacts", "companies", "deals", "tickets", "products", "quotes"]:
+            endpoint = table
+        else:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Hubspot source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        return hubspot(
+            api_key=api_key[0],
+        ).with_resources(endpoint)

ingestr/src/stripe_analytics/__init__.py

@@ -0,0 +1,99 @@
+"""This source uses Stripe API and dlt to load data such as Customer, Subscription, Event etc. to the database and to calculate the MRR and churn rate."""
+
+from typing import Any, Dict, Generator, Iterable, Optional, Tuple
+
+import dlt
+import stripe
+from dlt.sources import DltResource
+from pendulum import DateTime
+
+from .helpers import pagination, transform_date
+from .settings import ENDPOINTS, INCREMENTAL_ENDPOINTS
+
+
+@dlt.source
+def stripe_source(
+    endpoints: Tuple[str, ...] = ENDPOINTS,
+    stripe_secret_key: str = dlt.secrets.value,
+    start_date: Optional[DateTime] = None,
+    end_date: Optional[DateTime] = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from the Stripe API for the specified endpoints.
+
+    For all endpoints, Stripe API responses do not provide the key "updated",
+    so in most cases, we are forced to load the data in 'replace' mode.
+    This source is suitable for all types of endpoints, including 'Events', 'Invoice', etc.
+    but these endpoints can also be loaded in incremental mode (see source incremental_stripe_source).
+
+    Args:
+        endpoints (Tuple[str, ...]): A tuple of endpoint names to retrieve data from. Defaults to most popular Stripe API endpoints.
+        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
+        start_date (Optional[DateTime]): An optional start date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Defaults to None.
+        end_date (Optional[DateTime]): An optional end date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Defaults to None.
+
+    Returns:
+        Iterable[DltResource]: Resources with data that was created during the period greater than or equal to 'start_date' and less than 'end_date'.
+    """
+    stripe.api_key = stripe_secret_key
+    stripe.api_version = "2022-11-15"
+
+    def stripe_resource(
+        endpoint: str,
+    ) -> Generator[Dict[Any, Any], Any, None]:
+        yield from pagination(endpoint, start_date, end_date)
+
+    for endpoint in endpoints:
+        yield dlt.resource(
+            stripe_resource,
+            name=endpoint,
+            write_disposition="replace",
+        )(endpoint)
+
+
+@dlt.source
+def incremental_stripe_source(
+    endpoints: Tuple[str, ...] = INCREMENTAL_ENDPOINTS,
+    stripe_secret_key: str = dlt.secrets.value,
+    initial_start_date: Optional[DateTime] = None,
+    end_date: Optional[DateTime] = None,
+) -> Iterable[DltResource]:
+    """
+    As Stripe API does not include the "updated" key in its responses,
+    we are only able to perform incremental downloads from endpoints where all objects are uneditable.
+    This source yields the resources with incremental loading based on "append" mode.
+    You will load only the newest data without duplicating and without downloading a huge amount of data each time.
+
+    Args:
+        endpoints (tuple): A tuple of endpoint names to retrieve data from. Defaults to Stripe API endpoints with uneditable data.
+        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
+        initial_start_date (Optional[DateTime]): An optional parameter that specifies the initial value for dlt.sources.incremental.
+                            If parameter is not None, then load only data that were created after initial_start_date on the first run.
+                            Defaults to None. Format: datetime(YYYY, MM, DD).
+        end_date (Optional[DateTime]): An optional end date to limit the data retrieved.
+                  Defaults to None. Format: datetime(YYYY, MM, DD).
+    Returns:
+        Iterable[DltResource]: Resources with only that data has not yet been loaded.
+    """
+    stripe.api_key = stripe_secret_key
+    stripe.api_version = "2022-11-15"
+    start_date_unix = (
+        transform_date(initial_start_date) if initial_start_date is not None else -1
+    )
+
+    def incremental_resource(
+        endpoint: str,
+        created: Optional[Any] = dlt.sources.incremental(
+            "created", initial_value=start_date_unix
+        ),
+    ) -> Generator[Dict[Any, Any], Any, None]:
+        start_value = created.last_value
+        yield from pagination(endpoint, start_date=start_value, end_date=end_date)
+
+    for endpoint in endpoints:
+        yield dlt.resource(
+            incremental_resource,
+            name=endpoint,
+            write_disposition="append",
+            primary_key="id",
+        )(endpoint)

ingestr/src/stripe_analytics/helpers.py

@@ -0,0 +1,68 @@
+"""Stripe analytics source helpers"""
+
+from typing import Any, Dict, Iterable, Optional, Union
+
+import stripe
+from dlt.common import pendulum
+from dlt.common.typing import TDataItem
+from pendulum import DateTime
+
+
+def pagination(
+    endpoint: str, start_date: Optional[Any] = None, end_date: Optional[Any] = None
+) -> Iterable[TDataItem]:
+    """
+    Retrieves data from an endpoint with pagination.
+
+    Args:
+        endpoint (str): The endpoint to retrieve data from.
+        start_date (Optional[Any]): An optional start date to limit the data retrieved. Defaults to None.
+        end_date (Optional[Any]): An optional end date to limit the data retrieved. Defaults to None.
+
+    Returns:
+        Iterable[TDataItem]: Data items retrieved from the endpoint.
+    """
+    starting_after = None
+    while True:
+        response = stripe_get_data(
+            endpoint,
+            start_date=start_date,
+            end_date=end_date,
+            starting_after=starting_after,
+        )
+
+        if len(response["data"]) > 0:
+            starting_after = response["data"][-1]["id"]
+        yield response["data"]
+
+        if not response["has_more"]:
+            break
+
+
+def transform_date(date: Union[str, DateTime, int]) -> int:
+    if isinstance(date, str):
+        date = pendulum.from_format(date, "%Y-%m-%dT%H:%M:%SZ")
+    if isinstance(date, DateTime):
+        # convert to unix timestamp
+        date = int(date.timestamp())
+    return date
+
+
+def stripe_get_data(
+    resource: str,
+    start_date: Optional[Any] = None,
+    end_date: Optional[Any] = None,
+    **kwargs: Any,
+) -> Dict[Any, Any]:
+    if start_date:
+        start_date = transform_date(start_date)
+    if end_date:
+        end_date = transform_date(end_date)
+
+    if resource == "Subscription":
+        kwargs.update({"status": "all"})
+
+    resource_dict = getattr(stripe, resource).list(
+        created={"gte": start_date, "lt": end_date}, limit=100, **kwargs
+    )
+    return dict(resource_dict)

ingestr/src/stripe_analytics/settings.py

@@ -0,0 +1,14 @@
+"""Stripe analytics source settings and constants"""
+
+# the most popular endpoints
+# Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
+ENDPOINTS = (
+    "Subscription",
+    "Account",
+    "Coupon",
+    "Customer",
+    "Product",
+    "Price",
+)
+# possible incremental endpoints
+INCREMENTAL_ENDPOINTS = ("Event", "Invoice", "BalanceTransaction")

ingestr/src/version.py

@@ -1 +1 @@
-__version__ = "0.7.4"
+__version__ = "0.7.6"

{ingestr-0.7.4.dist-info → ingestr-0.7.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.4
+Version: 0.7.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -38,6 +38,7 @@ Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
+Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.66.2
 Requires-Dist: typer==0.12.3
 Description-Content-Type: text/markdown
@@ -172,25 +173,40 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <tr>
+        <td>Chess.com</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>HubSpot</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Shopify</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Stripe</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
 </table>
 

{ingestr-0.7.4.dist-info → ingestr-0.7.6.dist-info}/RECORD

@@ -1,9 +1,12 @@
-ingestr/main.py,sha256=j0pscsPbeJ9oYJiTCvymneZwg4Lc7KaR3GAMX0GG4To,16432
+ingestr/main.py,sha256=0J_bMCWLZT0tdRW0df8iKw05In55LJ_vUoN3X2TgXlc,17183
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/factory.py,sha256=XuT_8LvWd7gBxOjoD_NiG-jtPvHNQ9nqOeoCJzhRb6Y,3630
-ingestr/src/sources.py,sha256=QbSvECvGbHJKOpE9_dbq11343pA5ajsS9BPPPab1ivw,10007
+ingestr/src/factory.py,sha256=Fp_MaeiAhU7IHT6RMLTEhvXgmDyu6j1IHGnjC4qsPLI,3939
+ingestr/src/sources.py,sha256=7dy_KvoGI9vZSJwpkW3iNh8M1tjh1pBSz8qLpo0GTIo,14589
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=A6fZ_oURo3l_Fa_K29LgV21A4Onqu3NquwGYzL05E1Y,22
+ingestr/src/version.py,sha256=wu65dmVM9fKR1rBHH263ls8Ca2FZzb0ejYcrP_Ld0iY,22
+ingestr/src/chess/__init__.py,sha256=PaxT2DObudOGlhyoENE5LjR6rTdsxiqKKpAZeyzVLCA,6791
+ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
+ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -11,6 +14,9 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
 ingestr/src/gorgias/__init__.py,sha256=BzX9X1Yc_1Mch6NP1pn26hjRIiaadErgHxkdJHw4P3o,21227
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
+ingestr/src/hubspot/__init__.py,sha256=eSD_lEIEd16YijAtUATFG8FGO8YGPm-MtAk94KKsx6o,9740
+ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
+ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
 ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
 ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -27,6 +33,9 @@ ingestr/src/sql_database/arrow_helpers.py,sha256=yze1X3A9nUQA4HeuFDDWrfJVkCq8Uo5
 ingestr/src/sql_database/helpers.py,sha256=6o8e2_8MIuj3qlo40a2E6ns3gyK18ei1jCePONrMUjI,10191
 ingestr/src/sql_database/override.py,sha256=xbKGDztCzvrhJ5kJTXERal3LA56bEeVug4_rrTs8DgA,333
 ingestr/src/sql_database/schema_types.py,sha256=qXTanvFPE8wMCSDzQWPDi5yqaO-llfrFXjiGJALI4NA,5013
+ingestr/src/stripe_analytics/__init__.py,sha256=8yy6i4DAhUqY4ZForetQ0DWc_YQrY0FBH6yk0Z3m-Mw,4493
+ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
+ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
@@ -37,8 +46,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.7.4.dist-info/METADATA,sha256=VLL2Um1BU3x6Oz89Gx6d48O9ukAk4Ro7uy2dFIPTIo8,5829
-ingestr-0.7.4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.7.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.7.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.7.4.dist-info/RECORD,,
+ingestr-0.7.6.dist-info/METADATA,sha256=5QLxKZE65TQH_z6dH0g52FZmseLc2hIJBipDwcuCg5g,6104
+ingestr-0.7.6.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.7.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.7.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.7.6.dist-info/RECORD,,