ingestr 0.7.5__tar.gz → 0.7.6__tar.gz

{ingestr-0.7.5 → ingestr-0.7.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.5
+Version: 0.7.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -173,6 +173,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <tr>
+        <td>Chess.com</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
@@ -183,6 +188,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>HubSpot</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>

{ingestr-0.7.5 → ingestr-0.7.6}/README.md

@@ -128,6 +128,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
+    <tr>
+        <td>Chess.com</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
@@ -138,6 +143,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>HubSpot</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>

{ingestr-0.7.5 → ingestr-0.7.6}/docs/.vitepress/config.mjs

@@ -6,7 +6,13 @@ export default defineConfig({
   description: "Ingest & copy data between any source and any destination",
   base: "/ingestr/",
   head: [
-    ["script", { async: "", src: "https://www.googletagmanager.com/gtag/js?id=G-MZJ20PP4MJ" }],
+    [
+      "script",
+      {
+        async: "",
+        src: "https://www.googletagmanager.com/gtag/js?id=G-MZJ20PP4MJ",
+      },
+    ],
     [
       "script",
       {},
@@ -22,7 +28,7 @@ export default defineConfig({
       { text: "Home", link: "/" },
       { text: "Getting started", link: "/getting-started/quickstart.md" },
     ],
-    outline: 'deep',
+    outline: "deep",
 
     sidebar: [
       {
@@ -30,7 +36,10 @@ export default defineConfig({
         items: [
           { text: "Quickstart", link: "/getting-started/quickstart.md" },
           { text: "Core Concepts", link: "/getting-started/core-concepts.md" },
-          { text: "Incremental Loading", link: "/getting-started/incremental-loading.md" },
+          {
+            text: "Incremental Loading",
+            link: "/getting-started/incremental-loading.md",
+          },
           { text: "Telemetry", link: "/getting-started/telemetry.md" },
         ],
       },
@@ -44,7 +53,6 @@ export default defineConfig({
       {
         text: "Sources & Destinations",
         items: [
-          { text: "Overview", link: "/supported-sources/overview.md" },
           {
             text: "Databases",
             collapsed: false,
@@ -52,9 +60,15 @@ export default defineConfig({
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },
-              { text: "Google BigQuery", link: "/supported-sources/bigquery.md" },
+              {
+                text: "Google BigQuery",
+                link: "/supported-sources/bigquery.md",
+              },
               { text: "Local CSV Files", link: "/supported-sources/csv.md" },
-              { text: "Microsoft SQL Server", link: "/supported-sources/mssql.md" },
+              {
+                text: "Microsoft SQL Server",
+                link: "/supported-sources/mssql.md",
+              },
               { text: "MongoDB", link: "/supported-sources/mongodb.md" },
               { text: "MySQL", link: "/supported-sources/mysql.md" },
               { text: "Oracle", link: "/supported-sources/oracle.md" },
@@ -69,8 +83,10 @@ export default defineConfig({
             text: "Platforms",
             collapsed: false,
             items: [
-              { text: "Gorgias", link: "/supported-sources/gorgias.md" },
+              { text: "Chess.com", link: "/supported-sources/chess.md" },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
+              { text: "Gorgias", link: "/supported-sources/gorgias.md" },
+              { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Stripe", link: "/supported-sources/stripe.md" },
@@ -80,6 +96,8 @@ export default defineConfig({
       },
     ],
 
-    socialLinks: [{ icon: "github", link: "https://github.com/bruin-data/ingestr" }],
+    socialLinks: [
+      { icon: "github", link: "https://github.com/bruin-data/ingestr" },
+    ],
   },
 });

{ingestr-0.7.5 → ingestr-0.7.6}/docs/commands/example-uris.md

@@ -2,4 +2,4 @@
 
 This command is supposed to serve as a guide for the user to understand the various URI formats that are supported by the `ingestr` tool. The command will provide a list of supported sources and destinations, along with the URI format for each of them.
 
-For the detailed documentation, please refer to the [Sources & Destinations](../supported-sources/overview.md) section.
+For the detailed documentation, please refer to the Sources & Destinations section on the sidebar.

{ingestr-0.7.5 → ingestr-0.7.6}/docs/getting-started/quickstart.md

@@ -36,4 +36,4 @@ This command will:
 
 ## Supported Sources & Destinations
 
-See the [Supported Sources & Destinations](/supported-sources/overview.md) page for a list of all supported sources and destinations. More to come soon!
+See the Supported Sources & Destinations page for a list of all supported sources and destinations. More to come soon!

ingestr-0.7.6/docs/supported-sources/chess.md

@@ -0,0 +1,37 @@
+# Chess.com
+
+[Chess.com](https://www.chess.com/) is an online platform offering chess games, tournaments, lessons, and more.
+
+ingestr supports Chess.com as a source, primarily to play around with the data of players, games, and more since it doesn't require any authentication.
+
+## URI Format
+
+The URI format for Chess is as follows:
+
+```plaintext
+--source-uri 'chess://?players=<List[str]>'
+```
+
+URI parameter:
+
+- `players`: A list of players usernames for which you want to fetch data. If no usernames are provided, then data of 4 different players will be fetched.
+
+## Setting up a Chess Integration
+
+Let's say you have a list of player usernames: max2 and peter23. Here's a sample command that will copy the data from Chess into a DuckDB database:
+
+```sh
+ingestr ingest --source-uri 'chess://?players=max2,peter23' --source-table 'profiles' --dest-uri 'duckdb:///chess.duckdb' --dest-table 'players.profiles'
+```
+
+The result of this command will be a table in the `chess.duckdb` database.
+
+## Available Tables
+
+Chess source allows ingesting the following sources into separate tables:
+
+- `profiles`: Retrives player profiles based on a list of player usernames.
+- `games`: Retrives players' games for specified players.
+- `archives`: Retrives url to game archives for specified players.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.7.6/docs/supported-sources/hubspot.md

@@ -0,0 +1,45 @@
+# HubSpot
+
+[HubSpot](https://www.hubspot.com/) is a customer relationship management software that helps businesses attract visitors, connect with customers, and close deals.
+
+ingestr supports HubSpot as a source.
+
+## URI Format
+
+The URI format for HubSpot is as follows:
+
+```plaintext
+hubspot://?api_key=<api-key-here>
+```
+
+URI parameters:
+
+- `api_key`: The API key is used for authentication with the HubSpot API.
+
+The URI is used to connect to the HubSpot API for extracting data.
+
+## Setting up a HubSpot Integration
+
+Hubspot requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/hubspot#setup-guide).
+
+Once you complete the guide, you should have an API key. Let's say your API key is `pat_test_12345`, here's a sample command that will copy the data from HubSpot into a duckdb database:
+
+```sh
+ingestr ingest --source-uri 'hubspot://?api_key=pat_test_12345' --source-table 'companies' --dest-uri duckdb:///hubspot.duckdb --dest-table 'companies.data'
+```
+
+The result of this command will be a table in the `hubspot.duckdb` database.
+
+## Available Tables
+
+HubSpot source allows ingesting the following sources into separate tables:
+
+- `companies`: Retrieves information about organizations.
+- `deals`: Retrieves deal records and tracks deal progress.
+- `products`: Retrieves pricing information of products.
+- `tickets`: Handles requests for help from customers or users.
+- `quotes`: Retrieves price proposals that salespeople can create and send to their contacts.
+- `hubspot_events_for_objects`: Retrieves web analytics events for a given object type and object IDs.
+- `contacts`: Retrieves information about visitors, potential customers, and leads.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.

{ingestr-0.7.5 → ingestr-0.7.6}/ingestr/main.py

@@ -1,6 +1,7 @@
 import hashlib
 from datetime import datetime
 from enum import Enum
+import tempfile
 from typing import Optional
 
 import dlt
@@ -236,6 +237,13 @@ def ingest(
             envvar="SCHEMA_NAMING",
         ),
     ] = SchemaNaming.default,  # type: ignore
+    pipelines_dir: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The path to store dlt-related pipeline metadata. By default, ingestr will create a temporary directory and delete it after the execution is done in order to make retries stateless.",
+            envvar="PIPELINES_DIR",
+        ),
+    ] = None,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -280,13 +288,18 @@ def ingest(
         if progress == Progress.log:
             progressInstance = LogCollector(dump_system_stats=False)
 
+        is_pipelines_dir_temp = False
+        if pipelines_dir is None:
+            pipelines_dir = tempfile.mkdtemp()
+            is_pipelines_dir_temp = True
+
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),
             destination=destination.dlt_dest(
                 uri=dest_uri,
             ),
             progress=progressInstance,
-            pipelines_dir="pipeline_data",
+            pipelines_dir=pipelines_dir,
             refresh="drop_resources" if full_refresh else None,
         )
 
@@ -362,6 +375,8 @@ def ingest(
         if incremental_strategy != IncrementalStrategy.none:
             write_disposition = incremental_strategy.value
 
+        start_time = datetime.now()
+
         run_info: LoadInfo = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(
@@ -389,11 +404,17 @@ def ingest(
 
         destination.post_load()
 
+        end_time = datetime.now()
         elapsedHuman = ""
         if run_info.started_at:
-            elapsed = run_info.finished_at - run_info.started_at
+            elapsed = end_time - start_time
             elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
 
+        # remove the pipelines_dir folder if it was created by ingestr
+        if is_pipelines_dir_temp:
+            import shutil
+            shutil.rmtree(pipelines_dir)
+
         print(
             f"[bold green]Successfully finished loading data from '{factory.source_scheme}' to '{factory.destination_scheme}' {elapsedHuman} [/bold green]"
         )

ingestr-0.7.6/ingestr/src/chess/__init__.py

@@ -0,0 +1,166 @@
+"""A source loading player profiles and games from chess.com api"""
+
+from typing import Any, Callable, Dict, Iterator, List, Sequence
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers import requests
+
+from .helpers import get_path_with_retry, get_url_with_retry, validate_month_string
+from .settings import UNOFFICIAL_CHESS_API_URL
+
+
+@dlt.source(name="chess")
+def source(
+    players: List[str], start_month: str = None, end_month: str = None
+) -> Sequence[DltResource]:
+    """
+    A dlt source for the chess.com api. It groups several resources (in this case chess.com API endpoints) containing
+    various types of data: user profiles or chess match results
+    Args:
+        players (List[str]): A list of the player usernames for which to get the data.
+        start_month (str, optional): Filters out all the matches happening before `start_month`. Defaults to None.
+        end_month (str, optional): Filters out all the matches happening after `end_month`. Defaults to None.
+    Returns:
+        Sequence[DltResource]: A sequence of resources that can be selected from including players_profiles,
+        players_archives, players_games, players_online_status
+    """
+    return (
+        players_profiles(players),
+        players_archives(players),
+        players_games(players, start_month=start_month, end_month=end_month),
+        players_online_status(players),
+    )
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "last_online": {"data_type": "timestamp"},
+        "joined": {"data_type": "timestamp"},
+    },
+)
+def players_profiles(players: List[str]) -> Iterator[TDataItem]:
+    """
+    Yields player profiles for a list of player usernames.
+    Args:
+        players (List[str]): List of player usernames to retrieve profiles for.
+    Yields:
+        Iterator[TDataItem]: An iterator over player profiles data.
+    """
+
+    # get archives in parallel by decorating the http request with defer
+    @dlt.defer
+    def _get_profile(username: str) -> TDataItem:
+        return get_path_with_retry(f"player/{username}")
+
+    for username in players:
+        yield _get_profile(username)
+
+
+@dlt.resource(write_disposition="replace", selected=False)
+def players_archives(players: List[str]) -> Iterator[List[TDataItem]]:
+    """
+    Yields url to game archives for specified players.
+    Args:
+        players (List[str]): List of player usernames to retrieve archives for.
+    Yields:
+        Iterator[List[TDataItem]]: An iterator over list of player archive data.
+    """
+    for username in players:
+        data = get_path_with_retry(f"player/{username}/games/archives")
+        yield data.get("archives", [])
+
+
+@dlt.resource(
+    write_disposition="append", columns={"end_time": {"data_type": "timestamp"}}
+)
+def players_games(
+    players: List[str], start_month: str = None, end_month: str = None
+) -> Iterator[Callable[[], List[TDataItem]]]:
+    """
+    Yields `players` games that happened between `start_month` and `end_month`.
+    Args:
+        players (List[str]): List of player usernames to retrieve games for.
+        start_month (str, optional): The starting month in the format "YYYY/MM". Defaults to None.
+        end_month (str, optional): The ending month in the format "YYYY/MM". Defaults to None.
+    Yields:
+        Iterator[Callable[[], List[TDataItem]]]: An iterator over callables that return a list of games for each player.
+    """  # do a simple validation to prevent common mistakes in month format
+    validate_month_string(start_month)
+    validate_month_string(end_month)
+
+    # get a list of already checked archives
+    # from your point of view, the state is python dictionary that will have the same content the next time this function is called
+    checked_archives = dlt.current.resource_state().setdefault("archives", [])
+    # get player archives, note that you can call the resource like any other function and just iterate it like a list
+    archives = players_archives(players)
+
+    # get archives in parallel by decorating the http request with defer
+    @dlt.defer
+    def _get_archive(url: str) -> List[TDataItem]:
+        try:
+            games = get_url_with_retry(url).get("games", [])
+            return games  # type: ignore
+        except requests.HTTPError as http_err:
+            # sometimes archives are not available and the error seems to be permanent
+            if http_err.response.status_code == 404:
+                return []
+            raise
+
+    # enumerate the archives
+    for url in archives:
+        # the `url` format is https://api.chess.com/pub/player/{username}/games/{YYYY}/{MM}
+        if start_month and url[-7:] < start_month:
+            continue
+        if end_month and url[-7:] > end_month:
+            continue
+        # do not download archive again
+        if url in checked_archives:
+            continue
+        checked_archives.append(url)
+        # get the filtered archive
+        yield _get_archive(url)
+
+
+@dlt.resource(write_disposition="append")
+def players_online_status(players: List[str]) -> Iterator[TDataItem]:
+    """
+    Returns current online status for a list of players.
+    Args:
+        players (List[str]): List of player usernames to check online status for.
+    Yields:
+        Iterator[TDataItem]: An iterator over the online status of each player.
+    """
+    # we'll use unofficial endpoint to get online status, the official seems to be removed
+    for player in players:
+        status = get_url_with_retry(f"{UNOFFICIAL_CHESS_API_URL}user/popup/{player}")
+        # return just relevant selection
+        yield {
+            "username": player,
+            "onlineStatus": status["onlineStatus"],
+            "lastLoginDate": status["lastLoginDate"],
+            "check_time": pendulum.now(),  # dlt can deal with native python dates
+        }
+
+
+@dlt.source
+def chess_dlt_config_example(
+    secret_str: str = dlt.secrets.value,
+    secret_dict: Dict[str, Any] = dlt.secrets.value,
+    config_int: int = dlt.config.value,
+) -> DltResource:
+    """
+    An example of a source that uses dlt to provide secrets and config values.
+    Args:
+        secret_str (str, optional): Secret string provided by dlt.secrets.value. Defaults to dlt.secrets.value.
+        secret_dict (Dict[str, Any], optional): Secret dictionary provided by dlt.secrets.value. Defaults to dlt.secrets.value.
+        config_int (int, optional): Config integer provided by dlt.config.value. Defaults to dlt.config.value.
+    Returns:
+        DltResource: Returns a resource yielding the configured values.
+    """
+
+    # returns a resource yielding the configured values - it is just a test
+    return dlt.resource([secret_str, secret_dict, config_int], name="config_values")

ingestr-0.7.6/ingestr/src/chess/helpers.py

@@ -0,0 +1,21 @@
+"""Chess source helpers"""
+
+from dlt.common.typing import StrAny
+from dlt.sources.helpers import requests
+
+from .settings import OFFICIAL_CHESS_API_URL
+
+
+def get_url_with_retry(url: str) -> StrAny:
+    r = requests.get(url)
+    return r.json()  # type: ignore
+
+
+def get_path_with_retry(path: str) -> StrAny:
+    return get_url_with_retry(f"{OFFICIAL_CHESS_API_URL}{path}")
+
+
+def validate_month_string(string: str) -> None:
+    """Validates that the string is in YYYY/MM format"""
+    if string and string[4] != "/":
+        raise ValueError(string)

ingestr-0.7.6/ingestr/src/chess/settings.py

@@ -0,0 +1,4 @@
+"""Chess source settings and constants"""
+
+OFFICIAL_CHESS_API_URL = "https://api.chess.com/pub/"
+UNOFFICIAL_CHESS_API_URL = "https://www.chess.com/callback/"

{ingestr-0.7.5 → ingestr-0.7.6}/ingestr/src/factory.py

@@ -15,8 +15,10 @@ from ingestr.src.destinations import (
     SynapseDestination,
 )
 from ingestr.src.sources import (
+    ChessSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HubspotSource,
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
@@ -103,9 +105,12 @@ class SourceDestinationFactory:
             return ShopifySource()
         elif self.source_scheme == "gorgias":
             return GorgiasSource()
+        elif self.source_scheme == "chess":
+            return ChessSource()
         elif self.source_scheme == "stripe":
             return StripeAnalyticsSource()
-
+        elif self.source_scheme == "hubspot":
+            return HubspotSource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")