ingestr 0.14.4__tar.gz → 0.14.6__tar.gz

{ingestr-0.14.4 → ingestr-0.14.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.4
+Version: 0.14.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.4 → ingestr-0.14.6}/docs/.vitepress/config.mjs

@@ -19,18 +19,17 @@ export default defineConfig({
   head: [
     [
       "script",
-      {
-        async: "",
-        src: "https://www.googletagmanager.com/gtag/js?id=G-MZJ20PP4MJ",
-      },
+      {},
+      `(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
+new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
+j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
+'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+})(window,document,'script','dataLayer','GTM-K2L7S5FP');`,
     ],
     [
-      "script",
+      "noscript",
       {},
-      `window.dataLayer = window.dataLayer || [];
-      function gtag(){dataLayer.push(arguments);}
-      gtag('js', new Date());
-      gtag('config', 'G-MZJ20PP4MJ');`,
+      `<iframe src="https://www.googletagmanager.com/ns.html?id=GTM-K2L7S5FP" height="0" width="0" style="display:none;visibility:hidden"></iframe>`,
     ],
   ],
   themeConfig: {
@@ -157,11 +156,13 @@ export default defineConfig({
               { text: "Linear", link: "/supported-sources/linear.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Mixpanel", link: "/supported-sources/mixpanel.md" },
+              { text: "Monday", link: "/supported-sources/monday.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Personio", link: "/supported-sources/personio.md" },
               { text: "PhantomBuster", link: "/supported-sources/phantombuster.md" },
               { text: "Pinterest", link: "/supported-sources/pinterest.md" },
               { text: "Pipedrive", link: "/supported-sources/pipedrive.md" },
+              { text: "Plus Vibe AI", link: "/supported-sources/plusvibeai.md" },
               { text: "QuickBooks", link: "/supported-sources/quickbooks.md" },
               { text: "RevenueCat", link: "/supported-sources/revenuecat.md" },
               { text: "S3", link: "/supported-sources/s3.md" },

ingestr-0.14.6/docs/supported-sources/http.md

@@ -0,0 +1,111 @@
+# HTTP
+
+ingestr supports reading CSV, JSON, and Parquet files from public HTTP/HTTPS URLs. This allows you to ingest data from publicly accessible file URLs directly into your databases.
+
+## URI format
+
+The URI format for HTTP sources is as follows:
+
+```plaintext
+http://example.com/path/to/file.csv
+https://example.com/path/to/file.json
+https://example.com/path/to/file.parquet
+```
+
+## Supported file formats
+
+The HTTP source supports the following file formats:
+
+- **CSV** (`.csv`) - Comma-separated values files
+- **JSON** (`.json`, `.jsonl`) - JSON objects and JSON Lines format
+- **Parquet** (`.parquet`) - Apache Parquet columnar format
+
+The file format is automatically inferred from the URL extension. You can also explicitly specify the format using the `file_format` parameter.
+
+## Usage
+
+### Basic example
+
+```bash
+ingestr ingest \
+    --source-uri "https://example.com/data.csv" \
+    --dest-uri "duckdb:///local.duckdb" \
+    --dest-table "my_table"
+```
+
+### Example with Phantombuster CSV
+
+```bash
+ingestr ingest \
+    --source-uri "https://phantombuster.s3.amazonaws.com/hNQdB02WKv0/l7xqy7HlU0wyWIvPjqKk5Q/dts_size.csv" \
+    --dest-uri "postgres://user:pass@localhost:5432/mydb" \
+    --dest-table "phantombuster_data"
+```
+
+### Example with JSON file
+
+```bash
+ingestr ingest \
+    --source-uri "https://api.example.com/export/data.json" \
+    --dest-uri "snowflake://user:pass@account/database/schema" \
+    --dest-table "json_data"
+```
+
+### Example with Parquet file
+
+```bash
+ingestr ingest \
+    --source-uri "https://storage.example.com/data.parquet" \
+    --dest-uri "bigquery://project/dataset" \
+    --dest-table "parquet_data"
+```
+
+## Parameters
+
+### `file_format`
+
+Optional. Explicitly specify the file format if it cannot be inferred from the URL extension.
+
+**Valid values:** `csv`, `json`, `parquet`
+
+**Example:**
+
+```bash
+ingestr ingest \
+    --source-uri "https://example.com/data?format=csv" \
+    --source-file-format "csv" \
+    --dest-uri "duckdb:///local.duckdb" \
+    --dest-table "my_table"
+```
+
+### `chunksize`
+
+Optional. Number of records to process at once. This helps manage memory usage for large files.
+
+**Default values:**
+- CSV: 10,000 rows
+- JSON: 1,000 objects
+- Parquet: 10,000 rows
+
+**Example:**
+
+```bash
+ingestr ingest \
+    --source-uri "https://example.com/large-file.csv" \
+    --source-chunksize 5000 \
+    --dest-uri "postgres://user:pass@localhost:5432/mydb" \
+    --dest-table "chunked_data"
+```
+
+## Notes
+
+- The HTTP source downloads the entire file before processing, so ensure you have sufficient memory for large files
+- Authentication is not currently supported; only publicly accessible URLs can be used
+- The file must be accessible without requiring cookies, headers, or other authentication mechanisms
+- For very large files, consider using chunked transfer or a dedicated file storage source (e.g., S3, GCS)
+
+## Limitations
+
+- Only supports public URLs (no authentication)
+- The entire file is downloaded into memory before processing
+- No support for incremental loading

ingestr-0.14.6/docs/supported-sources/monday.md

@@ -0,0 +1,59 @@
+# Monday.com
+[Monday.com](https://monday.com/) is a Work OS that powers teams to run projects and workflows with confidence. It's a simple, yet powerful platform that enables people to manage work, meet deadlines, and build a culture of transparency.
+
+## URI format
+
+The URI format for Monday.com is as follows:
+```
+monday://?api_token=<api_token>
+```
+
+URI parameters:
+- `api_token` is your Monday.com API token for authentication.
+
+## Setting up a Monday.com Integration
+
+You can obtain an API token from the [Monday.com Developer Console](https://developer.monday.com/). For more information, see [Monday.com authentication documentation](https://developer.monday.com/api-reference/docs/authentication).
+
+To get your API token:
+1. Go to your Monday.com account
+2. Click on your profile picture in the top right corner
+3. Select "Admin" → "API"
+4. Generate a new API token
+
+## Example
+Let's say you want to ingest all boards into a DuckDB database called `monday.db`. For this example the value of `api_token` will be `fake_token`.
+
+You can run the following to achieve this:
+```sh
+ingestr ingest \
+  --source-uri "monday://?api_token=fake_token" \
+  --source-table "boards" \
+  --dest-uri "duckdb://./monday.db" \
+  --dest-table "public.boards"
+```
+
+## Tables
+
+Monday.com source allows ingesting the following resources into separate tables:
+
+| Table | Primary/Merge Key | Inc Key | Inc Strategy | Details |
+|-------|-------------------|---------|--------------|---------|
+| `account` | - | - | replace | Account information including name, slug, tier, and plan details. Full reload on each run. |
+| `account_roles` | - | - | replace | Account roles with their types and permissions. Full reload on each run. |
+| `users` | - | - | replace | Users in your Monday.com account with their profiles and permissions. Full reload on each run. |
+| `boards` | id | updated_at | merge | Boards with their metadata, state, and configuration. Incrementally loads only updated boards. |
+| `workspaces` | - | - | replace | Workspaces containing boards and their settings. Full reload on each run. |
+| `webhooks` | - | - | replace | Webhooks configured for boards with their events and configurations. Full reload on each run. |
+| `updates` | id | updated_at | merge | Updates (comments) on items with their content and metadata. Incrementally loads only updated entries. |
+| `teams` | - | - | replace | Teams in your account with their members. Full reload on each run. |
+| `tags` | - | - | replace | Tags used across your account for organizing items. Full reload on each run. |
+| `custom_activities` | - | - | replace | Custom activity types defined in your account. Full reload on each run. |
+| `board_columns` | - | - | replace | Columns defined in all boards with their types and settings. Full reload on each run. |
+| `board_views` | - | - | replace | Views configured for boards with their filters and settings. Full reload on each run. |
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!NOTE]
+> Monday.com has rate limits for API requests. The source handles pagination automatically and respects the API's maximum page size of 100 items.
+

ingestr-0.14.6/docs/supported-sources/plusvibeai.md

@@ -0,0 +1,93 @@
+# Plus Vibe AI
+
+[Plus Vibe AI](https://plusvibe.ai/) is an email marketing and outreach platform that helps businesses automate their email campaigns, manage leads, and track engagement metrics.
+
+ingestr supports Plus Vibe AI as a source.
+
+## URI format
+
+The URI format for Plus Vibe AI is as follows:
+
+```plaintext
+plusvibeai://?api_key=<api-key-here>&workspace_id=<workspace-id-here>
+```
+
+URI parameters:
+
+- `api_key`: API key for authentication (get from https://app.plusvibe.ai/v2/settings/api-access/)
+- `workspace_id`: Workspace ID to access your data
+
+The URI is used to connect to the Plus Vibe AI API for extracting data.
+
+## Setting up a Plus Vibe AI Integration
+
+To set up a Plus Vibe AI integration, you need to:
+
+1. Log in to your Plus Vibe AI account
+2. Navigate to Settings > API Access (https://app.plusvibe.ai/v2/settings/api-access/)
+3. Generate an API key
+4. Find your workspace ID in your account settings
+
+Once you have your API key and workspace ID, here's a sample command that will copy the data from Plus Vibe AI into a DuckDB database:
+
+```sh
+ingestr ingest --source-uri 'plusvibeai://?api_key=your_api_key&workspace_id=your_workspace_id' --source-table 'campaigns' --dest-uri duckdb:///plusvibeai.duckdb --dest-table 'campaigns.data'
+```
+
+The result of this command will be a table in the `plusvibeai.duckdb` database with JSON columns for nested objects.
+
+## Tables
+
+Plus Vibe AI source allows ingesting the following sources into separate tables:
+
+| Table | PK | Inc Key | Inc Strategy | Details |
+| ----- | -- | ------- | ------------ | ------- |
+| `campaigns` | id | modified_at | merge | Contains campaign information including configuration, schedules, sequences, and performance metrics. Nested objects (schedule, sequences) are stored as JSON columns. |
+| `leads` | _id | modified_at | merge | Contains lead information including contact details, campaign association, engagement metrics, and professional information. |
+| `email_accounts` | _id | timestamp_updated | merge | Contains email account configurations including SMTP/IMAP settings, warmup configurations, and analytics data stored in payload JSON. |
+| `emails` | id | timestamp_created | merge | Contains email data including message content, headers, thread information, and recipient details. Uses cursor-based pagination. |
+| `blocklist` | _id | created_at | merge | Contains blocklist entries for email addresses or domains that should be excluded from campaigns. |
+| `webhooks` | _id | modified_at | merge | Contains webhook configurations for receiving real-time notifications about campaign events and lead interactions. |
+| `tags` | _id | modified_at | merge | Contains tag information used for organizing and categorizing campaigns, leads, and other resources. |
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+## Features
+
+### Incremental Loading
+
+Plus Vibe AI source supports incremental loading based on modification timestamps. Each table uses its respective timestamp field to fetch only updated records since the last sync:
+
+- **Campaigns**: Uses `modified_at` field
+- **Leads**: Uses `modified_at` field  
+- **Email Accounts**: Uses `timestamp_updated` field
+- **Emails**: Uses `timestamp_created` field
+- **Blocklist**: Uses `created_at` field
+- **Webhooks**: Uses `modified_at` field
+- **Tags**: Uses `modified_at` field
+
+### Nested Data Handling
+
+The source preserves nested objects as JSON columns with `max_table_nesting=0` to maintain data structure integrity:
+
+- **Campaigns**: Schedule, sequences, and events are stored as JSON
+- **Email Accounts**: All configuration data is stored in the `payload` JSON field
+- **Emails**: Headers and address information are stored as JSON
+
+### Rate Limiting
+
+Plus Vibe AI API has a rate limit of 5 requests per second. The source automatically handles rate limiting with exponential backoff and retry logic.
+
+### Error Handling
+
+The source includes comprehensive error handling for:
+- Authentication failures (401/403)
+- Rate limiting (429) 
+- Server errors (5xx) with automatic retries
+- Network timeouts and connection issues
+
+> [!TIP]
+> For optimal performance, use incremental loading for regular syncs and full loading only for initial data extraction or when you need to capture all historical updates.
+
+> [!NOTE]
+> The emails endpoint uses cursor-based pagination with `page_trail` parameter, while other endpoints use standard offset-based pagination.

ingestr-0.14.6/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.6"

{ingestr-0.14.4 → ingestr-0.14.6}/ingestr/src/factory.py

@@ -52,6 +52,7 @@ from ingestr.src.sources import (
     GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HttpSource,
     HubspotSource,
     InfluxDBSource,
     IntercomSource,
@@ -64,12 +65,14 @@ from ingestr.src.sources import (
     LinkedInAdsSource,
     LocalCsvSource,
     MixpanelSource,
+    MondaySource,
     MongoDbSource,
     NotionSource,
     PersonioSource,
     PhantombusterSource,
     PinterestSource,
     PipedriveSource,
+    PlusVibeAISource,
     QuickBooksSource,
     RevenueCatSource,
     S3Source,
@@ -155,6 +158,8 @@ class SourceDestinationFactory:
         "anthropic": AnthropicSource,
         "csv": LocalCsvSource,
         "docebo": DoceboSource,
+        "http": HttpSource,
+        "https": HttpSource,
         "mongodb": MongoDbSource,
         "mongodb+srv": MongoDbSource,
         "notion": NotionSource,
@@ -212,6 +217,8 @@ class SourceDestinationFactory:
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,
         "wise": WiseSource,
+        "plusvibeai": PlusVibeAISource,
+        "monday": MondaySource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.14.6/ingestr/src/http/__init__.py

@@ -0,0 +1,35 @@
+"""HTTP source for reading CSV, JSON, and Parquet files from public URLs"""
+
+from typing import Any, Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .readers import HttpReader
+
+
+@dlt.source
+def http_source(
+    url: str,
+    file_format: Optional[str] = None,
+    **kwargs: Any,
+) -> DltResource:
+    """Source for reading files from HTTP URLs.
+
+    Supports CSV, JSON, and Parquet file formats.
+
+    Args:
+        url (str): The HTTP(S) URL to the file
+        file_format (str, optional): File format ('csv', 'json', 'parquet').
+            If not provided, will be inferred from URL extension.
+        **kwargs: Additional arguments passed to the reader functions
+
+    Returns:
+        DltResource: A dlt resource that yields the file data
+    """
+    reader = HttpReader(url, file_format)
+
+    return dlt.resource(
+        reader.read_file(**kwargs),
+        name="http_data",
+    )

ingestr-0.14.6/ingestr/src/http/readers.py

@@ -0,0 +1,114 @@
+"""Readers for HTTP file sources"""
+
+import io
+from typing import Any, Iterator, Optional
+from urllib.parse import urlparse
+
+import requests
+from dlt.sources import TDataItems
+
+
+class HttpReader:
+    """Reader for HTTP-based file sources"""
+
+    def __init__(self, url: str, file_format: Optional[str] = None):
+        self.url = url
+        self.file_format = file_format or self._infer_format(url)
+
+        if self.file_format not in ["csv", "json", "parquet"]:
+            raise ValueError(
+                f"Unsupported file format: {self.file_format}. "
+                "Supported formats: csv, json, parquet"
+            )
+
+    def _infer_format(self, url: str) -> str:
+        """Infer file format from URL extension"""
+        parsed = urlparse(url)
+        path = parsed.path.lower()
+
+        if path.endswith(".csv"):
+            return "csv"
+        elif path.endswith(".json") or path.endswith(".jsonl"):
+            return "json"
+        elif path.endswith(".parquet"):
+            return "parquet"
+        else:
+            raise ValueError(
+                f"Cannot infer file format from URL: {url}. "
+                "Please specify file_format parameter."
+            )
+
+    def _download_file(self) -> bytes:
+        """Download file from URL"""
+        response = requests.get(self.url, stream=True, timeout=30)
+        response.raise_for_status()
+        return response.content
+
+    def read_file(self, **kwargs: Any) -> Iterator[TDataItems]:
+        """Read file and yield data in chunks"""
+        content = self._download_file()
+
+        if self.file_format == "csv":
+            yield from self._read_csv(content, **kwargs)
+        elif self.file_format == "json":
+            yield from self._read_json(content, **kwargs)
+        elif self.file_format == "parquet":
+            yield from self._read_parquet(content, **kwargs)
+
+    def _read_csv(
+        self, content: bytes, chunksize: int = 10000, **pandas_kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read CSV file with Pandas chunk by chunk"""
+        import pandas as pd  # type: ignore
+
+        kwargs = {**{"header": "infer", "chunksize": chunksize}, **pandas_kwargs}
+
+        file_obj = io.BytesIO(content)
+        for df in pd.read_csv(file_obj, **kwargs):
+            yield df.to_dict(orient="records")
+
+    def _read_json(
+        self, content: bytes, chunksize: int = 1000, **kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read JSON or JSONL file"""
+        from dlt.common import json
+
+        file_obj = io.BytesIO(content)
+        text = file_obj.read().decode("utf-8")
+
+        # Try to detect if it's JSONL format (one JSON object per line)
+        lines = text.strip().split("\n")
+
+        if len(lines) > 1:
+            # Likely JSONL format
+            lines_chunk = []
+            for line in lines:
+                if line.strip():
+                    lines_chunk.append(json.loads(line))
+                    if len(lines_chunk) >= chunksize:
+                        yield lines_chunk
+                        lines_chunk = []
+            if lines_chunk:
+                yield lines_chunk
+        else:
+            # Single JSON object or array
+            data = json.loads(text)
+            if isinstance(data, list):
+                # Chunk the list
+                for i in range(0, len(data), chunksize):
+                    yield data[i : i + chunksize]
+            else:
+                # Single object
+                yield [data]
+
+    def _read_parquet(
+        self, content: bytes, chunksize: int = 10000, **kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read Parquet file"""
+        from pyarrow import parquet as pq  # type: ignore
+
+        file_obj = io.BytesIO(content)
+        parquet_file = pq.ParquetFile(file_obj)
+
+        for batch in parquet_file.iter_batches(batch_size=chunksize):
+            yield batch.to_pylist()

{ingestr-0.14.4 → ingestr-0.14.6}/ingestr/src/jira_source/__init__.py

@@ -37,6 +37,7 @@ def jira_source() -> Any:
         resolutions,
         project_versions,
         project_components,
+        events,
     ]
 
 
@@ -65,7 +66,11 @@ def projects(
     yield from client.get_projects(expand=expand, recent=recent)
 
 
-@dlt.resource(write_disposition="merge", primary_key="id")
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="id",
+    max_table_nesting=2,
+)
 def issues(
     base_url: str = dlt.secrets.value,
     email: str = dlt.secrets.value,
@@ -312,3 +317,24 @@ def project_components(
         return []
 
     return list(client.get_project_components(project_key))
+
+
+@dlt.resource(write_disposition="replace")
+def events(
+    base_url: str = dlt.secrets.value,
+    email: str = dlt.secrets.value,
+    api_token: str = dlt.secrets.value,
+) -> Iterable[TDataItem]:
+    """
+    Fetches all event types from Jira (e.g., Issue Created, Issue Updated, etc.).
+
+    Args:
+        base_url (str): Jira instance URL
+        email (str): User email for authentication
+        api_token (str): API token for authentication
+
+    Yields:
+        dict: The event data.
+    """
+    client = get_client(base_url, email, api_token)
+    yield from client.get_events()

{ingestr-0.14.4 → ingestr-0.14.6}/ingestr/src/jira_source/helpers.py

@@ -98,8 +98,6 @@ class JiraClient:
 
         for attempt in range(max_retries + 1):
             try:
-                logger.debug(f"Making request to {url} (attempt {attempt + 1})")
-
                 response = requests.request(
                     method=method,
                     url=url,
@@ -214,10 +212,6 @@ class JiraClient:
         consecutive_empty_pages = 0
         max_empty_pages = 3
 
-        logger.info(
-            f"Starting paginated request to {endpoint} with page_size={page_size}"
-        )
-
         while True:
             try:
                 response = self._make_request(endpoint, params)
@@ -238,7 +232,6 @@ class JiraClient:
                     is_last = True
                 else:
                     # Single item response
-                    logger.debug(f"Received single item response from {endpoint}")
                     yield response
                     break
 
@@ -253,27 +246,18 @@ class JiraClient:
                 else:
                     consecutive_empty_pages = 0
 
-                logger.debug(
-                    f"Retrieved {len(items)} items from {endpoint} (page {params['startAt'] // page_size + 1})"
-                )
-
                 for item in items:
                     if max_results and total_returned >= max_results:
-                        logger.info(f"Reached max_results limit of {max_results}")
                         return
                     yield item
                     total_returned += 1
 
                 # Check if we've reached the end
                 if is_last or len(items) < page_size:
-                    logger.debug(f"Reached end of pagination for {endpoint}")
                     break
 
                 # Check if we've got all available items
                 if total and total_returned >= total:
-                    logger.debug(
-                        f"Retrieved all {total} available items from {endpoint}"
-                    )
                     break
 
                 # Move to next page
@@ -295,10 +279,6 @@ class JiraClient:
                 )
                 raise JiraAPIError(f"Pagination failed: {str(e)}")
 
-        logger.info(
-            f"Completed pagination for {endpoint}, returned {total_returned} items"
-        )
-
     def search_issues(
         self,
         jql: str,
@@ -327,7 +307,7 @@ class JiraClient:
             params["expand"] = expand
 
         yield from self.get_paginated(
-            "search", params=params, page_size=page_size, max_results=max_results
+            "search/jql", params=params, page_size=page_size, max_results=max_results
         )
 
     def get_projects(
@@ -433,6 +413,13 @@ class JiraClient:
         """
         yield from self.get_paginated(f"project/{project_key}/component")
 
+    def get_events(self) -> Iterator[Dict[str, Any]]:
+        """Get all events (issue events like created, updated, etc.)."""
+        response = self._make_request("events")
+        if isinstance(response, list):
+            for event in response:
+                yield event
+
 
 def get_client(
     base_url: str, email: str, api_token: str, timeout: int = REQUEST_TIMEOUT