ingestr 0.13.73__tar.gz → 0.13.74__tar.gz

{ingestr-0.13.73 → ingestr-0.13.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.73
+Version: 0.13.74
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -96,6 +96,7 @@ Requires-Dist: ibm-db-sa==0.4.1
 Requires-Dist: ibm-db==3.2.6
 Requires-Dist: idna==3.10
 Requires-Dist: inflection==0.5.1
+Requires-Dist: influxdb-client==1.41.0
 Requires-Dist: intuit-oauth==1.2.4
 Requires-Dist: isodate==0.7.2
 Requires-Dist: jmespath==1.0.1
@@ -156,6 +157,7 @@ Requires-Dist: python-quickbooks==0.9.2
 Requires-Dist: pytz==2025.1
 Requires-Dist: pyyaml==6.0.2
 Requires-Dist: rauth==0.7.3
+Requires-Dist: reactivex==4.0.4
 Requires-Dist: requests-file==2.1.0
 Requires-Dist: requests-oauthlib==1.3.1
 Requires-Dist: requests-toolbelt==1.0.0

{ingestr-0.13.73 → ingestr-0.13.74}/docs/.vitepress/config.mjs

@@ -82,6 +82,7 @@ export default defineConfig({
               },
               { text: "GCP Spanner", link: "/supported-sources/spanner.md" },
               { text: "IBM Db2", link: "/supported-sources/db2.md" },
+              { text: "InfluxDB", link: "/supported-sources/influxdb.md" },
               { text: "Kafka", link: "/supported-sources/kafka.md" },
               { text: "Local CSV Files", link: "/supported-sources/csv.md" },
               {

ingestr-0.13.74/docs/supported-sources/influxdb.md

@@ -0,0 +1,38 @@
+# InfluxDB
+[InfluxDB](https://www.influxdata.com/) is a time series database optimized for storing high throughput metrics.
+
+ingestr supports InfluxDB as a source.
+
+## URI format
+
+```plaintext
+influxdb://<host>:<port>?token=<token>&org=<org>&bucket=<bucket>&secure=<secure>
+```
+
+URI parameters:
+- `host`: Hostname and optional port of your InfluxDB instance.
+- `token`: Authentication token.
+- `org`: Name of the organization.
+- `bucket`: Bucket that stores the measurements.
+- `secure`: Optional. Use HTTPS when `true` (default) or HTTP when `false`.
+
+The `<measurement>` name should be provided as the value for `--source-table`.
+
+## Example
+
+Copy cpu metrics from InfluxDB into DuckDB:
+
+```sh
+ingestr ingest \
+    --source-uri 'influxdb://eu-central-1-0.aws.cloud3.influxdata.com?token=my-token&org=my-org&bucket=metrics&secure=false' \
+    --source-table 'cpu' \
+    --dest-uri duckdb:///metrics.duckdb \
+    --dest-table 'dest.cpu'
+```
+
+## Tables
+
+The InfluxDB source accepts any measurement name. Specify the measurement as the `--source-table` option when running `ingestr`.
+
+> [!NOTE]
+> Primary Key Required for Merge: When using `--incremental-strategy merge`, you must also specify `--primary-key` Otherwise, the strategy defaults to append.

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/main.py

@@ -506,6 +506,7 @@ def ingest(
 
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
+    
 
         if (
             incremental_key
@@ -596,8 +597,13 @@ def ingest(
         if incremental_strategy != IncrementalStrategy.none:
             write_disposition = incremental_strategy.value
 
-        start_time = datetime.now()
+        if factory.source_scheme == "influxdb":
+            if primary_key:
+                write_disposition = "merge"
+            
 
+        start_time = datetime.now()
+     
         run_info: LoadInfo = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(

ingestr-0.13.74/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.74"

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/src/facebook_ads/__init__.py

@@ -1,14 +1,13 @@
 """Loads campaigns, ads sets, ads, leads and insight data from Facebook Marketing API"""
 
-from datetime import datetime
 from typing import Iterator, Sequence
 
 import dlt
 from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TDataItems
 from dlt.sources import DltResource
 from facebook_business.adobjects.ad import Ad
-from dlt.common.time import ensure_pendulum_datetime
 
 from .helpers import (
     execute_job,
@@ -167,14 +166,20 @@ def facebook_insights_source(
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
             "date_start",
-            initial_value=ensure_pendulum_datetime(start_date).start_of('day').date(),
-            end_value=ensure_pendulum_datetime(end_date).end_of('day').date() if end_date else None,
+            initial_value=ensure_pendulum_datetime(start_date).start_of("day").date(),
+            end_value=ensure_pendulum_datetime(end_date).end_of("day").date()
+            if end_date
+            else None,
             range_end="closed",
             range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
         start_date = date_start.last_value
-        end_date = pendulum.instance(date_start.end_value) if date_start.end_value else pendulum.now()
+        end_date = (
+            pendulum.instance(date_start.end_value)
+            if date_start.end_value
+            else pendulum.now()
+        )
 
         while start_date <= end_date:
             query = {

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/src/facebook_ads/helpers.py

@@ -6,12 +6,10 @@ import time
 from datetime import datetime
 from typing import Any, Iterator, Sequence
 
-import dlt
 import humanize
 import pendulum
 from dlt.common import logger
 from dlt.common.configuration.inject import with_config
-from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import DictStrAny, TDataItem, TDataItems
 from dlt.sources.helpers import requests
 from dlt.sources.helpers.requests import Client
@@ -34,7 +32,7 @@ def process_report_item(item: AbstractObject) -> DictStrAny:
         item["date_start"] = datetime.strptime(item["date_start"], "%Y-%m-%d").date()
     if "date_stop" in item:
         item["date_stop"] = datetime.strptime(item["date_stop"], "%Y-%m-%d").date()
-    
+
     d: DictStrAny = item.export_all_data()
     for pki in INSIGHTS_PRIMARY_KEY:
         if pki not in d:

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/src/factory.py

@@ -45,6 +45,7 @@ from ingestr.src.sources import (
     GoogleSheetsSource,
     GorgiasSource,
     HubspotSource,
+    InfluxDBSource,
     IsocPulseSource,
     KafkaSource,
     KinesisSource,
@@ -187,6 +188,7 @@ class SourceDestinationFactory:
         "pinterest": PinterestSource,
         "zoom": ZoomSource,
         "clickup": ClickupSource,
+        "influxdb": InfluxDBSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/src/github/__init__.py

@@ -4,6 +4,7 @@ import urllib.parse
 from typing import Iterator, Optional, Sequence
 
 import dlt
+import pendulum
 from dlt.common.typing import TDataItems
 from dlt.sources import DltResource
 
@@ -67,7 +68,11 @@ def github_reactions(
 
 @dlt.source(max_table_nesting=0)
 def github_repo_events(
-    owner: str, name: str, access_token: Optional[str] = None
+    owner: str,
+    name: str,
+    access_token: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime,
 ) -> DltResource:
     """Gets events for repository `name` with owner `owner` incrementally.
 
@@ -90,7 +95,8 @@ def github_repo_events(
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "created_at",
-            initial_value="1970-01-01T00:00:00Z",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat(),
             last_value_func=max,
             range_end="closed",
             range_start="closed",
@@ -100,8 +106,35 @@ def github_repo_events(
             f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
         )
 
+        # Get the date range from the incremental state
+        start_filter = pendulum.parse(
+            last_created_at.last_value or last_created_at.initial_value
+        )
+        end_filter = (
+            pendulum.parse(last_created_at.end_value)
+            if last_created_at.end_value
+            else None
+        )
+
         for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
-            yield page
+            # Filter events by date range
+            filtered_events = []
+            for event in page:
+                event_date = pendulum.parse(event["created_at"])
+
+                # Check if event is within the date range
+                if event_date >= start_filter:
+                    if end_filter is None or event_date <= end_filter:
+                        filtered_events.append(event)
+                    elif event_date > end_filter:
+                        # Skip events that are newer than our end date
+                        continue
+                else:
+                    # Events are ordered by date desc, so if we hit an older event, we can stop
+                    break
+
+            if filtered_events:
+                yield filtered_events
 
             # stop requesting pages if the last element was already older than initial value
             # note: incremental will skip those items anyway, we just do not want to use the api limits

ingestr-0.13.74/ingestr/src/influxdb/__init__.py

@@ -0,0 +1,45 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import InfluxClient
+
+@dlt.source(max_table_nesting=0)
+def influxdb_source(
+    measurement: str,
+    host: str,
+    org: str,
+    bucket: str,
+    token: str = dlt.secrets.value,
+    secure: bool = True,
+    start_date: pendulum.DateTime = pendulum.datetime(2024, 1, 1),
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = InfluxClient(
+        url=host, token=token, org=org, bucket=bucket, verify_ssl=secure
+    )
+
+    @dlt.resource(name=measurement)
+    def fetch_table(
+        timestamp=dlt.sources.incremental(
+            "time",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if timestamp.last_value is None:
+            start = start_date.isoformat()
+        else:
+            start = timestamp.last_value.isoformat()
+        if timestamp.end_value is None:
+            end = pendulum.now().isoformat()
+        else:
+            end = timestamp.end_value.isoformat()
+        yield from client.fetch_measurement(measurement, start, end)
+
+    return fetch_table

ingestr-0.13.74/ingestr/src/influxdb/client.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, Iterable
+
+from influxdb_client import InfluxDBClient  # type: ignore
+
+
+class InfluxClient:
+    def __init__(
+        self, url: str, token: str, org: str, bucket: str, verify_ssl: bool = True
+    ) -> None:
+        self.client = InfluxDBClient(
+            url=url, token=token, org=org, verify_ssl=verify_ssl
+        )
+        self.bucket = bucket
+
+    def fetch_measurement(
+        self, measurement: str, start: str, end: str | None = None
+    ) -> Iterable[Dict[str, Any]]:
+        query = f'from(bucket: "{self.bucket}") |> range(start: {start}'
+        if end:
+            query += f", stop: {end}"
+        query += f') |> filter(fn: (r) => r["_measurement"] == "{measurement}")'
+        query_api = self.client.query_api()
+
+        for record in query_api.query_stream(query):
+            cleaned_record = {}
+            exclude_keys = {"result", "table", "_start", "_stop"}
+            for key, value in record.values.items():
+                if key in exclude_keys:
+                    continue
+                if key.startswith("_"):
+                    cleaned_record[key[1:]] = value
+                else:
+                    cleaned_record[key] = value
+            yield cleaned_record

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/src/solidgate/helpers.py

@@ -6,7 +6,6 @@ import json
 import time
 from io import StringIO
 
-import pandas as pd  # type: ignore
 import pendulum
 
 from ingestr.src.http_client import create_client
@@ -132,9 +131,9 @@ class SolidgateClient:
                             else:
                                 row["created_at"] = None
 
-                            row2 = {k: v for k, v in row.items() if v != ''}
+                            row2 = {k: v for k, v in row.items() if v != ""}
                             rows.append(row2)
-                        
+
                         return rows
                     except Exception as e:
                         raise Exception(f"Error reading CSV: {e}")

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/src/sources.py

@@ -1814,7 +1814,23 @@ class GitHubSource:
                 owner=owner, name=repo, access_token=access_token
             ).with_resources(table)
         elif table == "repo_events":
-            return github_repo_events(owner=owner, name=repo, access_token=access_token)
+            start_date = kwargs.get("interval_start") or pendulum.now().subtract(
+                days=30
+            )
+            end_date = kwargs.get("interval_end") or pendulum.now()
+
+            if isinstance(start_date, str):
+                start_date = pendulum.parse(start_date)
+            if isinstance(end_date, str):
+                end_date = pendulum.parse(end_date)
+
+            return github_repo_events(
+                owner=owner,
+                name=repo,
+                access_token=access_token,
+                start_date=start_date,
+                end_date=end_date,
+            )
         elif table == "stargazers":
             return github_stargazers(owner=owner, name=repo, access_token=access_token)
         else:
@@ -3026,3 +3042,53 @@ class ZoomSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class InfluxDBSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        host = parsed_uri.netloc
+
+        secure = params.get("secure", ["true"])[0].lower() != "false"
+        scheme = "https" if secure else "http"
+        host_url = f"{scheme}://{host}"
+
+        token = params.get("token")
+        org = params.get("org")
+        bucket = params.get("bucket")
+
+        if not host:
+            raise MissingValueError("host", "InfluxDB")
+        if not token:
+            raise MissingValueError("token", "InfluxDB")
+        if not org:
+            raise MissingValueError("org", "InfluxDB")
+        if not bucket:
+            raise MissingValueError("bucket", "InfluxDB")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2024, 1, 1).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = ensure_pendulum_datetime(end_date)
+
+        from ingestr.src.influxdb import influxdb_source
+
+        return influxdb_source(
+            measurement=table,
+            host=host_url,
+            org=org[0],
+            bucket=bucket[0],
+            token=token[0],
+            secure=secure,
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

{ingestr-0.13.73 → ingestr-0.13.74}/ingestr/src/tiktok_ads/__init__.py

@@ -125,8 +125,12 @@ def tiktok_source(
         end_date_tz_adjusted = end_date.in_tz(timezone)
 
         if datetime is not None:
-            start_date_tz_adjusted = ensure_pendulum_datetime(datetime.last_value).in_tz(timezone)
-            end_date_tz_adjusted = ensure_pendulum_datetime(datetime.end_value).in_tz(timezone)
+            start_date_tz_adjusted = ensure_pendulum_datetime(
+                datetime.last_value
+            ).in_tz(timezone)
+            end_date_tz_adjusted = ensure_pendulum_datetime(datetime.end_value).in_tz(
+                timezone
+            )
 
         list_of_interval = find_intervals(
             current_date=start_date_tz_adjusted,

{ingestr-0.13.73 → ingestr-0.13.74}/pyproject.toml

@@ -92,6 +92,7 @@ exclude = [
     'src/linear/.*',
     'src/zoom/.*',
     'src/clickup/.*',
+    'src/influxdb/.*'
   ]
 
 [[tool.mypy.overrides]]
@@ -122,6 +123,7 @@ module = [
   "ingestr.src.linear.*",
   "ingestr.src.zoom.*",
   "ingestr.src.clickup.*",
+  "ingestr.src.influxdb",
 ]
 follow_imports = "skip"
 

{ingestr-0.13.73 → ingestr-0.13.74}/requirements.in

@@ -56,3 +56,4 @@ google-cloud-spanner==3.54.0
 smartsheet-python-sdk==3.0.5
 paramiko==3.5.1
 python-quickbooks==0.9.2
+influxdb-client==1.41.0

{ingestr-0.13.73 → ingestr-0.13.74}/requirements.txt

@@ -49,6 +49,7 @@ certifi==2025.1.31
     # via
     #   clickhouse-connect
     #   elastic-transport
+    #   influxdb-client
     #   requests
     #   smartsheet-python-sdk
     #   snowflake-connector-python
@@ -254,6 +255,8 @@ idna==3.10
     #   yarl
 inflection==0.5.1
     # via pyairtable
+influxdb-client==1.41.0
+    # via -r requirements.in
 intuit-oauth==1.2.4
     # via python-quickbooks
 isodate==0.7.2
@@ -418,6 +421,7 @@ python-dateutil==2.9.0.post0
     #   aiobotocore
     #   botocore
     #   google-cloud-bigquery
+    #   influxdb-client
     #   pandas
     #   pendulum
     #   pyathena
@@ -446,6 +450,8 @@ pyyaml==6.0.2
     #   google-ads
 rauth==0.7.3
     # via python-quickbooks
+reactivex==4.0.4
+    # via influxdb-client
 requests==2.32.3
     # via
     #   asana
@@ -506,6 +512,7 @@ semver==3.0.4
 setuptools==75.8.2
     # via
     #   dlt
+    #   influxdb-client
     #   python-quickbooks
     #   types-setuptools
 shellingham==1.5.4
@@ -597,6 +604,7 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   pyopenssl
+    #   reactivex
     #   simple-salesforce
     #   snowflake-connector-python
     #   sqlalchemy2-stubs
@@ -623,6 +631,7 @@ urllib3==2.3.0
     #   crate
     #   databricks-sql-connector
     #   elastic-transport
+    #   influxdb-client
     #   pyairtable
     #   requests
     #   types-requests

{ingestr-0.13.73 → ingestr-0.13.74}/requirements_arm64.txt

@@ -49,6 +49,7 @@ certifi==2025.1.31
     # via
     #   clickhouse-connect
     #   elastic-transport
+    #   influxdb-client
     #   requests
     #   smartsheet-python-sdk
     #   snowflake-connector-python
@@ -248,6 +249,8 @@ idna==3.10
     #   yarl
 inflection==0.5.1
     # via pyairtable
+influxdb-client==1.41.0
+    # via -r requirements.in
 intuit-oauth==1.2.4
     # via python-quickbooks
 isodate==0.7.2
@@ -412,6 +415,7 @@ python-dateutil==2.9.0.post0
     #   aiobotocore
     #   botocore
     #   google-cloud-bigquery
+    #   influxdb-client
     #   pandas
     #   pendulum
     #   pyathena
@@ -440,6 +444,8 @@ pyyaml==6.0.2
     #   google-ads
 rauth==0.7.3
     # via python-quickbooks
+reactivex==4.0.4
+    # via influxdb-client
 requests==2.32.4
     # via
     #   asana
@@ -500,6 +506,7 @@ semver==3.0.4
 setuptools==78.1.1
     # via
     #   dlt
+    #   influxdb-client
     #   python-quickbooks
     #   types-setuptools
 shellingham==1.5.4
@@ -590,6 +597,7 @@ typing-extensions==4.13.0
     #   pydantic
     #   pydantic-core
     #   pyopenssl
+    #   reactivex
     #   simple-salesforce
     #   snowflake-connector-python
     #   sqlalchemy2-stubs
@@ -619,6 +627,7 @@ urllib3==2.5.0
     #   crate
     #   databricks-sql-connector
     #   elastic-transport
+    #   influxdb-client
     #   pyairtable
     #   requests
     #   types-requests

ingestr-0.13.73/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.73"