ingestr 0.13.82__tar.gz → 0.13.84__tar.gz

{ingestr-0.13.82 → ingestr-0.13.84}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.82
+Version: 0.13.84
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -83,7 +83,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.3
+Requires-Dist: greenlet==3.2.4
 Requires-Dist: grpc-google-iam-v1==0.14.2
 Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3

{ingestr-0.13.82 → ingestr-0.13.84}/docs/.vitepress/config.mjs

@@ -155,6 +155,7 @@ export default defineConfig({
               { text: "Solidgate", link: "/supported-sources/solidgate.md" },
               { text: "Stripe", link: "/supported-sources/stripe.md" },
               { text: "TikTok Ads", link: "/supported-sources/tiktok-ads.md" },
+              { text: "Wise", link: "/supported-sources/wise.md" },
               { text: "Zendesk", link: "/supported-sources/zendesk.md" },
               { text: "Zoom", link: "/supported-sources/zoom.md" },
             ],

{ingestr-0.13.82 → ingestr-0.13.84}/docs/supported-sources/linear.md

@@ -47,14 +47,9 @@ Linear source allows ingesting the following tables:
 - `labels`: Fetches labels used for categorizing issues.
 - `organization`: Fetches organization-level information.
 - `project_updates`: Fetches updates posted to projects.
-- `roadmaps`: Fetches roadmap data for strategic planning.
-- `roadmap_to_projects`: Fetches relationships between roadmaps and projects.
 - `team_memberships`: Fetches team membership information.
 - `initiative_to_project`: Fetches relationships between initiatives and projects.
 - `project_milestone`: Fetches project milestone data.
 - `project_status`: Fetches project status information.
-- `team`: Fetches team-level data.
-- `project`: Fetches individual project information.
-
 
 Use these as the `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.13.84/docs/supported-sources/wise.md

@@ -0,0 +1,49 @@
+# Wise
+
+[Wise](https://wise.com/) is a global financial technology company that provides international money transfers, multi-currency accounts, and business payment solutions.
+
+ingestr supports Wise as a source.
+
+## URI format
+
+```plaintext
+wise://?api_key=<api_token>
+```
+
+URI parameters:
+
+- `api_key`: Wise API token.
+
+### Please follow the steps below to get a Wise API token
+- Go to wise.com and log in.
+- Click on your profile picture.
+- From the dropdown, choose "Settings".
+- In the left-hand menu, click "API Tokens".
+- Click "Create API Token".
+- Give it a name.
+- Click "Create".
+Copy the token and save it securely..
+
+## Example
+
+Copy transfers from Wise into a DuckDB database:
+
+```sh
+ingestr ingest \
+    --source-uri 'wise://?api_key=mytoken' \
+    --source-table transfers \
+    --dest-uri duckdb:///wise.duckdb \
+    --dest-table wise.transfers
+```
+
+## Tables
+
+Wise source allows ingesting the following sources into separate tables:
+| Table           | PK | Inc Key | Inc Strategy | Details                                                                                                                                        |
+| --------------- | ----------- | --------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
+| `profiles`        | `id` | –                | merge               | Retrieves all profiles  associated with the user. |
+| `transfers` | `id` |    `created`             | merge               | Lists all transfers for including status and details associated with the user.                                           |
+| `balances`   | `id` | `modificationTime`     | merge  | Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+
+
+<img alt="wise_ingestion_img" src="../media/wise_ingestion.png" />

ingestr-0.13.84/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.84"

{ingestr-0.13.82 → ingestr-0.13.84}/ingestr/src/factory.py

@@ -73,6 +73,7 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     TrustpilotSource,
+    WiseSource,
     ZendeskSource,
     ZoomSource,
 )
@@ -192,6 +193,7 @@ class SourceDestinationFactory:
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,
+        "wise": WiseSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.13.82 → ingestr-0.13.84}/ingestr/src/linear/__init__.py

@@ -485,9 +485,32 @@ query Projects($cursor: String) {
       description
       createdAt
       updatedAt
+      archivedAt
+      completedAt
+      canceledAt
+      startedAt
+      
+      color
+      icon
+      slugId
+      url
+      
       health
       priority
+      priorityLabel
+      state
+      
       targetDate
+      startDate
+      
+      progress
+      currentProgress
+      scope
+      
+      sortOrder
+      trashed
+      
+      creator { id }
       lead { id }
     }
     pageInfo { hasNextPage endCursor }
@@ -495,6 +518,58 @@ query Projects($cursor: String) {
 }
 """
 
+TEAMS_QUERY = """
+query Teams($cursor: String) {
+  teams(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      key
+      description
+      color
+      icon
+      private
+      archivedAt
+      createdAt
+      updatedAt
+      
+      organization { id }
+      parent { id }
+      
+      cyclesEnabled
+      cycleDuration
+      cycleStartDay
+      cycleCooldownTime
+      
+      issueCount
+      issueEstimationType
+      issueEstimationAllowZero
+      issueEstimationExtended
+      issueOrderingNoPriorityFirst
+      
+      autoArchivePeriod
+      autoClosePeriod
+      autoCloseChildIssues
+      autoCloseParentIssues
+      
+      groupIssueHistory
+      timezone
+      inviteHash
+      joinByDefault
+      
+      slackNewIssue
+      slackIssueComments
+      slackIssueStatuses
+      
+      triageEnabled
+      requirePriorityToLeaveTriage
+      upcomingCycleCount
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
 
 # Paginated resources configuration
 PAGINATED_RESOURCES = [
@@ -515,6 +590,7 @@ PAGINATED_RESOURCES = [
     ("project_milestone", PROJECT_MILESTONES_QUERY, "projectMilestones"),
     ("project_status", PROJECT_STATUSES_QUERY, "projectStatuses"),
     ("projects", PROJECTS_QUERY, "projects"),
+    ("teams", TEAMS_QUERY, "teams"),
 ]
 
 

{ingestr-0.13.82 → ingestr-0.13.84}/ingestr/src/sources.py

@@ -3386,3 +3386,41 @@ class InfluxDBSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class WiseSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        api_key = params.get("api_key")
+
+        if not api_key:
+            raise MissingValueError("api_key", "Wise")
+
+        if table not in ["profiles", "transfers", "balances"]:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Wise source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_timezone("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date).in_timezone("UTC")
+        else:
+            end_date = None
+
+        from ingestr.src.wise import wise_source
+
+        return wise_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

{ingestr-0.13.82 → ingestr-0.13.84}/ingestr/src/stripe_analytics/__init__.py

@@ -94,7 +94,7 @@ def async_stripe_source(
         )(endpoint)
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def incremental_stripe_source(
     endpoints: Tuple[str, ...],
     stripe_secret_key: str = dlt.secrets.value,

ingestr-0.13.84/ingestr/src/wise/__init__.py

@@ -0,0 +1,68 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import WiseClient
+
+
+@dlt.source(max_table_nesting=0)
+def wise_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = WiseClient(api_key)
+
+    # List of all profiles belonging to user.
+    @dlt.resource(write_disposition="merge", name="profiles", primary_key="id")
+    def profiles() -> Iterable[TDataItem]:
+        yield from client.fetch_profiles()
+
+    # List transfers for a profile.
+    @dlt.resource(write_disposition="merge", name="transfers", primary_key="id")
+    def transfers(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "created",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ):
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_transfers(profile["id"], start_dt, end_dt)
+
+    # Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+    @dlt.resource(write_disposition="merge", name="balances", primary_key="id")
+    def balances(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "modificationTime",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_balances(profile["id"], start_dt, end_dt)
+
+    return profiles, transfers, balances

ingestr-0.13.84/ingestr/src/wise/client.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class WiseClient:
+    BASE_URL = "https://api.transferwise.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    # https://docs.wise.com/api-docs/api-reference/profile#list-profiles
+    def fetch_profiles(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v2/profiles"
+        resp = self.session.get(url)
+        resp.raise_for_status()
+        for profile in resp.json():
+            yield profile
+
+    # https://docs.wise.com/api-docs/api-reference/transfer#list-transfers
+    def fetch_transfers(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ):
+        offset = 0
+
+        while True:
+            data = self.session.get(
+                f"{self.BASE_URL}/v1/transfers",
+                params={
+                    "profile": profile_id,
+                    "createdDateStart": start_time.to_date_string(),
+                    "createdDateEnd": end_time.to_date_string(),
+                    "limit": 100,
+                    "offset": offset,
+                },
+            )
+            response_data = data.json()
+
+            if not response_data or len(response_data) == 0:
+                break
+
+            for transfer in response_data:
+                transfer["created"] = pendulum.parse(transfer["created"])
+
+                yield transfer
+            offset += 100
+
+    # https://docs.wise.com/api-docs/api-reference/balance#list
+    def fetch_balances(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v4/profiles/{profile_id}/balances"
+        resp = self.session.get(url, params={"types": "STANDARD,SAVINGS"})
+        resp.raise_for_status()
+        for balance in resp.json():
+            balance["modificationTime"] = pendulum.parse(balance["modificationTime"])
+            if (
+                balance["modificationTime"] > start_time
+                and balance["modificationTime"] < end_time
+            ):
+                yield balance

{ingestr-0.13.82 → ingestr-0.13.84}/pyproject.toml

@@ -93,7 +93,8 @@ exclude = [
     'src/zoom/.*',
     'src/clickup/.*',
     'src/influxdb/.*',
-    'src/freshdesk/.*'
+    'src/freshdesk/.*',
+    'src/wise/.*'
   ]
 
 [[tool.mypy.overrides]]
@@ -125,7 +126,8 @@ module = [
   "ingestr.src.zoom.*",
   "ingestr.src.clickup.*",
   "ingestr.src.influxdb",
-  "ingestr.src.freshdesk"
+  "ingestr.src.freshdesk",
+  "ingestr.src.wise"
 ]
 follow_imports = "skip"
 

{ingestr-0.13.82 → ingestr-0.13.84}/requirements.txt

@@ -214,7 +214,7 @@ googleapis-common-protos==1.69.0
     #   google-api-core
     #   grpc-google-iam-v1
     #   grpcio-status
-greenlet==3.2.3
+greenlet==3.2.4
     # via sqlalchemy
 grpc-google-iam-v1==0.14.2
     # via google-cloud-spanner

ingestr-0.13.82/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.82"