ingestr 0.13.82__py3-none-any.whl → 0.13.84__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.82"
+version = "v0.13.84"

ingestr/src/factory.py

@@ -73,6 +73,7 @@ from ingestr.src.sources import (
     StripeAnalyticsSource,
     TikTokSource,
     TrustpilotSource,
+    WiseSource,
     ZendeskSource,
     ZoomSource,
 )
@@ -192,6 +193,7 @@ class SourceDestinationFactory:
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,
+        "wise": WiseSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/linear/__init__.py

@@ -485,9 +485,32 @@ query Projects($cursor: String) {
       description
       createdAt
       updatedAt
+      archivedAt
+      completedAt
+      canceledAt
+      startedAt
+      
+      color
+      icon
+      slugId
+      url
+      
       health
       priority
+      priorityLabel
+      state
+      
       targetDate
+      startDate
+      
+      progress
+      currentProgress
+      scope
+      
+      sortOrder
+      trashed
+      
+      creator { id }
       lead { id }
     }
     pageInfo { hasNextPage endCursor }
@@ -495,6 +518,58 @@ query Projects($cursor: String) {
 }
 """
 
+TEAMS_QUERY = """
+query Teams($cursor: String) {
+  teams(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      key
+      description
+      color
+      icon
+      private
+      archivedAt
+      createdAt
+      updatedAt
+      
+      organization { id }
+      parent { id }
+      
+      cyclesEnabled
+      cycleDuration
+      cycleStartDay
+      cycleCooldownTime
+      
+      issueCount
+      issueEstimationType
+      issueEstimationAllowZero
+      issueEstimationExtended
+      issueOrderingNoPriorityFirst
+      
+      autoArchivePeriod
+      autoClosePeriod
+      autoCloseChildIssues
+      autoCloseParentIssues
+      
+      groupIssueHistory
+      timezone
+      inviteHash
+      joinByDefault
+      
+      slackNewIssue
+      slackIssueComments
+      slackIssueStatuses
+      
+      triageEnabled
+      requirePriorityToLeaveTriage
+      upcomingCycleCount
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
 
 # Paginated resources configuration
 PAGINATED_RESOURCES = [
@@ -515,6 +590,7 @@ PAGINATED_RESOURCES = [
     ("project_milestone", PROJECT_MILESTONES_QUERY, "projectMilestones"),
     ("project_status", PROJECT_STATUSES_QUERY, "projectStatuses"),
     ("projects", PROJECTS_QUERY, "projects"),
+    ("teams", TEAMS_QUERY, "teams"),
 ]
 
 

ingestr/src/sources.py

@@ -3386,3 +3386,41 @@ class InfluxDBSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class WiseSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        api_key = params.get("api_key")
+
+        if not api_key:
+            raise MissingValueError("api_key", "Wise")
+
+        if table not in ["profiles", "transfers", "balances"]:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Wise source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_timezone("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date).in_timezone("UTC")
+        else:
+            end_date = None
+
+        from ingestr.src.wise import wise_source
+
+        return wise_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

ingestr/src/stripe_analytics/__init__.py

@@ -94,7 +94,7 @@ def async_stripe_source(
         )(endpoint)
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def incremental_stripe_source(
     endpoints: Tuple[str, ...],
     stripe_secret_key: str = dlt.secrets.value,

ingestr/src/wise/__init__.py

@@ -0,0 +1,68 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import WiseClient
+
+
+@dlt.source(max_table_nesting=0)
+def wise_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = WiseClient(api_key)
+
+    # List of all profiles belonging to user.
+    @dlt.resource(write_disposition="merge", name="profiles", primary_key="id")
+    def profiles() -> Iterable[TDataItem]:
+        yield from client.fetch_profiles()
+
+    # List transfers for a profile.
+    @dlt.resource(write_disposition="merge", name="transfers", primary_key="id")
+    def transfers(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "created",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ):
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_transfers(profile["id"], start_dt, end_dt)
+
+    # Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+    @dlt.resource(write_disposition="merge", name="balances", primary_key="id")
+    def balances(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "modificationTime",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_balances(profile["id"], start_dt, end_dt)
+
+    return profiles, transfers, balances

ingestr/src/wise/client.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class WiseClient:
+    BASE_URL = "https://api.transferwise.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    # https://docs.wise.com/api-docs/api-reference/profile#list-profiles
+    def fetch_profiles(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v2/profiles"
+        resp = self.session.get(url)
+        resp.raise_for_status()
+        for profile in resp.json():
+            yield profile
+
+    # https://docs.wise.com/api-docs/api-reference/transfer#list-transfers
+    def fetch_transfers(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ):
+        offset = 0
+
+        while True:
+            data = self.session.get(
+                f"{self.BASE_URL}/v1/transfers",
+                params={
+                    "profile": profile_id,
+                    "createdDateStart": start_time.to_date_string(),
+                    "createdDateEnd": end_time.to_date_string(),
+                    "limit": 100,
+                    "offset": offset,
+                },
+            )
+            response_data = data.json()
+
+            if not response_data or len(response_data) == 0:
+                break
+
+            for transfer in response_data:
+                transfer["created"] = pendulum.parse(transfer["created"])
+
+                yield transfer
+            offset += 100
+
+    # https://docs.wise.com/api-docs/api-reference/balance#list
+    def fetch_balances(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v4/profiles/{profile_id}/balances"
+        resp = self.session.get(url, params={"types": "STANDARD,SAVINGS"})
+        resp.raise_for_status()
+        for balance in resp.json():
+            balance["modificationTime"] = pendulum.parse(balance["modificationTime"])
+            if (
+                balance["modificationTime"] > start_time
+                and balance["modificationTime"] < end_time
+            ):
+                yield balance

{ingestr-0.13.82.dist-info → ingestr-0.13.84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.82
+Version: 0.13.84
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -83,7 +83,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.3
+Requires-Dist: greenlet==3.2.4
 Requires-Dist: grpc-google-iam-v1==0.14.2
 Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3

{ingestr-0.13.82.dist-info → ingestr-0.13.84.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qoWHNcHh0-xVnyQxbQ-SKuTxPb1RNV3ENkCpqO7CLrk,26694
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=S1Cgqn9Xc0avTk0r_rJ8fMXucqD7c5O68FsKjSWj2Pc,21
+ingestr/src/buildinfo.py,sha256=WN5lOieDcoxVlYgBMBodzZWgJZUv46uGz-LyWD1-F_M,21
 ingestr/src/destinations.py,sha256=M2Yni6wiWcrvZ8EPJemidqxN156l0rehgCc7xuil7mo,22840
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=rF5Ry4o4t8KulSPBtrd7ZKCI_0TH1DAetG0zs9H7oik,6792
+ingestr/src/factory.py,sha256=SQr7s-WCx_jL8DmPbeCc4VuE35z1J7gCAwKGAkOpv2M,6836
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=Yaej1Iy75JPOccw6jYL6kbhzYFq33t4YelGT-NiAA5I,119661
+ingestr/src/sources.py,sha256=NfD7pj0WOD1wQOV3oZkHmGOAwHMuCCO_Mzf8F8EpobU,120918
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -85,7 +85,7 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
 ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
-ingestr/src/linear/__init__.py,sha256=G2x4HaRl8WWReDJ5HElnNTfVIybdztrIB3acl1ickyo,11003
+ingestr/src/linear/__init__.py,sha256=f0f9MBEV3tK-tTT2DUOj-0-xl2QRJs1x0nYv2Lo1FZQ,12277
 ingestr/src/linear/helpers.py,sha256=pPqtJrY3D3U2IaqTsD6ChvwMLYiUVtmQ8wTvetz5U7A,3640
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
 ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
@@ -125,7 +125,7 @@ ingestr/src/solidgate/__init__.py,sha256=Ts83j-JSnFsFuF4tDhVOfZKg7H0-bIpfn3kg1ZO
 ingestr/src/solidgate/helpers.py,sha256=mAsW_1hpD7ab3Y2vw8fxHi4yD3aT1geLdIYZ7ycyxBc,5690
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
-ingestr/src/stripe_analytics/__init__.py,sha256=mK8dGKAlMPVqGE9gG30XfbvOvgVD0yWhNpt-D3iavDY,6385
+ingestr/src/stripe_analytics/__init__.py,sha256=6c4zNY5CgmyjbEF94Ch8zUefdl_w8sLI3FjHUyPcC2U,6406
 ingestr/src/stripe_analytics/helpers.py,sha256=KGtRcSrhKEqzJ3AWpgDV2o4cuBFaIwu2Gc1KgvVWTtg,11764
 ingestr/src/stripe_analytics/settings.py,sha256=xt1-ljwP4nLTNUa8l3KwFbtK8FtQHgHpzGF5uPKfRsw,2246
 ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
@@ -134,6 +134,8 @@ ingestr/src/tiktok_ads/__init__.py,sha256=_upvSv79uktELTzg78VavYgQftfhp4YM6OXfpk
 ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=jmWHvZzN1Vt_PWrJkgq5a2wIwon-OBEzXoZx0jEy-74,3905
 ingestr/src/trustpilot/__init__.py,sha256=ofhjep4qRPIi8q41qc97QVex8UbWF-Fd7gUsqeQlQX8,1279
 ingestr/src/trustpilot/client.py,sha256=zKYt5C7nrR83Id0KN49EPmtml8MEtlSPlAosEFU3VXY,1616
+ingestr/src/wise/__init__.py,sha256=EiXWE89Jb2Do1t8HK1-GZRS2KwgaiThxsHVEpaBcoh4,2087
+ingestr/src/wise/client.py,sha256=2hTCuOB24-HiwTp02o5i99KF5rjI0CXqgIb9V6gAjjE,2247
 ingestr/src/zendesk/__init__.py,sha256=tmJ_jdb6kpwmEKpcv6Im71-bOZI6h-Tcofe18OH4I24,17762
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
@@ -151,8 +153,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.82.dist-info/METADATA,sha256=4edZaTP7apukcZRUY4J00G99O3d7Qxz37csXXCNhaww,15182
-ingestr-0.13.82.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.82.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.82.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.82.dist-info/RECORD,,
+ingestr-0.13.84.dist-info/METADATA,sha256=RPTXlNxSvMJjGa4dE8VH_7gZsv9dqw51BI7E3dzJQbw,15182
+ingestr-0.13.84.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.84.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.84.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.84.dist-info/RECORD,,