ingestr 0.9.5__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/main.py +156 -40
- ingestr/src/adjust/__init__.py +1 -1
- ingestr/src/filters.py +21 -0
- ingestr/src/gorgias/__init__.py +17 -17
- ingestr/src/shopify/__init__.py +42 -42
- ingestr/src/slack/__init__.py +2 -2
- ingestr/src/sources.py +33 -6
- ingestr/src/version.py +1 -1
- ingestr/src/zendesk/__init__.py +2 -2
- {ingestr-0.9.5.dist-info → ingestr-0.10.0.dist-info}/METADATA +18 -18
- {ingestr-0.9.5.dist-info → ingestr-0.10.0.dist-info}/RECORD +14 -18
- ingestr/src/sql_database/__init__.py +0 -206
- ingestr/src/sql_database/arrow_helpers.py +0 -139
- ingestr/src/sql_database/helpers.py +0 -282
- ingestr/src/sql_database/override.py +0 -10
- ingestr/src/sql_database/schema_types.py +0 -139
- {ingestr-0.9.5.dist-info → ingestr-0.10.0.dist-info}/WHEEL +0 -0
- {ingestr-0.9.5.dist-info → ingestr-0.10.0.dist-info}/entry_points.txt +0 -0
- {ingestr-0.9.5.dist-info → ingestr-0.10.0.dist-info}/licenses/LICENSE.md +0 -0
ingestr/src/shopify/__init__.py
CHANGED
|
@@ -89,12 +89,12 @@ def shopify_source(
|
|
|
89
89
|
"description": "An unsigned 64-bit integer that's used as a unique identifier for the product.",
|
|
90
90
|
},
|
|
91
91
|
"images": {
|
|
92
|
-
"data_type": "
|
|
92
|
+
"data_type": "json",
|
|
93
93
|
"nullable": True,
|
|
94
94
|
"description": "A list of product image objects, each one representing an image associated with the product.",
|
|
95
95
|
},
|
|
96
96
|
"options": {
|
|
97
|
-
"data_type": "
|
|
97
|
+
"data_type": "json",
|
|
98
98
|
"nullable": True,
|
|
99
99
|
"description": "The custom product properties. For example, Size, Color, and Material.",
|
|
100
100
|
},
|
|
@@ -139,7 +139,7 @@ def shopify_source(
|
|
|
139
139
|
"description": "The date and time (ISO 8601 format) when the product was last modified.",
|
|
140
140
|
},
|
|
141
141
|
"variants": {
|
|
142
|
-
"data_type": "
|
|
142
|
+
"data_type": "json",
|
|
143
143
|
"nullable": True,
|
|
144
144
|
"description": "An array of product variants, each representing a different version of the product.",
|
|
145
145
|
},
|
|
@@ -191,7 +191,7 @@ def shopify_source(
|
|
|
191
191
|
"description": "The ID of the app that created the order.",
|
|
192
192
|
},
|
|
193
193
|
"billing_address": {
|
|
194
|
-
"data_type": "
|
|
194
|
+
"data_type": "json",
|
|
195
195
|
"nullable": True,
|
|
196
196
|
"description": "The mailing address associated with the payment method.",
|
|
197
197
|
},
|
|
@@ -226,7 +226,7 @@ def shopify_source(
|
|
|
226
226
|
"description": "A unique value referencing the checkout associated with the order.",
|
|
227
227
|
},
|
|
228
228
|
"client_details": {
|
|
229
|
-
"data_type": "
|
|
229
|
+
"data_type": "json",
|
|
230
230
|
"nullable": True,
|
|
231
231
|
"description": "Information about the browser the customer used when placing the order.",
|
|
232
232
|
},
|
|
@@ -236,7 +236,7 @@ def shopify_source(
|
|
|
236
236
|
"description": "The date and time when the order was closed.",
|
|
237
237
|
},
|
|
238
238
|
"company": {
|
|
239
|
-
"data_type": "
|
|
239
|
+
"data_type": "json",
|
|
240
240
|
"nullable": True,
|
|
241
241
|
"description": "Information about the purchasing company for the order.",
|
|
242
242
|
},
|
|
@@ -261,7 +261,7 @@ def shopify_source(
|
|
|
261
261
|
"description": "The three-letter code (ISO 4217 format) for the shop currency.",
|
|
262
262
|
},
|
|
263
263
|
"current_total_additional_fees_set": {
|
|
264
|
-
"data_type": "
|
|
264
|
+
"data_type": "json",
|
|
265
265
|
"nullable": True,
|
|
266
266
|
"description": "The current total additional fees on the order in shop and presentment currencies.",
|
|
267
267
|
},
|
|
@@ -271,12 +271,12 @@ def shopify_source(
|
|
|
271
271
|
"description": "The current total discounts on the order in the shop currency.",
|
|
272
272
|
},
|
|
273
273
|
"current_total_discounts_set": {
|
|
274
|
-
"data_type": "
|
|
274
|
+
"data_type": "json",
|
|
275
275
|
"nullable": True,
|
|
276
276
|
"description": "The current total discounts on the order in shop and presentment currencies.",
|
|
277
277
|
},
|
|
278
278
|
"current_total_duties_set": {
|
|
279
|
-
"data_type": "
|
|
279
|
+
"data_type": "json",
|
|
280
280
|
"nullable": True,
|
|
281
281
|
"description": "The current total duties charged on the order in shop and presentment currencies.",
|
|
282
282
|
},
|
|
@@ -286,7 +286,7 @@ def shopify_source(
|
|
|
286
286
|
"description": "The current total price of the order in the shop currency.",
|
|
287
287
|
},
|
|
288
288
|
"current_total_price_set": {
|
|
289
|
-
"data_type": "
|
|
289
|
+
"data_type": "json",
|
|
290
290
|
"nullable": True,
|
|
291
291
|
"description": "The current total price of the order in shop and presentment currencies.",
|
|
292
292
|
},
|
|
@@ -296,7 +296,7 @@ def shopify_source(
|
|
|
296
296
|
"description": "The sum of prices for all line items after discounts and returns in the shop currency.",
|
|
297
297
|
},
|
|
298
298
|
"current_subtotal_price_set": {
|
|
299
|
-
"data_type": "
|
|
299
|
+
"data_type": "json",
|
|
300
300
|
"nullable": True,
|
|
301
301
|
"description": "The sum of the prices for all line items after discounts and returns in shop and presentment currencies.",
|
|
302
302
|
},
|
|
@@ -306,12 +306,12 @@ def shopify_source(
|
|
|
306
306
|
"description": "The sum of the prices for all tax lines applied to the order in the shop currency.",
|
|
307
307
|
},
|
|
308
308
|
"current_total_tax_set": {
|
|
309
|
-
"data_type": "
|
|
309
|
+
"data_type": "json",
|
|
310
310
|
"nullable": True,
|
|
311
311
|
"description": "The sum of the prices for all tax lines applied to the order in shop and presentment currencies.",
|
|
312
312
|
},
|
|
313
313
|
"customer": {
|
|
314
|
-
"data_type": "
|
|
314
|
+
"data_type": "json",
|
|
315
315
|
"nullable": True,
|
|
316
316
|
"description": "Information about the customer.",
|
|
317
317
|
},
|
|
@@ -321,12 +321,12 @@ def shopify_source(
|
|
|
321
321
|
"description": "The two or three-letter language code, optionally followed by a region modifier.",
|
|
322
322
|
},
|
|
323
323
|
"discount_applications": {
|
|
324
|
-
"data_type": "
|
|
324
|
+
"data_type": "json",
|
|
325
325
|
"nullable": True,
|
|
326
326
|
"description": "An ordered list of stacked discount applications.",
|
|
327
327
|
},
|
|
328
328
|
"discount_codes": {
|
|
329
|
-
"data_type": "
|
|
329
|
+
"data_type": "json",
|
|
330
330
|
"nullable": True,
|
|
331
331
|
"description": "A list of discounts applied to the order.",
|
|
332
332
|
},
|
|
@@ -346,7 +346,7 @@ def shopify_source(
|
|
|
346
346
|
"description": "The status of payments associated with the order.",
|
|
347
347
|
},
|
|
348
348
|
"fulfillments": {
|
|
349
|
-
"data_type": "
|
|
349
|
+
"data_type": "json",
|
|
350
350
|
"nullable": True,
|
|
351
351
|
"description": "An array of fulfillments associated with the order.",
|
|
352
352
|
},
|
|
@@ -372,7 +372,7 @@ def shopify_source(
|
|
|
372
372
|
"description": "The URL for the page where the buyer landed when they entered the shop.",
|
|
373
373
|
},
|
|
374
374
|
"line_items": {
|
|
375
|
-
"data_type": "
|
|
375
|
+
"data_type": "json",
|
|
376
376
|
"nullable": True,
|
|
377
377
|
"description": "A list of line item objects containing information about an item in the order.",
|
|
378
378
|
},
|
|
@@ -397,7 +397,7 @@ def shopify_source(
|
|
|
397
397
|
"description": "An optional note that a shop owner can attach to the order.",
|
|
398
398
|
},
|
|
399
399
|
"note_attributes": {
|
|
400
|
-
"data_type": "
|
|
400
|
+
"data_type": "json",
|
|
401
401
|
"nullable": True,
|
|
402
402
|
"description": "Extra information added to the order as key-value pairs.",
|
|
403
403
|
},
|
|
@@ -412,22 +412,22 @@ def shopify_source(
|
|
|
412
412
|
"description": "The order's position in the shop's count of orders, starting at 1001.",
|
|
413
413
|
},
|
|
414
414
|
"original_total_additional_fees_set": {
|
|
415
|
-
"data_type": "
|
|
415
|
+
"data_type": "json",
|
|
416
416
|
"nullable": True,
|
|
417
417
|
"description": "The original total additional fees on the order in shop and presentment currencies.",
|
|
418
418
|
},
|
|
419
419
|
"original_total_duties_set": {
|
|
420
|
-
"data_type": "
|
|
420
|
+
"data_type": "json",
|
|
421
421
|
"nullable": True,
|
|
422
422
|
"description": "The original total duties charged on the order in shop and presentment currencies.",
|
|
423
423
|
},
|
|
424
424
|
"payment_terms": {
|
|
425
|
-
"data_type": "
|
|
425
|
+
"data_type": "json",
|
|
426
426
|
"nullable": True,
|
|
427
427
|
"description": "The terms and conditions under which a payment should be processed.",
|
|
428
428
|
},
|
|
429
429
|
"payment_gateway_names": {
|
|
430
|
-
"data_type": "
|
|
430
|
+
"data_type": "json",
|
|
431
431
|
"nullable": True,
|
|
432
432
|
"description": "The list of payment gateways used for the order.",
|
|
433
433
|
},
|
|
@@ -457,17 +457,17 @@ def shopify_source(
|
|
|
457
457
|
"description": "The website where the customer clicked a link to the shop.",
|
|
458
458
|
},
|
|
459
459
|
"refunds": {
|
|
460
|
-
"data_type": "
|
|
460
|
+
"data_type": "json",
|
|
461
461
|
"nullable": True,
|
|
462
462
|
"description": "A list of refunds applied to the order.",
|
|
463
463
|
},
|
|
464
464
|
"shipping_address": {
|
|
465
|
-
"data_type": "
|
|
465
|
+
"data_type": "json",
|
|
466
466
|
"nullable": True,
|
|
467
467
|
"description": "The mailing address where the order will be shipped.",
|
|
468
468
|
},
|
|
469
469
|
"shipping_lines": {
|
|
470
|
-
"data_type": "
|
|
470
|
+
"data_type": "json",
|
|
471
471
|
"nullable": True,
|
|
472
472
|
"description": "An array detailing the shipping methods used.",
|
|
473
473
|
},
|
|
@@ -492,7 +492,7 @@ def shopify_source(
|
|
|
492
492
|
"description": "The price of the order in the shop currency after discounts but before shipping, duties, taxes, and tips.",
|
|
493
493
|
},
|
|
494
494
|
"subtotal_price_set": {
|
|
495
|
-
"data_type": "
|
|
495
|
+
"data_type": "json",
|
|
496
496
|
"nullable": True,
|
|
497
497
|
"description": "The subtotal of the order in shop and presentment currencies after discounts but before shipping, duties, taxes, and tips.",
|
|
498
498
|
},
|
|
@@ -502,7 +502,7 @@ def shopify_source(
|
|
|
502
502
|
"description": "Tags attached to the order, formatted as a string of comma-separated values.",
|
|
503
503
|
},
|
|
504
504
|
"tax_lines": {
|
|
505
|
-
"data_type": "
|
|
505
|
+
"data_type": "json",
|
|
506
506
|
"nullable": True,
|
|
507
507
|
"description": "An array of tax line objects detailing taxes applied to the order.",
|
|
508
508
|
},
|
|
@@ -527,7 +527,7 @@ def shopify_source(
|
|
|
527
527
|
"description": "The total discounts applied to the price of the order in the shop currency.",
|
|
528
528
|
},
|
|
529
529
|
"total_discounts_set": {
|
|
530
|
-
"data_type": "
|
|
530
|
+
"data_type": "json",
|
|
531
531
|
"nullable": True,
|
|
532
532
|
"description": "The total discounts applied to the price of the order in shop and presentment currencies.",
|
|
533
533
|
},
|
|
@@ -537,7 +537,7 @@ def shopify_source(
|
|
|
537
537
|
"description": "The sum of all line item prices in the shop currency.",
|
|
538
538
|
},
|
|
539
539
|
"total_line_items_price_set": {
|
|
540
|
-
"data_type": "
|
|
540
|
+
"data_type": "json",
|
|
541
541
|
"nullable": True,
|
|
542
542
|
"description": "The total of all line item prices in shop and presentment currencies.",
|
|
543
543
|
},
|
|
@@ -552,12 +552,12 @@ def shopify_source(
|
|
|
552
552
|
"description": "The sum of all line item prices, discounts, shipping, taxes, and tips in the shop currency.",
|
|
553
553
|
},
|
|
554
554
|
"total_price_set": {
|
|
555
|
-
"data_type": "
|
|
555
|
+
"data_type": "json",
|
|
556
556
|
"nullable": True,
|
|
557
557
|
"description": "The total price of the order in shop and presentment currencies.",
|
|
558
558
|
},
|
|
559
559
|
"total_shipping_price_set": {
|
|
560
|
-
"data_type": "
|
|
560
|
+
"data_type": "json",
|
|
561
561
|
"nullable": True,
|
|
562
562
|
"description": "The total shipping price of the order in shop and presentment currencies.",
|
|
563
563
|
},
|
|
@@ -567,7 +567,7 @@ def shopify_source(
|
|
|
567
567
|
"description": "The sum of the prices for all tax lines applied to the order in the shop currency.",
|
|
568
568
|
},
|
|
569
569
|
"total_tax_set": {
|
|
570
|
-
"data_type": "
|
|
570
|
+
"data_type": "json",
|
|
571
571
|
"nullable": True,
|
|
572
572
|
"description": "The sum of the prices for all tax lines applied to the order in shop and presentment currencies.",
|
|
573
573
|
},
|
|
@@ -1669,27 +1669,27 @@ query discountNodes($after: String, $query: String, $first: Int) {
|
|
|
1669
1669
|
"description": "A globally unique ID for the product.",
|
|
1670
1670
|
},
|
|
1671
1671
|
"availablePublicationsCount": {
|
|
1672
|
-
"data_type": "
|
|
1672
|
+
"data_type": "json",
|
|
1673
1673
|
"nullable": False,
|
|
1674
1674
|
"description": "The number of publications that a resource is published to",
|
|
1675
1675
|
},
|
|
1676
1676
|
"category": {
|
|
1677
|
-
"data_type": "
|
|
1677
|
+
"data_type": "json",
|
|
1678
1678
|
"nullable": True,
|
|
1679
1679
|
"description": "The category of the product from Shopify's Standard Product Taxonomy.",
|
|
1680
1680
|
},
|
|
1681
1681
|
"combinedListing": {
|
|
1682
|
-
"data_type": "
|
|
1682
|
+
"data_type": "json",
|
|
1683
1683
|
"nullable": True,
|
|
1684
1684
|
"description": "A special product type that combines separate products into a single product listing.",
|
|
1685
1685
|
},
|
|
1686
1686
|
"combinedListingRole": {
|
|
1687
|
-
"data_type": "
|
|
1687
|
+
"data_type": "json",
|
|
1688
1688
|
"nullable": True,
|
|
1689
1689
|
"description": "The role of the product in a combined listing.",
|
|
1690
1690
|
},
|
|
1691
1691
|
"compareAtPriceRange": {
|
|
1692
|
-
"data_type": "
|
|
1692
|
+
"data_type": "json",
|
|
1693
1693
|
"nullable": True,
|
|
1694
1694
|
"description": "The compare-at price range of the product in the shop's default currency.",
|
|
1695
1695
|
},
|
|
@@ -1719,17 +1719,17 @@ query discountNodes($after: String, $query: String, $first: Int) {
|
|
|
1719
1719
|
"description": "A unique, human-readable string of the product's title.",
|
|
1720
1720
|
},
|
|
1721
1721
|
"metafields": {
|
|
1722
|
-
"data_type": "
|
|
1722
|
+
"data_type": "json",
|
|
1723
1723
|
"nullable": True,
|
|
1724
1724
|
"description": "A list of custom fields associated with the product.",
|
|
1725
1725
|
},
|
|
1726
1726
|
"options": {
|
|
1727
|
-
"data_type": "
|
|
1727
|
+
"data_type": "json",
|
|
1728
1728
|
"nullable": True,
|
|
1729
1729
|
"description": "A list of product options, e.g., size, color.",
|
|
1730
1730
|
},
|
|
1731
1731
|
"priceRangeV2": {
|
|
1732
|
-
"data_type": "
|
|
1732
|
+
"data_type": "json",
|
|
1733
1733
|
"nullable": False,
|
|
1734
1734
|
"description": "The minimum and maximum prices of a product.",
|
|
1735
1735
|
},
|
|
@@ -1784,12 +1784,12 @@ query discountNodes($after: String, $query: String, $first: Int) {
|
|
|
1784
1784
|
"description": "The date and time when the product was last modified.",
|
|
1785
1785
|
},
|
|
1786
1786
|
"variantsFirst250": {
|
|
1787
|
-
"data_type": "
|
|
1787
|
+
"data_type": "json",
|
|
1788
1788
|
"nullable": False,
|
|
1789
1789
|
"description": "A list of variants associated with the product, first 250.",
|
|
1790
1790
|
},
|
|
1791
1791
|
"variantsCount": {
|
|
1792
|
-
"data_type": "
|
|
1792
|
+
"data_type": "json",
|
|
1793
1793
|
"nullable": False,
|
|
1794
1794
|
"description": "The number of variants associated with the product.",
|
|
1795
1795
|
},
|
ingestr/src/slack/__init__.py
CHANGED
|
@@ -166,7 +166,7 @@ def slack_source(
|
|
|
166
166
|
@dlt.resource(
|
|
167
167
|
name="messages",
|
|
168
168
|
primary_key=("channel", "ts"),
|
|
169
|
-
columns={"blocks": {"data_type": "
|
|
169
|
+
columns={"blocks": {"data_type": "json"}},
|
|
170
170
|
write_disposition=write_disposition,
|
|
171
171
|
)
|
|
172
172
|
def messages_resource(
|
|
@@ -249,7 +249,7 @@ def slack_source(
|
|
|
249
249
|
table_name=table_name,
|
|
250
250
|
primary_key=("channel", "ts"),
|
|
251
251
|
write_disposition=write_disposition,
|
|
252
|
-
columns={"blocks": {"data_type": "
|
|
252
|
+
columns={"blocks": {"data_type": "json"}},
|
|
253
253
|
)(channel)
|
|
254
254
|
|
|
255
255
|
yield messages_channel
|
ingestr/src/sources.py
CHANGED
|
@@ -10,6 +10,10 @@ import pendulum
|
|
|
10
10
|
from dlt.common.configuration.specs import AwsCredentials
|
|
11
11
|
from dlt.common.time import ensure_pendulum_datetime
|
|
12
12
|
from dlt.common.typing import TSecretStrValue
|
|
13
|
+
from dlt.sources.credentials import ConnectionStringCredentials
|
|
14
|
+
from dlt.sources.sql_database import sql_table
|
|
15
|
+
from sqlalchemy import types as sa
|
|
16
|
+
from sqlalchemy.dialects import mysql
|
|
13
17
|
|
|
14
18
|
from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
|
|
15
19
|
from ingestr.src.adjust.adjust_helpers import parse_filters
|
|
@@ -19,6 +23,7 @@ from ingestr.src.arrow import memory_mapped_arrow
|
|
|
19
23
|
from ingestr.src.chess import source
|
|
20
24
|
from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
|
|
21
25
|
from ingestr.src.filesystem import readers
|
|
26
|
+
from ingestr.src.filters import table_adapter_exclude_columns
|
|
22
27
|
from ingestr.src.google_sheets import google_spreadsheet
|
|
23
28
|
from ingestr.src.gorgias import gorgias_source
|
|
24
29
|
from ingestr.src.hubspot import hubspot
|
|
@@ -29,7 +34,6 @@ from ingestr.src.mongodb import mongodb_collection
|
|
|
29
34
|
from ingestr.src.notion import notion_databases
|
|
30
35
|
from ingestr.src.shopify import shopify_source
|
|
31
36
|
from ingestr.src.slack import slack_source
|
|
32
|
-
from ingestr.src.sql_database import sql_table
|
|
33
37
|
from ingestr.src.stripe_analytics import stripe_source
|
|
34
38
|
from ingestr.src.table_definition import table_string_to_dataclass
|
|
35
39
|
from ingestr.src.zendesk import zendesk_chat, zendesk_support, zendesk_talk
|
|
@@ -66,14 +70,35 @@ class SqlSource:
|
|
|
66
70
|
if uri.startswith("mysql://"):
|
|
67
71
|
uri = uri.replace("mysql://", "mysql+pymysql://")
|
|
68
72
|
|
|
73
|
+
reflection_level = kwargs.get("sql_reflection_level")
|
|
74
|
+
|
|
75
|
+
query_adapter_callback = None
|
|
76
|
+
if kwargs.get("sql_limit"):
|
|
77
|
+
|
|
78
|
+
def query_adapter_callback(query, table):
|
|
79
|
+
query = query.limit(kwargs.get("sql_limit"))
|
|
80
|
+
if kwargs.get("incremental_key"):
|
|
81
|
+
query = query.order_by(kwargs.get("incremental_key"))
|
|
82
|
+
return query
|
|
83
|
+
|
|
84
|
+
def type_adapter_callback(sql_type):
|
|
85
|
+
if isinstance(sql_type, mysql.SET):
|
|
86
|
+
return sa.JSON
|
|
87
|
+
return sql_type
|
|
88
|
+
|
|
69
89
|
table_instance = self.table_builder(
|
|
70
|
-
credentials=uri,
|
|
90
|
+
credentials=ConnectionStringCredentials(uri),
|
|
71
91
|
schema=table_fields.dataset,
|
|
72
92
|
table=table_fields.table,
|
|
73
93
|
incremental=incremental,
|
|
74
|
-
merge_key=kwargs.get("merge_key"),
|
|
75
94
|
backend=kwargs.get("sql_backend", "sqlalchemy"),
|
|
76
95
|
chunk_size=kwargs.get("page_size", None),
|
|
96
|
+
reflection_level=reflection_level,
|
|
97
|
+
query_adapter_callback=query_adapter_callback,
|
|
98
|
+
type_adapter_callback=type_adapter_callback,
|
|
99
|
+
table_adapter_callback=table_adapter_exclude_columns(
|
|
100
|
+
kwargs.get("sql_exclude_columns", [])
|
|
101
|
+
),
|
|
77
102
|
)
|
|
78
103
|
|
|
79
104
|
return table_instance
|
|
@@ -722,7 +747,7 @@ class KafkaSource:
|
|
|
722
747
|
|
|
723
748
|
class AdjustSource:
|
|
724
749
|
def handles_incrementality(self) -> bool:
|
|
725
|
-
return
|
|
750
|
+
return True
|
|
726
751
|
|
|
727
752
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
728
753
|
if kwargs.get("incremental_key") and not table.startswith("custom:"):
|
|
@@ -784,7 +809,7 @@ class AdjustSource:
|
|
|
784
809
|
filters_raw = fields[3]
|
|
785
810
|
filters = parse_filters(filters_raw)
|
|
786
811
|
|
|
787
|
-
|
|
812
|
+
src = adjust_source(
|
|
788
813
|
start_date=start_date,
|
|
789
814
|
end_date=end_date,
|
|
790
815
|
api_key=api_key[0],
|
|
@@ -792,7 +817,9 @@ class AdjustSource:
|
|
|
792
817
|
metrics=metrics,
|
|
793
818
|
merge_key=kwargs.get("merge_key"),
|
|
794
819
|
filters=filters,
|
|
795
|
-
)
|
|
820
|
+
)
|
|
821
|
+
|
|
822
|
+
return src.with_resources(table)
|
|
796
823
|
|
|
797
824
|
|
|
798
825
|
class AppsflyerSource:
|
ingestr/src/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.10.0"
|
ingestr/src/zendesk/__init__.py
CHANGED
|
@@ -280,8 +280,8 @@ def zendesk_support(
|
|
|
280
280
|
primary_key="id",
|
|
281
281
|
write_disposition="merge",
|
|
282
282
|
columns={
|
|
283
|
-
"tags": {"data_type": "
|
|
284
|
-
"custom_fields": {"data_type": "
|
|
283
|
+
"tags": {"data_type": "json"},
|
|
284
|
+
"custom_fields": {"data_type": "json"},
|
|
285
285
|
},
|
|
286
286
|
)
|
|
287
287
|
def ticket_table(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.10.0
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -13,35 +13,35 @@ Classifier: Operating System :: OS Independent
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
14
14
|
Classifier: Topic :: Database
|
|
15
15
|
Requires-Python: >=3.9
|
|
16
|
-
Requires-Dist: confluent-kafka>=2.
|
|
16
|
+
Requires-Dist: confluent-kafka>=2.6.1
|
|
17
17
|
Requires-Dist: databricks-sql-connector==2.9.3
|
|
18
|
-
Requires-Dist: dlt==
|
|
19
|
-
Requires-Dist: duckdb-engine==0.
|
|
20
|
-
Requires-Dist: duckdb==
|
|
18
|
+
Requires-Dist: dlt==1.4.0
|
|
19
|
+
Requires-Dist: duckdb-engine==0.13.5
|
|
20
|
+
Requires-Dist: duckdb==1.1.3
|
|
21
21
|
Requires-Dist: facebook-business==20.0.0
|
|
22
22
|
Requires-Dist: google-api-python-client==2.130.0
|
|
23
23
|
Requires-Dist: google-cloud-bigquery-storage==2.24.0
|
|
24
|
-
Requires-Dist: mysql-connector-python==9.
|
|
24
|
+
Requires-Dist: mysql-connector-python==9.1.0
|
|
25
25
|
Requires-Dist: pendulum==3.0.0
|
|
26
|
-
Requires-Dist: psycopg2-binary==2.9.
|
|
27
|
-
Requires-Dist: py-machineid==0.
|
|
26
|
+
Requires-Dist: psycopg2-binary==2.9.10
|
|
27
|
+
Requires-Dist: py-machineid==0.6.0
|
|
28
28
|
Requires-Dist: pyairtable==2.3.3
|
|
29
|
-
Requires-Dist: pymongo==4.
|
|
30
|
-
Requires-Dist: pymysql==1.1.
|
|
31
|
-
Requires-Dist: pyrate-limiter==3.
|
|
32
|
-
Requires-Dist: redshift-connector==2.1.
|
|
33
|
-
Requires-Dist: rich==13.
|
|
34
|
-
Requires-Dist: rudder-sdk-python==2.1.
|
|
29
|
+
Requires-Dist: pymongo==4.10.1
|
|
30
|
+
Requires-Dist: pymysql==1.1.1
|
|
31
|
+
Requires-Dist: pyrate-limiter==3.7.0
|
|
32
|
+
Requires-Dist: redshift-connector==2.1.3
|
|
33
|
+
Requires-Dist: rich==13.9.4
|
|
34
|
+
Requires-Dist: rudder-sdk-python==2.1.4
|
|
35
35
|
Requires-Dist: s3fs==2024.9.0
|
|
36
|
-
Requires-Dist: snowflake-sqlalchemy==1.
|
|
37
|
-
Requires-Dist: sqlalchemy-bigquery==1.
|
|
36
|
+
Requires-Dist: snowflake-sqlalchemy==1.6.1
|
|
37
|
+
Requires-Dist: sqlalchemy-bigquery==1.12.0
|
|
38
38
|
Requires-Dist: sqlalchemy-hana==2.0.0
|
|
39
39
|
Requires-Dist: sqlalchemy-redshift==0.8.14
|
|
40
40
|
Requires-Dist: sqlalchemy2-stubs==0.0.2a38
|
|
41
41
|
Requires-Dist: sqlalchemy==1.4.52
|
|
42
42
|
Requires-Dist: stripe==10.7.0
|
|
43
|
-
Requires-Dist: tqdm==4.
|
|
44
|
-
Requires-Dist: typer==0.
|
|
43
|
+
Requires-Dist: tqdm==4.67.0
|
|
44
|
+
Requires-Dist: typer==0.13.1
|
|
45
45
|
Requires-Dist: types-requests==2.32.0.20240907
|
|
46
46
|
Provides-Extra: odbc
|
|
47
47
|
Requires-Dist: pyodbc==5.1.0; extra == 'odbc'
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
ingestr/main.py,sha256=
|
|
1
|
+
ingestr/main.py,sha256=OjYnhDR7zWMCsfHHKOqsh880rbT1ofq5QHin2dHPsIg,21724
|
|
2
2
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
3
3
|
ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
|
|
4
4
|
ingestr/src/factory.py,sha256=ft81B-YJgvEROkHAZjMjTIS7IYvle-uZQv45b7-Wfk0,4947
|
|
5
|
-
ingestr/src/
|
|
5
|
+
ingestr/src/filters.py,sha256=9ckkhNF0lGVvSbw24VKQGXdtvxDCOfnSIFjswMjIu-A,611
|
|
6
|
+
ingestr/src/sources.py,sha256=LGPg3wpANg4oX2tg42CSojN4LiF0t7UF2HUb8R3UTug,34816
|
|
6
7
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
7
|
-
ingestr/src/version.py,sha256=
|
|
8
|
-
ingestr/src/adjust/__init__.py,sha256=
|
|
8
|
+
ingestr/src/version.py,sha256=v4zmKjsKOPZbp6BrWoz7iK4ST0sdZdUh9bQSJmluZ5o,23
|
|
9
|
+
ingestr/src/adjust/__init__.py,sha256=I_G90D260OPIWCS716k0U4aeztlAieW9zi0R9-oW7TA,3007
|
|
9
10
|
ingestr/src/adjust/adjust_helpers.py,sha256=-tmmxy9k3wms-ZEIgxmlp2cAQ2X_O1lgjY1128bbMu4,3224
|
|
10
11
|
ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
|
|
11
12
|
ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
|
|
@@ -26,7 +27,7 @@ ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh
|
|
|
26
27
|
ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
|
|
27
28
|
ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
|
|
28
29
|
ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
|
|
29
|
-
ingestr/src/gorgias/__init__.py,sha256=
|
|
30
|
+
ingestr/src/gorgias/__init__.py,sha256=LZ3m6aGuhLVI3eNjvQE0rT4o_wbSPkY_SDKsM-g0V5U,21176
|
|
30
31
|
ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
|
|
31
32
|
ingestr/src/hubspot/__init__.py,sha256=eSD_lEIEd16YijAtUATFG8FGO8YGPm-MtAk94KKsx6o,9740
|
|
32
33
|
ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
|
|
@@ -43,24 +44,19 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
|
|
|
43
44
|
ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
44
45
|
ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
|
|
45
46
|
ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
|
|
46
|
-
ingestr/src/shopify/__init__.py,sha256=
|
|
47
|
+
ingestr/src/shopify/__init__.py,sha256=Hhv84zRfVsqAGP7pz-PmeopeX9CGu7TXSm3PSXHEwIA,62689
|
|
47
48
|
ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
|
|
48
49
|
ingestr/src/shopify/helpers.py,sha256=NfHD6lWXe88ybR0ri-FCQuh2Vf8l5WG0a0FVjmdoSC4,6296
|
|
49
50
|
ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
|
|
50
|
-
ingestr/src/slack/__init__.py,sha256=
|
|
51
|
+
ingestr/src/slack/__init__.py,sha256=UF-ficQ6K32u1EHytW3P35suACo9wuc6nMrAPViyZL8,9981
|
|
51
52
|
ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
|
|
52
53
|
ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
|
|
53
|
-
ingestr/src/sql_database/__init__.py,sha256=HEqY6U-YzzbeZ8avIthj-Fatm2C3i3jqYs5DAIAu4Ss,11511
|
|
54
|
-
ingestr/src/sql_database/arrow_helpers.py,sha256=yze1X3A9nUQA4HeuFDDWrfJVkCq8Uo5UyDo_zhJtI60,5699
|
|
55
|
-
ingestr/src/sql_database/helpers.py,sha256=6o8e2_8MIuj3qlo40a2E6ns3gyK18ei1jCePONrMUjI,10191
|
|
56
|
-
ingestr/src/sql_database/override.py,sha256=xbKGDztCzvrhJ5kJTXERal3LA56bEeVug4_rrTs8DgA,333
|
|
57
|
-
ingestr/src/sql_database/schema_types.py,sha256=qXTanvFPE8wMCSDzQWPDi5yqaO-llfrFXjiGJALI4NA,5013
|
|
58
54
|
ingestr/src/stripe_analytics/__init__.py,sha256=8yy6i4DAhUqY4ZForetQ0DWc_YQrY0FBH6yk0Z3m-Mw,4493
|
|
59
55
|
ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
|
|
60
56
|
ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
|
|
61
57
|
ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
|
|
62
58
|
ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
|
|
63
|
-
ingestr/src/zendesk/__init__.py,sha256=
|
|
59
|
+
ingestr/src/zendesk/__init__.py,sha256=C7HkN195DGdOHId2_Sa_kAlcBrUmnVYZUa_tPkiyf1Q,17564
|
|
64
60
|
ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
|
|
65
61
|
ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
|
|
66
62
|
ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
|
|
@@ -74,8 +70,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
|
|
|
74
70
|
ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
|
|
75
71
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
76
72
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
77
|
-
ingestr-0.
|
|
78
|
-
ingestr-0.
|
|
79
|
-
ingestr-0.
|
|
80
|
-
ingestr-0.
|
|
81
|
-
ingestr-0.
|
|
73
|
+
ingestr-0.10.0.dist-info/METADATA,sha256=CGw7IkjaPuBDADOEZ-7XVru3IiOcPyE2jXPHdII5xxE,7060
|
|
74
|
+
ingestr-0.10.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
|
|
75
|
+
ingestr-0.10.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
76
|
+
ingestr-0.10.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
77
|
+
ingestr-0.10.0.dist-info/RECORD,,
|