ingestr 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/main.py +16 -3
- ingestr/main_test.py +287 -7
- ingestr/src/factory.py +6 -0
- ingestr/src/gorgias/__init__.py +587 -0
- ingestr/src/gorgias/helpers.py +149 -0
- ingestr/src/gorgias/helpers_test.py +45 -0
- ingestr/src/sources.py +95 -3
- ingestr/src/version.py +1 -1
- ingestr/testdata/create_replace.csv +21 -0
- ingestr/testdata/delete_insert_expected.csv +6 -0
- ingestr/testdata/delete_insert_part1.csv +5 -0
- ingestr/testdata/delete_insert_part2.csv +6 -0
- ingestr/testdata/merge_expected.csv +5 -0
- ingestr/testdata/merge_part1.csv +4 -0
- ingestr/testdata/merge_part2.csv +5 -0
- {ingestr-0.6.0.dist-info → ingestr-0.6.1.dist-info}/METADATA +3 -2
- {ingestr-0.6.0.dist-info → ingestr-0.6.1.dist-info}/RECORD +20 -15
- {ingestr-0.6.0.dist-info → ingestr-0.6.1.dist-info}/WHEEL +1 -1
- ingestr/testdata/test_append.db +0 -0
- ingestr/testdata/test_create_replace.db +0 -0
- ingestr/testdata/test_delete_insert_with_timerange.db +0 -0
- ingestr/testdata/test_delete_insert_without_primary_key.db +0 -0
- ingestr/testdata/test_merge_with_primary_key.db +0 -0
- {ingestr-0.6.0.dist-info → ingestr-0.6.1.dist-info}/entry_points.txt +0 -0
- {ingestr-0.6.0.dist-info → ingestr-0.6.1.dist-info}/licenses/LICENSE.md +0 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Gorgias source helpers"""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Iterable, Optional, Tuple
|
|
4
|
+
|
|
5
|
+
from dlt.common.pendulum import pendulum
|
|
6
|
+
from dlt.common.time import ensure_pendulum_datetime
|
|
7
|
+
from dlt.common.typing import Dict, TDataItems
|
|
8
|
+
from dlt.sources.helpers import requests
|
|
9
|
+
from pyrate_limiter import Duration, Limiter, Rate
|
|
10
|
+
from requests.auth import HTTPBasicAuth
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_max_datetime_from_datetime_fields(
|
|
14
|
+
item: Dict[str, Any],
|
|
15
|
+
) -> Tuple[str, Optional[pendulum.DateTime]]:
|
|
16
|
+
"""Get the maximum datetime from any field that ends with _datetime"""
|
|
17
|
+
|
|
18
|
+
max_field_name = None
|
|
19
|
+
max_field_value = None
|
|
20
|
+
for field in item:
|
|
21
|
+
if field.endswith("_datetime") and item[field] is not None:
|
|
22
|
+
dt = ensure_pendulum_datetime(item[field])
|
|
23
|
+
if not max_field_name or dt > max_field_value:
|
|
24
|
+
max_field_name = field
|
|
25
|
+
max_field_value = dt
|
|
26
|
+
|
|
27
|
+
return max_field_name, max_field_value
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def convert_datetime_fields(item: Dict[str, Any]) -> Dict[str, Any]:
|
|
31
|
+
for field in item:
|
|
32
|
+
if field.endswith("_datetime") and item[field] is not None:
|
|
33
|
+
item[field] = ensure_pendulum_datetime(item[field])
|
|
34
|
+
|
|
35
|
+
if "updated_datetime" not in item:
|
|
36
|
+
_, max_datetime = get_max_datetime_from_datetime_fields(item)
|
|
37
|
+
item["updated_datetime"] = max_datetime
|
|
38
|
+
|
|
39
|
+
return item
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def find_latest_timestamp_from_page(
|
|
43
|
+
items: list[Dict[str, Any]],
|
|
44
|
+
) -> Optional[Dict[str, Any]]:
|
|
45
|
+
latest_time = None
|
|
46
|
+
for item in items:
|
|
47
|
+
_, max_field_value = get_max_datetime_from_datetime_fields(item)
|
|
48
|
+
if not latest_time or ensure_pendulum_datetime(max_field_value) > latest_time:
|
|
49
|
+
latest_time = max_field_value
|
|
50
|
+
|
|
51
|
+
return latest_time
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class GorgiasApi:
|
|
55
|
+
"""
|
|
56
|
+
A Gorgias API client that can be used to get pages of data from Gorgias.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(
|
|
60
|
+
self,
|
|
61
|
+
domain: str,
|
|
62
|
+
email: str,
|
|
63
|
+
api_key: str,
|
|
64
|
+
) -> None:
|
|
65
|
+
"""
|
|
66
|
+
Args:
|
|
67
|
+
domain: The domain of your Gorgias account.
|
|
68
|
+
email: The email associated with your Gorgias account.
|
|
69
|
+
api_key: The API key for accessing the Gorgias API.
|
|
70
|
+
"""
|
|
71
|
+
self.domain = domain
|
|
72
|
+
self.email = email
|
|
73
|
+
self.api_key = api_key
|
|
74
|
+
|
|
75
|
+
def get_pages(
|
|
76
|
+
self,
|
|
77
|
+
resource: str,
|
|
78
|
+
params: Optional[Dict[str, Any]] = None,
|
|
79
|
+
start_date: Optional[str] = None,
|
|
80
|
+
end_date: Optional[str] = None,
|
|
81
|
+
) -> Iterable[TDataItems]:
|
|
82
|
+
"""Get all pages from Gorgias using requests.
|
|
83
|
+
Iterates through all pages and yield each page items.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
resource: The resource to get pages for (e.g. products, orders, customers).
|
|
87
|
+
params: Query params to include in the request.
|
|
88
|
+
|
|
89
|
+
Yields:
|
|
90
|
+
List of data items from the page
|
|
91
|
+
"""
|
|
92
|
+
url = f"https://{self.domain}.gorgias.com/api/{resource}"
|
|
93
|
+
rate = Rate(2, Duration.SECOND)
|
|
94
|
+
limiter = Limiter(rate, raise_when_fail=False)
|
|
95
|
+
|
|
96
|
+
start_date_obj = ensure_pendulum_datetime(start_date) if start_date else None
|
|
97
|
+
|
|
98
|
+
if not params:
|
|
99
|
+
params = {}
|
|
100
|
+
|
|
101
|
+
params["limit"] = 100
|
|
102
|
+
if "order_by" not in params:
|
|
103
|
+
params["order_by"] = "updated_datetime:desc"
|
|
104
|
+
|
|
105
|
+
while True:
|
|
106
|
+
limiter.try_acquire(f"gorgias-{self.domain}")
|
|
107
|
+
response = requests.get(
|
|
108
|
+
url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
|
|
109
|
+
)
|
|
110
|
+
response.raise_for_status()
|
|
111
|
+
if len(response.json()["data"]) == 0:
|
|
112
|
+
break
|
|
113
|
+
|
|
114
|
+
json = response.json()
|
|
115
|
+
|
|
116
|
+
items = self.__filter_items_in_range(json["data"], start_date, end_date)
|
|
117
|
+
if len(items) > 0:
|
|
118
|
+
yield items
|
|
119
|
+
|
|
120
|
+
# if there is no cursor, yield the items first and then break the loop
|
|
121
|
+
cursor = json.get("meta", {}).get("next_cursor")
|
|
122
|
+
params["cursor"] = cursor
|
|
123
|
+
if not cursor:
|
|
124
|
+
break
|
|
125
|
+
|
|
126
|
+
if start_date_obj:
|
|
127
|
+
max_datetime = find_latest_timestamp_from_page(json["data"])
|
|
128
|
+
if start_date_obj > ensure_pendulum_datetime(max_datetime):
|
|
129
|
+
break
|
|
130
|
+
|
|
131
|
+
def __filter_items_in_range(
|
|
132
|
+
self,
|
|
133
|
+
items: list[Dict[str, Any]],
|
|
134
|
+
start_date: Optional[str],
|
|
135
|
+
end_date: Optional[str],
|
|
136
|
+
) -> list[Dict[str, Any]]:
|
|
137
|
+
start_date_obj = ensure_pendulum_datetime(start_date) if start_date else None
|
|
138
|
+
end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
|
|
139
|
+
|
|
140
|
+
filtered = []
|
|
141
|
+
for item in items:
|
|
142
|
+
converted_item = convert_datetime_fields(item)
|
|
143
|
+
if start_date_obj and item["updated_datetime"] < start_date_obj:
|
|
144
|
+
continue
|
|
145
|
+
if end_date_obj and item["updated_datetime"] > end_date_obj:
|
|
146
|
+
continue
|
|
147
|
+
filtered.append(converted_item)
|
|
148
|
+
|
|
149
|
+
return filtered
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from dlt.common.pendulum import pendulum
|
|
2
|
+
|
|
3
|
+
from .helpers import convert_datetime_fields, find_latest_timestamp_from_page
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_convert_datetime_fields():
|
|
7
|
+
item = {
|
|
8
|
+
"key1": "val1",
|
|
9
|
+
"created_datetime": "2024-06-20T07:39:36.514848+00:00",
|
|
10
|
+
"sent_datetime": "2024-06-20T07:40:20.166593+00:00",
|
|
11
|
+
"should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
actual = convert_datetime_fields(item)
|
|
15
|
+
|
|
16
|
+
assert actual == {
|
|
17
|
+
"key1": "val1",
|
|
18
|
+
"created_datetime": pendulum.datetime(2024, 6, 20, 7, 39, 36, 514848, tz="UTC"),
|
|
19
|
+
"sent_datetime": pendulum.datetime(2024, 6, 20, 7, 40, 20, 166593, tz="UTC"),
|
|
20
|
+
"should_send_datetime": pendulum.datetime(
|
|
21
|
+
2024, 6, 20, 7, 39, 37, 514848, tz="UTC"
|
|
22
|
+
),
|
|
23
|
+
"updated_datetime": pendulum.datetime(2024, 6, 20, 7, 40, 20, 166593, tz="UTC"),
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_find_latest_timestamp_from_page():
|
|
28
|
+
items = [
|
|
29
|
+
{
|
|
30
|
+
"key1": "val1",
|
|
31
|
+
"created_datetime": "2024-06-20T07:39:36.514848+00:00",
|
|
32
|
+
"sent_datetime": "2024-06-20T07:40:20.166593+00:00",
|
|
33
|
+
"should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"key1": "val2",
|
|
37
|
+
"created_datetime": "2024-06-20T07:39:36.514848+00:00",
|
|
38
|
+
"sent_datetime": "2024-06-20T07:40:21.123123+00:00",
|
|
39
|
+
"should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
|
|
40
|
+
},
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
actual = find_latest_timestamp_from_page(items)
|
|
44
|
+
|
|
45
|
+
assert actual == pendulum.datetime(2024, 6, 20, 7, 40, 21, 123123, tz="UTC")
|
ingestr/src/sources.py
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import base64
|
|
2
2
|
import csv
|
|
3
3
|
import json
|
|
4
|
-
from typing import Callable
|
|
4
|
+
from typing import Any, Callable, Optional
|
|
5
5
|
from urllib.parse import parse_qs, urlparse
|
|
6
6
|
|
|
7
7
|
import dlt
|
|
8
8
|
|
|
9
9
|
from ingestr.src.google_sheets import google_spreadsheet
|
|
10
|
+
from ingestr.src.gorgias import gorgias_source
|
|
10
11
|
from ingestr.src.mongodb import mongodb_collection
|
|
11
12
|
from ingestr.src.notion import notion_databases
|
|
12
13
|
from ingestr.src.shopify import shopify_source
|
|
@@ -19,6 +20,9 @@ class SqlSource:
|
|
|
19
20
|
def __init__(self, table_builder=sql_table) -> None:
|
|
20
21
|
self.table_builder = table_builder
|
|
21
22
|
|
|
23
|
+
def handles_incrementality(self) -> bool:
|
|
24
|
+
return False
|
|
25
|
+
|
|
22
26
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
23
27
|
table_fields = table.split(".")
|
|
24
28
|
if len(table_fields) != 2:
|
|
@@ -57,6 +61,9 @@ class MongoDbSource:
|
|
|
57
61
|
def __init__(self, table_builder=mongodb_collection) -> None:
|
|
58
62
|
self.table_builder = table_builder
|
|
59
63
|
|
|
64
|
+
def handles_incrementality(self) -> bool:
|
|
65
|
+
return False
|
|
66
|
+
|
|
60
67
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
61
68
|
table_fields = table.split(".")
|
|
62
69
|
if len(table_fields) != 2:
|
|
@@ -85,18 +92,43 @@ class MongoDbSource:
|
|
|
85
92
|
|
|
86
93
|
|
|
87
94
|
class LocalCsvSource:
|
|
95
|
+
def handles_incrementality(self) -> bool:
|
|
96
|
+
return False
|
|
97
|
+
|
|
88
98
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
89
|
-
def csv_file(
|
|
99
|
+
def csv_file(
|
|
100
|
+
incremental: Optional[dlt.sources.incremental[Any]] = None,
|
|
101
|
+
):
|
|
90
102
|
file_path = uri.split("://")[1]
|
|
91
103
|
myFile = open(file_path, "r")
|
|
92
104
|
reader = csv.DictReader(myFile)
|
|
93
|
-
|
|
105
|
+
if not reader.fieldnames:
|
|
106
|
+
raise RuntimeError(
|
|
107
|
+
"failed to extract headers from the CSV, are you sure the given file contains a header row?"
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
incremental_key = kwargs.get("incremental_key")
|
|
111
|
+
if incremental_key and incremental_key not in reader.fieldnames:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
f"incremental_key '{incremental_key}' not found in the CSV file"
|
|
114
|
+
)
|
|
94
115
|
|
|
95
116
|
page_size = 1000
|
|
96
117
|
page = []
|
|
97
118
|
current_items = 0
|
|
98
119
|
for dictionary in reader:
|
|
99
120
|
if current_items < page_size:
|
|
121
|
+
if incremental_key and incremental and incremental.start_value:
|
|
122
|
+
inc_value = dictionary.get(incremental_key)
|
|
123
|
+
if inc_value is None:
|
|
124
|
+
raise ValueError(
|
|
125
|
+
f"incremental_key '{incremental_key}' not found in the CSV file"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
print("BURAYA GELLDIII")
|
|
129
|
+
if inc_value < incremental.start_value:
|
|
130
|
+
continue
|
|
131
|
+
|
|
100
132
|
page.append(dictionary)
|
|
101
133
|
current_items += 1
|
|
102
134
|
else:
|
|
@@ -110,6 +142,12 @@ class LocalCsvSource:
|
|
|
110
142
|
return dlt.resource(
|
|
111
143
|
csv_file,
|
|
112
144
|
merge_key=kwargs.get("merge_key"), # type: ignore
|
|
145
|
+
)(
|
|
146
|
+
incremental=dlt.sources.incremental(
|
|
147
|
+
kwargs.get("incremental_key", ""),
|
|
148
|
+
initial_value=kwargs.get("interval_start"),
|
|
149
|
+
end_value=kwargs.get("interval_end"),
|
|
150
|
+
)
|
|
113
151
|
)
|
|
114
152
|
|
|
115
153
|
|
|
@@ -119,6 +157,9 @@ class NotionSource:
|
|
|
119
157
|
def __init__(self, table_builder=notion_databases) -> None:
|
|
120
158
|
self.table_builder = table_builder
|
|
121
159
|
|
|
160
|
+
def handles_incrementality(self) -> bool:
|
|
161
|
+
return True
|
|
162
|
+
|
|
122
163
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
123
164
|
if kwargs.get("incremental_key"):
|
|
124
165
|
raise ValueError("Incremental loads are not supported for Notion")
|
|
@@ -136,6 +177,9 @@ class NotionSource:
|
|
|
136
177
|
|
|
137
178
|
|
|
138
179
|
class ShopifySource:
|
|
180
|
+
def handles_incrementality(self) -> bool:
|
|
181
|
+
return True
|
|
182
|
+
|
|
139
183
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
140
184
|
if kwargs.get("incremental_key"):
|
|
141
185
|
raise ValueError(
|
|
@@ -172,12 +216,60 @@ class ShopifySource:
|
|
|
172
216
|
).with_resources(resource)
|
|
173
217
|
|
|
174
218
|
|
|
219
|
+
class GorgiasSource:
|
|
220
|
+
def handles_incrementality(self) -> bool:
|
|
221
|
+
return True
|
|
222
|
+
|
|
223
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
224
|
+
if kwargs.get("incremental_key"):
|
|
225
|
+
raise ValueError(
|
|
226
|
+
"Gorgias takes care of incrementality on its own, you should not provide incremental_key"
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
# gorgias://domain?api_key=<api_key>&email=<email>
|
|
230
|
+
|
|
231
|
+
source_fields = urlparse(uri)
|
|
232
|
+
source_params = parse_qs(source_fields.query)
|
|
233
|
+
api_key = source_params.get("api_key")
|
|
234
|
+
if not api_key:
|
|
235
|
+
raise ValueError("api_key in the URI is required to connect to Gorgias")
|
|
236
|
+
|
|
237
|
+
email = source_params.get("email")
|
|
238
|
+
if not email:
|
|
239
|
+
raise ValueError("email in the URI is required to connect to Gorgias")
|
|
240
|
+
|
|
241
|
+
resource = None
|
|
242
|
+
if table in ["customers", "tickets", "ticket_messages", "satisfaction_surveys"]:
|
|
243
|
+
resource = table
|
|
244
|
+
else:
|
|
245
|
+
raise ValueError(
|
|
246
|
+
f"Resource '{table}' is not supported for Gorgias source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
date_args = {}
|
|
250
|
+
if kwargs.get("interval_start"):
|
|
251
|
+
date_args["start_date"] = kwargs.get("interval_start")
|
|
252
|
+
|
|
253
|
+
if kwargs.get("interval_end"):
|
|
254
|
+
date_args["end_date"] = kwargs.get("interval_end")
|
|
255
|
+
|
|
256
|
+
return gorgias_source(
|
|
257
|
+
domain=source_fields.netloc,
|
|
258
|
+
email=email[0],
|
|
259
|
+
api_key=api_key[0],
|
|
260
|
+
**date_args,
|
|
261
|
+
).with_resources(resource)
|
|
262
|
+
|
|
263
|
+
|
|
175
264
|
class GoogleSheetsSource:
|
|
176
265
|
table_builder: Callable
|
|
177
266
|
|
|
178
267
|
def __init__(self, table_builder=google_spreadsheet) -> None:
|
|
179
268
|
self.table_builder = table_builder
|
|
180
269
|
|
|
270
|
+
def handles_incrementality(self) -> bool:
|
|
271
|
+
return False
|
|
272
|
+
|
|
181
273
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
182
274
|
if kwargs.get("incremental_key"):
|
|
183
275
|
raise ValueError("Incremental loads are not supported for Google Sheets")
|
ingestr/src/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.6.
|
|
1
|
+
__version__ = "0.6.1"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"symbol","date","isEnabled","name"
|
|
2
|
+
"A","2024-04-19","True","AGILENT TECHNOLOGIES INC"
|
|
3
|
+
"AA","2024-04-19","True","ALCOA CORP"
|
|
4
|
+
"AAA","2024-04-19","True","ALTERNATIVE ACCESS FIRST PRI"
|
|
5
|
+
"AAAU","2024-04-19","True","GOLDMAN SACHS PHYSICAL GOLD"
|
|
6
|
+
"AACG","2024-04-19","True","ATA CREATIVITY GLOBAL - ADR"
|
|
7
|
+
"AACI","2024-04-19","True","ARMADA ACQUISITION CORP I"
|
|
8
|
+
"AACIU","2024-04-19","True","ARMADA ACQUISITION CORP I"
|
|
9
|
+
"AACIW","2024-04-19","True",""
|
|
10
|
+
"AACT","2024-04-19","True","ARES ACQUISITION CORP II"
|
|
11
|
+
"AACT+","2024-04-19","True",""
|
|
12
|
+
"AACT=","2024-04-19","True","ARES ACQUISITION CORP II"
|
|
13
|
+
"AADI","2024-04-19","True","AADI BIOSCIENCE INC"
|
|
14
|
+
"AADR","2024-04-19","True","ADVISORSHARES DORSEY WRIGHT"
|
|
15
|
+
"AAGR","2024-04-19","True","AFRICAN AGRICULTURE HOLDINGS"
|
|
16
|
+
"AAGRW","2024-04-19","True",""
|
|
17
|
+
"AAL","2024-04-19","True","AMERICAN AIRLINES GROUP INC"
|
|
18
|
+
"AAMC","2024-04-19","True","ALTISOURCE ASSET MANAGEMENT"
|
|
19
|
+
"AAME","2024-04-19","True","ATLANTIC AMERICAN CORP"
|
|
20
|
+
"AAN","2024-04-19","True","AARON'S CO INC/THE"
|
|
21
|
+
"AAOI","2024-04-19","True","APPLIED OPTOELECTRONICS INC"
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"symbol","date","isEnabled","name"
|
|
2
|
+
"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
|
|
3
|
+
"AA","2024-04-19","True","ALCOA CORP____updated"
|
|
4
|
+
"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
|
|
5
|
+
"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"
|
|
6
|
+
"B","2024-04-18","True","SOME TECHNOLOGIES INC"
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"symbol","date","isEnabled","name"
|
|
2
|
+
"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
|
|
3
|
+
"AA","2024-04-19","True","ALCOA CORP____updated"
|
|
4
|
+
"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
|
|
5
|
+
"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"
|
|
6
|
+
"BBB","2024-04-18","True","SOME CORP____updated"
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"symbol","date","isEnabled","name"
|
|
2
|
+
"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
|
|
3
|
+
"AA","2024-04-19","True","ALCOA CORP"
|
|
4
|
+
"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
|
|
5
|
+
"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"symbol","date","isEnabled","name"
|
|
2
|
+
"A","2024-04-20","True","AGILENT TECHNOLOGIES INC____updated"
|
|
3
|
+
"AA","2024-04-19","True","ALCOA CORP____updated"
|
|
4
|
+
"AAA","2024-04-21","True","ALTERNATIVE ACCESS FIRST PRI____updated"
|
|
5
|
+
"AAAU","2024-04-22","True","GOLDMAN SACHS PHYSICAL GOLD____updated"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.1
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -16,7 +16,7 @@ Classifier: Topic :: Database
|
|
|
16
16
|
Requires-Python: >=3.9
|
|
17
17
|
Requires-Dist: cx-oracle==8.3.0
|
|
18
18
|
Requires-Dist: databricks-sql-connector==2.9.3
|
|
19
|
-
Requires-Dist: dlt==0.4.
|
|
19
|
+
Requires-Dist: dlt==0.4.12
|
|
20
20
|
Requires-Dist: duckdb-engine==0.11.5
|
|
21
21
|
Requires-Dist: duckdb==0.10.2
|
|
22
22
|
Requires-Dist: google-api-python-client==2.130.0
|
|
@@ -27,6 +27,7 @@ Requires-Dist: py-machineid==0.5.1
|
|
|
27
27
|
Requires-Dist: pymongo==4.6.3
|
|
28
28
|
Requires-Dist: pymysql==1.1.0
|
|
29
29
|
Requires-Dist: pyodbc==5.1.0
|
|
30
|
+
Requires-Dist: pyrate-limiter==3.6.1
|
|
30
31
|
Requires-Dist: redshift-connector==2.1.0
|
|
31
32
|
Requires-Dist: rich==13.7.1
|
|
32
33
|
Requires-Dist: rudder-sdk-python==2.1.0
|
|
@@ -1,17 +1,20 @@
|
|
|
1
|
-
ingestr/main.py,sha256=
|
|
2
|
-
ingestr/main_test.py,sha256=
|
|
1
|
+
ingestr/main.py,sha256=jok8jEPLCOXlhKyKzR2mWA36A9DauRc4DI4AY_r4ubM,14861
|
|
2
|
+
ingestr/main_test.py,sha256=MDV2Eo86W_CcxGgEkYYoBc6xIXjVMER4hMhgAdxXYMc,28464
|
|
3
3
|
ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
|
|
4
4
|
ingestr/src/destinations_test.py,sha256=rgEk8EpAntFbSOwXovC4prv3RA22mwq8pIO6sZ_rYzg,4212
|
|
5
|
-
ingestr/src/factory.py,sha256=
|
|
5
|
+
ingestr/src/factory.py,sha256=7skwetBXFIwAuKyTFfffGPgSo_PRRZ5uEnxOHUv28yQ,3517
|
|
6
6
|
ingestr/src/factory_test.py,sha256=X9sFkvNByWChIcyeDt1QiIPMIzGNKb7M5A_GUE0-nnI,664
|
|
7
|
-
ingestr/src/sources.py,sha256=
|
|
7
|
+
ingestr/src/sources.py,sha256=v0oFu-Pt_Zv06FypBb9_rHKP_iIMpkgm9MMaVpOUjPg,10313
|
|
8
8
|
ingestr/src/sources_test.py,sha256=t94u1lYAspxzfe-DkxVtq5vw6xrLWphipvwntrwrzqg,3930
|
|
9
|
-
ingestr/src/version.py,sha256=
|
|
9
|
+
ingestr/src/version.py,sha256=baAcEjLSYFIeNZF51tOMmA_zAMhN8HvKael-UU-Ruec,22
|
|
10
10
|
ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
|
|
11
11
|
ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
|
|
12
12
|
ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
|
|
13
13
|
ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
|
|
14
14
|
ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
|
|
15
|
+
ingestr/src/gorgias/__init__.py,sha256=el_rJOTurK3P2tw-CtwjTpTcc7q6w2nLlJPXCU0yEqY,21228
|
|
16
|
+
ingestr/src/gorgias/helpers.py,sha256=dQ56CpZJobBVZP-vCM56vKHZdrfCHqla5peUt0bVr1c,4905
|
|
17
|
+
ingestr/src/gorgias/helpers_test.py,sha256=kSR2nhB8U8HZ8pgDnd7HvXlzojmBnpOm8fTKHJvvKGY,1580
|
|
15
18
|
ingestr/src/mongodb/__init__.py,sha256=E7SDeCyYNkYZZ_RFhjCRDZUGpKtaxpPG5sFSmKJV62U,4336
|
|
16
19
|
ingestr/src/mongodb/helpers.py,sha256=80vtAeNyUn1iMN0CeLrTlKqYN6I6fHF81Kd2UuE8Kns,5653
|
|
17
20
|
ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
|
|
@@ -30,13 +33,15 @@ ingestr/src/sql_database/schema_types.py,sha256=foGHh4iGagGLfS7nF3uGYhBjqgX0jlrj
|
|
|
30
33
|
ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
|
|
31
34
|
ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
|
|
32
35
|
ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
|
|
33
|
-
ingestr/testdata/
|
|
34
|
-
ingestr/testdata/
|
|
35
|
-
ingestr/testdata/
|
|
36
|
-
ingestr/testdata/
|
|
37
|
-
ingestr/testdata/
|
|
38
|
-
ingestr
|
|
39
|
-
ingestr
|
|
40
|
-
ingestr-0.6.
|
|
41
|
-
ingestr-0.6.
|
|
42
|
-
ingestr-0.6.
|
|
36
|
+
ingestr/testdata/create_replace.csv,sha256=TQDbOSkRKq9ZZv1d68Qjwh94aIyUQ-oEwxpJIrd3YK8,1060
|
|
37
|
+
ingestr/testdata/delete_insert_expected.csv,sha256=wbj7uboVWwm3sNMh1n7f4-OKFEQJv1s96snjEHp9nkg,336
|
|
38
|
+
ingestr/testdata/delete_insert_part1.csv,sha256=mdLFGu6ZOU6zwAigLRq4hFkAbVgyIaoEE39UCeeRc7s,234
|
|
39
|
+
ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmaySYCgtLsNqSrk,337
|
|
40
|
+
ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
|
|
41
|
+
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
42
|
+
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
43
|
+
ingestr-0.6.1.dist-info/METADATA,sha256=SzZWTcIS64RXkeC_qMSPRUrm_PzD9q7e-11bxF0j2F4,5699
|
|
44
|
+
ingestr-0.6.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
45
|
+
ingestr-0.6.1.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
46
|
+
ingestr-0.6.1.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
47
|
+
ingestr-0.6.1.dist-info/RECORD,,
|
ingestr/testdata/test_append.db
DELETED
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
File without changes
|
|
File without changes
|