ingestr 0.13.23__py3-none-any.whl → 0.13.25__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/conftest.py +63 -0
- ingestr/main.py +4 -2
- ingestr/src/buildinfo.py +1 -1
- ingestr/src/factory.py +2 -0
- ingestr/src/filters.py +21 -0
- ingestr/src/frankfurter/__init__.py +142 -0
- ingestr/src/frankfurter/helpers.py +32 -0
- ingestr/src/github/helpers.py +5 -5
- ingestr/src/google_sheets/__init__.py +4 -4
- ingestr/src/google_sheets/helpers/data_processing.py +2 -2
- ingestr/src/sources.py +34 -0
- {ingestr-0.13.23.dist-info → ingestr-0.13.25.dist-info}/METADATA +5 -4
- {ingestr-0.13.23.dist-info → ingestr-0.13.25.dist-info}/RECORD +16 -13
- {ingestr-0.13.23.dist-info → ingestr-0.13.25.dist-info}/WHEEL +0 -0
- {ingestr-0.13.23.dist-info → ingestr-0.13.25.dist-info}/entry_points.txt +0 -0
- {ingestr-0.13.23.dist-info → ingestr-0.13.25.dist-info}/licenses/LICENSE.md +0 -0
ingestr/conftest.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
4
|
+
|
|
5
|
+
from main_test import DESTINATIONS, SOURCES # type: ignore
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def pytest_configure(config):
|
|
9
|
+
if is_master(config):
|
|
10
|
+
config.shared_directory = tempfile.mkdtemp()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def pytest_configure_node(node):
|
|
14
|
+
"""xdist hook"""
|
|
15
|
+
node.workerinput["shared_directory"] = node.config.shared_directory
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def is_master(config):
|
|
19
|
+
"""True if the code running the given pytest.config object is running in a xdist master
|
|
20
|
+
node or not running xdist at all.
|
|
21
|
+
"""
|
|
22
|
+
return not hasattr(config, "workerinput")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def start_containers(config):
|
|
26
|
+
if hasattr(config, "workerinput"):
|
|
27
|
+
return
|
|
28
|
+
|
|
29
|
+
unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
|
|
30
|
+
for container in unique_containers:
|
|
31
|
+
container.container_lock_dir = config.shared_directory
|
|
32
|
+
|
|
33
|
+
with ThreadPoolExecutor() as executor:
|
|
34
|
+
for container in unique_containers:
|
|
35
|
+
executor.submit(container.start_fully)
|
|
36
|
+
# futures = [
|
|
37
|
+
# executor.submit(container.start_fully) for container in unique_containers
|
|
38
|
+
# ]
|
|
39
|
+
# # Wait for all futures to complete
|
|
40
|
+
# for future in futures:
|
|
41
|
+
# future.result()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def stop_containers(config):
|
|
45
|
+
if hasattr(config, "workerinput"):
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
should_manage_containers = os.environ.get("PYTEST_XDIST_WORKER", "gw0") == "gw0"
|
|
49
|
+
if not should_manage_containers:
|
|
50
|
+
return
|
|
51
|
+
|
|
52
|
+
unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
|
|
53
|
+
|
|
54
|
+
for container in unique_containers:
|
|
55
|
+
container.stop_fully()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def pytest_sessionstart(session):
|
|
59
|
+
start_containers(session.config)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def pytest_sessionfinish(session, exitstatus):
|
|
63
|
+
stop_containers(session.config)
|
ingestr/main.py
CHANGED
|
@@ -11,7 +11,7 @@ from typing_extensions import Annotated
|
|
|
11
11
|
import ingestr.src.partition as partition
|
|
12
12
|
import ingestr.src.resource as resource
|
|
13
13
|
from ingestr.src.destinations import AthenaDestination
|
|
14
|
-
from ingestr.src.filters import cast_set_to_list
|
|
14
|
+
from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
|
|
15
15
|
from ingestr.src.telemetry.event import track
|
|
16
16
|
|
|
17
17
|
app = typer.Typer(
|
|
@@ -35,7 +35,7 @@ DATE_FORMATS = [
|
|
|
35
35
|
|
|
36
36
|
# https://dlthub.com/docs/dlt-ecosystem/file-formats/parquet#supported-destinations
|
|
37
37
|
PARQUET_SUPPORTED_DESTINATIONS = [
|
|
38
|
-
"
|
|
38
|
+
"athenabigquery",
|
|
39
39
|
"duckdb",
|
|
40
40
|
"snowflake",
|
|
41
41
|
"databricks",
|
|
@@ -553,6 +553,8 @@ def ingest(
|
|
|
553
553
|
)
|
|
554
554
|
|
|
555
555
|
resource.for_each(dlt_source, lambda x: x.add_map(cast_set_to_list))
|
|
556
|
+
if factory.source_scheme.startswith("mysql"):
|
|
557
|
+
resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
|
|
556
558
|
|
|
557
559
|
def col_h(x):
|
|
558
560
|
if column_hints:
|
ingestr/src/buildinfo.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version = "v0.13.
|
|
1
|
+
version = "v0.13.25"
|
ingestr/src/factory.py
CHANGED
|
@@ -28,6 +28,7 @@ from ingestr.src.sources import (
|
|
|
28
28
|
ChessSource,
|
|
29
29
|
DynamoDBSource,
|
|
30
30
|
FacebookAdsSource,
|
|
31
|
+
FrankfurterSource,
|
|
31
32
|
GCSSource,
|
|
32
33
|
GitHubSource,
|
|
33
34
|
GoogleAdsSource,
|
|
@@ -146,6 +147,7 @@ class SourceDestinationFactory:
|
|
|
146
147
|
"personio": PersonioSource,
|
|
147
148
|
"kinesis": KinesisSource,
|
|
148
149
|
"pipedrive": PipedriveSource,
|
|
150
|
+
"frankfurter": FrankfurterSource,
|
|
149
151
|
}
|
|
150
152
|
destinations: Dict[str, Type[DestinationProtocol]] = {
|
|
151
153
|
"bigquery": BigQueryDestination,
|
ingestr/src/filters.py
CHANGED
|
@@ -10,6 +10,27 @@ def cast_set_to_list(row):
|
|
|
10
10
|
return row
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
def handle_mysql_empty_dates(row):
|
|
14
|
+
# MySQL returns empty dates as 0000-00-00, which is not a valid date, we handle them here.
|
|
15
|
+
if not isinstance(row, dict):
|
|
16
|
+
return row
|
|
17
|
+
|
|
18
|
+
for key in row.keys():
|
|
19
|
+
if not isinstance(row[key], str):
|
|
20
|
+
continue
|
|
21
|
+
|
|
22
|
+
if row[key] == "0000-00-00":
|
|
23
|
+
from datetime import date
|
|
24
|
+
|
|
25
|
+
row[key] = date(1970, 1, 1)
|
|
26
|
+
|
|
27
|
+
elif row[key] == "0000-00-00 00:00:00":
|
|
28
|
+
from datetime import datetime
|
|
29
|
+
|
|
30
|
+
row[key] = datetime(1970, 1, 1, 0, 0, 0)
|
|
31
|
+
return row
|
|
32
|
+
|
|
33
|
+
|
|
13
34
|
def table_adapter_exclude_columns(cols: list[str]):
|
|
14
35
|
def excluder(table: Table):
|
|
15
36
|
cols_to_remove = [col for col in table._columns if col.name in cols] # type: ignore
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from typing import Any, Iterator, Optional
|
|
2
|
+
|
|
3
|
+
import dlt
|
|
4
|
+
from dlt.common.pendulum import pendulum
|
|
5
|
+
from dlt.common.time import ensure_pendulum_datetime
|
|
6
|
+
from dlt.common.typing import TAnyDateTime
|
|
7
|
+
|
|
8
|
+
from ingestr.src.frankfurter.helpers import get_path_with_retry
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dlt.source(
|
|
12
|
+
name="frankfurter",
|
|
13
|
+
max_table_nesting=0,
|
|
14
|
+
)
|
|
15
|
+
def frankfurter_source(
|
|
16
|
+
table: str,
|
|
17
|
+
start_date: Optional[TAnyDateTime] = None,
|
|
18
|
+
end_date: Optional[TAnyDateTime] = None,
|
|
19
|
+
) -> Any:
|
|
20
|
+
"""
|
|
21
|
+
A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
|
|
22
|
+
various types of data: currencies, latest rates, historical rates.
|
|
23
|
+
|
|
24
|
+
Returns the appropriate resource based on the provided parameters.
|
|
25
|
+
"""
|
|
26
|
+
# Determine which resource to return based on the `table` parameter
|
|
27
|
+
if table == "currencies":
|
|
28
|
+
return currencies()
|
|
29
|
+
|
|
30
|
+
elif table == "latest":
|
|
31
|
+
return latest()
|
|
32
|
+
|
|
33
|
+
elif table == "exchange_rates":
|
|
34
|
+
return exchange_rates(start_date=start_date, end_date=end_date)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dlt.resource(
|
|
38
|
+
write_disposition="replace",
|
|
39
|
+
columns={
|
|
40
|
+
"currency_code": {"data_type": "text"},
|
|
41
|
+
"currency_name": {"data_type": "text"},
|
|
42
|
+
},
|
|
43
|
+
)
|
|
44
|
+
def currencies() -> Iterator[dict]:
|
|
45
|
+
"""
|
|
46
|
+
Yields each currency as a separate row with two columns: currency_code and currency_name.
|
|
47
|
+
"""
|
|
48
|
+
# Retrieve the list of currencies from the API
|
|
49
|
+
currencies_data = get_path_with_retry("currencies")
|
|
50
|
+
|
|
51
|
+
for currency_code, currency_name in currencies_data.items():
|
|
52
|
+
yield {"currency_code": currency_code, "currency_name": currency_name}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dlt.resource(
|
|
56
|
+
write_disposition="replace",
|
|
57
|
+
columns={
|
|
58
|
+
"date": {"data_type": "text"},
|
|
59
|
+
"currency_name": {"data_type": "text"},
|
|
60
|
+
"rate": {"data_type": "double"},
|
|
61
|
+
},
|
|
62
|
+
primary_key=["date", "currency_name"], # Composite primary key
|
|
63
|
+
)
|
|
64
|
+
def latest() -> Iterator[dict]:
|
|
65
|
+
"""
|
|
66
|
+
Fetches the latest exchange rates and yields them as rows.
|
|
67
|
+
"""
|
|
68
|
+
# Base URL
|
|
69
|
+
url = "latest?"
|
|
70
|
+
|
|
71
|
+
# Fetch data
|
|
72
|
+
latest_data = get_path_with_retry(url)
|
|
73
|
+
|
|
74
|
+
# Extract rates and base currency
|
|
75
|
+
rates = latest_data["rates"]
|
|
76
|
+
|
|
77
|
+
# Prepare the date
|
|
78
|
+
date = pendulum.now().to_date_string()
|
|
79
|
+
|
|
80
|
+
# Add the base currency (EUR) with a rate of 1.0
|
|
81
|
+
yield {
|
|
82
|
+
"date": date,
|
|
83
|
+
"currency_name": "EUR",
|
|
84
|
+
"rate": 1.0,
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
# Add all currencies and their rates
|
|
88
|
+
for currency_name, rate in rates.items():
|
|
89
|
+
yield {
|
|
90
|
+
"date": date,
|
|
91
|
+
"currency_name": currency_name,
|
|
92
|
+
"rate": rate,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dlt.resource(
|
|
97
|
+
write_disposition="replace",
|
|
98
|
+
columns={
|
|
99
|
+
"date": {"data_type": "text"},
|
|
100
|
+
"currency_name": {"data_type": "text"},
|
|
101
|
+
"rate": {"data_type": "double"},
|
|
102
|
+
},
|
|
103
|
+
primary_key=["date", "currency_name"], # Composite primary key
|
|
104
|
+
)
|
|
105
|
+
def exchange_rates(
|
|
106
|
+
start_date: TAnyDateTime,
|
|
107
|
+
end_date: TAnyDateTime,
|
|
108
|
+
) -> Iterator[dict]:
|
|
109
|
+
"""
|
|
110
|
+
Fetches exchange rates for a specified date range.
|
|
111
|
+
If only start_date is provided, fetches data for that date.
|
|
112
|
+
If both start_date and end_date are provided, fetches data for each day in the range.
|
|
113
|
+
"""
|
|
114
|
+
start_date_str = ensure_pendulum_datetime(start_date).format("YYYY-MM-DD")
|
|
115
|
+
end_date_str = ensure_pendulum_datetime(end_date).format("YYYY-MM-DD")
|
|
116
|
+
|
|
117
|
+
# Compose the URL
|
|
118
|
+
url = f"{start_date_str}..{end_date_str}?"
|
|
119
|
+
|
|
120
|
+
# Fetch data from the API
|
|
121
|
+
data = get_path_with_retry(url)
|
|
122
|
+
|
|
123
|
+
# Extract base currency and rates from the API response
|
|
124
|
+
base_currency = data["base"]
|
|
125
|
+
rates = data["rates"]
|
|
126
|
+
|
|
127
|
+
# Iterate over the rates dictionary (one entry per date)
|
|
128
|
+
for date, daily_rates in rates.items():
|
|
129
|
+
# Add the base currency with a rate of 1.0
|
|
130
|
+
yield {
|
|
131
|
+
"date": date,
|
|
132
|
+
"currency_name": base_currency,
|
|
133
|
+
"rate": 1.0,
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
# Add all other currencies and their rates
|
|
137
|
+
for currency_name, rate in daily_rates.items():
|
|
138
|
+
yield {
|
|
139
|
+
"date": date,
|
|
140
|
+
"currency_name": currency_name,
|
|
141
|
+
"rate": rate,
|
|
142
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
|
|
3
|
+
from dlt.common.pendulum import pendulum
|
|
4
|
+
from dlt.common.typing import StrAny
|
|
5
|
+
from dlt.sources.helpers import requests
|
|
6
|
+
|
|
7
|
+
FRANKFURTER_API_URL = "https://api.frankfurter.dev/v1/"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_url_with_retry(url: str) -> StrAny:
|
|
11
|
+
r = requests.get(url)
|
|
12
|
+
return r.json() # type: ignore
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_path_with_retry(path: str) -> StrAny:
|
|
16
|
+
return get_url_with_retry(f"{FRANKFURTER_API_URL}{path}")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def validate_dates(start_date: datetime, end_date: datetime) -> None:
|
|
20
|
+
current_date = pendulum.now()
|
|
21
|
+
|
|
22
|
+
# Check if start_date is in the future
|
|
23
|
+
if start_date > current_date:
|
|
24
|
+
raise ValueError("Interval-start cannot be in the future.")
|
|
25
|
+
|
|
26
|
+
# Check if end_date is in the future
|
|
27
|
+
if end_date > current_date:
|
|
28
|
+
raise ValueError("Interval-end cannot be in the future.")
|
|
29
|
+
|
|
30
|
+
# Check if start_date is before end_date
|
|
31
|
+
if start_date > end_date:
|
|
32
|
+
raise ValueError("Interval-end cannot be before interval-start.")
|
ingestr/src/github/helpers.py
CHANGED
|
@@ -103,9 +103,9 @@ def get_reactions_data(
|
|
|
103
103
|
|
|
104
104
|
|
|
105
105
|
def _extract_top_connection(data: StrAny, node_type: str) -> StrAny:
|
|
106
|
-
assert (
|
|
107
|
-
|
|
108
|
-
)
|
|
106
|
+
assert isinstance(data, dict) and len(data) == 1, (
|
|
107
|
+
f"The data with list of {node_type} must be a dictionary and contain only one element"
|
|
108
|
+
)
|
|
109
109
|
data = next(iter(data.values()))
|
|
110
110
|
return data[node_type] # type: ignore
|
|
111
111
|
|
|
@@ -158,7 +158,7 @@ def _get_graphql_pages(
|
|
|
158
158
|
)
|
|
159
159
|
items_count += len(data_items)
|
|
160
160
|
print(
|
|
161
|
-
f
|
|
161
|
+
f"Got {len(data_items)}/{items_count} {node_type}s, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
|
|
162
162
|
)
|
|
163
163
|
if data_items:
|
|
164
164
|
yield data_items
|
|
@@ -187,7 +187,7 @@ def _get_comment_reaction(comment_ids: List[str], access_token: str) -> StrAny:
|
|
|
187
187
|
# print(query)
|
|
188
188
|
page, rate_limit = _run_graphql_query(access_token, query, {})
|
|
189
189
|
print(
|
|
190
|
-
f
|
|
190
|
+
f"Got {len(page)} comments, query cost {rate_limit['cost']}, remaining credits: {rate_limit['remaining']}"
|
|
191
191
|
)
|
|
192
192
|
data.update(page)
|
|
193
193
|
return data
|
|
@@ -70,9 +70,9 @@ def google_spreadsheet(
|
|
|
70
70
|
spreadsheet_id=spreadsheet_id,
|
|
71
71
|
range_names=list(all_range_names),
|
|
72
72
|
)
|
|
73
|
-
assert len(all_range_names) == len(
|
|
74
|
-
|
|
75
|
-
)
|
|
73
|
+
assert len(all_range_names) == len(all_range_data), (
|
|
74
|
+
"Google Sheets API must return values for all requested ranges"
|
|
75
|
+
)
|
|
76
76
|
|
|
77
77
|
# get metadata for two first rows of each range
|
|
78
78
|
# first should contain headers
|
|
@@ -126,7 +126,7 @@ def google_spreadsheet(
|
|
|
126
126
|
headers = get_range_headers(headers_metadata, name)
|
|
127
127
|
if headers is None:
|
|
128
128
|
# generate automatic headers and treat the first row as data
|
|
129
|
-
headers = [f"col_{idx+1}" for idx in range(len(headers_metadata))]
|
|
129
|
+
headers = [f"col_{idx + 1}" for idx in range(len(headers_metadata))]
|
|
130
130
|
data_row_metadata = headers_metadata
|
|
131
131
|
rows_data = values[0:]
|
|
132
132
|
logger.warning(
|
|
@@ -149,12 +149,12 @@ def get_range_headers(headers_metadata: List[DictStrAny], range_name: str) -> Li
|
|
|
149
149
|
header_val = str(f"col_{idx + 1}")
|
|
150
150
|
else:
|
|
151
151
|
logger.warning(
|
|
152
|
-
f"In range {range_name}, header value: {header_val} at position {idx+1} is not a string!"
|
|
152
|
+
f"In range {range_name}, header value: {header_val} at position {idx + 1} is not a string!"
|
|
153
153
|
)
|
|
154
154
|
return None
|
|
155
155
|
else:
|
|
156
156
|
logger.warning(
|
|
157
|
-
f"In range {range_name}, header at position {idx+1} is not missing!"
|
|
157
|
+
f"In range {range_name}, header at position {idx + 1} is not missing!"
|
|
158
158
|
)
|
|
159
159
|
return None
|
|
160
160
|
headers.append(header_val)
|
ingestr/src/sources.py
CHANGED
|
@@ -67,6 +67,8 @@ from ingestr.src.errors import (
|
|
|
67
67
|
from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
|
|
68
68
|
from ingestr.src.filesystem import readers
|
|
69
69
|
from ingestr.src.filters import table_adapter_exclude_columns
|
|
70
|
+
from ingestr.src.frankfurter import frankfurter_source
|
|
71
|
+
from ingestr.src.frankfurter.helpers import validate_dates
|
|
70
72
|
from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
|
|
71
73
|
from ingestr.src.google_ads import google_ads
|
|
72
74
|
from ingestr.src.google_analytics import google_analytics
|
|
@@ -2041,3 +2043,35 @@ class PipedriveSource:
|
|
|
2041
2043
|
return pipedrive_source(
|
|
2042
2044
|
pipedrive_api_key=api_key, since_timestamp=start_date
|
|
2043
2045
|
).with_resources(table)
|
|
2046
|
+
|
|
2047
|
+
|
|
2048
|
+
class FrankfurterSource:
|
|
2049
|
+
def handles_incrementality(self) -> bool:
|
|
2050
|
+
return True
|
|
2051
|
+
|
|
2052
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2053
|
+
# start and end dates only assigned and validated for exchange_rates table
|
|
2054
|
+
# Note: if an end date but no start date is provided, start date and end date will be set to current date
|
|
2055
|
+
if table == "exchange_rates":
|
|
2056
|
+
if kwargs.get("interval_start"):
|
|
2057
|
+
start_date = ensure_pendulum_datetime(str(kwargs.get("interval_start")))
|
|
2058
|
+
if kwargs.get("interval_end"):
|
|
2059
|
+
end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
|
|
2060
|
+
else:
|
|
2061
|
+
end_date = start_date
|
|
2062
|
+
else:
|
|
2063
|
+
start_date = pendulum.now()
|
|
2064
|
+
end_date = pendulum.now()
|
|
2065
|
+
validate_dates(start_date=start_date, end_date=end_date)
|
|
2066
|
+
|
|
2067
|
+
# Validate table
|
|
2068
|
+
if table not in ["currencies", "latest", "exchange_rates"]:
|
|
2069
|
+
raise ValueError(
|
|
2070
|
+
f"Table '{table}' is not supported for Frankfurter source."
|
|
2071
|
+
)
|
|
2072
|
+
|
|
2073
|
+
return frankfurter_source(
|
|
2074
|
+
table=table,
|
|
2075
|
+
start_date=start_date,
|
|
2076
|
+
end_date=end_date,
|
|
2077
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.13.
|
|
3
|
+
Version: 0.13.25
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -46,10 +46,10 @@ Requires-Dist: databricks-sqlalchemy==1.0.2
|
|
|
46
46
|
Requires-Dist: dataclasses-json==0.6.7
|
|
47
47
|
Requires-Dist: decorator==5.2.1
|
|
48
48
|
Requires-Dist: deprecation==2.1.0
|
|
49
|
-
Requires-Dist: dlt==1.
|
|
49
|
+
Requires-Dist: dlt==1.9.0
|
|
50
50
|
Requires-Dist: dnspython==2.7.0
|
|
51
|
-
Requires-Dist: duckdb-engine==0.
|
|
52
|
-
Requires-Dist: duckdb==1.2.
|
|
51
|
+
Requires-Dist: duckdb-engine==0.17.0
|
|
52
|
+
Requires-Dist: duckdb==1.2.1
|
|
53
53
|
Requires-Dist: et-xmlfile==2.0.0
|
|
54
54
|
Requires-Dist: facebook-business==20.0.0
|
|
55
55
|
Requires-Dist: filelock==3.17.0
|
|
@@ -168,6 +168,7 @@ Requires-Dist: sqlalchemy-hana==2.0.0
|
|
|
168
168
|
Requires-Dist: sqlalchemy-redshift==0.8.14
|
|
169
169
|
Requires-Dist: sqlalchemy2-stubs==0.0.2a38
|
|
170
170
|
Requires-Dist: sqlalchemy==1.4.52
|
|
171
|
+
Requires-Dist: sqlglot==26.12.1
|
|
171
172
|
Requires-Dist: stripe==10.7.0
|
|
172
173
|
Requires-Dist: tenacity==9.0.0
|
|
173
174
|
Requires-Dist: thrift==0.16.0
|
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
ingestr/
|
|
1
|
+
ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
|
|
2
|
+
ingestr/main.py,sha256=wvbRCJ2--M0Zw2cYtSH874TxTtlD0wadHREeLG3anOY,25618
|
|
2
3
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
3
4
|
ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
|
|
4
|
-
ingestr/src/buildinfo.py,sha256=
|
|
5
|
+
ingestr/src/buildinfo.py,sha256=lkdWOsV5VTf0j0DW9a0BdkPbOZMZEYAUvs8aGZ2V-uE,21
|
|
5
6
|
ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
|
|
6
7
|
ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
|
|
7
|
-
ingestr/src/factory.py,sha256=
|
|
8
|
-
ingestr/src/filters.py,sha256=
|
|
8
|
+
ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
|
|
9
|
+
ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
|
|
9
10
|
ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
|
|
10
11
|
ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
|
|
11
12
|
ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
|
|
12
|
-
ingestr/src/sources.py,sha256=
|
|
13
|
+
ingestr/src/sources.py,sha256=u2Kh2K9v1YhrKXss9yGGxFXr-fMp1E6pvbcwjZqYiGM,73394
|
|
13
14
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
14
15
|
ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
|
|
15
16
|
ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
|
|
@@ -40,8 +41,10 @@ ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_
|
|
|
40
41
|
ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
|
|
41
42
|
ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
|
|
42
43
|
ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
|
|
44
|
+
ingestr/src/frankfurter/__init__.py,sha256=xJUicENGYtOPsGznKP8IA_5Jt-_gJP29onrByBgUf-g,4259
|
|
45
|
+
ingestr/src/frankfurter/helpers.py,sha256=RSqI-WAAJfunWnLqiBRmPuonRg7rDOqmY76beb8a6rM,967
|
|
43
46
|
ingestr/src/github/__init__.py,sha256=xVijF-Wi4p88hkVJnKH-oTixismjD3aUcGqGa6Wr4e4,5889
|
|
44
|
-
ingestr/src/github/helpers.py,sha256=
|
|
47
|
+
ingestr/src/github/helpers.py,sha256=rpv_3HzuOl4PQ-FUeA66pev-pgze9SaE8RUHIPYfZ_A,6759
|
|
45
48
|
ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
|
|
46
49
|
ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
|
|
47
50
|
ingestr/src/google_ads/__init__.py,sha256=bH0TtnRWcOUESezpvoA7VEUHAq_0ITGQeX4GGVBfl1I,3725
|
|
@@ -52,10 +55,10 @@ ingestr/src/google_ads/reports.py,sha256=AVY1pPt5yaIFskQe1k5VW2Dhlux3bzewsHlDrdG
|
|
|
52
55
|
ingestr/src/google_analytics/__init__.py,sha256=8Evpmoy464YpNbCI_NmvFHIzWCu7J7SjJw-RrPZ6AL8,3674
|
|
53
56
|
ingestr/src/google_analytics/helpers.py,sha256=vLmFyQ_IEJEK5LlxBJQeJw0VHaE5gRRZdBa54U72CaQ,5965
|
|
54
57
|
ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
|
|
55
|
-
ingestr/src/google_sheets/__init__.py,sha256=
|
|
58
|
+
ingestr/src/google_sheets/__init__.py,sha256=CL0HfY74uxX8-ge0ucI0VhWMYZVAfoX7WRPBitRi-CI,6647
|
|
56
59
|
ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
|
|
57
60
|
ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9kowZvQUCUxz0G6SlI,5404
|
|
58
|
-
ingestr/src/google_sheets/helpers/data_processing.py,sha256=
|
|
61
|
+
ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
|
|
59
62
|
ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
|
|
60
63
|
ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
|
|
61
64
|
ingestr/src/hubspot/__init__.py,sha256=NYgSIAPXQh2Qp1eKun7TgcerKogq6pWtNkr-_f0FXbI,9464
|
|
@@ -118,8 +121,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
|
|
|
118
121
|
ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
|
|
119
122
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
120
123
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
121
|
-
ingestr-0.13.
|
|
122
|
-
ingestr-0.13.
|
|
123
|
-
ingestr-0.13.
|
|
124
|
-
ingestr-0.13.
|
|
125
|
-
ingestr-0.13.
|
|
124
|
+
ingestr-0.13.25.dist-info/METADATA,sha256=IiZGRzg-cRXN10snt3Y_97yODJ9KkDF-x9XjP_cx9vo,13659
|
|
125
|
+
ingestr-0.13.25.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
126
|
+
ingestr-0.13.25.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
127
|
+
ingestr-0.13.25.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
128
|
+
ingestr-0.13.25.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|