ingestr 0.13.93__py3-none-any.whl → 0.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/src/adjust/__init__.py +4 -4
- ingestr/src/anthropic/__init__.py +277 -0
- ingestr/src/anthropic/helpers.py +525 -0
- ingestr/src/applovin/__init__.py +1 -1
- ingestr/src/buildinfo.py +1 -1
- ingestr/src/destinations.py +145 -7
- ingestr/src/docebo/__init__.py +589 -0
- ingestr/src/docebo/client.py +435 -0
- ingestr/src/docebo/helpers.py +97 -0
- ingestr/src/elasticsearch/helpers.py +112 -0
- ingestr/src/facebook_ads/__init__.py +10 -5
- ingestr/src/factory.py +14 -0
- ingestr/src/fluxx/__init__.py +4175 -0
- ingestr/src/frankfurter/__init__.py +157 -157
- ingestr/src/fundraiseup/__init__.py +49 -0
- ingestr/src/fundraiseup/client.py +81 -0
- ingestr/src/google_analytics/__init__.py +1 -1
- ingestr/src/mongodb/__init__.py +1 -1
- ingestr/src/mongodb/helpers.py +101 -79
- ingestr/src/sources.py +151 -4
- ingestr/tests/unit/test_smartsheets.py +1 -1
- {ingestr-0.13.93.dist-info → ingestr-0.14.0.dist-info}/METADATA +12 -7
- {ingestr-0.13.93.dist-info → ingestr-0.14.0.dist-info}/RECORD +26 -18
- {ingestr-0.13.93.dist-info → ingestr-0.14.0.dist-info}/WHEEL +0 -0
- {ingestr-0.13.93.dist-info → ingestr-0.14.0.dist-info}/entry_points.txt +0 -0
- {ingestr-0.13.93.dist-info → ingestr-0.14.0.dist-info}/licenses/LICENSE.md +0 -0
ingestr/src/mongodb/helpers.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Mongo database source helpers"""
|
|
1
|
+
"""Mongo database source helpers and destination utilities"""
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
4
|
from itertools import islice
|
|
@@ -23,6 +23,7 @@ from bson.timestamp import Timestamp
|
|
|
23
23
|
from dlt.common import logger
|
|
24
24
|
from dlt.common.configuration.specs import BaseConfiguration, configspec
|
|
25
25
|
from dlt.common.data_writers import TDataItemFormat
|
|
26
|
+
from dlt.common.schema import TTableSchema
|
|
26
27
|
from dlt.common.time import ensure_pendulum_datetime
|
|
27
28
|
from dlt.common.typing import TDataItem
|
|
28
29
|
from dlt.common.utils import map_nested_in_place
|
|
@@ -870,124 +871,145 @@ class MongoDbCollectionResourceConfiguration(BaseConfiguration):
|
|
|
870
871
|
def convert_mongo_shell_to_extended_json(query_string: str) -> str:
|
|
871
872
|
"""
|
|
872
873
|
Convert MongoDB shell syntax to MongoDB Extended JSON v2 format.
|
|
873
|
-
|
|
874
|
+
|
|
874
875
|
This function handles common MongoDB shell constructs like ISODate, ObjectId,
|
|
875
876
|
NumberLong, NumberDecimal, etc. and converts them to their Extended JSON equivalents
|
|
876
877
|
that can be parsed by bson.json_util.
|
|
877
|
-
|
|
878
|
+
|
|
878
879
|
Args:
|
|
879
880
|
query_string: A string containing MongoDB shell syntax
|
|
880
|
-
|
|
881
|
+
|
|
881
882
|
Returns:
|
|
882
883
|
A string with MongoDB Extended JSON v2 format
|
|
883
|
-
|
|
884
|
+
|
|
884
885
|
Examples:
|
|
885
886
|
>>> convert_mongo_shell_to_extended_json('ISODate("2010-01-01T00:00:00.000Z")')
|
|
886
887
|
'{"$date": "2010-01-01T00:00:00.000Z"}'
|
|
887
|
-
|
|
888
|
+
|
|
888
889
|
>>> convert_mongo_shell_to_extended_json('ObjectId("507f1f77bcf86cd799439011")')
|
|
889
890
|
'{"$oid": "507f1f77bcf86cd799439011"}'
|
|
890
891
|
"""
|
|
891
892
|
converted = query_string
|
|
892
|
-
|
|
893
|
+
|
|
893
894
|
# Convert ISODate("...") to {"$date": "..."}
|
|
894
895
|
# Pattern matches ISODate("2010-01-01T00:00:00.000+0000") or similar
|
|
895
|
-
converted = re.sub(
|
|
896
|
-
|
|
897
|
-
r'{"$date": "\1"}',
|
|
898
|
-
converted
|
|
899
|
-
)
|
|
900
|
-
|
|
896
|
+
converted = re.sub(r'ISODate\("([^"]+)"\)', r'{"$date": "\1"}', converted)
|
|
897
|
+
|
|
901
898
|
# Convert ObjectId("...") to {"$oid": "..."}
|
|
902
|
-
converted = re.sub(
|
|
903
|
-
|
|
904
|
-
r'{"$oid": "\1"}',
|
|
905
|
-
converted
|
|
906
|
-
)
|
|
907
|
-
|
|
899
|
+
converted = re.sub(r'ObjectId\("([^"]+)"\)', r'{"$oid": "\1"}', converted)
|
|
900
|
+
|
|
908
901
|
# Convert NumberLong(...) to {"$numberLong": "..."}
|
|
909
902
|
# Note: NumberLong can have quotes or not: NumberLong(123) or NumberLong("123")
|
|
910
|
-
converted = re.sub(
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
converted
|
|
914
|
-
)
|
|
915
|
-
converted = re.sub(
|
|
916
|
-
r'NumberLong\(([^)]+)\)',
|
|
917
|
-
r'{"$numberLong": "\1"}',
|
|
918
|
-
converted
|
|
919
|
-
)
|
|
920
|
-
|
|
903
|
+
converted = re.sub(r'NumberLong\("([^"]+)"\)', r'{"$numberLong": "\1"}', converted)
|
|
904
|
+
converted = re.sub(r"NumberLong\(([^)]+)\)", r'{"$numberLong": "\1"}', converted)
|
|
905
|
+
|
|
921
906
|
# Convert NumberInt(...) to {"$numberInt": "..."}
|
|
922
|
-
converted = re.sub(
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
converted
|
|
926
|
-
)
|
|
927
|
-
converted = re.sub(
|
|
928
|
-
r'NumberInt\(([^)]+)\)',
|
|
929
|
-
r'{"$numberInt": "\1"}',
|
|
930
|
-
converted
|
|
931
|
-
)
|
|
932
|
-
|
|
907
|
+
converted = re.sub(r'NumberInt\("([^"]+)"\)', r'{"$numberInt": "\1"}', converted)
|
|
908
|
+
converted = re.sub(r"NumberInt\(([^)]+)\)", r'{"$numberInt": "\1"}', converted)
|
|
909
|
+
|
|
933
910
|
# Convert NumberDecimal("...") to {"$numberDecimal": "..."}
|
|
934
911
|
converted = re.sub(
|
|
935
|
-
r'NumberDecimal\("([^"]+)"\)',
|
|
936
|
-
r'{"$numberDecimal": "\1"}',
|
|
937
|
-
converted
|
|
912
|
+
r'NumberDecimal\("([^"]+)"\)', r'{"$numberDecimal": "\1"}', converted
|
|
938
913
|
)
|
|
939
|
-
|
|
914
|
+
|
|
940
915
|
# Convert Timestamp(..., ...) to {"$timestamp": {"t": ..., "i": ...}}
|
|
941
916
|
# Timestamp(1234567890, 1) -> {"$timestamp": {"t": 1234567890, "i": 1}}
|
|
942
917
|
converted = re.sub(
|
|
943
|
-
r
|
|
944
|
-
r'{"$timestamp": {"t": \1, "i": \2}}',
|
|
945
|
-
converted
|
|
918
|
+
r"Timestamp\((\d+),\s*(\d+)\)", r'{"$timestamp": {"t": \1, "i": \2}}', converted
|
|
946
919
|
)
|
|
947
|
-
|
|
920
|
+
|
|
948
921
|
# Convert BinData(..., "...") to {"$binary": {"base64": "...", "subType": "..."}}
|
|
949
922
|
converted = re.sub(
|
|
950
923
|
r'BinData\((\d+),\s*"([^"]+)"\)',
|
|
951
924
|
r'{"$binary": {"base64": "\2", "subType": "\1"}}',
|
|
952
|
-
converted
|
|
925
|
+
converted,
|
|
953
926
|
)
|
|
954
|
-
|
|
927
|
+
|
|
955
928
|
# Convert MinKey() to {"$minKey": 1}
|
|
956
|
-
converted = re.sub(
|
|
957
|
-
|
|
958
|
-
r'{"$minKey": 1}',
|
|
959
|
-
converted
|
|
960
|
-
)
|
|
961
|
-
|
|
929
|
+
converted = re.sub(r"MinKey\(\)", r'{"$minKey": 1}', converted)
|
|
930
|
+
|
|
962
931
|
# Convert MaxKey() to {"$maxKey": 1}
|
|
963
|
-
converted = re.sub(
|
|
964
|
-
|
|
965
|
-
r'{"$maxKey": 1}',
|
|
966
|
-
converted
|
|
967
|
-
)
|
|
968
|
-
|
|
932
|
+
converted = re.sub(r"MaxKey\(\)", r'{"$maxKey": 1}', converted)
|
|
933
|
+
|
|
969
934
|
# Convert UUID("...") to {"$uuid": "..."}
|
|
970
|
-
converted = re.sub(
|
|
971
|
-
|
|
972
|
-
r'{"$uuid": "\1"}',
|
|
973
|
-
converted
|
|
974
|
-
)
|
|
975
|
-
|
|
935
|
+
converted = re.sub(r'UUID\("([^"]+)"\)', r'{"$uuid": "\1"}', converted)
|
|
936
|
+
|
|
976
937
|
# Convert DBRef("collection", "id") to {"$ref": "collection", "$id": "id"}
|
|
977
938
|
converted = re.sub(
|
|
978
|
-
r'DBRef\("([^"]+)",\s*"([^"]+)"\)',
|
|
979
|
-
r'{"$ref": "\1", "$id": "\2"}',
|
|
980
|
-
converted
|
|
939
|
+
r'DBRef\("([^"]+)",\s*"([^"]+)"\)', r'{"$ref": "\1", "$id": "\2"}', converted
|
|
981
940
|
)
|
|
982
|
-
|
|
941
|
+
|
|
983
942
|
# Convert Code("...") to {"$code": "..."}
|
|
984
|
-
converted = re.sub(
|
|
985
|
-
|
|
986
|
-
r'{"$code": "\1"}',
|
|
987
|
-
converted
|
|
988
|
-
)
|
|
989
|
-
|
|
943
|
+
converted = re.sub(r'Code\("([^"]+)"\)', r'{"$code": "\1"}', converted)
|
|
944
|
+
|
|
990
945
|
return converted
|
|
991
946
|
|
|
992
947
|
|
|
993
948
|
__source_name__ = "mongodb"
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
# MongoDB destination helper functions
|
|
952
|
+
def process_file_items(file_path: str) -> list[dict]:
|
|
953
|
+
"""Process items from a file path (JSONL format)."""
|
|
954
|
+
import json
|
|
955
|
+
|
|
956
|
+
documents = []
|
|
957
|
+
with open(file_path, "r") as f:
|
|
958
|
+
for line in f:
|
|
959
|
+
if line.strip():
|
|
960
|
+
doc = json.loads(line.strip())
|
|
961
|
+
documents.append(doc) # Include all fields including DLT metadata
|
|
962
|
+
return documents
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def mongodb_insert(uri: str, database: str):
|
|
966
|
+
"""Creates a dlt.destination for inserting data into a MongoDB collection.
|
|
967
|
+
|
|
968
|
+
Args:
|
|
969
|
+
uri (str): MongoDB connection URI.
|
|
970
|
+
database (str): Name of the MongoDB database.
|
|
971
|
+
|
|
972
|
+
Returns:
|
|
973
|
+
dlt.destination: A DLT destination object configured for MongoDB.
|
|
974
|
+
"""
|
|
975
|
+
|
|
976
|
+
state = {"first_batch": True}
|
|
977
|
+
|
|
978
|
+
def destination(items: TDataItem, table: TTableSchema) -> None:
|
|
979
|
+
import pyarrow
|
|
980
|
+
from pymongo import MongoClient
|
|
981
|
+
|
|
982
|
+
# Extract database name from connection string
|
|
983
|
+
# Get collection name from table metadata
|
|
984
|
+
collection_name = table["name"]
|
|
985
|
+
|
|
986
|
+
# Connect to MongoDB
|
|
987
|
+
client: MongoClient
|
|
988
|
+
|
|
989
|
+
with MongoClient(uri) as client:
|
|
990
|
+
db = client[database]
|
|
991
|
+
collection = db[collection_name]
|
|
992
|
+
|
|
993
|
+
# Process and insert documents
|
|
994
|
+
if isinstance(items, str):
|
|
995
|
+
documents = process_file_items(items)
|
|
996
|
+
elif isinstance(items, pyarrow.RecordBatch):
|
|
997
|
+
documents = [item for item in items.to_pylist()]
|
|
998
|
+
else:
|
|
999
|
+
documents = [item for item in items if isinstance(item, dict)]
|
|
1000
|
+
|
|
1001
|
+
if state["first_batch"] and documents:
|
|
1002
|
+
collection.delete_many({})
|
|
1003
|
+
state["first_batch"] = False
|
|
1004
|
+
|
|
1005
|
+
if documents:
|
|
1006
|
+
collection.insert_many(documents) # Insert all new data
|
|
1007
|
+
|
|
1008
|
+
return dlt.destination(
|
|
1009
|
+
destination,
|
|
1010
|
+
name="mongodb",
|
|
1011
|
+
loader_file_format="typed-jsonl",
|
|
1012
|
+
batch_size=1000,
|
|
1013
|
+
naming_convention="snake_case",
|
|
1014
|
+
loader_parallelism_strategy="sequential",
|
|
1015
|
+
)
|
ingestr/src/sources.py
CHANGED
|
@@ -237,6 +237,9 @@ class SqlSource:
|
|
|
237
237
|
backend_kwargs: Dict[str, Any] = None, # type: ignore
|
|
238
238
|
type_adapter_callback: Optional[TTypeAdapter] = None,
|
|
239
239
|
included_columns: Optional[List[str]] = None,
|
|
240
|
+
excluded_columns: Optional[
|
|
241
|
+
List[str]
|
|
242
|
+
] = None, # Added for dlt 1.16.0 compatibility
|
|
240
243
|
query_adapter_callback: Optional[TQueryAdapter] = None,
|
|
241
244
|
resolve_foreign_keys: bool = False,
|
|
242
245
|
) -> Iterator[TDataItem]:
|
|
@@ -430,16 +433,17 @@ class MongoDbSource:
|
|
|
430
433
|
# Parse the query using MongoDB's extended JSON parser
|
|
431
434
|
# First, convert MongoDB shell syntax to Extended JSON format
|
|
432
435
|
from bson import json_util
|
|
436
|
+
|
|
433
437
|
from ingestr.src.mongodb.helpers import convert_mongo_shell_to_extended_json
|
|
434
|
-
|
|
438
|
+
|
|
435
439
|
# Convert MongoDB shell constructs to Extended JSON v2 format
|
|
436
440
|
converted_query = convert_mongo_shell_to_extended_json(query_json)
|
|
437
|
-
|
|
441
|
+
|
|
438
442
|
try:
|
|
439
443
|
query = json_util.loads(converted_query)
|
|
440
444
|
except Exception as e:
|
|
441
445
|
raise ValueError(f"Invalid MongoDB query format: {e}")
|
|
442
|
-
|
|
446
|
+
|
|
443
447
|
# Validate that it's a list for aggregation pipeline
|
|
444
448
|
if not isinstance(query, list):
|
|
445
449
|
raise ValueError(
|
|
@@ -1883,6 +1887,72 @@ class DynamoDBSource:
|
|
|
1883
1887
|
return dynamodb(table, creds, incremental)
|
|
1884
1888
|
|
|
1885
1889
|
|
|
1890
|
+
class DoceboSource:
|
|
1891
|
+
def handles_incrementality(self) -> bool:
|
|
1892
|
+
return False
|
|
1893
|
+
|
|
1894
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1895
|
+
# docebo://?base_url=https://yourcompany.docebosaas.com&client_id=xxx&client_secret=xxx
|
|
1896
|
+
# Optional: &username=xxx&password=xxx for password grant type
|
|
1897
|
+
|
|
1898
|
+
if kwargs.get("incremental_key"):
|
|
1899
|
+
raise ValueError("Incremental loads are not yet supported for Docebo")
|
|
1900
|
+
|
|
1901
|
+
parsed_uri = urlparse(uri)
|
|
1902
|
+
source_params = parse_qs(parsed_uri.query)
|
|
1903
|
+
|
|
1904
|
+
base_url = source_params.get("base_url")
|
|
1905
|
+
if not base_url:
|
|
1906
|
+
raise ValueError("base_url is required to connect to Docebo")
|
|
1907
|
+
|
|
1908
|
+
client_id = source_params.get("client_id")
|
|
1909
|
+
if not client_id:
|
|
1910
|
+
raise ValueError("client_id is required to connect to Docebo")
|
|
1911
|
+
|
|
1912
|
+
client_secret = source_params.get("client_secret")
|
|
1913
|
+
if not client_secret:
|
|
1914
|
+
raise ValueError("client_secret is required to connect to Docebo")
|
|
1915
|
+
|
|
1916
|
+
# Username and password are optional (uses client_credentials grant if not provided)
|
|
1917
|
+
username = source_params.get("username", [None])[0]
|
|
1918
|
+
password = source_params.get("password", [None])[0]
|
|
1919
|
+
|
|
1920
|
+
# Supported tables
|
|
1921
|
+
supported_tables = [
|
|
1922
|
+
"users",
|
|
1923
|
+
"courses",
|
|
1924
|
+
"user_fields",
|
|
1925
|
+
"branches",
|
|
1926
|
+
"groups",
|
|
1927
|
+
"group_members",
|
|
1928
|
+
"course_fields",
|
|
1929
|
+
"learning_objects",
|
|
1930
|
+
"learning_plans",
|
|
1931
|
+
"learning_plan_enrollments",
|
|
1932
|
+
"learning_plan_course_enrollments",
|
|
1933
|
+
"course_enrollments",
|
|
1934
|
+
"sessions",
|
|
1935
|
+
"categories",
|
|
1936
|
+
"certifications",
|
|
1937
|
+
"external_training",
|
|
1938
|
+
"survey_answers",
|
|
1939
|
+
]
|
|
1940
|
+
if table not in supported_tables:
|
|
1941
|
+
raise ValueError(
|
|
1942
|
+
f"Resource '{table}' is not supported for Docebo source. Supported tables: {', '.join(supported_tables)}"
|
|
1943
|
+
)
|
|
1944
|
+
|
|
1945
|
+
from ingestr.src.docebo import docebo_source
|
|
1946
|
+
|
|
1947
|
+
return docebo_source(
|
|
1948
|
+
base_url=base_url[0],
|
|
1949
|
+
client_id=client_id[0],
|
|
1950
|
+
client_secret=client_secret[0],
|
|
1951
|
+
username=username,
|
|
1952
|
+
password=password,
|
|
1953
|
+
).with_resources(table)
|
|
1954
|
+
|
|
1955
|
+
|
|
1886
1956
|
class GoogleAnalyticsSource:
|
|
1887
1957
|
def handles_incrementality(self) -> bool:
|
|
1888
1958
|
return True
|
|
@@ -2537,7 +2607,7 @@ class SalesforceSource:
|
|
|
2537
2607
|
|
|
2538
2608
|
if table not in src.resources:
|
|
2539
2609
|
raise UnsupportedResourceError(table, "Salesforce")
|
|
2540
|
-
|
|
2610
|
+
|
|
2541
2611
|
return src.with_resources(table)
|
|
2542
2612
|
|
|
2543
2613
|
|
|
@@ -3556,3 +3626,80 @@ class WiseSource:
|
|
|
3556
3626
|
start_date=start_date,
|
|
3557
3627
|
end_date=end_date,
|
|
3558
3628
|
).with_resources(table)
|
|
3629
|
+
|
|
3630
|
+
|
|
3631
|
+
class FundraiseupSource:
|
|
3632
|
+
def handles_incrementality(self) -> bool:
|
|
3633
|
+
return False
|
|
3634
|
+
|
|
3635
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3636
|
+
parsed_uri = urlparse(uri)
|
|
3637
|
+
params = parse_qs(parsed_uri.query)
|
|
3638
|
+
|
|
3639
|
+
api_key = params.get("api_key")
|
|
3640
|
+
if api_key is None:
|
|
3641
|
+
raise MissingValueError("api_key", "Fundraiseup")
|
|
3642
|
+
|
|
3643
|
+
if table not in [
|
|
3644
|
+
"donations",
|
|
3645
|
+
"events",
|
|
3646
|
+
"fundraisers",
|
|
3647
|
+
"recurring_plans",
|
|
3648
|
+
"supporters",
|
|
3649
|
+
]:
|
|
3650
|
+
raise UnsupportedResourceError(table, "Fundraiseup")
|
|
3651
|
+
|
|
3652
|
+
from ingestr.src.fundraiseup import fundraiseup_source
|
|
3653
|
+
|
|
3654
|
+
return fundraiseup_source(
|
|
3655
|
+
api_key=api_key[0],
|
|
3656
|
+
).with_resources(table)
|
|
3657
|
+
|
|
3658
|
+
|
|
3659
|
+
class AnthropicSource:
|
|
3660
|
+
def handles_incrementality(self) -> bool:
|
|
3661
|
+
return True
|
|
3662
|
+
|
|
3663
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3664
|
+
# anthropic://?api_key=<admin_api_key>
|
|
3665
|
+
parsed_uri = urlparse(uri)
|
|
3666
|
+
params = parse_qs(parsed_uri.query)
|
|
3667
|
+
|
|
3668
|
+
api_key = params.get("api_key")
|
|
3669
|
+
if api_key is None:
|
|
3670
|
+
raise MissingValueError("api_key", "Anthropic")
|
|
3671
|
+
|
|
3672
|
+
if table not in [
|
|
3673
|
+
"claude_code_usage",
|
|
3674
|
+
"usage_report",
|
|
3675
|
+
"cost_report",
|
|
3676
|
+
"organization",
|
|
3677
|
+
"workspaces",
|
|
3678
|
+
"api_keys",
|
|
3679
|
+
"invites",
|
|
3680
|
+
"users",
|
|
3681
|
+
"workspace_members",
|
|
3682
|
+
]:
|
|
3683
|
+
raise UnsupportedResourceError(table, "Anthropic")
|
|
3684
|
+
|
|
3685
|
+
# Get start and end dates from kwargs
|
|
3686
|
+
start_date = kwargs.get("interval_start")
|
|
3687
|
+
if start_date:
|
|
3688
|
+
start_date = ensure_pendulum_datetime(start_date)
|
|
3689
|
+
else:
|
|
3690
|
+
# Default to 2023-01-01
|
|
3691
|
+
start_date = pendulum.datetime(2023, 1, 1)
|
|
3692
|
+
|
|
3693
|
+
end_date = kwargs.get("interval_end")
|
|
3694
|
+
if end_date:
|
|
3695
|
+
end_date = ensure_pendulum_datetime(end_date)
|
|
3696
|
+
else:
|
|
3697
|
+
end_date = None
|
|
3698
|
+
|
|
3699
|
+
from ingestr.src.anthropic import anthropic_source
|
|
3700
|
+
|
|
3701
|
+
return anthropic_source(
|
|
3702
|
+
api_key=api_key[0],
|
|
3703
|
+
initial_start_date=start_date,
|
|
3704
|
+
end_date=end_date,
|
|
3705
|
+
).with_resources(table)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.14.0
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -48,7 +48,7 @@ Requires-Dist: dataclasses-json==0.6.7
|
|
|
48
48
|
Requires-Dist: decorator==5.2.1
|
|
49
49
|
Requires-Dist: deprecation==2.1.0
|
|
50
50
|
Requires-Dist: dlt-cratedb==0.0.2
|
|
51
|
-
Requires-Dist: dlt==1.
|
|
51
|
+
Requires-Dist: dlt==1.16.0
|
|
52
52
|
Requires-Dist: dnspython==2.7.0
|
|
53
53
|
Requires-Dist: duckdb-engine==0.17.0
|
|
54
54
|
Requires-Dist: duckdb==1.2.1
|
|
@@ -104,7 +104,6 @@ Requires-Dist: jsonpath-ng==1.7.0
|
|
|
104
104
|
Requires-Dist: leb128==1.0.8
|
|
105
105
|
Requires-Dist: lxml==5.3.1
|
|
106
106
|
Requires-Dist: lz4==4.4.3
|
|
107
|
-
Requires-Dist: makefun==1.15.6
|
|
108
107
|
Requires-Dist: mako==1.3.9
|
|
109
108
|
Requires-Dist: markdown-it-py==3.0.0
|
|
110
109
|
Requires-Dist: markupsafe==3.0.2
|
|
@@ -118,7 +117,7 @@ Requires-Dist: mysql-connector-python==9.2.0
|
|
|
118
117
|
Requires-Dist: numpy==2.2.3
|
|
119
118
|
Requires-Dist: oauthlib==3.2.2
|
|
120
119
|
Requires-Dist: openpyxl==3.1.5
|
|
121
|
-
Requires-Dist: orjson==3.
|
|
120
|
+
Requires-Dist: orjson==3.11.3
|
|
122
121
|
Requires-Dist: packaging==24.2
|
|
123
122
|
Requires-Dist: pandas==2.2.3
|
|
124
123
|
Requires-Dist: paramiko==3.5.1
|
|
@@ -161,7 +160,7 @@ Requires-Dist: reactivex==4.0.4
|
|
|
161
160
|
Requires-Dist: requests-file==2.1.0
|
|
162
161
|
Requires-Dist: requests-oauthlib==1.3.1
|
|
163
162
|
Requires-Dist: requests-toolbelt==1.0.0
|
|
164
|
-
Requires-Dist: requests==2.32.
|
|
163
|
+
Requires-Dist: requests==2.32.5
|
|
165
164
|
Requires-Dist: requirements-parser==0.11.0
|
|
166
165
|
Requires-Dist: rich-argparse==1.7.0
|
|
167
166
|
Requires-Dist: rich==13.9.4
|
|
@@ -195,6 +194,7 @@ Requires-Dist: thrift==0.16.0
|
|
|
195
194
|
Requires-Dist: time-machine==2.16.0
|
|
196
195
|
Requires-Dist: tomlkit==0.13.2
|
|
197
196
|
Requires-Dist: tqdm==4.67.1
|
|
197
|
+
Requires-Dist: trino==0.336.0
|
|
198
198
|
Requires-Dist: typer==0.13.1
|
|
199
199
|
Requires-Dist: types-requests==2.32.0.20240907
|
|
200
200
|
Requires-Dist: types-setuptools==75.8.2.20250305
|
|
@@ -333,7 +333,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
|
|
|
333
333
|
<tr>
|
|
334
334
|
<td>Elasticsearch</td>
|
|
335
335
|
<td>✅</td>
|
|
336
|
-
<td
|
|
336
|
+
<td>✅</td>
|
|
337
337
|
</tr>
|
|
338
338
|
<tr>
|
|
339
339
|
<td>GCP Spanner</td>
|
|
@@ -353,7 +353,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
|
|
|
353
353
|
<tr>
|
|
354
354
|
<td>MongoDB</td>
|
|
355
355
|
<td>✅</td>
|
|
356
|
-
<td
|
|
356
|
+
<td>✅</td>
|
|
357
357
|
</tr>
|
|
358
358
|
<tr>
|
|
359
359
|
<td>MotherDuck</td>
|
|
@@ -395,6 +395,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
|
|
|
395
395
|
<td>✅</td>
|
|
396
396
|
<td>❌</td>
|
|
397
397
|
</tr>
|
|
398
|
+
<tr>
|
|
399
|
+
<td>Trino</td>
|
|
400
|
+
<td>✅</td>
|
|
401
|
+
<td>✅</td>
|
|
402
|
+
</tr>
|
|
398
403
|
<tr>
|
|
399
404
|
<td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
|
|
400
405
|
</tr>
|
|
@@ -2,24 +2,26 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
|
|
|
2
2
|
ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
|
|
3
3
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
4
4
|
ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
|
|
5
|
-
ingestr/src/buildinfo.py,sha256=
|
|
6
|
-
ingestr/src/destinations.py,sha256=
|
|
5
|
+
ingestr/src/buildinfo.py,sha256=wnJ_CPlYtFn5F7P7HRnnyrCPI2u2P6bB5qMLZpfYxkc,20
|
|
6
|
+
ingestr/src/destinations.py,sha256=BE8tK69Aq7FLBV_j6ijMhZCRIUYpSTnk7FTZMZW5Q5M,28557
|
|
7
7
|
ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
|
|
8
|
-
ingestr/src/factory.py,sha256=
|
|
8
|
+
ingestr/src/factory.py,sha256=sT5x4R95Qa0QIjkS7a5lFKUFwFpVpdb1nv8HIxOu_RM,7376
|
|
9
9
|
ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
|
|
10
10
|
ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
|
|
11
11
|
ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
|
|
12
12
|
ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
|
|
13
13
|
ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
|
|
14
14
|
ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
|
|
15
|
-
ingestr/src/sources.py,sha256=
|
|
15
|
+
ingestr/src/sources.py,sha256=ADnE41x6-9U0fK1CYqPc6XRfGqg_6mZGqP10tlLUxDU,130361
|
|
16
16
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
17
17
|
ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
|
|
18
18
|
ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
|
|
19
|
-
ingestr/src/adjust/__init__.py,sha256
|
|
19
|
+
ingestr/src/adjust/__init__.py,sha256=-DkqpkCuwohw7BlwB9ZvtpbwQAY1Gr8J1T4KyFwsA8E,3315
|
|
20
20
|
ingestr/src/adjust/adjust_helpers.py,sha256=IHSS94A7enOWkZ8cP5iW3RdYt0Xl3qZGAmDc1Xy4qkI,3802
|
|
21
21
|
ingestr/src/airtable/__init__.py,sha256=XzRsS39xszUlh_s7P1_zq5v8vLfjz3m-NtTPaa8TTZU,2818
|
|
22
|
-
ingestr/src/
|
|
22
|
+
ingestr/src/anthropic/__init__.py,sha256=D23oY20fE_RP9yPVkx7i6l3G1IfRLrJ2XwA8y2ot7JM,8482
|
|
23
|
+
ingestr/src/anthropic/helpers.py,sha256=Co8kmWQwKMHxcUwDU9959LTU6rFxqDIIbIvVSMGatrc,16105
|
|
24
|
+
ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
|
|
23
25
|
ingestr/src/applovin_max/__init__.py,sha256=fxXqsIibJarp5NOGe08G964HftwLDymTtYS_LqPJht4,3315
|
|
24
26
|
ingestr/src/appsflyer/__init__.py,sha256=QoK-B3cYYMD3bqzQaLWNH6FkJyjRbzRkBF2n6urxubs,8071
|
|
25
27
|
ingestr/src/appsflyer/client.py,sha256=E6xPW4KlbBnQZ0K4eq2Xgb3AmGrtrzIX9bX8EnQr-D4,3615
|
|
@@ -40,9 +42,13 @@ ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k
|
|
|
40
42
|
ingestr/src/clickup/__init__.py,sha256=uvfAqNturT4bMvU4NS3E8BdL6nvDFzNuh7bMlih8HJk,2547
|
|
41
43
|
ingestr/src/clickup/helpers.py,sha256=RzDKMUAHccuDhocIQ2ToBXfCERo8CBJqA3t-IPltBCE,1519
|
|
42
44
|
ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
|
|
45
|
+
ingestr/src/docebo/__init__.py,sha256=RBBjlt405PIIDOLEt78g9yBNJfhUMeJxR5DZD7oufXY,27543
|
|
46
|
+
ingestr/src/docebo/client.py,sha256=nki0kNQhN8VDz5cdqlQQPhr1JMPlcNEYKnWK3umAyOc,15663
|
|
47
|
+
ingestr/src/docebo/helpers.py,sha256=SaEjta6k3Lj-S5fvrheA5_xj7zfASMdOc_ihsqno5ko,3238
|
|
43
48
|
ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
|
|
44
49
|
ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
|
|
45
|
-
ingestr/src/
|
|
50
|
+
ingestr/src/elasticsearch/helpers.py,sha256=gbLpkLSZhUfCUIdFeETBx-9D3eVR4WL79fC3mQfFTNM,3515
|
|
51
|
+
ingestr/src/facebook_ads/__init__.py,sha256=d0pTXmtNp6Qh65aY-qF2DPz3jgHkZm3pkQn1_a3G5v0,9892
|
|
46
52
|
ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
|
|
47
53
|
ingestr/src/facebook_ads/helpers.py,sha256=c-WG008yU_zIdhFwljtqE2jfjVYuaVoNKldxcnJN3U4,9761
|
|
48
54
|
ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
|
|
@@ -50,13 +56,15 @@ ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5v
|
|
|
50
56
|
ingestr/src/filesystem/__init__.py,sha256=42YAOHQxZ7TkTXC1eeaLUJpjqJ3l7DH7C8j927pV4pc,4353
|
|
51
57
|
ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
|
|
52
58
|
ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
|
|
53
|
-
ingestr/src/fluxx/__init__.py,sha256=
|
|
59
|
+
ingestr/src/fluxx/__init__.py,sha256=Ua_QVKfeNWzpJRaGgjgR50Ot52s0f5hY-VN-a53gY2Y,566245
|
|
54
60
|
ingestr/src/fluxx/helpers.py,sha256=dCNgvMMTSEO4LNp6luNZ-XrV4NPW-_OUfmp0k3jFhuc,6602
|
|
55
|
-
ingestr/src/frankfurter/__init__.py,sha256=
|
|
61
|
+
ingestr/src/frankfurter/__init__.py,sha256=gOdL8ZqgHHYZByjtfE3WX3BTRHdYqyn9FpQwzDHSAx0,5089
|
|
56
62
|
ingestr/src/frankfurter/helpers.py,sha256=SpRr992OcSf7IDI5y-ToUdO6m6sGpqFz59LTY0ojchI,1502
|
|
57
63
|
ingestr/src/freshdesk/__init__.py,sha256=ukyorgCNsW_snzsYBDsr3Q0WB8f-to9Fk0enqHHFQlk,3087
|
|
58
64
|
ingestr/src/freshdesk/freshdesk_client.py,sha256=1nFf0K4MQ0KZbWwk4xSbYHaykVqmPLfN39miOFDpWVc,4385
|
|
59
65
|
ingestr/src/freshdesk/settings.py,sha256=0Wr_OMnUZcTlry7BmALssLxD2yh686JW4moLNv12Jnw,409
|
|
66
|
+
ingestr/src/fundraiseup/__init__.py,sha256=q3TQeP1HdbWNeXFMq-0-BdSo82Fq4Io1OEYOY1cAYcU,1743
|
|
67
|
+
ingestr/src/fundraiseup/client.py,sha256=klU57l8iJ5NAS1nTb_4UyVAerbPWpTa8PMHlpp9Riz0,2453
|
|
60
68
|
ingestr/src/github/__init__.py,sha256=C7b5j6CrxmTItS4tyDa3OYzdAw5c__xboOtoEJYe3wQ,7217
|
|
61
69
|
ingestr/src/github/helpers.py,sha256=rpv_3HzuOl4PQ-FUeA66pev-pgze9SaE8RUHIPYfZ_A,6759
|
|
62
70
|
ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
|
|
@@ -66,7 +74,7 @@ ingestr/src/google_ads/field.py,sha256=uc8KEaYQrwgQoQPUdxIQWZxpFeZHbiV98FM0ZSael
|
|
|
66
74
|
ingestr/src/google_ads/metrics.py,sha256=tAqpBpm-8l95oPT9cBxMWaEoDTNHVXnqUphYDHWKDiE,12099
|
|
67
75
|
ingestr/src/google_ads/predicates.py,sha256=K4wTuqfmJ9ko1RKeHTBDfQO_mUADVyuRqtywBPP-72w,683
|
|
68
76
|
ingestr/src/google_ads/reports.py,sha256=AVY1pPt5yaIFskQe1k5VW2Dhlux3bzewsHlDrdGEems,12686
|
|
69
|
-
ingestr/src/google_analytics/__init__.py,sha256=
|
|
77
|
+
ingestr/src/google_analytics/__init__.py,sha256=1JkT2OnZAVzNVH06br2Rrh5WyjO6GrNyf9F83rAKsrw,4343
|
|
70
78
|
ingestr/src/google_analytics/helpers.py,sha256=tM7h_yughca3l7tnS_2XGIBM37mVm-Uewv7tK7XHVbc,10111
|
|
71
79
|
ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
|
|
72
80
|
ingestr/src/google_sheets/__init__.py,sha256=CL0HfY74uxX8-ge0ucI0VhWMYZVAfoX7WRPBitRi-CI,6647
|
|
@@ -95,8 +103,8 @@ ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffW
|
|
|
95
103
|
ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
|
|
96
104
|
ingestr/src/mixpanel/__init__.py,sha256=s1QtqMP0BTGW6YtdCabJFWj7lEn7KujzELwGpBOQgfs,1796
|
|
97
105
|
ingestr/src/mixpanel/client.py,sha256=c_reouegOVYBOwHLfgYFwpmkba0Sxro1Zkml07NCYf0,3602
|
|
98
|
-
ingestr/src/mongodb/__init__.py,sha256=
|
|
99
|
-
ingestr/src/mongodb/helpers.py,sha256
|
|
106
|
+
ingestr/src/mongodb/__init__.py,sha256=6-DvvaKL7XOPPRwItI7lSpoMQLEPzYubV6dKhpzbuME,7494
|
|
107
|
+
ingestr/src/mongodb/helpers.py,sha256=mxSMBrgDrjH7D60aLHE6j6THPmXyTmm6iFdJ0zmnPC0,36963
|
|
100
108
|
ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
|
|
101
109
|
ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
|
|
102
110
|
ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -157,9 +165,9 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
|
|
|
157
165
|
ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
|
|
158
166
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
159
167
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
160
|
-
ingestr/tests/unit/test_smartsheets.py,sha256=
|
|
161
|
-
ingestr-0.
|
|
162
|
-
ingestr-0.
|
|
163
|
-
ingestr-0.
|
|
164
|
-
ingestr-0.
|
|
165
|
-
ingestr-0.
|
|
168
|
+
ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
|
|
169
|
+
ingestr-0.14.0.dist-info/METADATA,sha256=rRaZKV5Ik4cZgl0s6BPRvnrSd5fs7FN0ptg1hNCjKT0,15265
|
|
170
|
+
ingestr-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
171
|
+
ingestr-0.14.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
172
|
+
ingestr-0.14.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
173
|
+
ingestr-0.14.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|