ingestr 0.13.39__py3-none-any.whl → 0.13.40__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.39"
+version = "v0.13.40"

ingestr/src/elasticsearch/__init__.py

@@ -0,0 +1,80 @@
+from datetime import date, datetime
+from typing import Any, Optional
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from pendulum import parse
+
+from elasticsearch import Elasticsearch
+
+
+@dlt.source
+def elasticsearch_source(
+    connection_url: str,
+    index: str,
+    verify_certs: bool,
+    incremental: Optional[dlt.sources.incremental] = None,
+):
+    client = Elasticsearch(connection_url, verify_certs=verify_certs)
+
+    @dlt.resource(
+        name=index, primary_key="id", write_disposition="merge", incremental=incremental
+    )
+    def get_documents(incremental=incremental):
+        body = {"query": {"match_all": {}}}
+
+        if incremental:
+            start_value = incremental.last_value
+            range_filter = {"gte": start_value}
+            if incremental.end_value is not None:
+                range_filter["lt"] = incremental.end_value
+            body = {"query": {"range": {incremental.cursor_path: range_filter}}}
+
+        page = client.search(index=index, scroll="5m", size=5, body=body)
+
+        sid = page["_scroll_id"]
+        hits = page["hits"]["hits"]
+
+        if not hits:
+            return
+
+        # fetching first page (via .search)
+        for doc in hits:
+            doc_data = {"id": doc["_id"], **doc["_source"]}
+            if incremental:
+                doc_data[incremental.cursor_path] = convert_elasticsearch_objs(
+                    doc_data[incremental.cursor_path]
+                )
+            yield doc_data
+
+        while True:
+            # fetching page 2 and other pages (via .scroll)
+            page = client.scroll(scroll_id=sid, scroll="5m")
+            sid = page["_scroll_id"]
+            hits = page["hits"]["hits"]
+            if not hits:
+                break
+            for doc in hits:
+                doc_data = {"id": doc["_id"], **doc["_source"]}
+                if incremental:
+                    doc_data[incremental.cursor_path] = convert_elasticsearch_objs(
+                        doc_data[incremental.cursor_path]
+                    )
+                yield doc_data
+
+        client.clear_scroll(scroll_id=sid)
+
+    return get_documents
+
+
+def convert_elasticsearch_objs(value: Any) -> Any:
+    if isinstance(value, str):
+        parsed_date = parse(value, strict=False)
+        if parsed_date is not None:
+            if isinstance(
+                parsed_date,
+                (pendulum.DateTime, pendulum.Date, datetime, date, str, float, int),
+            ):
+                return ensure_pendulum_datetime(parsed_date)
+    return value

ingestr/src/factory.py

@@ -28,6 +28,7 @@ from ingestr.src.sources import (
     AsanaSource,
     ChessSource,
     DynamoDBSource,
+    ElasticsearchSource,
     FacebookAdsSource,
     FrankfurterSource,
     FreshdeskSource,
@@ -153,6 +154,7 @@ class SourceDestinationFactory:
         "frankfurter": FrankfurterSource,
         "freshdesk": FreshdeskSource,
         "phantombuster": PhantombusterSource,
+        "elasticsearch": ElasticsearchSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/sources.py

@@ -2298,3 +2298,53 @@ class PhantombusterSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table_name)
+
+
+class ElasticsearchSource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        from ingestr.src.elasticsearch import elasticsearch_source
+
+        incremental = None
+        if kwargs.get("incremental_key"):
+            start_value = kwargs.get("interval_start")
+            end_value = kwargs.get("interval_end")
+
+            incremental = dlt_incremental(
+                kwargs.get("incremental_key", ""),
+                initial_value=start_value,
+                end_value=end_value,
+                range_end="closed",
+                range_start="closed",
+            )
+
+        # elasticsearch://localhost:9200?secure=true&verify_certs=false
+        parsed = urlparse(uri)
+
+        index = table
+        if not index:
+            raise ValueError("Table name must be provided which is the index name in elasticsearch")
+
+        query_params = parsed.query
+        params = parse_qs(query_params)
+
+        secure = True
+        if "secure" in params:
+            secure = params["secure"][0].capitalize() == "True"
+
+        verify_certs = True
+        if "verify_certs" in params:
+            verify_certs = params["verify_certs"][0].capitalize() == "True"
+
+        scheme = "https" if secure else "http"
+        netloc = parsed.netloc
+        connection_url = f"{scheme}://{netloc}"
+
+        return elasticsearch_source(
+            connection_url=connection_url,
+            index=index,
+            verify_certs=verify_certs,
+            incremental=incremental,
+        ).with_resources(table)

{ingestr-0.13.39.dist-info → ingestr-0.13.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.39
+Version: 0.13.40
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -50,6 +50,8 @@ Requires-Dist: dlt==1.10.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1
+Requires-Dist: elastic-transport==8.17.1
+Requires-Dist: elasticsearch==8.10.1
 Requires-Dist: et-xmlfile==2.0.0
 Requires-Dist: facebook-business==20.0.0
 Requires-Dist: filelock==3.17.0

{ingestr-0.13.39.dist-info → ingestr-0.13.40.dist-info}/RECORD

@@ -2,15 +2,15 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=Pe_rzwcDRKIYa7baEVUAAPOHyqQbX29RUexMl0F_S1k,25273
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=edyodue-Rkn4zTwWVR9OU0dSsDXVokKw_KKllrI1amM,21
+ingestr/src/buildinfo.py,sha256=kx4THAPAkQ2P32re7w8VrOWSESl3Fz3mnN83MSygHeE,21
 ingestr/src/destinations.py,sha256=MctbeJUyNr0DRB0XYt2xAbEKkHZ40-nXXEOYCs4KuoE,15420
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=j-FKRBEBZVLT_DEn-SCu9KEvaab3BchEV5hzTjpree8,5511
+ingestr/src/factory.py,sha256=x-Ym3uHMgzj_svUk7Lopn3Jj-IhcQLCuDqA_eUPFLAI,5582
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=vppNI75ucM0EtW2kP5ldKyhc4Pij_hGVmKlZ9DNL4g0,79181
+ingestr/src/sources.py,sha256=RitbAjFVnq1I7MsjbD7hrn6Akd_92P6OCEg--YHivDw,80770
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -35,6 +35,7 @@ ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
 ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
 ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
+ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
 ingestr/src/facebook_ads/__init__.py,sha256=reEpSr4BaKA1wO3qVgCH51gW-TgWkbJ_g24UIhJWbac,9286
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
 ingestr/src/facebook_ads/helpers.py,sha256=ZLbNHiKer5lPb4g3_435XeBJr57Wv0o1KTyBA1mQ100,9068
@@ -127,8 +128,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.39.dist-info/METADATA,sha256=goY5MW5AzJwYQ0cbwTmlNxZgCP1QRSt6ROmBOImESIM,13575
-ingestr-0.13.39.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.39.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.39.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.39.dist-info/RECORD,,
+ingestr-0.13.40.dist-info/METADATA,sha256=DV_PkyMFlK4isa37puXrTKAfFPb4oQ4_cKv-b1lojI4,13653
+ingestr-0.13.40.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.40.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.40.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.40.dist-info/RECORD,,