ingestr 0.13.62__py3-none-any.whl → 0.13.64__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.62"
+version = "v0.13.64"

ingestr/src/destinations.py

@@ -1,19 +1,31 @@
 import abc
 import base64
 import csv
+import datetime
 import json
 import os
 import shutil
+import struct
 import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
 import dlt.destinations.impl.filesystem.filesystem
 from dlt.common.configuration.specs import AwsCredentials
+from dlt.common.destination.capabilities import DestinationCapabilitiesContext
+from dlt.common.schema import Schema
 from dlt.common.storages.configuration import FileSystemCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
 )
+from dlt.destinations.impl.mssql.configuration import MsSqlClientConfiguration
+from dlt.destinations.impl.mssql.mssql import (
+    HINT_TO_MSSQL_ATTR,
+    MsSqlJobClient,
+)
+from dlt.destinations.impl.mssql.sql_client import (
+    PyOdbcMsSqlClient,
+)
 
 from ingestr.src.errors import MissingValueError
 from ingestr.src.loader import load_dlt_file
@@ -143,9 +155,89 @@ class DuckDBDestination(GenericSqlDestination):
         return dlt.destinations.duckdb(uri, **kwargs)
 
 
+def handle_datetimeoffset(dto_value: bytes) -> datetime.datetime:
+    # ref: https://github.com/mkleehammer/pyodbc/issues/134#issuecomment-281739794
+    tup = struct.unpack(
+        "<6hI2h", dto_value
+    )  # e.g., (2017, 3, 16, 10, 35, 18, 500000000, -6, 0)
+    return datetime.datetime(
+        tup[0],
+        tup[1],
+        tup[2],
+        tup[3],
+        tup[4],
+        tup[5],
+        tup[6] // 1000,
+        datetime.timezone(datetime.timedelta(hours=tup[7], minutes=tup[8])),
+    )
+
+
+class OdbcMsSqlClient(PyOdbcMsSqlClient):
+    SQL_COPT_SS_ACCESS_TOKEN = 1256
+    SKIP_CREDENTIALS = {"PWD", "AUTHENTICATION", "UID"}
+
+    def open_connection(self):
+        cfg = self.credentials._get_odbc_dsn_dict()
+        if (
+            cfg.get("AUTHENTICATION", "").strip().lower()
+            != "activedirectoryaccesstoken"
+        ):
+            return super().open_connection()
+
+        import pyodbc  # type: ignore
+
+        dsn = ";".join(
+            [f"{k}={v}" for k, v in cfg.items() if k not in self.SKIP_CREDENTIALS]
+        )
+
+        self._conn = pyodbc.connect(
+            dsn,
+            timeout=self.credentials.connect_timeout,
+            attrs_before={
+                self.SQL_COPT_SS_ACCESS_TOKEN: self.serialize_token(cfg["PWD"]),
+            },
+        )
+
+        # https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function
+        self._conn.add_output_converter(-155, handle_datetimeoffset)
+        self._conn.autocommit = True
+        return self._conn
+
+    def serialize_token(self, token):
+        # https://github.com/mkleehammer/pyodbc/issues/228#issuecomment-494773723
+        encoded = token.encode("utf_16_le")
+        return struct.pack("<i", len(encoded)) + encoded
+
+
+class MsSqlClient(MsSqlJobClient):
+    def __init__(
+        self,
+        schema: Schema,
+        config: MsSqlClientConfiguration,
+        capabilities: DestinationCapabilitiesContext,
+    ) -> None:
+        sql_client = OdbcMsSqlClient(
+            config.normalize_dataset_name(schema),
+            config.normalize_staging_dataset_name(schema),
+            config.credentials,
+            capabilities,
+        )
+        super(MsSqlJobClient, self).__init__(schema, config, sql_client)
+        self.config: MsSqlClientConfiguration = config
+        self.sql_client = sql_client
+        self.active_hints = HINT_TO_MSSQL_ATTR if self.config.create_indexes else {}
+        self.type_mapper = capabilities.get_type_mapper()
+
+
+class MsSqlDestImpl(dlt.destinations.mssql):
+    @property
+    def client_class(self):
+        return MsSqlClient
+
+
 class MsSQLDestination(GenericSqlDestination):
     def dlt_dest(self, uri: str, **kwargs):
-        return dlt.destinations.mssql(credentials=uri, **kwargs)
+        return MsSqlDestImpl(credentials=uri, **kwargs)
 
 
 class DatabricksDestination(GenericSqlDestination):

ingestr/src/factory.py

@@ -79,6 +79,7 @@ SQL_SOURCE_SCHEMES = [
     "crate",
     "duckdb",
     "mssql",
+    "mssql+pyodbc",
     "mysql",
     "mysql+pymysql",
     "mysql+mysqlconnector",

ingestr/src/sources.py

@@ -258,8 +258,53 @@ class SqlSource:
             # override the query adapters, the only one we want is the one here in the case of custom queries
             query_adapters = [custom_query_variable_subsitution(query_value, kwargs)]
 
+        credentials = ConnectionStringCredentials(uri)
+        if uri.startswith("mssql://"):
+            parsed_uri = urlparse(uri)
+            params = parse_qs(parsed_uri.query)
+            params = {k.lower(): v for k, v in params.items()}
+            if params.get("authentication") == ["ActiveDirectoryAccessToken"]:
+                import pyodbc  # type: ignore
+                from sqlalchemy import create_engine
+
+                from ingestr.src.destinations import (
+                    OdbcMsSqlClient,
+                    handle_datetimeoffset,
+                )
+
+                cfg = {
+                    "DRIVER": params.get("driver", ["ODBC Driver 18 for SQL Server"])[
+                        0
+                    ],
+                    "SERVER": f"{parsed_uri.hostname},{parsed_uri.port or 1433}",
+                    "DATABASE": parsed_uri.path.lstrip("/"),
+                }
+                for k, v in params.items():
+                    if k.lower() not in ["driver", "authentication", "connect_timeout"]:
+                        cfg[k.upper()] = v[0]
+
+                token = OdbcMsSqlClient.serialize_token(None, parsed_uri.password)  # type: ignore[arg-type]
+                dsn = ";".join([f"{k}={v}" for k, v in cfg.items()])
+
+                def creator():
+                    connection = pyodbc.connect(
+                        dsn,
+                        autocommit=True,
+                        timeout=kwargs.get("connect_timeout", 30),
+                        attrs_before={
+                            OdbcMsSqlClient.SQL_COPT_SS_ACCESS_TOKEN: token,
+                        },
+                    )
+                    connection.add_output_converter(-155, handle_datetimeoffset)
+                    return connection
+
+                credentials = create_engine(
+                    "mssql+pyodbc://",
+                    creator=creator,
+                )
+
         builder_res = self.table_builder(
-            credentials=ConnectionStringCredentials(uri),
+            credentials=credentials,
             schema=table_fields.dataset,
             table=table_fields.table,
             incremental=incremental,
@@ -2812,7 +2857,7 @@ class IsocPulseSource:
         src = pulse_source(
             token=token[0],
             start_date=start_date.strftime("%Y-%m-%d"),
-            end_date=str(end_date) if end_date else None,
+            end_date=end_date.strftime("%Y-%m-%d") if end_date else None,
             metric=metric,
             opts=opts,
         )
@@ -2915,7 +2960,7 @@ class ZoomSource:
 
         from ingestr.src.zoom import zoom_source
 
-        if table not in {"meetings"}:
+        if table not in {"meetings", "users", "participants"}:
             raise UnsupportedResourceError(table, "Zoom")
 
         return zoom_source(

ingestr/src/zoom/__init__.py

@@ -42,14 +42,58 @@ def zoom_source(
             end_dt = pendulum.now("UTC")
         else:
             end_dt = pendulum.parse(datetime.end_value)
+        
         base_params: Dict[str, Any] = {
             "type": "scheduled",
             "page_size": 300,
             "from": start_dt.to_date_string(),
             "to": end_dt.to_date_string(),
         }
+        
         for user in client.get_users():
             user_id = user["id"]
             yield from client.get_meetings(user_id, base_params)
 
-    return meetings
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def users() -> Iterable[TDataItem]:
+        yield from client.get_users()
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def participants(
+        datetime: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental(
+            "join_time",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date is not None else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.last_value:
+            start_dt = pendulum.parse(datetime.last_value)
+        else:
+            start_dt = pendulum.parse(start_date)
+
+        if end_date is None:
+            end_dt = pendulum.now("UTC")
+        else:
+            end_dt = pendulum.parse(datetime.end_value)
+
+        participant_params: Dict[str, Any] = {
+            "page_size": 300,
+        }
+        meeting_params = {
+            "type": "previous_meetings",
+            "page_size": 300,
+        }
+        for user in client.get_users():
+            user_id = user["id"]
+            for meeting in client.get_meetings(user_id=user_id, params=meeting_params):
+                meeting_id = meeting["id"]
+                yield from client.get_participants(
+                    meeting_id=meeting_id,
+                    params=participant_params,
+                    start_date=start_dt,
+                    end_date=end_dt,
+                )
+
+    return meetings, users, participants

ingestr/src/zoom/helpers.py

@@ -1,6 +1,8 @@
 import time
 from typing import Any, Dict, Iterator, Optional
 
+import pendulum
+
 from ingestr.src.http_client import create_client
 
 
@@ -47,6 +49,7 @@ class ZoomClient:
 
     def get_users(self) -> Iterator[Dict[str, Any]]:
         url = f"{self.base_url}/users"
+
         params = {"page_size": 1000}
         while True:
             response = self.session.get(url, headers=self._headers(), params=params)
@@ -59,6 +62,7 @@ class ZoomClient:
                 break
             params["next_page_token"] = token
 
+#https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/meetings
     def get_meetings(
         self, user_id: str, params: Dict[str, Any]
     ) -> Iterator[Dict[str, Any]]:
@@ -74,3 +78,25 @@ class ZoomClient:
             if not token:
                 break
             params["next_page_token"] = token
+
+    # https://developers.zoom.us/docs/api/rest/reference/zoom-api/methods/#operation/reportMeetingParticipants
+    def get_participants(
+        self,
+        meeting_id: str,
+        params: Dict[str, Any],
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+    ) -> Iterator[Dict[str, Any]]:
+        url = f"{self.base_url}/report/meetings/{meeting_id}/participants"
+        while True:
+            response = self.session.get(url, headers=self._headers(), params=params)
+            response.raise_for_status()
+            data = response.json()
+            for item in data.get("participants", []):
+                join_time = pendulum.parse(item["join_time"])
+                if join_time >= start_date and join_time <= end_date:
+                    yield item
+            token = data.get("next_page_token")
+            if not token:
+                break
+            params["next_page_token"] = token

{ingestr-0.13.62.dist-info → ingestr-0.13.64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.62
+Version: 0.13.64
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.62.dist-info → ingestr-0.13.64.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=taDyHyaVSpB17iNLl8zA0gmr4CqDO-MSTQX1CaRBB9U,26364
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=TQVet9YuexpAbyia8_nOwytK_io6rEVmB-flmAr4z8E,21
-ingestr/src/destinations.py,sha256=YU7c5cNqViCIJ9NDhgGuorh5jMvLi7yViPWpEJ57Xx0,18788
+ingestr/src/buildinfo.py,sha256=PuIV-cer5xdwwdtvFJ-yA_CjyfoO-SckFPSRHGfPM4I,21
+ingestr/src/destinations.py,sha256=ZJTbTn1K9oXinL19dTGQDUrft5C9fjrpSlTw1CLQhuM,21749
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=nYXINj54QtMGuQv9oCXbHO7DFuAudqZfijkSvSXEVj8,6526
+ingestr/src/factory.py,sha256=AJCvlK4M1sIpAAks1K-xsR_uxziIxru74mj572zixhg,6546
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=nTzCi_RZhUesBkXyUTAIoAUgj5iZSQJ_D8mId8rq8mE,101467
+ingestr/src/sources.py,sha256=K554sV-RW29vj0c84r_rR0yZTl5HVvQ9vqbQOkJ9E7M,103386
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -135,8 +135,8 @@ ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1
 ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
 ingestr/src/zendesk/helpers/credentials.py,sha256=EWyi0ZlxWFgd1huD86KNF4dApLHgmabqWksFpEg1cf0,1332
 ingestr/src/zendesk/helpers/talk_api.py,sha256=TSVSOErsBZvxcX91LMhAgvy6yLSYvpuVfOyKViOHtvA,4718
-ingestr/src/zoom/__init__.py,sha256=6NdHXLv438FLD-cUEgmgnuJi__70-C88bTGOLqYHqbQ,1736
-ingestr/src/zoom/helpers.py,sha256=Y6fjIpQTWUAkKXJKfwjJnZc6wlFlqzJfWqq34WZhrcU,2669
+ingestr/src/zoom/__init__.py,sha256=xkkqeWeNkD3K45j1WvCrn0YAwA1U04z01193YJMcqwU,3280
+ingestr/src/zoom/helpers.py,sha256=63DypaYdeEUpq7W9AJkKRVpE0z0YmlzOZP-9c1La6cc,3710
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
 ingestr/testdata/create_replace.csv,sha256=TQDbOSkRKq9ZZv1d68Qjwh94aIyUQ-oEwxpJIrd3YK8,1060
 ingestr/testdata/delete_insert_expected.csv,sha256=wbj7uboVWwm3sNMh1n7f4-OKFEQJv1s96snjEHp9nkg,336
@@ -146,8 +146,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.62.dist-info/METADATA,sha256=GdxhHlw_v3U_5vL1ZTbDFxRMr7RLBnWDA-dVwG74OtQ,15027
-ingestr-0.13.62.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.62.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.62.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.62.dist-info/RECORD,,
+ingestr-0.13.64.dist-info/METADATA,sha256=QaBM1vQbQPYKgzDga2AY6uKibQbd1SkavE3dw92Pw-o,15027
+ingestr-0.13.64.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.64.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.64.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.64.dist-info/RECORD,,