ingestr 0.6.6__py3-none-any.whl → 0.7.1__py3-none-any.whl

ingestr/src/destinations_test.py

@@ -1,113 +0,0 @@
-import json
-import os
-import unittest
-
-import dlt
-import pytest
-
-from ingestr.src.destinations import (
-    BigQueryDestination,
-    DatabricksDestination,
-    DuckDBDestination,
-    MsSQLDestination,
-    PostgresDestination,
-    RedshiftDestination,
-    SnowflakeDestination,
-)
-
-
-class BigQueryDestinationTest(unittest.TestCase):
-    destination = BigQueryDestination()
-    abs_path_to_credentials = os.path.abspath(
-        os.path.join(os.path.dirname(__file__), "./testdata/fakebqcredentials.json")
-    )
-    actual_credentials: dict = {}
-
-    def setUp(self):
-        with open(self.abs_path_to_credentials, "r") as f:
-            self.actual_credentials = json.load(f)
-
-    def test_bq_destination_cred_path_required(self):
-        with pytest.raises(ValueError):
-            uri = "bigquery://my-project"
-            self.destination.dlt_dest(uri)
-
-    def test_bq_destination_simple_uri(self):
-        uri = f"bigquery://my-project?credentials_path={self.abs_path_to_credentials}"
-        result = self.destination.dlt_dest(uri)
-
-        self.assertTrue(isinstance(result, dlt.destinations.bigquery))
-        self.assertEqual(result.config_params["credentials"], self.actual_credentials)
-        self.assertTrue("location" not in result.config_params)
-
-    def test_bq_destination_with_location(self):
-        uri = f"bigquery://my-project?credentials_path={self.abs_path_to_credentials}&location=EU"
-        result = self.destination.dlt_dest(uri)
-
-        self.assertTrue(isinstance(result, dlt.destinations.bigquery))
-        self.assertEqual(result.config_params["credentials"], self.actual_credentials)
-        self.assertEqual(result.config_params["location"], "EU")
-
-    def test_bq_destination_run_params_require_two_or_three_fields(self):
-        with pytest.raises(ValueError):
-            self.destination.dlt_run_params("", "sometable")
-
-        with pytest.raises(ValueError):
-            self.destination.dlt_run_params("", "sometable.with.extra.fields")
-
-    def test_bq_destination_run_params_parse_table_names_correctly(self):
-        result = self.destination.dlt_run_params("", "dataset.sometable")
-        self.assertEqual(result, {"dataset_name": "dataset", "table_name": "sometable"})
-
-        result = self.destination.dlt_run_params("", "project.dataset.sometable")
-        self.assertEqual(result, {"dataset_name": "dataset", "table_name": "sometable"})
-
-
-class GenericSqlDestinationFixture(object):
-    def test_credentials_are_passed_correctly(self):
-        uri = "some-uri"
-        result = self.destination.dlt_dest(uri)
-
-        self.assertTrue(isinstance(result, self.expected_class))
-        self.assertEqual(result.config_params["credentials"], uri)
-
-    def test_destination_run_params_require_two_fields(self):
-        with pytest.raises(ValueError):
-            self.destination.dlt_run_params("", "sometable")
-
-        with pytest.raises(ValueError):
-            self.destination.dlt_run_params("", "sometable.with.extra")
-
-    def test_destination_run_params_parse_table_names_correctly(self):
-        result = self.destination.dlt_run_params("", "dataset.sometable")
-        self.assertEqual(result, {"dataset_name": "dataset", "table_name": "sometable"})
-
-
-class PostgresDestinationTest(unittest.TestCase, GenericSqlDestinationFixture):
-    destination = PostgresDestination()
-    expected_class = dlt.destinations.postgres
-
-
-class SnowflakeDestinationTest(unittest.TestCase, GenericSqlDestinationFixture):
-    destination = SnowflakeDestination()
-    expected_class = dlt.destinations.snowflake
-
-
-class RedshiftDestinationTest(unittest.TestCase, GenericSqlDestinationFixture):
-    destination = RedshiftDestination()
-    expected_class = dlt.destinations.redshift
-
-
-class DuckDBDestinationTest(unittest.TestCase, GenericSqlDestinationFixture):
-    destination = DuckDBDestination()
-    expected_class = dlt.destinations.duckdb
-
-
-class MsSQLDestinationTest(unittest.TestCase, GenericSqlDestinationFixture):
-    destination = MsSQLDestination()
-    expected_class = dlt.destinations.mssql
-
-
-class DatabricksDestinationTest(unittest.TestCase, GenericSqlDestinationFixture):
-    destination = DatabricksDestination()
-    expected_class = dlt.destinations.databricks

ingestr/src/factory_test.py

@@ -1,13 +0,0 @@
-from ingestr.src.factory import parse_scheme_from_uri
-
-
-def test_scheme_is_parsed_from_uri_correctly():
-    assert parse_scheme_from_uri("bigquery://my-project") == "bigquery"
-    assert parse_scheme_from_uri("http://localhost:8080") == "http"
-    assert parse_scheme_from_uri("file:///tmp/myfile") == "file"
-    assert parse_scheme_from_uri("https://example.com?query=123") == "https"
-    assert parse_scheme_from_uri("ftp://ftp.example.com/downloads/file.zip") == "ftp"
-    assert (
-        parse_scheme_from_uri("redshift+psycopg2://user:pw@host") == "redshift+psycopg2"
-    )
-    assert parse_scheme_from_uri("mysql+pymysql://user:pw@host") == "mysql+pymysql"

ingestr/src/gorgias/helpers_test.py

@@ -1,45 +0,0 @@
-from dlt.common.pendulum import pendulum
-
-from .helpers import convert_datetime_fields, find_latest_timestamp_from_page
-
-
-def test_convert_datetime_fields():
-    item = {
-        "key1": "val1",
-        "created_datetime": "2024-06-20T07:39:36.514848+00:00",
-        "sent_datetime": "2024-06-20T07:40:20.166593+00:00",
-        "should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
-    }
-
-    actual = convert_datetime_fields(item)
-
-    assert actual == {
-        "key1": "val1",
-        "created_datetime": pendulum.datetime(2024, 6, 20, 7, 39, 36, 514848, tz="UTC"),
-        "sent_datetime": pendulum.datetime(2024, 6, 20, 7, 40, 20, 166593, tz="UTC"),
-        "should_send_datetime": pendulum.datetime(
-            2024, 6, 20, 7, 39, 37, 514848, tz="UTC"
-        ),
-        "updated_datetime": pendulum.datetime(2024, 6, 20, 7, 40, 20, 166593, tz="UTC"),
-    }
-
-
-def test_find_latest_timestamp_from_page():
-    items = [
-        {
-            "key1": "val1",
-            "created_datetime": "2024-06-20T07:39:36.514848+00:00",
-            "sent_datetime": "2024-06-20T07:40:20.166593+00:00",
-            "should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
-        },
-        {
-            "key1": "val2",
-            "created_datetime": "2024-06-20T07:39:36.514848+00:00",
-            "sent_datetime": "2024-06-20T07:40:21.123123+00:00",
-            "should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
-        },
-    ]
-
-    actual = find_latest_timestamp_from_page(items)
-
-    assert actual == pendulum.datetime(2024, 6, 20, 7, 40, 21, 123123, tz="UTC")

ingestr/src/sources_test.py

@@ -1,96 +0,0 @@
-import unittest
-
-import dlt
-import pytest
-
-from ingestr.src.sources import MongoDbSource, SqlSource
-
-
-class SqlSourceTest(unittest.TestCase):
-    def test_sql_source_requires_two_fields_in_table(self):
-        source = SqlSource()
-        with pytest.raises(ValueError):
-            uri = "bigquery://my-project"
-            source.dlt_source(uri, "onetable")
-
-    def test_table_instance_is_created(self):
-        uri = "bigquery://my-project"
-        table = "schema.table"
-
-        # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key, backend):
-            self.assertEqual(credentials, uri)
-            self.assertEqual(schema, "schema")
-            self.assertEqual(table, "table")
-            self.assertEqual(backend, "sqlalchemy")
-            self.assertIsNone(incremental)
-            self.assertIsNone(merge_key)
-            return dlt.resource()
-
-        source = SqlSource(table_builder=sql_table)
-        res = source.dlt_source(uri, table)
-        self.assertIsNotNone(res)
-
-    def test_table_instance_is_created_with_incremental(self):
-        uri = "bigquery://my-project"
-        table = "schema.table"
-        incremental_key = "id"
-
-        # monkey patch the sql_table function
-        def sql_table(credentials, schema, table, incremental, merge_key, backend):
-            self.assertEqual(credentials, uri)
-            self.assertEqual(schema, "schema")
-            self.assertEqual(table, "table")
-            self.assertEqual(backend, "sqlalchemy")
-            self.assertIsInstance(incremental, dlt.sources.incremental)
-            self.assertEqual(incremental.cursor_path, incremental_key)
-            self.assertIsNone(merge_key)
-            return dlt.resource()
-
-        source = SqlSource(table_builder=sql_table)
-        res = source.dlt_source(uri, table, incremental_key=incremental_key)
-        self.assertIsNotNone(res)
-
-
-class MongoDbSourceTest(unittest.TestCase):
-    def test_sql_source_requires_two_fields_in_table(self):
-        source = MongoDbSource()
-        with pytest.raises(ValueError):
-            uri = "mongodb://my-project"
-            source.dlt_source(uri, "onetable")
-
-    def test_table_instance_is_created(self):
-        uri = "mongodb://my-project"
-        table = "schema.table"
-
-        # monkey patch the mongo function
-        def mongo(connection_url, database, collection, incremental, parallel):
-            self.assertEqual(connection_url, uri)
-            self.assertEqual(database, "schema")
-            self.assertEqual(collection, "table")
-            self.assertIsNone(incremental)
-            self.assertTrue(parallel)
-            return dlt.resource()
-
-        source = MongoDbSource(table_builder=mongo)
-        res = source.dlt_source(uri, table)
-        self.assertIsNotNone(res)
-
-    def test_table_instance_is_created_with_incremental(self):
-        uri = "mongodb://my-project"
-        table = "schema.table"
-        incremental_key = "id"
-
-        # monkey patch the mongo function
-        def mongo(connection_url, database, collection, incremental, parallel):
-            self.assertEqual(connection_url, uri)
-            self.assertEqual(database, "schema")
-            self.assertEqual(collection, "table")
-            self.assertIsInstance(incremental, dlt.sources.incremental)
-            self.assertEqual(incremental.cursor_path, incremental_key)
-            self.assertTrue(parallel)
-            return dlt.resource()
-
-        source = MongoDbSource(table_builder=mongo)
-        res = source.dlt_source(uri, table, incremental_key=incremental_key)
-        self.assertIsNotNone(res)