PyPI - airbyte-source-iterable - Versions diffs - 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl - Mend

airbyte-source-iterable 0.2.1py3-none-any.whl → 0.3.0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (23) hide show

airbyte_source_iterable-0.3.0.dist-info/METADATA +113 -0
{airbyte_source_iterable-0.2.1.dist-info → airbyte_source_iterable-0.3.0.dist-info}/RECORD +9 -26
{airbyte_source_iterable-0.2.1.dist-info → airbyte_source_iterable-0.3.0.dist-info}/WHEEL +1 -2
airbyte_source_iterable-0.3.0.dist-info/entry_points.txt +3 -0
airbyte_source_iterable-0.2.1.dist-info/METADATA +0 -118
airbyte_source_iterable-0.2.1.dist-info/entry_points.txt +0 -2
airbyte_source_iterable-0.2.1.dist-info/top_level.txt +0 -3
integration_tests/__init__.py +0 -0
integration_tests/abnormal_state.json +0 -74
integration_tests/acceptance.py +0 -13
integration_tests/catalog.json +0 -186
integration_tests/configured_catalog.json +0 -186
integration_tests/configured_catalog_additional_events.json +0 -291
integration_tests/invalid_config.json +0 -4
unit_tests/__init__.py +0 -3
unit_tests/conftest.py +0 -49
unit_tests/test_export_adjustable_range.py +0 -117
unit_tests/test_exports_stream.py +0 -35
unit_tests/test_slice_generator.py +0 -94
unit_tests/test_source.py +0 -28
unit_tests/test_stream_events.py +0 -202
unit_tests/test_streams.py +0 -270
unit_tests/test_utils.py +0 -12

integration_tests/configured_catalog.json DELETED Viewed

@@ -1,186 +0,0 @@
-{
-  "streams": [
-    {
-      "stream": {
-        "name": "campaigns",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "campaigns_metrics",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "channels",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "email_bounce",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "email_click",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "email_complaint",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "email_open",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "email_send",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "email_send_skip",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "email_subscribe",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "email_unsubscribe",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "events",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "lists",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "list_users",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "message_types",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "metadata",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
-    {
-      "stream": {
-        "name": "users",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["profileUpdatedAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "templates",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    }
-  ]
-}

integration_tests/configured_catalog_additional_events.json DELETED Viewed

@@ -1,291 +0,0 @@
-{
-  "streams": [
-    {
-      "stream": {
-        "name": "push_send",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "push_send_skip",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "push_open",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "push_uninstall",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "push_bounce",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "web_push_send",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "web_push_click",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "web_push_send_skip",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "in_app_send",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "in_app_open",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "in_app_click",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "in_app_close",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "in_app_delete",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "in_app_delivery",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "in_app_send_skip",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "inbox_session",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "inbox_message_impression",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "sms_send",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "sms_bounce",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "sms_click",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "sms_received",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "sms_send_skip",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "sms_usage_info",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "purchase",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "custom_event",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "hosted_unsubscribe_click",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["createdAt"]
-      },
-      "sync_mode": "incremental",
-      "destination_sync_mode": "append"
-    }
-  ]
-}

integration_tests/invalid_config.json DELETED Viewed

@@ -1,4 +0,0 @@
-{
-  "api_key": "test-api-key",
-  "start_date": "2020-12-12T00:00:00Z"
-}

unit_tests/__init__.py DELETED Viewed

@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#

unit_tests/conftest.py DELETED Viewed

@@ -1,49 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-import pytest
-from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream
-@pytest.fixture(autouse=True)
-def disable_cache(mocker):
-    mocker.patch("source_iterable.streams.ListUsers.use_cache", False)
-@pytest.fixture
-def catalog(request):
-    return ConfiguredAirbyteCatalog(
-        streams=[
-            ConfiguredAirbyteStream(
-                stream=AirbyteStream(name=request.param, json_schema={}, supported_sync_modes=["full_refresh"]),
-                sync_mode="full_refresh",
-                destination_sync_mode="append",
-            )
-        ]
-    )
-@pytest.fixture(name="config")
-def config_fixture():
-    return {"api_key": 123, "start_date": "2019-10-10T00:00:00"}
-@pytest.fixture()
-def mock_lists_resp(mocker):
-    mocker.patch("source_iterable.streams.Lists.read_records", return_value=iter([{"id": 1}, {"id": 2}]))
-@pytest.fixture(name="lists_stream")
-def lists_stream():
-    # local imports
-    from source_iterable.streams import Lists
-    # return the instance of the stream so we could make global tests on it,
-    # to cover the different `should_retry` logic
-    return Lists(authenticator=None)
-@pytest.fixture(autouse=True)
-def mock_sleep(mocker):
-    mocker.patch("time.sleep")

unit_tests/test_export_adjustable_range.py DELETED Viewed

@@ -1,117 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-import datetime
-import json
-import urllib.parse
-from typing import List
-from unittest import mock
-import freezegun
-import pendulum
-import pytest
-import responses
-from airbyte_cdk.models import Type as MessageType
-from requests.exceptions import ChunkedEncodingError
-from source_iterable.slice_generators import AdjustableSliceGenerator
-from source_iterable.source import SourceIterable
-TEST_START_DATE = "2020"
-@pytest.fixture
-def time_mock(request):
-    with freezegun.freeze_time() as time_mock:
-        yield time_mock
-def get_range_days_from_request(request):
-    query = urllib.parse.urlsplit(request.url).query
-    query = urllib.parse.parse_qs(query)
-    return (pendulum.parse(query["endDateTime"][0]) - pendulum.parse(query["startDateTime"][0])).days
-@mock.patch("logging.getLogger", mock.MagicMock())
-def read_from_source(catalog):
-    return list(
-        SourceIterable().read(
-            mock.MagicMock(),
-            {"start_date": TEST_START_DATE, "api_key": "api_key"},
-            catalog,
-            None,
-        )
-    )
-@responses.activate
-@pytest.mark.parametrize("catalog", (["email_send"]), indirect=True)
-def test_email_stream(mock_lists_resp, catalog, time_mock):
-    DAYS_DURATION = 100
-    DAYS_PER_MINUTE_RATE = 8
-    time_mock.move_to(pendulum.parse(TEST_START_DATE) + pendulum.Duration(days=DAYS_DURATION))
-    ranges: List[int] = []
-    def response_cb(req):
-        days = get_range_days_from_request(req)
-        ranges.append(days)
-        time_mock.tick(delta=datetime.timedelta(minutes=days / DAYS_PER_MINUTE_RATE))
-        return (200, {}, json.dumps({"createdAt": "2020"}))
-    responses.add(responses.GET, "https://api.iterable.com/api/lists/getUsers?listId=1", json={"lists": [{"id": 1}]}, status=200)
-    responses.add_callback("GET", "https://api.iterable.com/api/export/data.json", callback=response_cb)
-    records = read_from_source(catalog)
-    assert records
-    assert sum(ranges) == DAYS_DURATION
-    # since read is called on source instance, under the hood .streams() is called which triggers one more http call
-    assert len(responses.calls) == len(ranges) + 1
-    assert ranges == [
-        AdjustableSliceGenerator.INITIAL_RANGE_DAYS,
-        *([int(DAYS_PER_MINUTE_RATE / AdjustableSliceGenerator.REQUEST_PER_MINUTE_LIMIT)] * 35),
-    ]
-@responses.activate
-@pytest.mark.parametrize(
-    "catalog, days_duration, days_per_minute_rate",
-    [
-        ("email_send", 10, 200),
-        ("email_send", 100, 200000),
-        ("email_send", 10000, 200000),
-        ("email_click", 1000, 20),
-        ("email_open", 1000, 1),
-        ("email_open", 1, 1000),
-        ("email_open", 0, 1000000),
-    ],
-    indirect=["catalog"],
-)
-def test_email_stream_chunked_encoding(mocker, mock_lists_resp, catalog, days_duration, days_per_minute_rate, time_mock):
-    mocker.patch("time.sleep")
-    time_mock.move_to(pendulum.parse(TEST_START_DATE) + pendulum.Duration(days=days_duration))
-    ranges: List[int] = []
-    encoding_throw = 0
-    def response_cb(req):
-        nonlocal encoding_throw
-        # Every request fails with 2 ChunkedEncodingError exception but works well on third time.
-        if encoding_throw < 2:
-            encoding_throw += 1
-            raise ChunkedEncodingError()
-        encoding_throw = 0
-        days = get_range_days_from_request(req)
-        ranges.append(days)
-        time_mock.tick(delta=datetime.timedelta(minutes=days / days_per_minute_rate))
-        return (200, {}, json.dumps({"createdAt": "2020"}))
-    responses.add(responses.GET, "https://api.iterable.com/api/lists/getUsers?listId=1", json={"lists": [{"id": 1}]}, status=200)
-    responses.add_callback("GET", "https://api.iterable.com/api/export/data.json", callback=response_cb)
-    # added condition because read_from_source also returns LOG messages
-    records = [record for record in read_from_source(catalog) if record.type == MessageType.RECORD]
-    assert sum(ranges) == days_duration
-    assert len(ranges) == len(records)
-    # since read is called on source instance, under the hood .streams() is called which triggers one more http call
-    assert len(responses.calls) == 3 * len(ranges) + 1

unit_tests/test_exports_stream.py DELETED Viewed

@@ -1,35 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-import json
-from unittest import mock
-import pendulum
-import pytest
-import responses
-from airbyte_cdk.models import SyncMode
-from source_iterable.slice_generators import StreamSlice
-from source_iterable.streams import Users
-@pytest.fixture
-def session_mock():
-    with mock.patch("airbyte_cdk.sources.streams.http.http.requests") as requests_mock:
-        session_mock = mock.MagicMock()
-        response_mock = mock.MagicMock()
-        requests_mock.Session.return_value = session_mock
-        session_mock.send.return_value = response_mock
-        response_mock.status_code = 200
-        yield session_mock
-@responses.activate
-def test_stream_correct():
-    stream_slice = StreamSlice(start_date=pendulum.parse("2020"), end_date=pendulum.parse("2021"))
-    record_js = {"profileUpdatedAt": "2020"}
-    NUMBER_OF_RECORDS = 10**2
-    resp_body = "\n".join([json.dumps(record_js)] * NUMBER_OF_RECORDS)
-    responses.add("GET", "https://api.iterable.com/api/export/data.json", body=resp_body)
-    stream = Users(start_date="2020", authenticator=None)
-    records = list(stream.read_records(sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=stream_slice, stream_state={}))
-    assert len(records) == NUMBER_OF_RECORDS

airbyte-source-iterable 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

airbyte-source-iterable 0.2.1py3-none-any.whl → 0.3.0py3-none-any.whl