airbyte-source-iterable 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

unit_tests/test_slice_generator.py

@@ -1,94 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import freezegun
-import pendulum
-import pytest
-from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator
-
-TEST_DATE = pendulum.parse("2020-01-01")
-
-
-def test_slice_gen():
-    start_date = TEST_DATE
-    generator = AdjustableSliceGenerator(start_date)
-    dates = []
-    for i in generator:
-        dates.append(i)
-        generator.adjust_range(pendulum.Period(start=start_date, end=start_date))
-    assert dates
-    days = [(slice.end_date - slice.start_date).total_days() for slice in dates]
-    assert days[1] == AdjustableSliceGenerator.DEFAULT_RANGE_DAYS
-
-
-@freezegun.freeze_time(TEST_DATE + pendulum.Duration(days=1000))
-def test_slice_gen_no_range_adjust():
-    start_date = TEST_DATE
-    generator = AdjustableSliceGenerator(start_date)
-    dates = []
-    for i in generator:
-        dates.append(i)
-    assert dates
-    days = [(slice.end_date - slice.start_date).total_days() for slice in dates]
-    assert days
-    assert days[1] == AdjustableSliceGenerator.MAX_RANGE_DAYS
-
-
-@pytest.mark.parametrize(
-    "start_day,end_day,days,range",
-    [
-        (
-            "2020-01-01",
-            "2020-01-10",
-            5,
-            [
-                (pendulum.parse("2020-01-01"), pendulum.parse("2020-01-06")),
-                (pendulum.parse("2020-01-06"), pendulum.parse("2020-01-10")),
-            ],
-        ),
-        (
-            "2020-01-01",
-            "2020-01-10 20:00:12",
-            5,
-            [
-                (pendulum.parse("2020-01-01"), pendulum.parse("2020-01-06")),
-                (pendulum.parse("2020-01-06"), pendulum.parse("2020-01-10 20:00:12")),
-            ],
-        ),
-        (
-            "2020-01-01",
-            "2020-01-01 20:00:12",
-            5,
-            [
-                (pendulum.parse("2020-01-01"), pendulum.parse("2020-01-01 20:00:12")),
-            ],
-        ),
-        (
-            "2020-01-01",
-            "2020-01-10",
-            50,
-            [(pendulum.parse("2020-01-01"), pendulum.parse("2020-01-10"))],
-        ),
-        (
-            "2020-01-01",
-            "2020-01-01",
-            50,
-            [],
-        ),
-    ],
-)
-def test_datetime_ranges(start_day, end_day, days, range):
-    start_day = pendulum.parse(start_day)
-    with freezegun.freeze_time(end_day):
-        end_day = pendulum.parse(end_day)
-        RangeSliceGenerator.RANGE_LENGTH_DAYS = days
-        generator = RangeSliceGenerator(start_day)
-        assert [(slice.start_date, slice.end_date) for slice in generator] == range
-
-
-def test_datetime_wrong_range():
-    start_day = pendulum.parse("2020")
-    end_day = pendulum.parse("2000")
-    with pytest.raises(StopIteration):
-        next(RangeSliceGenerator.make_datetime_ranges(start_day, end_day, 1))

unit_tests/test_source.py

@@ -1,28 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from unittest.mock import MagicMock, patch
-
-import pytest
-import responses
-from source_iterable.source import SourceIterable
-from source_iterable.streams import Lists
-
-
-@responses.activate
-@pytest.mark.parametrize("body, status, expected_streams", ((b"", 401, 7), (b"alpha@gmail.com\nbeta@gmail.com", 200, 44)))
-def test_source_streams(mock_lists_resp, config, body, status, expected_streams):
-    responses.add(responses.GET, "https://api.iterable.com/api/lists/getUsers?listId=1", body=body, status=status)
-    streams = SourceIterable().streams(config=config)
-    assert len(streams) == expected_streams
-
-
-def test_source_check_connection_ok(config):
-    with patch.object(Lists, "read_records", return_value=iter([{"id": 1}])):
-        assert SourceIterable().check_connection(MagicMock(), config=config) == (True, None)
-
-
-def test_source_check_connection_failed(config):
-    with patch.object(Lists, "read_records", return_value=iter([])):
-        assert SourceIterable().check_connection(MagicMock(), config=config)[0] is False

unit_tests/test_stream_events.py

@@ -1,202 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-
-import pytest
-import requests
-import responses
-from source_iterable.streams import Events
-
-
-@responses.activate
-@pytest.mark.parametrize(
-    "response_objects,expected_objects,jsonl_body",
-    [
-        (
-            [
-                {
-                    "createdAt": "2021",
-                    "signupSource": "str",
-                    "emailListIds": [1],
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"},
-                    "_type": "str",
-                    "messageTypeIds": [],
-                    "channelIds": [],
-                    "email": "test@mail.com",
-                    "profileUpdatedAt": "2021",
-                },
-                {
-                    "productRecommendationCount": 1,
-                    "campaignId": 1,
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"},
-                    "contentId": 1,
-                    "_type": "1",
-                    "messageId": "1",
-                    "messageBusId": "1",
-                    "templateId": 1,
-                    "createdAt": "2021",
-                    "messageTypeId": 1,
-                    "catalogCollectionCount": 1,
-                    "catalogLookupCount": 0,
-                    "email": "test@mail.com",
-                    "channelId": 1,
-                },
-                {
-                    "createdAt": "2021",
-                    "campaignId": 1,
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"},
-                    "_type": "str",
-                    "messageId": "1",
-                    "templateId": 1,
-                    "recipientState": "str",
-                    "email": "test@mail.com",
-                },
-                {
-                    "unsubSource": "str",
-                    "createdAt": "2021",
-                    "emailListIds": [],
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"},
-                    "_type": "str",
-                    "messageId": "1",
-                    "messageTypeIds": [],
-                    "channelIds": [1],
-                    "templateId": 1,
-                    "recipientState": "str",
-                    "email": "test@mail.com",
-                },
-            ],
-            [],
-            False,
-        ),
-        (
-            [
-                {
-                    "createdAt": "2021",
-                    "signupSource": "str",
-                    "emailListIds": [1],
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"},
-                    "_type": "str",
-                    "messageTypeIds": [],
-                    "channelIds": [],
-                    "email": "test@mail.com",
-                    "profileUpdatedAt": "2021",
-                }
-            ],
-            [],
-            False,
-        ),
-        (
-            [
-                {
-                    "createdAt": "2021",
-                    "signupSource": "str",
-                    "emailListIds": [1],
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"},
-                    "_type": "str",
-                    "messageTypeIds": [],
-                    "channelIds": [],
-                    "email": "test@mail.com",
-                    "profileUpdatedAt": "2021",
-                },
-                {
-                    "productRecommendationCount": 1,
-                    "campaignId": 1,
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"},
-                    "contentId": 1,
-                    "_type": "1",
-                    "messageId": "1",
-                    "messageBusId": "1",
-                    "templateId": 1,
-                    "createdAt": "2021",
-                    "messageTypeId": 1,
-                    "catalogCollectionCount": 1,
-                    "catalogLookupCount": 0,
-                    "email": "test@mail.com",
-                    "channelId": 1,
-                },
-            ],
-            [
-                {
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"},
-                    "_type": "str",
-                    "createdAt": "2021",
-                    "email": "test@mail.com",
-                    "data": {
-                        "signupSource": "str",
-                        "emailListIds": [1],
-                        "messageTypeIds": [],
-                        "channelIds": [],
-                        "profileUpdatedAt": "2021",
-                    },
-                },
-                {
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"},
-                    "_type": "1",
-                    "createdAt": "2021",
-                    "email": "test@mail.com",
-                    "data": {
-                        "productRecommendationCount": 1,
-                        "campaignId": 1,
-                        "contentId": 1,
-                        "messageId": "1",
-                        "messageBusId": "1",
-                        "templateId": 1,
-                        "messageTypeId": 1,
-                        "catalogCollectionCount": 1,
-                        "catalogLookupCount": 0,
-                        "channelId": 1,
-                    },
-                },
-            ],
-            True,
-        ),
-        (
-            [
-                {
-                    "createdAt": "2021",
-                    "signupSource": "str",
-                    "emailListIds": [1],
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"},
-                    "_type": "str",
-                    "messageTypeIds": [],
-                    "channelIds": [],
-                    "email": "test@mail.com",
-                    "profileUpdatedAt": "2021",
-                }
-            ],
-            [
-                {
-                    "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"},
-                    "_type": "str",
-                    "createdAt": "2021",
-                    "email": "test@mail.com",
-                    "data": {
-                        "signupSource": "str",
-                        "emailListIds": [1],
-                        "messageTypeIds": [],
-                        "channelIds": [],
-                        "profileUpdatedAt": "2021",
-                    },
-                }
-            ],
-            True,
-        ),
-    ],
-)
-def test_events_parse_response(response_objects, expected_objects, jsonl_body):
-    if jsonl_body:
-        response_body = "\n".join([json.dumps(obj) for obj in response_objects])
-    else:
-        response_body = json.dumps(response_objects)
-    responses.add(responses.GET, "https://example.com", body=response_body)
-    response = requests.get("https://example.com")
-    stream = Events(authenticator=None)
-
-    if jsonl_body:
-        records = [record for record in stream.parse_response(response)]
-        assert records == expected_objects
-    else:
-        with pytest.raises(TypeError):
-            [record for record in stream.parse_response(response)]

unit_tests/test_streams.py

@@ -1,270 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-from unittest.mock import MagicMock
-
-import pendulum
-import pytest
-import requests
-import responses
-from airbyte_cdk.models import SyncMode
-from source_iterable.streams import (
-    Campaigns,
-    CampaignsMetrics,
-    Channels,
-    Events,
-    Lists,
-    ListUsers,
-    MessageTypes,
-    Metadata,
-    Templates,
-    Users,
-)
-from source_iterable.utils import dateutil_parse, read_full_refresh
-
-
-@pytest.mark.parametrize(
-    "stream,date,slice,expected_path",
-    [
-        (Lists, False, {}, "lists"),
-        (Campaigns, False, {}, "campaigns"),
-        (Channels, False, {}, "channels"),
-        (Events, False, {}, "export/userEvents"),
-        (MessageTypes, False, {}, "messageTypes"),
-        (Metadata, False, {}, "metadata"),
-        (ListUsers, False, {"list_id": 1}, "lists/getUsers?listId=1"),
-        (CampaignsMetrics, True, {}, "campaigns/metrics"),
-        (Templates, True, {}, "templates"),
-    ],
-)
-def test_path(config, stream, date, slice, expected_path):
-    args = {"authenticator": None}
-    if date:
-        args["start_date"] = "2019-10-10T00:00:00"
-
-    assert stream(**args).path(stream_slice=slice) == expected_path
-
-
-def test_campaigns_metrics_csv():
-    csv_string = "a,b,c,d\n1, 2,,3\n6,,1, 2\n"
-    output = [{"a": 1, "b": 2, "d": 3}, {"a": 6, "c": 1, "d": 2}]
-
-    assert CampaignsMetrics._parse_csv_string_to_dict(csv_string) == output
-
-
-@pytest.mark.parametrize(
-    "url,id",
-    [
-        ("http://google.com?listId=1&another=another", 1),
-        ("http://google.com?another=another", None),
-    ],
-)
-def test_list_users_get_list_id(url, id):
-    assert ListUsers._get_list_id(url) == id
-
-
-def test_campaigns_metrics_request_params():
-    stream = CampaignsMetrics(authenticator=None, start_date="2019-10-10T00:00:00")
-    params = stream.request_params(stream_slice={"campaign_ids": "c101"}, stream_state=None)
-    assert params == {"campaignId": "c101", "startDateTime": "2019-10-10T00:00:00"}
-
-
-def test_events_request_params():
-    stream = Events(authenticator=None)
-    params = stream.request_params(stream_slice={"email": "a@a.a"}, stream_state=None)
-    assert params == {"email": "a@a.a", "includeCustomEvents": "true"}
-
-
-def test_templates_parse_response():
-    stream = Templates(authenticator=None, start_date="2019-10-10T00:00:00")
-    with responses.RequestsMock() as rsps:
-        rsps.add(
-            responses.GET,
-            "https://api.iterable.com/api/1/foobar",
-            json={"templates": [{"createdAt": "2022-01-01", "id": 1}]},
-            status=200,
-            content_type="application/json",
-        )
-        resp = requests.get("https://api.iterable.com/api/1/foobar")
-
-        records = stream.parse_response(response=resp)
-
-        assert list(records) == [{"id": 1, "createdAt": dateutil_parse("2022-01-01")}]
-
-
-def test_list_users_parse_response():
-    stream = ListUsers(authenticator=None)
-    with responses.RequestsMock() as rsps:
-        rsps.add(
-            responses.GET,
-            "https://api.iterable.com/lists/getUsers?listId=100",
-            body="user100",
-            status=200,
-            content_type="application/json",
-        )
-        resp = requests.get("https://api.iterable.com/lists/getUsers?listId=100")
-
-        records = stream.parse_response(response=resp)
-
-        assert list(records) == [{"email": "user100", "listId": 100}]
-
-
-def test_campaigns_metrics_parse_response():
-
-    stream = CampaignsMetrics(authenticator=None, start_date="2019-10-10T00:00:00")
-    with responses.RequestsMock() as rsps:
-        rsps.add(
-            responses.GET,
-            "https://api.iterable.com/lists/getUsers?listId=100",
-            body="""a,b,c,d
-1, 2,, 3
-6,, 1, 2
-""",
-            status=200,
-            content_type="application/json",
-        )
-        resp = requests.get("https://api.iterable.com/lists/getUsers?listId=100")
-
-        records = stream.parse_response(response=resp)
-
-        assert list(records) == [
-            {"data": {"a": 1, "b": 2, "d": 3}},
-            {"data": {"a": 6, "c": 1, "d": 2}},
-        ]
-
-
-def test_iterable_stream_parse_response():
-    stream = Lists(authenticator=None)
-    with responses.RequestsMock() as rsps:
-        rsps.add(
-            responses.GET,
-            "https://api.iterable.com/lists/getUsers?listId=100",
-            json={"lists": [{"id": 1}, {"id": 2}]},
-            status=200,
-            content_type="application/json",
-        )
-        resp = requests.get("https://api.iterable.com/lists/getUsers?listId=100")
-
-        records = stream.parse_response(response=resp)
-
-        assert list(records) == [{"id": 1}, {"id": 2}]
-
-
-def test_iterable_stream_backoff_time():
-    stream = Lists(authenticator=None)
-    assert stream.backoff_time(response=None) is None
-
-
-def test_iterable_export_stream_backoff_time():
-    stream = Users(authenticator=None, start_date="2019-10-10T00:00:00")
-    assert stream.backoff_time(response=None) is None
-
-
-@pytest.mark.parametrize(
-    "current_state,record_date,expected_state",
-    [
-        ({}, "2022", {"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}),
-        ({"profileUpdatedAt": "2020-01-01T00:00:00+00:00"}, "2022", {"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}),
-        ({"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}, "2020", {"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}),
-    ],
-)
-def test_get_updated_state(current_state, record_date, expected_state):
-    stream = Users(authenticator=None, start_date="2019-10-10T00:00:00")
-    state = stream.get_updated_state(
-        current_stream_state=current_state,
-        latest_record={"profileUpdatedAt": pendulum.parse(record_date)},
-    )
-    assert state == expected_state
-
-
-@responses.activate
-def test_stream_stops_on_401(mock_lists_resp):
-    # no requests should be made after getting 401 error despite the multiple slices
-    users_stream = ListUsers(authenticator=None)
-    responses.add(responses.GET, "https://api.iterable.com/api/lists/getUsers?listId=1", json={}, status=401)
-    slices = 0
-    for slice_ in users_stream.stream_slices(sync_mode=SyncMode.full_refresh):
-        slices += 1
-        _ = list(users_stream.read_records(stream_slice=slice_, sync_mode=SyncMode.full_refresh))
-    assert len(responses.calls) == 1
-    assert slices > 1
-
-
-@responses.activate
-def test_listuser_stream_keep_working_on_500():
-    users_stream = ListUsers(authenticator=None)
-
-    msg_error = "An error occurred. Please try again later. If problem persists, please contact your CSM"
-    generic_error1 = {"msg": msg_error, "code": "GenericError"}
-    generic_error2 = {"msg": msg_error, "code": "Generic Error"}
-
-    responses.get("https://api.iterable.com/api/lists", json={"lists": [{"id": 1000}, {"id": 2000}, {"id": 3000}]})
-    responses.get("https://api.iterable.com/api/lists/getUsers?listId=1000", json=generic_error1, status=500)
-    responses.get("https://api.iterable.com/api/lists/getUsers?listId=2000", body="one@d1.com\ntwo@d1.com\nthree@d1.com")
-    responses.get("https://api.iterable.com/api/lists/getUsers?listId=3000", json=generic_error2, status=500)
-    responses.get("https://api.iterable.com/api/lists/getUsers?listId=3000", body="one@d2.com\ntwo@d2.com\nthree@d2.com")
-
-    expected_records = [
-        {"email": "one@d1.com", "listId": 2000},
-        {"email": "two@d1.com", "listId": 2000},
-        {"email": "three@d1.com", "listId": 2000},
-        {"email": "one@d2.com", "listId": 3000},
-        {"email": "two@d2.com", "listId": 3000},
-        {"email": "three@d2.com", "listId": 3000},
-    ]
-
-    records = list(read_full_refresh(users_stream))
-    assert records == expected_records
-
-
-@responses.activate
-def test_events_read_full_refresh():
-    stream = Events(authenticator=None)
-    responses.get("https://api.iterable.com/api/lists", json={"lists": [{"id": 1}]})
-    responses.get("https://api.iterable.com/api/lists/getUsers?listId=1", body="user1\nuser2\nuser3\nuser4\nuser5\nuser6")
-
-    def get_body(emails):
-        return "\n".join([json.dumps({"email": email}) for email in emails]) + "\n"
-
-    msg_error = "An error occurred. Please try again later. If problem persists, please contact your CSM"
-    generic_error1 = {"msg": msg_error, "code": "GenericError"}
-    generic_error2 = {"msg": msg_error, "code": "Generic Error"}
-
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user1&includeCustomEvents=true", body=get_body(["user1"]))
-
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user2&includeCustomEvents=true", json=generic_error1, status=500)
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user2&includeCustomEvents=true", body=get_body(["user2"]))
-
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user3&includeCustomEvents=true", body=get_body(["user3"]))
-
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user4&includeCustomEvents=true", json=generic_error1, status=500)
-
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user5&includeCustomEvents=true", json=generic_error2, status=500)
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user5&includeCustomEvents=true", json=generic_error2, status=500)
-    responses.get("https://api.iterable.com/api/export/userEvents?email=user5&includeCustomEvents=true", body=get_body(["user5"]))
-
-    m = responses.get(
-        "https://api.iterable.com/api/export/userEvents?email=user6&includeCustomEvents=true", json=generic_error2, status=500
-    )
-
-    records = list(read_full_refresh(stream))
-    assert [r["email"] for r in records] == ["user1", "user2", "user3", "user5"]
-    assert m.call_count == 3
-
-
-def test_retry_read_timeout():
-    stream = Lists(authenticator=None)
-    stream._session.send = MagicMock(side_effect=requests.exceptions.ReadTimeout)
-    with pytest.raises(requests.exceptions.ReadTimeout):
-        list(read_full_refresh(stream))
-    stream._session.send.call_args[1] == {"timeout": (60, 300)}
-    assert stream._session.send.call_count == stream.max_retries + 1
-
-    stream = Campaigns(authenticator=None)
-    stream._session.send = MagicMock(side_effect=requests.exceptions.ConnectionError)
-    with pytest.raises(requests.exceptions.ConnectionError):
-        list(read_full_refresh(stream))
-    stream._session.send.call_args[1] == {"timeout": (60, 300)}
-    assert stream._session.send.call_count == stream.max_retries + 1

unit_tests/test_utils.py

@@ -1,12 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import pendulum
-from source_iterable.utils import dateutil_parse
-
-
-def test_dateutil_parse():
-    assert pendulum.parse("2021-04-08 14:23:30 +00:00", strict=False) == dateutil_parse("2021-04-08 14:23:30 +00:00")
-    assert pendulum.parse("2021-04-14T16:51:23+00:00", strict=False) == dateutil_parse("2021-04-14T16:51:23+00:00")
-    assert pendulum.parse("2021-04-14T16:23:30.700000+00:00", strict=False) == dateutil_parse("2021-04-14T16:23:30.700000+00:00")