ingestr 0.13.16__py3-none-any.whl → 0.13.18__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.16"
+version = "v0.13.18"

ingestr/src/factory.py

@@ -36,6 +36,7 @@ from ingestr.src.sources import (
     GorgiasSource,
     HubspotSource,
     KafkaSource,
+    KinesisSource,
     KlaviyoSource,
     LinkedInAdsSource,
     LocalCsvSource,
@@ -141,6 +142,7 @@ class SourceDestinationFactory:
         "applovinmax": ApplovinMaxSource,
         "salesforce": SalesforceSource,
         "personio": PersonioSource,
+        "kinesis": KinesisSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/hubspot/__init__.py

@@ -199,14 +199,6 @@ def crm_objects(
 
     props = ",".join(sorted(list(set(props))))
 
-    if len(props) > 2000:
-        raise ValueError(
-            "Your request to Hubspot is too long to process. "
-            "Maximum allowed query length is 2000 symbols, while "
-            f"your list of properties `{props[:200]}`... is {len(props)} "
-            "symbols long. Use the `props` argument of the resource to "
-            "set the list of properties to extract from the endpoint."
-        )
 
     params = {"properties": props, "limit": 100}
 

ingestr/src/kinesis/__init__.py

@@ -0,0 +1,139 @@
+"""Reads messages from Kinesis queue."""
+
+from typing import Iterable, List, Optional
+
+import dlt
+from dlt.common import json, pendulum
+from dlt.common.configuration.specs import AwsCredentials
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import StrStr, TAnyDateTime, TDataItem
+from dlt.common.utils import digest128
+
+from .helpers import get_shard_iterator, max_sequence_by_shard
+
+
+@dlt.resource(
+    name=lambda args: args["stream_name"],
+    primary_key="kinesis_msg_id",
+    standalone=True,
+)
+def kinesis_stream(
+    stream_name: str,
+    initial_at_timestamp: TAnyDateTime,
+    credentials: AwsCredentials,
+    last_msg: Optional[dlt.sources.incremental[StrStr]] = dlt.sources.incremental(
+        "kinesis", last_value_func=max_sequence_by_shard
+    ),
+    max_number_of_messages: int = None,  # type: ignore
+    milliseconds_behind_latest: int = 1000,
+    parse_json: bool = True,
+    chunk_size: int = 1000,
+) -> Iterable[TDataItem]:
+    """Reads a kinesis stream and yields messages. Supports incremental loading. Parses messages as json by default.
+
+    Args:
+        stream_name (str): The name of the stream to read from. If not provided, the
+            value must be present in config/secrets
+        credentials (AwsCredentials): The credentials to use to connect to kinesis. If not provided,
+            the value from secrets or credentials present on the device will be used.
+        last_msg (Optional[dlt.sources.incremental]): An incremental over a mapping from shard_id to message sequence
+            that will be used to create shard iterators of type AFTER_SEQUENCE_NUMBER when loading incrementally.
+        initial_at_timestamp (TAnyDateTime): An initial timestamp used to generate AT_TIMESTAMP or LATEST iterator when timestamp value is 0
+        max_number_of_messages (int): Maximum number of messages to read in one run. Actual read may exceed that number by up to chunk_size. Defaults to None (no limit).
+        milliseconds_behind_latest (int): The number of milliseconds behind the top of the shard to stop reading messages, defaults to 1000.
+        parse_json (bool): If True, assumes that messages are json strings, parses them and returns instead of `data` (otherwise). Defaults to False.
+        chunk_size (int): The number of records to fetch at once. Defaults to 1000.
+    Yields:
+            Iterable[TDataItem]: Messages. Contain Kinesis envelope in `kinesis` and bytes data in `data` (if `parse_json` disabled)
+
+    """
+    session = credentials._to_botocore_session()
+    # the default timeouts are (60, 60) which is fine
+    kinesis_client = session.create_client("kinesis")
+    # normalize at_timestamp to pendulum
+    initial_at_datetime = (
+        None
+        if initial_at_timestamp is None
+        else ensure_pendulum_datetime(initial_at_timestamp)
+    )
+    # set it in state
+    resource_state = dlt.current.resource_state()
+    initial_at_datetime = resource_state.get(
+        "initial_at_timestamp", initial_at_datetime
+    )
+    # so next time we request shards at AT_TIMESTAMP that is now
+    resource_state["initial_at_timestamp"] = pendulum.now("UTC").subtract(seconds=1)
+
+    shards_list = kinesis_client.list_shards(StreamName=stream_name)
+    shards: List[StrStr] = shards_list["Shards"]
+    while next_token := shards_list.get("NextToken"):
+        shards_list = kinesis_client.list_shards(NextToken=next_token)
+        shards.extend(shards_list)
+
+    shard_ids = [shard["ShardId"] for shard in shards]
+
+    # get next shard to fetch messages from
+    while shard_id := shard_ids.pop(0) if shard_ids else None:
+     
+        shard_iterator, _ = get_shard_iterator(
+            kinesis_client,
+            stream_name,
+            shard_id,
+            last_msg,  # type: ignore
+            initial_at_datetime,  # type: ignore
+        )
+       
+        while shard_iterator:
+            records = []
+            records_response = kinesis_client.get_records(
+                ShardIterator=shard_iterator,
+                Limit=chunk_size,  # The size of data can be up to 1 MB, it must be controlled by the user
+            )
+            
+            for record in records_response["Records"]:
+                sequence_number = record["SequenceNumber"]
+                content = record["Data"]
+
+                arrival_time = record["ApproximateArrivalTimestamp"]
+                arrival_timestamp = arrival_time.astimezone(pendulum.UTC)
+
+                message = {
+                    "kinesis": {
+                        "shard_id": shard_id,
+                        "seq_no": sequence_number,
+                        "ts": ensure_pendulum_datetime(arrival_timestamp),
+                        "partition": record["PartitionKey"],
+                        "stream_name": stream_name,
+                    },
+                    "kinesis_msg_id": digest128(shard_id + sequence_number),
+                }
+
+                if parse_json:
+                    message.update(json.loadb(content))
+                else:
+                    message["data"] = content
+                records.append(message)
+            yield records
+
+            # do not load more  max_number_of_messages
+            if max_number_of_messages is not None:
+                max_number_of_messages -= len(records)
+                if max_number_of_messages <= 0:
+                    return
+
+            # add child shards so we can request messages from them
+            child_shards = records_response.get("ChildShards", None)
+            if child_shards:
+                for child_shard in child_shards:
+                    child_shard_id = child_shard["ShardId"]
+                    if child_shard_id not in shards:
+                        shard_ids.append(child_shard_id)
+
+            # gets 0 when no messages so we cutoff empty shards
+            records_ms_behind_latest = records_response.get("MillisBehindLatest", 0)
+            if records_ms_behind_latest < milliseconds_behind_latest:
+                # stop taking messages from shard
+                shard_iterator = None  # type: ignore
+            else:
+                # continue taking messages
+                shard_iterator = records_response["NextShardIterator"]

ingestr/src/kinesis/helpers.py

@@ -0,0 +1,65 @@
+from typing import Any, Sequence, Tuple
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.typing import DictStrAny, StrAny, StrStr
+
+
+def get_shard_iterator(
+    kinesis_client: Any,
+    stream_name: str,
+    shard_id: str,
+    last_msg: dlt.sources.incremental[StrStr],
+    initial_at_timestamp: pendulum.DateTime | None,
+) -> Tuple[str, StrAny]:
+    """Gets shard `shard_id` of `stream_name` iterator. If `last_msg` incremental is present it may
+    contain last message sequence for shard_id. in that case AFTER_SEQUENCE_NUMBER is created.
+    If no message sequence is present, `initial_at_timestamp` is used for AT_TIMESTAMP or LATEST.
+    The final fallback is TRIM_HORIZON
+    """
+    sequence_state = (
+        {} if last_msg is None else last_msg.last_value or last_msg.initial_value or {}
+    )
+    iterator_params: DictStrAny
+    msg_sequence = sequence_state.get(shard_id, None)
+    if msg_sequence:
+        iterator_params = dict(
+            ShardIteratorType="AFTER_SEQUENCE_NUMBER",
+            StartingSequenceNumber=msg_sequence,
+        )
+    elif initial_at_timestamp is None:
+        # Fetch all records from the beginning
+        iterator_params = dict(ShardIteratorType="TRIM_HORIZON")
+
+    elif initial_at_timestamp.timestamp() == 0.0:
+        # will sets to latest i.e only the messages at the tip of the stream are read
+        iterator_params = dict(ShardIteratorType="LATEST")
+    else:
+        iterator_params = dict(
+            ShardIteratorType="AT_TIMESTAMP", Timestamp=initial_at_timestamp.timestamp()
+        )
+
+    shard_iterator: StrStr = kinesis_client.get_shard_iterator(
+        StreamName=stream_name, ShardId=shard_id, **iterator_params
+    )
+    return shard_iterator["ShardIterator"], iterator_params
+
+
+def max_sequence_by_shard(values: Sequence[StrStr]) -> StrStr:
+    """A last_value_function that operates on mapping of shard_id:msg_sequence defining the max"""
+    last_value = None
+    # if tuple/list contains only one element then return it
+    if len(values) == 1:
+        item = values[0]
+    else:
+        # item is kinesis metadata, last_value is previous state of the shards
+        item, last_value = values
+
+    if last_value is None:
+        last_value = {}
+    else:
+        last_value = dict(last_value)  # always make a copy
+    shard_id = item["shard_id"]
+    # we compare message sequence at shard_id
+    last_value[shard_id] = max(item["seq_no"], last_value.get(shard_id, ""))
+    return last_value

ingestr/src/sources.py

@@ -75,6 +75,7 @@ from ingestr.src.gorgias import gorgias_source
 from ingestr.src.hubspot import hubspot
 from ingestr.src.kafka import kafka_consumer
 from ingestr.src.kafka.helpers import KafkaCredentials
+from ingestr.src.kinesis import kinesis_stream
 from ingestr.src.klaviyo._init_ import klaviyo_source
 from ingestr.src.linkedin_ads import linked_in_ads_source
 from ingestr.src.linkedin_ads.dimension_time_enum import (
@@ -1969,3 +1970,39 @@ class PersonioSource:
             start_date=interval_start_date,
             end_date=interval_end_date,
         ).with_resources(table)
+
+
+class KinesisSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        # kinesis://?aws_access_key_id=<AccessKeyId>&aws_secret_access_key=<SecretAccessKey>&region_name=<Region>
+        # source table = stream name
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        aws_access_key_id = params.get("aws_access_key_id")
+        if aws_access_key_id is None:
+            raise MissingValueError("aws_access_key_id", "Kinesis")
+
+        aws_secret_access_key = params.get("aws_secret_access_key")
+        if aws_secret_access_key is None:
+            raise MissingValueError("aws_secret_access_key", "Kinesis")
+
+        region_name = params.get("region_name")
+        if region_name is None:
+            raise MissingValueError("region_name", "Kinesis")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            # the resource will read all messages after this timestamp.
+            start_date = ensure_pendulum_datetime(start_date)
+        credentials = AwsCredentials(
+            aws_access_key_id=aws_access_key_id[0],
+            aws_secret_access_key=aws_secret_access_key[0],
+            region_name=region_name[0],
+        )
+        return kinesis_stream(
+            stream_name=table, credentials=credentials, initial_at_timestamp=start_date
+        )

{ingestr-0.13.16.dist-info → ingestr-0.13.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.16
+Version: 0.13.18
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,52 +14,178 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: aiobotocore==2.21.1
+Requires-Dist: aiohappyeyeballs==2.4.8
+Requires-Dist: aiohttp==3.11.13
+Requires-Dist: aioitertools==0.12.0
+Requires-Dist: aiosignal==1.3.2
+Requires-Dist: alembic==1.15.1
+Requires-Dist: annotated-types==0.7.0
 Requires-Dist: asana==3.2.3
+Requires-Dist: asn1crypto==1.5.1
+Requires-Dist: asynch==0.2.4
+Requires-Dist: attrs==25.1.0
+Requires-Dist: backoff==2.2.1
+Requires-Dist: beautifulsoup4==4.13.3
+Requires-Dist: boto3==1.37.1
+Requires-Dist: botocore==1.37.1
+Requires-Dist: cachetools==5.5.2
+Requires-Dist: certifi==2025.1.31
+Requires-Dist: cffi==1.17.1
+Requires-Dist: charset-normalizer==3.4.1
+Requires-Dist: ciso8601==2.3.2
+Requires-Dist: click==8.1.8
+Requires-Dist: clickhouse-connect==0.8.14
+Requires-Dist: clickhouse-driver==0.2.9
+Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: cryptography==44.0.2
+Requires-Dist: curlify==2.2.1
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
+Requires-Dist: decorator==5.2.1
+Requires-Dist: deprecation==2.1.0
 Requires-Dist: dlt==1.6.1
+Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.15.0
 Requires-Dist: duckdb==1.2.0
+Requires-Dist: et-xmlfile==2.0.0
 Requires-Dist: facebook-business==20.0.0
+Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
+Requires-Dist: frozenlist==1.5.0
+Requires-Dist: fsspec==2024.10.0
 Requires-Dist: gcsfs==2024.10.0
+Requires-Dist: gitdb==4.0.12
+Requires-Dist: gitpython==3.1.44
+Requires-Dist: giturlparse==0.12.0
 Requires-Dist: google-ads==25.1.0
 Requires-Dist: google-analytics-data==0.18.17
+Requires-Dist: google-api-core==2.24.1
 Requires-Dist: google-api-python-client==2.130.0
+Requires-Dist: google-auth-httplib2==0.2.0
+Requires-Dist: google-auth-oauthlib==1.2.1
+Requires-Dist: google-auth==2.38.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
+Requires-Dist: google-cloud-bigquery==3.30.0
+Requires-Dist: google-cloud-core==2.4.2
+Requires-Dist: google-cloud-storage==3.1.0
+Requires-Dist: google-crc32c==1.6.0
+Requires-Dist: google-resumable-media==2.7.2
+Requires-Dist: googleapis-common-protos==1.69.0
+Requires-Dist: greenlet==3.1.1
+Requires-Dist: grpcio-status==1.62.3
+Requires-Dist: grpcio==1.70.0
+Requires-Dist: hdbcli==2.23.27
+Requires-Dist: hexbytes==1.3.0
+Requires-Dist: httplib2==0.22.0
+Requires-Dist: humanize==4.12.1
+Requires-Dist: idna==3.10
+Requires-Dist: inflection==0.5.1
+Requires-Dist: isodate==0.7.2
+Requires-Dist: jmespath==1.0.1
+Requires-Dist: jsonpath-ng==1.7.0
+Requires-Dist: leb128==1.0.8
+Requires-Dist: lxml==5.3.1
+Requires-Dist: lz4==4.4.3
+Requires-Dist: makefun==1.15.6
+Requires-Dist: mako==1.3.9
+Requires-Dist: markdown-it-py==3.0.0
+Requires-Dist: markupsafe==3.0.2
+Requires-Dist: marshmallow==3.26.1
+Requires-Dist: mdurl==0.1.2
+Requires-Dist: monotonic==1.6
+Requires-Dist: more-itertools==10.6.0
+Requires-Dist: multidict==6.1.0
+Requires-Dist: mypy-extensions==1.0.0
 Requires-Dist: mysql-connector-python==9.2.0
+Requires-Dist: numpy==2.2.3
+Requires-Dist: oauthlib==3.2.2
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: orjson==3.10.15
+Requires-Dist: packaging==24.2
+Requires-Dist: pandas==2.2.3
+Requires-Dist: pathvalidate==3.2.3
 Requires-Dist: pendulum==3.0.0
+Requires-Dist: platformdirs==4.3.6
+Requires-Dist: pluggy==1.5.0
+Requires-Dist: ply==3.11
+Requires-Dist: propcache==0.3.0
+Requires-Dist: proto-plus==1.26.0
+Requires-Dist: protobuf==4.25.6
 Requires-Dist: psutil==6.1.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
 Requires-Dist: pyarrow==18.1.0
+Requires-Dist: pyasn1-modules==0.4.1
+Requires-Dist: pyasn1==0.6.1
 Requires-Dist: pyathena==3.12.2
+Requires-Dist: pycountry==24.6.1
+Requires-Dist: pycparser==2.22
+Requires-Dist: pydantic-core==2.27.2
+Requires-Dist: pydantic==2.10.6
+Requires-Dist: pygments==2.19.1
+Requires-Dist: pyjwt==2.10.1
 Requires-Dist: pymongo==4.11.1
 Requires-Dist: pymysql==1.1.1
+Requires-Dist: pyopenssl==25.0.0
+Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: pytz==2025.1
+Requires-Dist: pyyaml==6.0.2
 Requires-Dist: redshift-connector==2.1.5
+Requires-Dist: requests-file==2.1.0
+Requires-Dist: requests-oauthlib==1.3.1
+Requires-Dist: requests-toolbelt==1.0.0
+Requires-Dist: requests==2.32.3
+Requires-Dist: requirements-parser==0.11.0
+Requires-Dist: rich-argparse==1.7.0
 Requires-Dist: rich==13.9.4
+Requires-Dist: rsa==4.9
 Requires-Dist: rudder-sdk-python==2.1.4
 Requires-Dist: s3fs==2024.10.0
+Requires-Dist: s3transfer==0.11.3
+Requires-Dist: scramp==1.4.5
+Requires-Dist: semver==3.0.4
+Requires-Dist: setuptools==75.8.2
+Requires-Dist: shellingham==1.5.4
 Requires-Dist: simple-salesforce==1.12.6
+Requires-Dist: simplejson==3.20.1
+Requires-Dist: six==1.17.0
+Requires-Dist: smmap==5.0.2
+Requires-Dist: snowflake-connector-python==3.14.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
+Requires-Dist: sortedcontainers==2.4.0
+Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: stripe==10.7.0
+Requires-Dist: tenacity==9.0.0
+Requires-Dist: thrift==0.16.0
+Requires-Dist: time-machine==2.16.0
+Requires-Dist: tomlkit==0.13.2
 Requires-Dist: tqdm==4.67.1
 Requires-Dist: typer==0.13.1
 Requires-Dist: types-requests==2.32.0.20240907
-Provides-Extra: clickhouse
-Requires-Dist: clickhouse-connect==0.8.14; extra == 'clickhouse'
-Requires-Dist: clickhouse-driver==0.2.9; extra == 'clickhouse'
-Requires-Dist: clickhouse-sqlalchemy==0.2.7; extra == 'clickhouse'
-Requires-Dist: zstd==1.5.6.1; extra == 'clickhouse'
+Requires-Dist: types-setuptools==75.8.2.20250305
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: typing-inspect==0.9.0
+Requires-Dist: tzdata==2025.1
+Requires-Dist: tzlocal==5.3
+Requires-Dist: uritemplate==4.1.1
+Requires-Dist: urllib3==2.3.0
+Requires-Dist: wrapt==1.17.2
+Requires-Dist: yarl==1.18.3
+Requires-Dist: zeep==4.3.1
+Requires-Dist: zstandard==0.23.0
+Requires-Dist: zstd==1.5.6.5
 Provides-Extra: odbc
 Requires-Dist: pyodbc==5.1.0; extra == 'odbc'
 Provides-Extra: oracle
@@ -231,6 +357,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Amazon Kinesis</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Apache Kafka</td>
         <td>✅</td>

{ingestr-0.13.16.dist-info → ingestr-0.13.18.dist-info}/RECORD

@@ -1,13 +1,13 @@
 ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=LtEZWoUhm5i2aKerdgEpLtNCf3fdhGGMM4td-LRZVbY,1407
-ingestr/src/buildinfo.py,sha256=Rj60_siwT8KglZcq6523cm-L69t-jzt5DrLSvlcq3ok,21
+ingestr/src/buildinfo.py,sha256=Ph2-6uM0ocE2IFQ-YyOxltsHqhrIhOawp3Pr6vUGaWE,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=or0EL2_2wk6Rz41n63q7pjG3Ug_6ZGjxE7q8OjGdjBA,5145
+ingestr/src/factory.py,sha256=Si3xQuaqiwR_LMtxg2rA93MkDYpq_BnVWTfBsNVnFIA,5198
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
-ingestr/src/sources.py,sha256=vCZ20jVD5jCCNyTxR7VxNXpJFCE218t0ydbaZwls48M,69510
+ingestr/src/sources.py,sha256=y6zFGGbi5FvrdQ89e0t1ud3BWNN2kvrNu2iuXb9wu6g,70977
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -56,11 +56,13 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=DXvn1yGToFUKk-1mMqqoN0OCLNpD16-2mPyEmkhyoVY,9876
+ingestr/src/hubspot/__init__.py,sha256=jDI9PIgz2cvRMbTtX2tF1FcATFpBOMI7M8Ua4G7qDCc,9465
 ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
 ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
+ingestr/src/kinesis/__init__.py,sha256=Bm0S9BvWDHZUhOc8WKTeawORRgldmJsb0Y3XNHpuJ-c,6205
+ingestr/src/kinesis/helpers.py,sha256=aF0GCDKSectaaW8XPdERY_6bUs0ky19dcBs24ZFn-o0,2473
 ingestr/src/klaviyo/_init_.py,sha256=ucWHqBe8DQvXVpbmxKFAV5ljpCFb4ps_2QTD0OSiWxY,7905
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
@@ -108,8 +110,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.16.dist-info/METADATA,sha256=XiCcUlZub3pjTfhHgrqeimVPsk1qz0_AgvL5kDIgGq8,9363
-ingestr-0.13.16.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.16.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.16.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.16.dist-info/RECORD,,
+ingestr-0.13.18.dist-info/METADATA,sha256=HYn4_5kTZd5RoQgkz06WF4-x12T8RVJtoQe6fyHQwYk,13569
+ingestr-0.13.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.18.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.18.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.18.dist-info/RECORD,,