ingestr 0.14.4__py3-none-any.whl → 0.14.6__py3-none-any.whl

ingestr/src/plusvibeai/settings.py

@@ -0,0 +1,252 @@
+"""PlusVibeAI source settings and constants"""
+
+# Default start date for PlusVibeAI API requests
+DEFAULT_START_DATE = "2020-01-01"
+
+# PlusVibeAI API request timeout in seconds
+REQUEST_TIMEOUT = 300
+
+# Default page size for paginated requests
+DEFAULT_PAGE_SIZE = 100
+
+# Maximum page size (adjust based on API limits)
+MAX_PAGE_SIZE = 1000
+
+# Base API path for PlusVibeAI
+API_BASE_PATH = "/api/v1"
+
+# Campaign fields to retrieve from PlusVibeAI API
+CAMPAIGN_FIELDS = (
+    # Basic Information
+    "id",
+    "camp_name",
+    "parent_camp_id",
+    "campaign_type",
+    "organization_id",
+    "workspace_id",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_lead_sent",
+    "last_paused_at_bounced",
+    # Campaign Configuration
+    "tags",
+    "template_id",
+    "email_accounts",
+    "daily_limit",
+    "interval_limit_in_min",
+    "send_priority",
+    "send_as_txt",
+    # Tracking & Settings
+    "is_emailopened_tracking",
+    "is_unsubscribed_link",
+    "exclude_ooo",
+    "is_acc_based_sending",
+    "send_risky_email",
+    "unsub_blocklist",
+    "other_email_acc",
+    "is_esp_match",
+    "stop_on_lead_replied",
+    # Bounce Settings
+    "is_pause_on_bouncerate",
+    "bounce_rate_limit",
+    "is_paused_at_bounced",
+    # Schedule
+    "schedule",
+    "first_wait_time",
+    "camp_st_date",
+    "camp_end_date",
+    # Events & Sequences
+    "events",
+    "sequences",
+    "sequence_steps",
+    "camp_emails",
+    # Lead Statistics
+    "lead_count",
+    "completed_lead_count",
+    "lead_contacted_count",
+    # Email Performance Metrics
+    "sent_count",
+    "opened_count",
+    "unique_opened_count",
+    "replied_count",
+    "bounced_count",
+    "unsubscribed_count",
+    # Reply Classification
+    "positive_reply_count",
+    "negative_reply_count",
+    "neutral_reply_count",
+    # Daily & Business Metrics
+    "email_sent_today",
+    "opportunity_val",
+    "open_rate",
+    "replied_rate",
+    # Custom Data
+    "custom_fields",
+)
+
+# Lead fields to retrieve from PlusVibeAI API
+LEAD_FIELDS = (
+    # Basic Information
+    "_id",
+    "organization_id",
+    "campaign_id",
+    "workspace_id",
+    # Lead Status & Progress
+    "is_completed",
+    "current_step",
+    "status",
+    "label",
+    # Email Account Info
+    "email_account_id",
+    "email_acc_name",
+    # Campaign Info
+    "camp_name",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_sent_at",
+    # Email Engagement Metrics
+    "sent_step",
+    "replied_count",
+    "opened_count",
+    # Email Verification
+    "is_mx",
+    "mx",
+    # Contact Information
+    "email",
+    "first_name",
+    "last_name",
+    "phone_number",
+    # Address Information
+    "address_line",
+    "city",
+    "state",
+    "country",
+    "country_code",
+    # Professional Information
+    "job_title",
+    "department",
+    "company_name",
+    "company_website",
+    "industry",
+    # Social Media
+    "linkedin_person_url",
+    "linkedin_company_url",
+    # Workflow
+    "total_steps",
+    # Bounce Information
+    "bounce_msg",
+)
+
+# Email Account fields to retrieve from PlusVibeAI API
+EMAIL_ACCOUNT_FIELDS = (
+    # Basic Information
+    "_id",
+    "email",
+    "status",
+    "warmup_status",
+    # Timestamps
+    "timestamp_created",
+    "timestamp_updated",
+    # Payload - nested object containing all configuration
+    "payload",
+    # Payload sub-fields (for reference, stored in payload object):
+    # - name (first_name, last_name)
+    # - warmup (limit, warmup_custom_words, warmup_signature, advanced, increment, reply_rate)
+    # - imap_host, imap_port
+    # - smtp_host, smtp_port
+    # - daily_limit, sending_gap
+    # - reply_to, custom_domain, signature
+    # - tags, cmps
+    # - analytics (health_scores, reply_rates, daily_counters)
+)
+
+# Email fields to retrieve from PlusVibeAI API
+EMAIL_FIELDS = (
+    # Basic Information
+    "id",
+    "message_id",
+    "is_unread",
+    # Lead Information
+    "lead",
+    "lead_id",
+    "campaign_id",
+    # From Address
+    "from_address_email",
+    "from_address_json",
+    # Subject & Content
+    "subject",
+    "content_preview",
+    "body",
+    # Headers & Metadata
+    "headers",
+    "label",
+    "thread_id",
+    "eaccount",
+    # To/CC/BCC Addresses
+    "to_address_email_list",
+    "to_address_json",
+    "cc_address_email_list",
+    "cc_address_json",
+    "bcc_address_email_list",
+    # Timestamps
+    "timestamp_created",
+    "source_modified_at",
+)
+
+# Blocklist fields to retrieve from PlusVibeAI API
+BLOCKLIST_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "value",
+    "created_by_label",
+    # Timestamps
+    "created_at",
+)
+
+# Webhook fields to retrieve from PlusVibeAI API
+WEBHOOK_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "url",
+    "name",
+    "secret",
+    # Configuration
+    "camp_ids",
+    "evt_types",
+    "status",
+    "integration_type",
+    # Settings
+    "ignore_ooo",
+    "ignore_automatic",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_run",
+    # Response Data
+    "last_resp",
+    "last_recv_resp",
+    # User Information
+    "created_by",
+    "modified_by",
+)
+
+# Tag fields to retrieve from PlusVibeAI API
+TAG_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "name",
+    "color",
+    "description",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+)

ingestr/src/sources.py

@@ -1829,6 +1829,7 @@ class JiraSource:
         "resolutions",
         "project_versions",
         "project_components",
+        "events",
     ]
 
     def handles_incrementality(self) -> bool:
@@ -3760,6 +3761,52 @@ class AnthropicSource:
         ).with_resources(table)
 
 
+class PlusVibeAISource:
+    resources = [
+        "campaigns",
+        "leads",
+        "email_accounts",
+        "emails",
+        "blocklist",
+        "webhooks",
+        "tags",
+    ]
+
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        # plusvibeai://?api_key=<key>&workspace_id=<id>
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key")
+        workspace_id = params.get("workspace_id")
+
+        if not api_key:
+            raise MissingValueError("api_key", "PlusVibeAI")
+
+        if not workspace_id:
+            raise MissingValueError("workspace_id", "PlusVibeAI")
+
+        if table not in self.resources:
+            raise UnsupportedResourceError(table, "PlusVibeAI")
+
+        import dlt
+
+        from ingestr.src.plusvibeai import plusvibeai_source
+
+        dlt.secrets["sources.plusvibeai.api_key"] = api_key[0]
+        dlt.secrets["sources.plusvibeai.workspace_id"] = workspace_id[0]
+
+        # Handle custom base URL if provided
+        base_url = params.get("base_url", ["https://api.plusvibe.ai"])[0]
+        dlt.secrets["sources.plusvibeai.base_url"] = base_url
+
+        src = plusvibeai_source()
+        return src.with_resources(table)
+
+
 class IntercomSource:
     def handles_incrementality(self) -> bool:
         return True
@@ -3831,3 +3878,86 @@ class IntercomSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class HttpSource:
+    """Source for reading CSV, JSON, and Parquet files from HTTP URLs"""
+
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        """
+        Create a dlt source for reading files from HTTP URLs.
+
+        URI format: http://example.com/file.csv or https://example.com/file.json
+
+        Args:
+            uri: HTTP(S) URL to the file
+            table: Not used for HTTP source (files are read directly)
+            **kwargs: Additional arguments:
+                - file_format: Optional file format override ('csv', 'json', 'parquet')
+                - chunksize: Number of records to process at once (default varies by format)
+                - merge_key: Merge key for the resource
+
+        Returns:
+            DltResource for the HTTP file
+        """
+        from ingestr.src.http import http_source
+
+        # Extract the actual URL (remove the http:// or https:// scheme if duplicated)
+        url = uri
+        if uri.startswith("http://http://") or uri.startswith("https://https://"):
+            url = uri.split("://", 1)[1]
+
+        file_format = kwargs.get("file_format")
+        chunksize = kwargs.get("chunksize")
+        merge_key = kwargs.get("merge_key")
+
+        reader_kwargs = {}
+        if chunksize is not None:
+            reader_kwargs["chunksize"] = chunksize
+
+        source = http_source(url=url, file_format=file_format, **reader_kwargs)
+
+        if merge_key:
+            source.apply_hints(merge_key=merge_key)
+
+        return source
+
+
+class MondaySource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        api_token = query_params.get("api_token")
+
+        if api_token is None:
+            raise MissingValueError("api_token", "Monday")
+
+        parts = table.replace(" ", "").split(":")
+        table_name = parts[0]
+        params = parts[1:]
+
+        # Get interval_start and interval_end from kwargs (command line args)
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        # Convert datetime to string format YYYY-MM-DD
+        start_date = interval_start.strftime("%Y-%m-%d") if interval_start else None
+        end_date = interval_end.strftime("%Y-%m-%d") if interval_end else None
+
+        from ingestr.src.monday import monday_source
+
+        try:
+            return monday_source(
+                api_token=api_token[0],
+                params=params,
+                start_date=start_date,
+                end_date=end_date,
+            ).with_resources(table_name)
+        except ResourcesNotFoundError:
+            raise UnsupportedResourceError(table_name, "Monday")

{ingestr-0.14.4.dist-info → ingestr-0.14.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.4
+Version: 0.14.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.4.dist-info → ingestr-0.14.6.dist-info}/RECORD

@@ -2,17 +2,17 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=CMElSN6kWpX1OuAi0ov5fFYJdYbHWxQl5AOE3NOS_u4,20
+ingestr/src/buildinfo.py,sha256=dazsjzReTYtam8X7FVSN4WAYUmvlPNZ0XztT57SOHTU,20
 ingestr/src/destinations.py,sha256=QtjE0AGs0WkPHaI2snWPHJ8HHi4lwXUBYLJPklz8Mvk,27772
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=7hOPrQSQowCvrVYBLTXE7BJl2pI7Eo5jdLkdpKSbkaw,7476
+ingestr/src/factory.py,sha256=03eGDe2rL6qyT5sGmTGZi-XIwJbbdoedE_KjW3ZF7QY,7661
 ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=edVfJA4OpjNFMyn6oB2pifVHA1j-z7YYXTiLr_FQHzA,134324
+ingestr/src/sources.py,sha256=yQhmgIIfzMr8qHxQr-yDmzowti_q59khRzBDPY0Kw-I,138486
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -83,6 +83,8 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
+ingestr/src/http/__init__.py,sha256=Y9mQIE0RolHOh6dPjW41qzYXSG8BC0GPKxEtz2CJGpU,902
+ingestr/src/http/readers.py,sha256=rgBwYG5SOQ7P2uzBAFMOQIevKxV51ZW41VSiRTZ0Xvo,3863
 ingestr/src/hubspot/__init__.py,sha256=FCqjLeOjijdc9JC_NoDwtRqy3FDyY-szDi6UV7CdDN0,11548
 ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
@@ -92,8 +94,8 @@ ingestr/src/intercom/__init__.py,sha256=rqorWFwcfcTYrCrpSsPPM2sGOc7qq5XbYZRCDVJX
 ingestr/src/intercom/helpers.py,sha256=IljM0x4K70nuahidZaP7mtIlsHkPIcZq56j9mmuSck4,21074
 ingestr/src/intercom/settings.py,sha256=u_4P2tZiYsnlGjMTN4Ttr4bLHh1b272Pu5Q9YFq3ZCE,7053
 ingestr/src/isoc_pulse/__init__.py,sha256=9b4eN4faatpiwTuRNPuYcEt1hEFDEjua9XhfakUigBk,4648
-ingestr/src/jira_source/__init__.py,sha256=fy0lYAFVA9Skmdbx_gAI2iF8CXpgBEKSBsuNOQLaNo8,9178
-ingestr/src/jira_source/helpers.py,sha256=obkFnVH45rDFTzen0L9y8UVxo3gdm2LFs21m_oB5ow0,15209
+ingestr/src/jira_source/__init__.py,sha256=Hy-JUdo9Nu1Goti6zqWjG1GNnCzfv9cY85MFb-sqo-I,9814
+ingestr/src/jira_source/helpers.py,sha256=vJ7fTGSQnLHKiwLQUPEpGNsHBRUyBZswL8R1A3Tz0GY,14636
 ingestr/src/jira_source/settings.py,sha256=Ufb-sGS-x_STtWJ-y6k69hP1CVtat_J5MtFPn51TUGE,2861
 ingestr/src/kafka/__init__.py,sha256=QUHsGmdv5_E-3z0GDHXvbk39puwuGDBsyYSDhvbA89E,3595
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
@@ -109,6 +111,9 @@ ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffW
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
 ingestr/src/mixpanel/__init__.py,sha256=s1QtqMP0BTGW6YtdCabJFWj7lEn7KujzELwGpBOQgfs,1796
 ingestr/src/mixpanel/client.py,sha256=c_reouegOVYBOwHLfgYFwpmkba0Sxro1Zkml07NCYf0,3602
+ingestr/src/monday/__init__.py,sha256=ZNdGCC_1CEYlgxAef-5QO56Drm9IMP82-rZpEvbD8aY,6918
+ingestr/src/monday/helpers.py,sha256=xkAYTFIwjbU-dQTa4d41oQm6kFvCHv74AhCmN-H8aPE,11572
+ingestr/src/monday/settings.py,sha256=5TC0OrTHQO52AifwP3Z2xsh4D8SDUq0WxqY5AQMjcns,5667
 ingestr/src/mongodb/__init__.py,sha256=6-DvvaKL7XOPPRwItI7lSpoMQLEPzYubV6dKhpzbuME,7494
 ingestr/src/mongodb/helpers.py,sha256=BKb0F-AUWjFJikE9OPP9z5wFuMmJsf8YsyWhvQ9dC1k,38076
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -127,6 +132,9 @@ ingestr/src/pipedrive/typing.py,sha256=lEMXu4hhAA3XkhVSlBUa-juqyupisd3c-qSQKxFvz
 ingestr/src/pipedrive/helpers/__init__.py,sha256=UX1K_qnGXB0ShtnBOfp2XuVbK8RRoCK8TdEmIjRckgg,710
 ingestr/src/pipedrive/helpers/custom_fields_munger.py,sha256=rZ4AjdITHfJE2NNomCR7vMBS1KnWpEGVF6fADwsIHUE,4488
 ingestr/src/pipedrive/helpers/pages.py,sha256=Klpjw2OnMuhzit3PpiHKsfzGcJ3rQPSQBl3HhE3-6eA,3358
+ingestr/src/plusvibeai/__init__.py,sha256=Uo-N2-1kbq5RJw8ym5tm8rqVchVbJJ2hOd6bwsg1zdM,10125
+ingestr/src/plusvibeai/helpers.py,sha256=5hxxA2-XUtkZA1xrstZ39ilzUh4EouNDOiiL-NzGu9w,17939
+ingestr/src/plusvibeai/settings.py,sha256=3Hb7jcUNshSlGO4E27yUe_8n3f0VArX9XTmkTkN-Tvo,5366
 ingestr/src/quickbooks/__init__.py,sha256=cZUuVCOTGPHTscRj6i0DytO63_fWF-4ieMxoU4PcyTg,3727
 ingestr/src/revenuecat/__init__.py,sha256=5HbyZuEOekkbeeT72sM_bnGygSyYdmd_vczfAUz7xoM,4029
 ingestr/src/revenuecat/helpers.py,sha256=CYU6l79kplnfL87GfdxyGeEBrBSWEZfGP0GyjPHuVDk,9619
@@ -172,8 +180,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
-ingestr-0.14.4.dist-info/METADATA,sha256=iktFVAzjjXbgReL__B3RnjolZsexAhwo-w3kEAseCFk,15265
-ingestr-0.14.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.14.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.14.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.14.4.dist-info/RECORD,,
+ingestr-0.14.6.dist-info/METADATA,sha256=3akmbk91m4xi1AMYFkuMPKmAtlcUSzwCOsYbeXwFlsk,15265
+ingestr-0.14.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.14.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.14.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.14.6.dist-info/RECORD,,