ingestr 0.14.5__py3-none-any.whl → 0.14.6__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.14.5"
+version = "v0.14.6"

ingestr/src/factory.py

@@ -52,6 +52,7 @@ from ingestr.src.sources import (
     GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HttpSource,
     HubspotSource,
     InfluxDBSource,
     IntercomSource,
@@ -64,6 +65,7 @@ from ingestr.src.sources import (
     LinkedInAdsSource,
     LocalCsvSource,
     MixpanelSource,
+    MondaySource,
     MongoDbSource,
     NotionSource,
     PersonioSource,
@@ -156,6 +158,8 @@ class SourceDestinationFactory:
         "anthropic": AnthropicSource,
         "csv": LocalCsvSource,
         "docebo": DoceboSource,
+        "http": HttpSource,
+        "https": HttpSource,
         "mongodb": MongoDbSource,
         "mongodb+srv": MongoDbSource,
         "notion": NotionSource,
@@ -214,6 +218,7 @@ class SourceDestinationFactory:
         "influxdb": InfluxDBSource,
         "wise": WiseSource,
         "plusvibeai": PlusVibeAISource,
+        "monday": MondaySource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/http/__init__.py

@@ -0,0 +1,35 @@
+"""HTTP source for reading CSV, JSON, and Parquet files from public URLs"""
+
+from typing import Any, Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .readers import HttpReader
+
+
+@dlt.source
+def http_source(
+    url: str,
+    file_format: Optional[str] = None,
+    **kwargs: Any,
+) -> DltResource:
+    """Source for reading files from HTTP URLs.
+
+    Supports CSV, JSON, and Parquet file formats.
+
+    Args:
+        url (str): The HTTP(S) URL to the file
+        file_format (str, optional): File format ('csv', 'json', 'parquet').
+            If not provided, will be inferred from URL extension.
+        **kwargs: Additional arguments passed to the reader functions
+
+    Returns:
+        DltResource: A dlt resource that yields the file data
+    """
+    reader = HttpReader(url, file_format)
+
+    return dlt.resource(
+        reader.read_file(**kwargs),
+        name="http_data",
+    )

ingestr/src/http/readers.py

@@ -0,0 +1,114 @@
+"""Readers for HTTP file sources"""
+
+import io
+from typing import Any, Iterator, Optional
+from urllib.parse import urlparse
+
+import requests
+from dlt.sources import TDataItems
+
+
+class HttpReader:
+    """Reader for HTTP-based file sources"""
+
+    def __init__(self, url: str, file_format: Optional[str] = None):
+        self.url = url
+        self.file_format = file_format or self._infer_format(url)
+
+        if self.file_format not in ["csv", "json", "parquet"]:
+            raise ValueError(
+                f"Unsupported file format: {self.file_format}. "
+                "Supported formats: csv, json, parquet"
+            )
+
+    def _infer_format(self, url: str) -> str:
+        """Infer file format from URL extension"""
+        parsed = urlparse(url)
+        path = parsed.path.lower()
+
+        if path.endswith(".csv"):
+            return "csv"
+        elif path.endswith(".json") or path.endswith(".jsonl"):
+            return "json"
+        elif path.endswith(".parquet"):
+            return "parquet"
+        else:
+            raise ValueError(
+                f"Cannot infer file format from URL: {url}. "
+                "Please specify file_format parameter."
+            )
+
+    def _download_file(self) -> bytes:
+        """Download file from URL"""
+        response = requests.get(self.url, stream=True, timeout=30)
+        response.raise_for_status()
+        return response.content
+
+    def read_file(self, **kwargs: Any) -> Iterator[TDataItems]:
+        """Read file and yield data in chunks"""
+        content = self._download_file()
+
+        if self.file_format == "csv":
+            yield from self._read_csv(content, **kwargs)
+        elif self.file_format == "json":
+            yield from self._read_json(content, **kwargs)
+        elif self.file_format == "parquet":
+            yield from self._read_parquet(content, **kwargs)
+
+    def _read_csv(
+        self, content: bytes, chunksize: int = 10000, **pandas_kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read CSV file with Pandas chunk by chunk"""
+        import pandas as pd  # type: ignore
+
+        kwargs = {**{"header": "infer", "chunksize": chunksize}, **pandas_kwargs}
+
+        file_obj = io.BytesIO(content)
+        for df in pd.read_csv(file_obj, **kwargs):
+            yield df.to_dict(orient="records")
+
+    def _read_json(
+        self, content: bytes, chunksize: int = 1000, **kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read JSON or JSONL file"""
+        from dlt.common import json
+
+        file_obj = io.BytesIO(content)
+        text = file_obj.read().decode("utf-8")
+
+        # Try to detect if it's JSONL format (one JSON object per line)
+        lines = text.strip().split("\n")
+
+        if len(lines) > 1:
+            # Likely JSONL format
+            lines_chunk = []
+            for line in lines:
+                if line.strip():
+                    lines_chunk.append(json.loads(line))
+                    if len(lines_chunk) >= chunksize:
+                        yield lines_chunk
+                        lines_chunk = []
+            if lines_chunk:
+                yield lines_chunk
+        else:
+            # Single JSON object or array
+            data = json.loads(text)
+            if isinstance(data, list):
+                # Chunk the list
+                for i in range(0, len(data), chunksize):
+                    yield data[i : i + chunksize]
+            else:
+                # Single object
+                yield [data]
+
+    def _read_parquet(
+        self, content: bytes, chunksize: int = 10000, **kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read Parquet file"""
+        from pyarrow import parquet as pq  # type: ignore
+
+        file_obj = io.BytesIO(content)
+        parquet_file = pq.ParquetFile(file_obj)
+
+        for batch in parquet_file.iter_batches(batch_size=chunksize):
+            yield batch.to_pylist()

ingestr/src/monday/__init__.py

@@ -0,0 +1,246 @@
+"""
+Monday.com source for data extraction via GraphQL API.
+
+This source provides access to Monday.com app installation data.
+"""
+
+from typing import Any, Iterable, Iterator, Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import MondayClient, normalize_dict
+
+
+@dlt.source(max_table_nesting=0, name="monday_source")
+def monday_source(
+    api_token: str,
+    params: list[str],
+    start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+) -> Iterable[DltResource]:
+    """
+    Monday.com data source.
+
+    Args:
+        api_token: Monday.com API token for authentication
+        params: Table-specific parameters in format [table_type, ...params]
+        start_date: Optional start date for date-filtered queries (YYYY-MM-DD)
+        end_date: Optional end date for date-filtered queries (YYYY-MM-DD)
+
+    Yields:
+        DltResource: Data resource for the requested table
+    """
+    monday_client = MondayClient(api_token)
+
+    @dlt.resource(
+        name="account",
+        write_disposition="replace",
+    )
+    def fetch_account() -> Iterator[dict[str, Any]]:
+        """
+        Fetch account information from Monday.com.
+
+        Table format: account (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Account table must be in the format `account`")
+
+        yield normalize_dict(monday_client.get_account())
+
+    @dlt.resource(
+        name="account_roles",
+        write_disposition="replace",
+    )
+    def fetch_account_roles() -> Iterator[dict[str, Any]]:
+        """
+        Fetch account roles from Monday.com.
+
+        Table format: account_roles (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError(
+                "Account roles table must be in the format `account_roles`"
+            )
+
+        yield from monday_client.get_account_roles()
+
+    @dlt.resource(
+        name="users",
+        write_disposition="replace",
+    )
+    def fetch_users() -> Iterator[dict[str, Any]]:
+        """
+        Fetch users from Monday.com.
+
+        Table format: users (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Users table must be in the format `users`")
+
+        yield from monday_client.get_users()
+
+    @dlt.resource(
+        name="boards",
+        write_disposition="merge",
+        primary_key="id",
+    )
+    def fetch_boards(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updated_at", initial_value=start_date
+        ),
+    ) -> Iterator[dict[str, Any]]:
+        """
+        Fetch boards from Monday.com.
+
+        Table format: boards (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Boards table must be in the format `boards`")
+
+        yield from monday_client.get_boards()
+
+    @dlt.resource(
+        name="workspaces",
+        write_disposition="replace",
+    )
+    def fetch_workspaces() -> Iterator[dict[str, Any]]:
+        """
+        Fetch workspaces from Monday.com.
+
+        Table format: workspaces (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Workspaces table must be in the format `workspaces`")
+
+        yield from monday_client.get_workspaces()
+
+    @dlt.resource(
+        name="webhooks",
+        write_disposition="replace",
+    )
+    def fetch_webhooks() -> Iterator[dict[str, Any]]:
+        """
+        Fetch webhooks from Monday.com.
+
+        Table format: webhooks (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Webhooks table must be in the format `webhooks`")
+
+        yield from monday_client.get_webhooks()
+
+    @dlt.resource(
+        name="updates",
+        write_disposition="merge",
+        primary_key="id",
+    )
+    def fetch_updates(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updated_at", initial_value=start_date
+        ),
+    ) -> Iterator[dict[str, Any]]:
+        """
+        Fetch updates from Monday.com.
+
+        Table format: updates (no parameters needed)
+        Requires start_date and end_date parameters
+        """
+        if len(params) != 0:
+            raise ValueError("Updates table must be in the format `updates`")
+
+        yield from monday_client.get_updates(start_date=start_date, end_date=end_date)
+
+    @dlt.resource(
+        name="teams",
+        write_disposition="replace",
+    )
+    def fetch_teams() -> Iterator[dict[str, Any]]:
+        """
+        Fetch teams from Monday.com.
+
+        Table format: teams (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Teams table must be in the format `teams`")
+
+        yield from monday_client.get_teams()
+
+    @dlt.resource(
+        name="tags",
+        write_disposition="replace",
+    )
+    def fetch_tags() -> Iterator[dict[str, Any]]:
+        """
+        Fetch tags from Monday.com.
+
+        Table format: tags (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Tags table must be in the format `tags`")
+
+        yield from monday_client.get_tags()
+
+    @dlt.resource(
+        name="custom_activities",
+        write_disposition="replace",
+    )
+    def fetch_custom_activities() -> Iterator[dict[str, Any]]:
+        """
+        Fetch custom activities from Monday.com.
+
+        Table format: custom_activities (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError(
+                "Custom activities table must be in the format `custom_activities`"
+            )
+
+        yield from monday_client.get_custom_activities()
+
+    @dlt.resource(
+        name="board_columns",
+        write_disposition="replace",
+    )
+    def fetch_board_columns() -> Iterator[dict[str, Any]]:
+        """
+        Fetch board columns from Monday.com.
+
+        Table format: board_columns (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError(
+                "Board columns table must be in the format `board_columns`"
+            )
+
+        yield from monday_client.get_board_columns()
+
+    @dlt.resource(
+        name="board_views",
+        write_disposition="replace",
+    )
+    def fetch_board_views() -> Iterator[dict[str, Any]]:
+        """
+        Fetch board views from Monday.com.
+
+        Table format: board_views (no parameters needed)
+        """
+        if len(params) != 0:
+            raise ValueError("Board views table must be in the format `board_views`")
+
+        yield from monday_client.get_board_views()
+
+    return (
+        fetch_account,
+        fetch_account_roles,
+        fetch_users,
+        fetch_boards,
+        fetch_workspaces,
+        fetch_webhooks,
+        fetch_updates,
+        fetch_teams,
+        fetch_tags,
+        fetch_custom_activities,
+        fetch_board_columns,
+        fetch_board_views,
+    )

ingestr/src/monday/helpers.py

@@ -0,0 +1,392 @@
+from typing import Any, Dict, Iterator, Optional
+
+from ingestr.src.http_client import create_client
+
+from .settings import (
+    ACCOUNT_QUERY,
+    ACCOUNT_ROLES_QUERY,
+    BOARD_COLUMNS_QUERY,
+    BOARD_VIEWS_QUERY,
+    BOARDS_QUERY,
+    CUSTOM_ACTIVITIES_QUERY,
+    MAX_PAGE_SIZE,
+    TAGS_QUERY,
+    TEAMS_QUERY,
+    UPDATES_QUERY,
+    USERS_QUERY,
+    WEBHOOKS_QUERY,
+    WORKSPACES_QUERY,
+)
+
+
+def _paginate(
+    client: "MondayClient",
+    query: str,
+    field_name: str,
+    limit: int = 100,
+    extra_variables: Optional[Dict[str, Any]] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Helper function to paginate through Monday.com API results.
+
+    Args:
+        client: MondayClient instance
+        query: GraphQL query with $limit and $page variables
+        field_name: Name of the field in the response to extract
+        limit: Number of results per page
+        extra_variables: Additional variables to pass to the query
+
+    Yields:
+        Normalized dictionaries from the API response
+    """
+    page = 1
+
+    while True:
+        variables = {
+            "limit": min(limit, MAX_PAGE_SIZE),
+            "page": page,
+        }
+
+        if extra_variables:
+            variables.update(extra_variables)
+
+        data = client._execute_query(query, variables)
+        items = data.get(field_name, [])
+
+        if not items:
+            break
+
+        for item in items:
+            yield normalize_dict(item)
+
+        if len(items) < limit:
+            break
+
+        page += 1
+
+
+def _get_all_board_ids(client: "MondayClient") -> list[str]:
+    """
+    Collect all board IDs from the Monday.com API.
+
+    Args:
+        client: MondayClient instance
+
+    Returns:
+        List of board IDs as strings
+    """
+    board_ids = []
+    for board in _paginate(client, BOARDS_QUERY, "boards", MAX_PAGE_SIZE):
+        board_id = board.get("id")
+        if board_id:
+            board_ids.append(str(board_id))
+    return board_ids
+
+
+def _fetch_nested_board_data(
+    client: "MondayClient", query: str, nested_field: str
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch nested data from boards (columns, views, etc).
+
+    Args:
+        client: MondayClient instance
+        query: GraphQL query to execute
+        nested_field: Name of the nested field to extract (e.g., "columns", "views")
+
+    Yields:
+        Dict containing nested data with board_id added
+    """
+    board_ids = _get_all_board_ids(client)
+
+    if not board_ids:
+        return
+
+    for board_id in board_ids:
+        variables = {"board_ids": [board_id]}
+        data = client._execute_query(query, variables)
+        boards = data.get("boards", [])
+
+        for board in boards:
+            nested_items = board.get(nested_field, [])
+
+            if nested_items and isinstance(nested_items, list):
+                for item in nested_items:
+                    item_data = item.copy()
+                    item_data["board_id"] = board.get("id")
+                    yield normalize_dict(item_data)
+
+
+def _fetch_simple_list(
+    client: "MondayClient", query: str, field_name: str
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch a simple list of items from Monday.com API without pagination.
+
+    Args:
+        client: MondayClient instance
+        query: GraphQL query to execute
+        field_name: Name of the field in the response to extract
+
+    Yields:
+        Normalized dictionaries from the API response
+    """
+    data = client._execute_query(query)
+    items = data.get(field_name, [])
+
+    for item in items:
+        yield normalize_dict(item)
+
+
+def normalize_dict(data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Normalize dictionary fields by detecting their structure:
+    - Convert nested objects with 'id' field to {field_name}_id
+    - Convert objects with other fields to flattened {field_name}_{subfield}
+    - Convert arrays to JSON strings for storage
+    - Preserve null values
+
+    Args:
+        data: The dictionary to normalize
+
+    Returns:
+        Normalized dictionary with flattened structure
+
+    Example:
+        >>> normalize_dict({"user": {"id": "123"}, "plan": {"tier": "pro"}})
+        {"user_id": "123", "plan_tier": "pro"}
+    """
+    import json
+
+    normalized: Dict[str, Any] = {}
+
+    for key, value in data.items():
+        if value is None:
+            # Keep null values as-is
+            normalized[key] = None
+        elif isinstance(value, dict):
+            # If the dict has only an 'id' field, replace with {key}_id
+            if "id" in value and len(value) == 1:
+                normalized[f"{key}_id"] = value["id"]
+            # If dict has multiple fields, flatten them
+            elif value:
+                for subkey, subvalue in value.items():
+                    normalized[f"{key}_{subkey}"] = subvalue
+        elif isinstance(value, list):
+            # If list contains dicts with only 'id' field, extract ids
+            if value and isinstance(value[0], dict) and list(value[0].keys()) == ["id"]:
+                normalized[key] = [item["id"] for item in value]
+            else:
+                # Convert other lists to JSON strings for storage
+                normalized[key] = json.dumps(value)
+        else:
+            # Add scalar values directly
+            normalized[key] = value
+
+    return normalized
+
+
+class MondayClient:
+    """Monday.com GraphQL API client."""
+
+    def __init__(self, api_token: str) -> None:
+        self.api_token = api_token
+        self.base_url = "https://api.monday.com/v2"
+        self.session = create_client()
+
+    def _headers(self) -> Dict[str, str]:
+        return {
+            "Authorization": self.api_token,
+            "Content-Type": "application/json",
+        }
+
+    def _execute_query(
+        self, query: str, variables: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """Execute a GraphQL query against Monday.com API."""
+        payload: Dict[str, Any] = {"query": query}
+        if variables:
+            payload["variables"] = variables
+
+        response = self.session.post(
+            self.base_url,
+            headers=self._headers(),
+            json=payload,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        if "errors" in data:
+            raise Exception(f"GraphQL errors: {data['errors']}")
+
+        return data.get("data", {})
+
+    def get_account(self) -> Dict[str, Any]:
+        """
+        Fetch account information from Monday.com API.
+
+        Returns:
+            Dict containing account data
+        """
+        data = self._execute_query(ACCOUNT_QUERY)
+        account = data.get("account", {})
+
+        if not account:
+            raise Exception("No account data returned from Monday.com API")
+
+        return normalize_dict(account)
+
+    def get_account_roles(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch account roles from Monday.com API.
+
+        Yields:
+            Dict containing account role data
+        """
+        yield from _fetch_simple_list(self, ACCOUNT_ROLES_QUERY, "account_roles")
+
+    def get_users(self, limit: int = MAX_PAGE_SIZE) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch users from Monday.com API with pagination.
+
+        Args:
+            limit: Number of results per page (max 100)
+
+        Yields:
+            Dict containing user data
+        """
+        yield from _paginate(self, USERS_QUERY, "users", limit)
+
+    def get_boards(self, limit: int = MAX_PAGE_SIZE) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch boards from Monday.com API with pagination.
+
+        Args:
+            limit: Number of results per page (max 100)
+
+        Yields:
+            Dict containing board data
+        """
+        yield from _paginate(self, BOARDS_QUERY, "boards", limit)
+
+    def get_workspaces(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch workspaces from Monday.com API.
+        First gets all boards to extract unique workspace IDs,
+        then fetches workspace details.
+
+        Yields:
+            Dict containing workspace data
+        """
+        # Collect unique workspace IDs from boards
+        workspace_ids = set()
+        for board in _paginate(self, BOARDS_QUERY, "boards", MAX_PAGE_SIZE):
+            workspace_id = board.get("workspace_id")
+            if workspace_id:
+                workspace_ids.add(str(workspace_id))
+
+        if not workspace_ids:
+            return
+
+        # Fetch workspace details
+        variables = {"ids": list(workspace_ids)}
+        data = self._execute_query(WORKSPACES_QUERY, variables)
+        workspaces = data.get("workspaces", [])
+
+        for workspace in workspaces:
+            yield normalize_dict(workspace)
+
+    def get_webhooks(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch webhooks from Monday.com API.
+        First gets all board IDs, then fetches webhooks for each board.
+
+        Yields:
+            Dict containing webhook data
+        """
+        board_ids = _get_all_board_ids(self)
+
+        for board_id in board_ids:
+            variables = {"board_id": board_id}
+            data = self._execute_query(WEBHOOKS_QUERY, variables)
+            webhooks = data.get("webhooks", [])
+
+            for webhook in webhooks:
+                yield normalize_dict(webhook)
+
+    def get_updates(
+        self,
+        limit: int = MAX_PAGE_SIZE,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+    ) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch updates from Monday.com API.
+
+        Args:
+            limit: Number of results (max 100)
+            start_date: Start date in YYYY-MM-DD format
+            end_date: End date in YYYY-MM-DD format
+
+        Yields:
+            Dict containing update data
+        """
+        variables: Dict[str, Any] = {"limit": min(limit, MAX_PAGE_SIZE)}
+
+        if start_date:
+            variables["from_date"] = start_date
+        if end_date:
+            variables["to_date"] = end_date
+
+        data = self._execute_query(UPDATES_QUERY, variables)
+        updates = data.get("updates", [])
+
+        for update in updates:
+            yield normalize_dict(update)
+
+    def get_teams(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch teams from Monday.com API.
+
+        Yields:
+            Dict containing team data
+        """
+        yield from _fetch_simple_list(self, TEAMS_QUERY, "teams")
+
+    def get_tags(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch tags from Monday.com API.
+
+        Yields:
+            Dict containing tag data
+        """
+        yield from _fetch_simple_list(self, TAGS_QUERY, "tags")
+
+    def get_custom_activities(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch custom activities from Monday.com API.
+
+        Yields:
+            Dict containing custom activity data
+        """
+        yield from _fetch_simple_list(self, CUSTOM_ACTIVITIES_QUERY, "custom_activity")
+
+    def get_board_columns(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch board columns from Monday.com API.
+        First gets all board IDs, then fetches columns for each board.
+
+        Yields:
+            Dict containing board column data with board_id
+        """
+        yield from _fetch_nested_board_data(self, BOARD_COLUMNS_QUERY, "columns")
+
+    def get_board_views(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fetch board views from Monday.com API.
+        First gets all board IDs, then fetches views for each board.
+
+        Yields:
+            Dict containing board view data with board_id
+        """
+        yield from _fetch_nested_board_data(self, BOARD_VIEWS_QUERY, "views")

ingestr/src/monday/settings.py

@@ -0,0 +1,328 @@
+"""Monday.com source settings and constants"""
+
+# GraphQL query for fetching app installs
+APP_INSTALLS_QUERY = """
+query ($app_id: ID!, $account_id: ID, $limit: Int!, $page: Int!) {
+    app_installs(
+        app_id: $app_id
+        account_id: $account_id
+        limit: $limit
+        page: $page
+    ) {
+        app_id
+        timestamp
+        app_install_account {
+            id
+        }
+        app_install_user {
+            id
+        }
+        app_version {
+            major
+            minor
+            patch
+            type
+            text
+        }
+        permissions {
+            approved_scopes
+            required_scopes
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching account information
+ACCOUNT_QUERY = """
+query {
+    account {
+        id
+        name
+        slug
+        tier
+        country_code
+        first_day_of_the_week
+        show_timeline_weekends
+        sign_up_product_kind
+        active_members_count
+        logo
+        plan {
+            max_users
+            period
+            tier
+            version
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching account roles
+ACCOUNT_ROLES_QUERY = """
+query {
+    account_roles {
+        id
+        name
+        roleType
+    }
+}
+"""
+
+# GraphQL query for fetching users
+USERS_QUERY = """
+query ($limit: Int!, $page: Int!) {
+    users(limit: $limit, page: $page) {
+        id
+        name
+        email
+        enabled
+        is_admin
+        is_guest
+        is_pending
+        is_view_only
+        created_at
+        birthday
+        country_code
+        join_date
+        location
+        mobile_phone
+        phone
+        photo_original
+        photo_thumb
+        photo_tiny
+        time_zone_identifier
+        title
+        url
+        utc_hours_diff
+        current_language
+        account {
+            id
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching boards
+BOARDS_QUERY = """
+query ($limit: Int!, $page: Int!) {
+    boards(limit: $limit, page: $page) {
+        id
+        name
+        description
+        state
+        board_kind
+        board_folder_id
+        workspace_id
+        permissions
+        item_terminology
+        items_count
+        updated_at
+        url
+        communication
+        object_type_unique_key
+        type
+        creator {
+            id
+        }
+        owners {
+            id
+        }
+        subscribers {
+            id
+        }
+        team_owners {
+            id
+        }
+        team_subscribers {
+            id
+        }
+        tags {
+            id
+            
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching custom activities
+CUSTOM_ACTIVITIES_QUERY = """
+query {
+    custom_activity {
+        id
+        name
+        type
+        color
+        icon_id
+    }
+}
+"""
+
+# GraphQL query for fetching board columns
+BOARD_COLUMNS_QUERY = """
+query ($board_ids: [ID!]) {
+    boards(ids: $board_ids) {
+        id
+        columns {
+            id
+            title
+            type
+            archived
+            description
+            settings_str
+            width
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching board views
+BOARD_VIEWS_QUERY = """
+query ($board_ids: [ID!]) {
+    boards(ids: $board_ids) {
+        id
+        views {
+            id
+            name
+            type
+            settings_str
+            view_specific_data_str
+            source_view_id
+            access_level
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching workspaces by IDs
+WORKSPACES_QUERY = """
+query ($ids: [ID!]) {
+    workspaces(ids: $ids) {
+        id
+        name
+        kind
+        description
+        created_at
+        is_default_workspace
+        state
+        account_product {
+            id
+        }
+        owners_subscribers {
+            id
+        }
+        team_owners_subscribers {
+            id
+        }
+        teams_subscribers {
+            id
+        }
+        users_subscribers {
+            id
+        }
+        settings {
+            icon
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching webhooks by board ID
+WEBHOOKS_QUERY = """
+query ($board_id: ID!) {
+    webhooks(board_id: $board_id) {
+        id
+        event
+        board_id
+        config
+    }
+}
+"""
+
+# GraphQL query for fetching updates
+UPDATES_QUERY = """
+query ($limit: Int!, $from_date: String, $to_date: String) {
+    updates(limit: $limit, from_date: $from_date, to_date: $to_date) {
+        id
+        body
+        text_body
+        created_at
+        updated_at
+        edited_at
+        creator_id
+        item_id
+        creator {
+            id
+        }
+        item {
+            id
+        }
+        assets {
+            id
+            name
+            file_extension
+            file_size
+            public_url
+            url
+            url_thumbnail
+            created_at
+            original_geometry
+            uploaded_by {
+                id
+            }
+        }
+        replies {
+            id
+            body
+            text_body
+            created_at
+            updated_at
+            creator_id
+            creator {
+                id
+            }
+        }
+        likes {
+            id
+        }
+        pinned_to_top {
+            item_id
+        }
+        viewers {
+            medium
+            user_id
+            user {
+                id
+            }
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching teams
+TEAMS_QUERY = """
+query {
+    teams {
+        id
+        name
+        picture_url
+        users {
+            id
+            created_at
+            phone
+        }
+    }
+}
+"""
+
+# GraphQL query for fetching tags
+TAGS_QUERY = """
+query {
+    tags {
+        id
+        name
+        color
+    }
+}
+"""
+
+# Maximum number of results per page
+MAX_PAGE_SIZE = 100

ingestr/src/sources.py

@@ -3878,3 +3878,86 @@ class IntercomSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class HttpSource:
+    """Source for reading CSV, JSON, and Parquet files from HTTP URLs"""
+
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        """
+        Create a dlt source for reading files from HTTP URLs.
+
+        URI format: http://example.com/file.csv or https://example.com/file.json
+
+        Args:
+            uri: HTTP(S) URL to the file
+            table: Not used for HTTP source (files are read directly)
+            **kwargs: Additional arguments:
+                - file_format: Optional file format override ('csv', 'json', 'parquet')
+                - chunksize: Number of records to process at once (default varies by format)
+                - merge_key: Merge key for the resource
+
+        Returns:
+            DltResource for the HTTP file
+        """
+        from ingestr.src.http import http_source
+
+        # Extract the actual URL (remove the http:// or https:// scheme if duplicated)
+        url = uri
+        if uri.startswith("http://http://") or uri.startswith("https://https://"):
+            url = uri.split("://", 1)[1]
+
+        file_format = kwargs.get("file_format")
+        chunksize = kwargs.get("chunksize")
+        merge_key = kwargs.get("merge_key")
+
+        reader_kwargs = {}
+        if chunksize is not None:
+            reader_kwargs["chunksize"] = chunksize
+
+        source = http_source(url=url, file_format=file_format, **reader_kwargs)
+
+        if merge_key:
+            source.apply_hints(merge_key=merge_key)
+
+        return source
+
+
+class MondaySource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        api_token = query_params.get("api_token")
+
+        if api_token is None:
+            raise MissingValueError("api_token", "Monday")
+
+        parts = table.replace(" ", "").split(":")
+        table_name = parts[0]
+        params = parts[1:]
+
+        # Get interval_start and interval_end from kwargs (command line args)
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+
+        # Convert datetime to string format YYYY-MM-DD
+        start_date = interval_start.strftime("%Y-%m-%d") if interval_start else None
+        end_date = interval_end.strftime("%Y-%m-%d") if interval_end else None
+
+        from ingestr.src.monday import monday_source
+
+        try:
+            return monday_source(
+                api_token=api_token[0],
+                params=params,
+                start_date=start_date,
+                end_date=end_date,
+            ).with_resources(table_name)
+        except ResourcesNotFoundError:
+            raise UnsupportedResourceError(table_name, "Monday")

{ingestr-0.14.5.dist-info → ingestr-0.14.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.5
+Version: 0.14.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.5.dist-info → ingestr-0.14.6.dist-info}/RECORD

@@ -2,17 +2,17 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=8EpHAODYK-dLsm_OMa9EmwMw0Qxn4aTYbjDN-PvxxBQ,20
+ingestr/src/buildinfo.py,sha256=dazsjzReTYtam8X7FVSN4WAYUmvlPNZ0XztT57SOHTU,20
 ingestr/src/destinations.py,sha256=QtjE0AGs0WkPHaI2snWPHJ8HHi4lwXUBYLJPklz8Mvk,27772
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=NNUwks5V66iaWfOblF9MtIpclBasFjYZsMXOKpXUKq4,7538
+ingestr/src/factory.py,sha256=03eGDe2rL6qyT5sGmTGZi-XIwJbbdoedE_KjW3ZF7QY,7661
 ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=MuMD7r8_ojCAMVix5p5BD461biZdT5AKE4YZspikrHw,135647
+ingestr/src/sources.py,sha256=yQhmgIIfzMr8qHxQr-yDmzowti_q59khRzBDPY0Kw-I,138486
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -83,6 +83,8 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
+ingestr/src/http/__init__.py,sha256=Y9mQIE0RolHOh6dPjW41qzYXSG8BC0GPKxEtz2CJGpU,902
+ingestr/src/http/readers.py,sha256=rgBwYG5SOQ7P2uzBAFMOQIevKxV51ZW41VSiRTZ0Xvo,3863
 ingestr/src/hubspot/__init__.py,sha256=FCqjLeOjijdc9JC_NoDwtRqy3FDyY-szDi6UV7CdDN0,11548
 ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
@@ -109,6 +111,9 @@ ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffW
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
 ingestr/src/mixpanel/__init__.py,sha256=s1QtqMP0BTGW6YtdCabJFWj7lEn7KujzELwGpBOQgfs,1796
 ingestr/src/mixpanel/client.py,sha256=c_reouegOVYBOwHLfgYFwpmkba0Sxro1Zkml07NCYf0,3602
+ingestr/src/monday/__init__.py,sha256=ZNdGCC_1CEYlgxAef-5QO56Drm9IMP82-rZpEvbD8aY,6918
+ingestr/src/monday/helpers.py,sha256=xkAYTFIwjbU-dQTa4d41oQm6kFvCHv74AhCmN-H8aPE,11572
+ingestr/src/monday/settings.py,sha256=5TC0OrTHQO52AifwP3Z2xsh4D8SDUq0WxqY5AQMjcns,5667
 ingestr/src/mongodb/__init__.py,sha256=6-DvvaKL7XOPPRwItI7lSpoMQLEPzYubV6dKhpzbuME,7494
 ingestr/src/mongodb/helpers.py,sha256=BKb0F-AUWjFJikE9OPP9z5wFuMmJsf8YsyWhvQ9dC1k,38076
 ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
@@ -175,8 +180,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
-ingestr-0.14.5.dist-info/METADATA,sha256=YSfv9AqmjLd5qjotu6cNcKxFQT8FMQu32EsY60B07Tw,15265
-ingestr-0.14.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.14.5.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.14.5.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.14.5.dist-info/RECORD,,
+ingestr-0.14.6.dist-info/METADATA,sha256=3akmbk91m4xi1AMYFkuMPKmAtlcUSzwCOsYbeXwFlsk,15265
+ingestr-0.14.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.14.6.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.14.6.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.14.6.dist-info/RECORD,,