rapidpro-api 0.1.0__py3-none-any.whl

rapidpro_api/field_processors.py

@@ -0,0 +1,34 @@
+import copy
+
+def process_field(field, metadata:dict=None):
+    """
+    Processes a field object. Merges with metadata.
+    
+    Args:
+        field (dict): The field object to process.
+        metadata (dict, optional): Additional metadata to merge with the field object.
+        
+    Returns:
+        dict: The processed field object.
+    Example: 
+        group ={
+            "key": "reporters",
+            "name": "Reporters",
+            "type": text,
+        }
+        metadata = {
+            "workspace_code": "test_workspace"
+        }
+        processed_group = process_group(group, metadata)
+        # will return {
+            "key": "reporters",
+            "name": "Reporters",
+            "type": text,
+            "workspace_code": "test_workspace"
+        }
+    """
+    result = copy.deepcopy(field)
+    # Merge with metadata
+    if metadata:
+        result.update(metadata)
+    return result

rapidpro_api/flow_processors.py

@@ -0,0 +1,140 @@
+import copy
+import logging
+from .time_utils import get_iso_date_strings
+
+def process_flow(flow, metadata=None):
+    """
+    Process a flow object to extract relevant information.
+    
+    Args:
+        flow (dict): The flow object containing details about the flow.
+        
+    Returns:
+        dict: Processed flow information with relevant fields.
+    Example:
+        >>> flow = {
+        ...     "uuid": "12345",
+        ...     "name": "Sample Flow",
+                "type": "message",
+                "archived": False,
+        ...     "labels": [],
+        ...     "expires": 4320,
+        ...     "created_on": "2023-10-01T12:00:00Z",
+        ...     "runs": {
+        ...         "active": 1,
+        ...         "completed": 2,
+        ...         "waiting": 0,
+        ...         "interrupted": 0,
+        ...         "expired": 0,
+        ...         "failed": 0
+        ...     }
+        ...     "results": []
+        ... }
+        >>> metadata = {"source": "API"}
+        >>> processed_flow = process_flow(flow, metadata)
+        >>> print(processed_flow)
+        {
+            'uuid': '12345',
+            'name': 'Sample Flow',
+            'type': 'message',
+            'archived': False,
+            'labels': [],
+            'expires': 4320,
+            'created_on': '2023-10-01T12:00:00Z',
+            'created_on_year': '2023',
+            'created_on_month': '2023-10',
+            'created_on_day': '2023-10-01',
+            'active': 1,
+            'completed': 2,
+            'waiting': 0,
+            'interrupted': 0,
+            'expired': 0,
+            'failed': 0,
+            'source': 'API' 
+        }
+
+    """
+    # Validate the contact is a dictionary
+    if not isinstance(flow, dict):
+        logging.error("Contact is not a dictionary")
+        raise ValueError("Contact is not a dictionary")
+
+    # Validate the contact has a uuid
+    if not flow.get("uuid"):
+        logging.warning("Contact does not have a uuid")
+
+    # flattern runs
+    processed = copy.deepcopy(flow)
+    processed.update(flow.get("runs", {}))
+    # remove runs, node_uuids and results
+    processed.pop("runs", None)
+
+    # add metadata if provided
+    if metadata:
+        processed.update(metadata)
+    # add the dates for created_ond
+    try:
+      y, m, d = get_iso_date_strings(flow.get("created_on", ""))
+      processed["created_on_year"] = y
+      processed["created_on_month"] = m 
+      processed["created_on_day"] = d 
+    except (ValueError, KeyError) as e:
+      logging.warning("created_on (%s) is invalid for uuid %s. Error: %s", flow.get("created_on", "<not found>"), flow.get("uuid", ""), e)
+    return processed
+
+
+def process_flow_results(flow, metadata=None):
+    """
+    Process the results of a flow to extract relevant information.
+    Args:
+        flow (dict): The flow object containing details about the flow.
+        metadata (dict, optional): Additional metadata to include in the processed results.
+    Returns:
+        list: Processed flow results with relevant fields.
+    Example:
+    >>> flow = {
+          "uuid": "12345",
+          "name": "Sample Flow",
+          "type": "message",
+          "archived: False,
+          "labels": [],
+          "expires": 4320,
+          "created_on": "2023-10-01T12:00:00Z",
+        "results": [
+              {
+                "key": "report",
+                "name": "Report",
+                "categories": ["Report", "Other"],
+                "node_uuids": ["12345"]
+              }
+              {
+                "key": "report",
+                "name": "Report",
+                "categories": ["Report", "Other"],
+                "node_uuids": ["12345"]
+              }
+            }
+        ]  
+        >>> metadata = {"source": "API"}
+        >>> processed_results = process_flow_results(flow, metadata) 
+        >>> print(processed_results)
+        [
+            {
+                'uuid': '12345',
+                'name': 'Sample Flow',
+                'key': 'report',
+                'name': 'Report',
+                'categories': ['Report', 'Other'],
+                'node_uuids': ['12345'],
+                'source': 'API'
+            }
+        ]
+    """
+    processed_results = []
+    for result in flow.get("results", []):
+        processed_result = process_flow(result, metadata)
+        # add the flow uuid and name to the result
+        processed_result["uuid"] = flow.get("uuid", "")
+        processed_result["name"] = flow.get("name", "")
+        processed_results.append(processed_result)
+    return processed_results

rapidpro_api/group_processors.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+@file   group_processors.py
+@brief  Group processors
+@details
+This module contains functions to process groups in RapidPro API format.
+"""
+import logging
+import copy
+from datetime import datetime
+
+def is_in_groups(groups, groups_to_check):
+    """
+    For each group in the groups_check list checks if the contact is in the group.
+   
+    Args:
+        groups (list): A list of groups in RapidPro API format, a dictionary with the group name and uuid. (e.g. [{"name": "group1", "uuid": "12345678-1234-1234-1234-123456789012"}]).
+        groups_to_check (list): A list of groups to check if the contact is in (e.g. ["group1", "group2"]).
+    Returns:
+        dict : A dictionary with the group name as key and a boolean value as value. The value is True if the contact is in the group, False otherwise.
+    Example:
+        contact_groups = [{"name": "group1", "uuid": "12345678-1234-1234-1234-123456789011"}, {"name": "group2", "uuid": "12345678-1234-1234-1234-123456789012"}]
+        groups_to_check = ["group1", "group3"]
+        is_contact_in_groups(contact_groups, groups_to_check)
+        # will return {"group1": True, "group3": False}
+    """
+
+    if groups_to_check is None:
+        return {}
+
+    if not isinstance(groups_to_check, list):
+        logging.error("Contact groups is not a list")
+        raise ValueError("groups_to_check is not a list")
+
+    groups_checked = {}
+    # get groups names
+    groups_names = [group["name"] for group in groups]
+    for group in groups_to_check:
+        if group in groups_names:
+            groups_checked[group] = True
+        else:
+            groups_checked[group] = False
+    return groups_checked
+
+
+def process_group(group, metadata:dict=None, ingestion_date:datetime=None):
+    """
+    Processes a group object. Merges with metadata.
+    
+    Args:
+        group (dict): The group object to process.
+        metadata (dict, optional): Additional metadata to merge with the group object.
+        ingestion_date (datetime, optional): The date when the group was ingested. If provided, it will add fields for year, month, and day.
+        
+    Returns:
+        dict: The processed group object.
+    Example: 
+        group ={
+            "uuid": "5f05311e-8f81-4a67-a5b5-1501b6d6496a",
+            "name": "Reporters",
+            "query": null,
+            "status": "ready",
+            "system": false,
+            "count": 315
+        }
+        metadata = {
+            "workspace_code": "test_workspace"
+        }
+        processed_group = process_group(group, metadata)
+        # will return {
+            "uuid": "5f05311e-8f81-4a67-a5b5-1501b6d6496a",
+            "name": "Reporters",
+            "query": null,
+            "status": "ready",
+            "system": false,
+            "count": 315,
+            "workspace_code": "test_workspace"
+            "ingestion_year": 2023,
+            "ingestion_month": "2023-10",
+            "ingestion_day": "2023-10-01"
+        }
+    """
+    result = copy.deepcopy(group)
+
+    # Add the ingestion date if provided
+    if ingestion_date:
+        result['ingestion_year'] = ingestion_date.year
+        result['ingestion_month'] = ingestion_date.strftime('%Y-%m')
+        result['ingestion_day'] = ingestion_date.strftime('%Y-%m-%d')
+        result['ingestion_date'] = ingestion_date
+    else:
+        # If no ingestion date is provided, set default values
+        result['ingestion_year'] = None
+        result['ingestion_month'] = None
+        result['ingestion_day'] = None
+        result['ingestion_date'] = None
+
+    # Merge with metadata
+    if metadata:
+        result.update(metadata)
+    return result

rapidpro_api/message_processors.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+"""
+This module processes message objects from RapidPro API.
+It flattens nested fields, checks for labels, and merges metadata.
+"""
+import logging
+from .contact_processors import get_contact_urn_type
+
+
+def is_in_labels(message_labels, labels=None):
+    """
+    Checks if any of the specified labels are present in the message's labels.
+
+    Args:
+        message_labels (list of dict): A list of label dictionaries, each expected to have a "name" key.
+        labels (list of str, optional): A list of label names to check for presence in message_labels.
+
+    Returns:
+        list of dict : with the label name as key and True if the label in `labels` is present in `message_labels`, False otherwise.
+    Example:
+        message_labels = [{"name": "Important"}, {"name": "Urgent"}]
+        labels = ["Important", "Urgent", "NotInMessage"]
+        is_in_labels(message_labels, labels)
+        # will return {"Important": True, "Urgent": True, "NotInMessage": False}
+    """
+    if not labels:
+        return {}
+    message_label_names = {label.get("name") for label in message_labels or []}
+    return {label: label in message_label_names for label in labels}
+
+
+def flatten(obj, prefix):
+    """
+    Flattens a dictionary by prefixing its keys.
+    This function takes a dictionary and a prefix string, and returns a new dictionary
+    where each key is prefixed with the provided prefix followed by an underscore.
+    Args:
+        obj (dict): The dictionary to flatten.
+        prefix (str): The prefix to add to each key.
+    Returns:
+        dict : A new dictionary with flattened keys.
+    Example:
+        flatten({"name": "Alice", "age": 30}, "user")
+        # will return {"user_name": "Alice", "user_age": 30}
+    """
+    if not isinstance(obj, dict):
+        return {}
+    return {f"{prefix}_{k}": v for k, v in obj.items()}
+
+def process_message(message, labels=None, metadata=None):
+    """
+    Processes a message object:
+    - Flattens contact, channel, and flow fields.
+    - Removes attachments, quick_replies, and labels.
+    - Adds is_in_labels attribute.
+    - Merges with metadata.
+    - Adds is_broadcast attribute.
+    Args:
+        message (dict): The message object to process.
+        metadata (dict, optional): Additional metadata to merge into the result.
+        labels (list of str, optional): Labels to check against the message's labels.
+    Returns:
+        dict : Processed message with flattened fields and additional attributes.
+    Example:
+        message = {
+            "id": 123,
+            "contact": {"uuid": "d33e9ad5-5c35-414c-abd4-e7451c69ff1d", "name": "Bob McFlow"},
+            "channel": {"uuid": "channel-uuid"},
+            "flow": {"uuid": "flow-uuid"},
+            "labels": [{"name": "Important", "uuid": "d33e9ad5-5c35-414c-abd4-e7451c69ff1d""}, 
+                       {"name": "Urgent", "uuid": "d33e9ad5-5c35-414c-abd4-e7451c69ff1e"}],
+            "text": "Hello, this is a test message.",
+            "attachments": [{"content_type": "image/jpeg", "url": "http://example.com/image.jpg"}],
+            "quick_replies": [{"title": "Reply", "payload": "reply_payload"}],
+            "urn": "tel:+1234567890",
+            "broadcast": 123
+        }
+        metadata = {"workspace_code": "test_workspace"}
+        labels = ["Important", "Urgent", "NotInMessage"]
+        processed_message = process_message(message, metadata, labels)
+        # will return a processed message with flattened fields and additional attributes.
+        
+    """
+    result = dict(message)  # shallow copy
+
+    # Flatten fields
+    for field in ("contact", "channel", "flow"):
+        if field in result and isinstance(result[field], dict):
+            # Flatten the field with the prefix
+            result.update(flatten(result[field], field))
+            result.pop(field, None)
+
+    # Add is_in_labels
+    if labels:
+        result.update(is_in_labels(message.get("labels", None), labels))
+
+    # Process urn if present
+    # Add the urn type to the root object
+    try:
+        result["urn_type"] = get_contact_urn_type(result.get("urn",))[0]
+    except ValueError:
+        result["urn_type"] = None
+        logging.debug("Message %s has no URN. Set as None", result.get("id", "<ID not found>"))
+    # Remove attachments and quick_replies
+    # Add is_broadcast
+    result["is_broadcast"] = bool(message.get("broadcast"))
+    # Merge with metadata
+    if metadata:
+        result.update(metadata)
+    return result

rapidpro_api/run_processors_pl.py

@@ -0,0 +1,87 @@
+import polars as pl
+from rapidpro_api.utils import flatten_pl_struct
+from rapidpro_api.contact_processors import get_contact_urn_type
+   
+
+
+def process_runs_pl(run_batch: list, metadata: dict = None) -> pl.DataFrame:
+    """
+    Process a batch of runs using native Polars operations.
+    Args:
+        run_batch (list): A list of run dicts as returned by the RapidPro API.
+        metadata (dict): Optional metadata columns to add to each run.
+    Returns:
+        pl.DataFrame: A Polars DataFrame of processed runs.
+        
+    """
+    if not run_batch:
+        return pl.DataFrame()
+
+    # First of all, convert the values into a list of dicts:
+    for run in run_batch:
+        # If values is empty, we create a dummy entry to maintain structure
+        values = run.get("values", {})
+
+        if not values:
+            run["run_responses"] = 0
+            run["values"] = [{ 
+                    "key": "empty_run",
+                    "name": "**empty_run**",
+                    "value": "",
+                    "category": "",
+                    "node": "",
+                    "time": ""
+            }
+            ]
+        # Convert values to a list of dicts if it's not already
+        elif isinstance(values, dict):
+            run["run_responses"] = len(values)
+            value_array = []
+            for key, value in values.items():
+                value["key"] = key  # Add the key to the value dict
+                value_array.append(value)
+            run["values"] = value_array
+    # Convert to Polars DataFrame
+    df = pl.DataFrame(run_batch, infer_schema_length=None)
+     # Apply flatten_pl_struct to flow and contact columns
+    df = flatten_pl_struct(df, "flow")
+    df = flatten_pl_struct(df, "contact")
+
+    if "contact_urn" in df.columns:
+        df = df.with_columns([
+            pl.col("contact_urn").map_elements(
+                lambda x: get_contact_urn_type(x) if x else None,
+                return_dtype=pl.Utf8
+            ).alias("urn_type")
+        ])
+    # Process created_on date components
+
+    if "created_on" in df.columns:
+        df = df.with_columns([
+            pl.col("created_on").str.to_datetime().dt.year().cast(pl.Utf8).alias("created_on_year"),
+            pl.col("created_on").str.to_datetime().dt.strftime("%Y-%m").alias("created_on_month"),
+            pl.col("created_on").str.to_datetime().dt.strftime("%Y-%m-%d").alias("created_on_day")
+        ])
+    
+    # exited_on can be null in all the cols which makes it a bit tricker to parse.
+    if "exited_on" in df.columns:
+        # ensure exited_on is string and not null
+        df = df.with_columns(pl.col("exited_on").fill_null("").cast(pl.Utf8))
+        # create date components, allowing for empty strings
+        df = df.with_columns([
+            pl.col("exited_on").str.to_datetime(format="%Y-%m-%dT%H:%M:%S%.fZ", strict=False).dt.year().cast(pl.Utf8).alias("exited_on_year"),
+            pl.col("exited_on").str.to_datetime(format="%Y-%m-%dT%H:%M:%S%.fZ", strict=False).dt.strftime("%Y-%m").alias("exited_on_month"),
+            pl.col("exited_on").str.to_datetime(format="%Y-%m-%dT%H:%M:%S%.fZ", strict=False).dt.strftime("%Y-%m-%d").alias("exited_on_day")
+        ])
+
+    
+    # Explode and normalize the values column
+    if "values" in df.columns:
+        df = df.explode("values").unnest("values")
+    
+    # Add metadata columns if provided
+    if metadata and isinstance(metadata, dict):
+        for k, v in metadata.items():
+            df = df.with_columns(pl.lit(v).alias(k))
+  
+    return df

rapidpro_api/stats/__init__.py

@@ -0,0 +1,5 @@
+from .stats import Stats
+from .download_stats import DownloadStats
+from .download_stats_list import DownloadStatsList
+from .process_stats import ProcessStats
+from .process_stats_list import ProcessStatsList

rapidpro_api/stats/download_stats.py

@@ -0,0 +1,112 @@
+import logging
+import pandas as pd
+from typing import List, Dict, Any, Optional, Tuple
+from deltalake import write_deltalake, DeltaTable
+from rapidpro_api.stats import Stats
+
+class DownloadStats:
+    """
+    Encapsulates statistics for a single workspace download.
+
+    Example usage:
+        # Initialize workspace and parameters
+        workspace = some_workspace_object  # Your workspace object
+        max_retries = 3
+        data_folder = "/path/to/data"
+        
+        # Create download stats tracker
+        download_stats = DownloadStats(workspace, max_retries, data_folder)
+        
+        # Track download attempts
+        for attempt in range(1, max_retries + 1):
+            download_stats.update_attempt(attempt)
+            try:
+                # Perform download operation here
+                # ... download logic ...
+                
+                # On success
+                download_stats.record_success()
+                download_stats.set_comment("Download completed successfully")
+                break
+                
+            except Exception as e:
+                if attempt == max_retries:
+                    # Final failure
+                    download_stats.record_failure(str(e))
+                    download_stats.set_comment("Max retries exceeded")
+                
+        # Get final statistics
+        stats_dict = download_stats.to_dict()
+        print(f"Download completed: {download_stats.completed}")
+        print(f"Stats: {stats_dict}")
+    """
+    
+    def __init__(self, ws, retries: int, base_folder: str):
+        """
+        Initialize stats for a single workspace download.
+        
+        Args:
+            ws: Workspace object
+            retries: Maximum number of retries
+            base_folder: Base folder for data storage
+        """
+        self.stats = Stats()
+        self.ws_code = ws.code
+        self.completed = False
+        
+        # Set initial attributes
+        self.stats.set_attribute("workspace_code", ws.code)
+        self.stats.set_attribute("workspace_active", ws.active)
+        self.stats.set_attribute("workspace_vendor", ws.vendor)
+        self.stats.set_attribute("data_folder", base_folder)
+        self.stats.set_attribute("max_retries", retries)
+        self.stats.set_attribute("completed", "False")
+        self.stats.set_attribute("comments", "")
+        self.stats.start_timer("download_time")
+        self.stats.set_attribute("download_attemps", 0)
+        self.stats.set_attribute("exception", "")
+        self.stats.set_attribute("exception_type", "")
+    
+    def update_attempt(self, attempt: int):
+        """Update the download attempt count."""
+        self.stats.set_attribute("download_attemps", attempt)
+    
+    def record_success(self):
+        """Record a successful download completion."""
+        self.completed = True
+        self.stats.set_attribute("completed", "True")
+        self.stats.stop_timer("download_time")
+    
+    def record_failure(self, error_msg: str = "", exception_type: type = None):
+        """Record a failed download."""
+        self.completed = False
+        self.stats.set_attribute("completed", "False")
+        self.stats.stop_timer("download_time")
+        if error_msg:
+            self.set_exception(error_msg, exception_type)
+    
+    def set_comment(self, comment: str):
+        """Set a comment for this download."""
+        self.stats.set_attribute("comments", comment)
+    
+    def set_exception(self, comment: str, exception_type: type = None):
+        """Set a comment for this download."""
+        self.stats.set_attribute("exception", comment)
+        if exception_type:
+            self.stats.set_attribute("exception_type", str(exception_type))
+
+   
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert stats to dictionary."""
+        return self.stats.to_dict()
+    
+    def to_delta(self, delta_path: str):
+        """
+        Save the stats to a Delta Lake table.
+        
+        Args:
+            delta_path: Path to the Delta Lake table
+        """
+        df = pd.DataFrame([self.to_dict()])
+        write_deltalake(table_or_uri=delta_path, data=df, mode="append")
+