rapidpro-api 0.1.1__tar.gz → 0.2.1__tar.gz

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rapidpro-api
-Version: 0.1.1
+Version: 0.2.1
 Summary: A library for extracting and transforming data from RapidPro
 Home-page: https://github.com/unicef/magasin-primero-paquet
 Author: merlos
@@ -20,6 +20,7 @@ Requires-Dist: fsspec==2025.3.2
 Requires-Dist: pyarrow==18.1.0
 Requires-Dist: adlfs==2024.12.0
 Requires-Dist: deltalake==0.25.5
+Requires-Dist: ijson==3.3.0
 Provides-Extra: dev
 Requires-Dist: pytest==8.3.5; extra == "dev"
 Requires-Dist: pytest-cov==6.1.1; extra == "dev"

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/rapidpro_api/contact_processors.py

@@ -96,11 +96,9 @@ def get_age_group(born: int):
     Returns:
         str: The age group as a string. Possible values are:
             - "None" if the input is invalid, empty, or represents a negative age.
-            - "0-14" for ages between 0 and 14 (inclusive).
-            - "15-19" for ages between 15 and 19 (inclusive).
-            - "20-24" for ages between 20 and 24 (inclusive).
-            - "25-30" for ages between 25 and 30 (inclusive).
-            - "31-34" for ages between 31 and 34 (inclusive).
+            - "0-17" for ages between 0 and 17 (inclusive).
+            - "18-24" for ages between 18 and 24 (inclusive).
+            - "25-34" for ages between 25 and 34 (inclusive).
             - "35+" for ages 35 and above.
     """
     if not born:
@@ -119,16 +117,12 @@ def get_age_group(born: int):
         return "None"
     if age < 0:
         return "None"
-    elif 0 <= age <= 14:
-        return "0-14"
-    elif 15 <= age <= 19:
-        return "15-19"
-    elif 20 <= age <= 24:
-        return "20-24"
-    elif 25 <= age <= 30:
-        return "25-30"
-    elif 31 <= age <= 34:
-        return "31-34"
+    elif 0 <= age <= 17:
+        return "0-17"
+    elif 18 <= age <= 24:
+        return "18-24"
+    elif 25 <= age <= 34:
+        return "25-34"
     else:
         return "35+"
 def parse_born(born: str) -> Union[int, None]:

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/rapidpro_api/datasets/BaseDataset.py

@@ -417,30 +417,10 @@ class BaseDataset:
         # Successfull download
         return True
     
-    def iter_dated_batch_raw_files(self, after_date: datetime=None, before_date: datetime=None, with_file_info:bool=False) -> [list, dict]:
+    def _iter_dated_file_rows(self, after_date: datetime=None, before_date: datetime=None):
         """
-        Iterate over the existing dataset file batches.
-        Args:
-            after_date: The date to start the iteration from. If not set will iterate from the first date in the dataset.
-            before_date: The date to end the iteration. If not set will iterate until the last date in the dataset.
-            with_file_info: If True, it will yield a tuple with the file info and the raw data. If False, it will yield only the raw data.
-        Yields:
-            A tuple with the file info (dict) and the raw data (list of dict where each dict is a record from RproAPI) if with_file_info is True, otherwise
-            it will yield only the raw data.
-        Returns:
-            A generator that yields batches of data (list of dicts).
-        Raises:
-            FileNotFoundError: If no data has been downloaded yet or if the workspace is not found.
-        Example:
-            >>> ds = BaseDataset(name="contacts", workspace=Workspace(code="code"))
-            >>> for file_info, raw_data in ds.iter_dated_batch_raw_files(after_date=datetime(2023, 10, 1), before_date=datetime(2023, 10, 31), with_file_info=True):
-            >>>     print(file_info)  # file info will be a dict of RawDataset file info
-            >>>     print(raw_data)  # raw_data will be a list of dicts with the raw data from RapidPro API
-            >>>     # process the raw_data here
-            >>>     # e.g. process_contacts_pl(raw_data, metadata=metadata)
-            >>> # also getting the index of the file (explore the whole dataset)
-            >>> for index, (file_info, raw_data) in enumerate(ds.iter_dated_batch_raw_files(with_file_info=True)):
-            >>>     print(f"Processing file {index+1}: {file_info['path']}")
+        Yield (index, row, file_path, num_files) for each raw file of the workspace within the date range.
+        Shared by iter_dated_batch_raw_files and iter_dated_batch_raw_files_stream.
         """
         if after_date is None:
             after_date = self.INIT_DATE
@@ -450,13 +430,12 @@ class BaseDataset:
         after_date_str = after_date.strftime('%Y-%m-%d')
         before_date_str = before_date.strftime('%Y-%m-%d')
 
-
         logging.info("[%s] Processing workspace %s from %s until %s...",
                     self.workspace.code, self.workspace.code, after_date_str, before_date_str)
 
         # get the list of files
         files_df = self.raw_dataset.files(partition=[self.workspace.code], as_dataframe=True)
-        
+
         # print (files_df.head())
         if files_df.empty:
             logging.warning("[%s] No files found for workspace %s",
@@ -483,7 +462,35 @@ class BaseDataset:
                 logging.info("%s - %s: [%s/%s] Skipping file  %s as it is newer than the before date (last date) %s",
                             self.name, self.workspace.code, index, num_files, file_path, before_date_str)
                 continue
-            
+
+            yield index, row, file_path, num_files
+
+    def iter_dated_batch_raw_files(self, after_date: datetime=None, before_date: datetime=None, with_file_info:bool=False) -> [list, dict]:
+        """
+        Iterate over the existing dataset file batches.
+        Args:
+            after_date: The date to start the iteration from. If not set will iterate from the first date in the dataset.
+            before_date: The date to end the iteration. If not set will iterate until the last date in the dataset.
+            with_file_info: If True, it will yield a tuple with the file info and the raw data. If False, it will yield only the raw data.
+        Yields:
+            A tuple with the file info (dict) and the raw data (list of dict where each dict is a record from RproAPI) if with_file_info is True, otherwise
+            it will yield only the raw data.
+        Returns:
+            A generator that yields batches of data (list of dicts).
+        Raises:
+            FileNotFoundError: If no data has been downloaded yet or if the workspace is not found.
+        Example:
+            >>> ds = BaseDataset(name="contacts", workspace=Workspace(code="code"))
+            >>> for file_info, raw_data in ds.iter_dated_batch_raw_files(after_date=datetime(2023, 10, 1), before_date=datetime(2023, 10, 31), with_file_info=True):
+            >>>     print(file_info)  # file info will be a dict of RawDataset file info
+            >>>     print(raw_data)  # raw_data will be a list of dicts with the raw data from RapidPro API
+            >>>     # process the raw_data here
+            >>>     # e.g. process_contacts_pl(raw_data, metadata=metadata)
+            >>> # also getting the index of the file (explore the whole dataset)
+            >>> for index, (file_info, raw_data) in enumerate(ds.iter_dated_batch_raw_files(with_file_info=True)):
+            >>>     print(f"Processing file {index+1}: {file_info['path']}")
+        """
+        for index, row, file_path, num_files in self._iter_dated_file_rows(after_date, before_date):
             logging.info("%s - %s: [%s/%s] Iterating file %s (%s bytes)...",
                         self.name, self.workspace.code, index, num_files, file_path, row['filesize'])
             raw_data = self.raw_dataset.get_file_data(file_path)
@@ -491,6 +498,37 @@ class BaseDataset:
                 yield row, raw_data
             else:
                 yield raw_data
+
+    def iter_dated_batch_raw_files_stream(self, after_date: datetime=None, before_date: datetime=None,
+                                           with_file_info:bool=False, chunk_size:int=None):
+        """
+        Like iter_dated_batch_raw_files, but never materializes a full file's records in memory:
+        each file is parsed lazily (via RawDataset.iter_file_records), record by record.
+        Args:
+            after_date: The date to start the iteration from. If not set will iterate from the first date in the dataset.
+            before_date: The date to end the iteration. If not set will iterate until the last date in the dataset.
+            with_file_info: If True, it will yield a tuple with the file info and the records. If False, it will yield only the records.
+            chunk_size (int, optional): If set, records within a file are grouped into lists of at most this size.
+                If not set, individual records are yielded one at a time.
+        Yields:
+            A tuple with the file info (dict) and a generator of records (or record chunks, if chunk_size is set)
+            if with_file_info is True, otherwise just the generator.
+        Raises:
+            FileNotFoundError: If no data has been downloaded yet or if the workspace is not found.
+        Example:
+            >>> ds = BaseDataset(name="contacts", workspace=Workspace(code="code"))
+            >>> for file_info, chunks in ds.iter_dated_batch_raw_files_stream(with_file_info=True, chunk_size=10000):
+            >>>     for chunk in chunks:
+            >>>         process_ureport_contacts_pl(chunk, metadata=metadata)
+        """
+        for index, row, file_path, num_files in self._iter_dated_file_rows(after_date, before_date):
+            logging.info("%s - %s: [%s/%s] Streaming file %s (%s bytes)...",
+                        self.name, self.workspace.code, index, num_files, file_path, row['filesize'])
+            records = self.raw_dataset.iter_file_records(file_path, chunk_size=chunk_size)
+            if with_file_info:
+                yield row, records
+            else:
+                yield records
     
     def iter_last_date_raw_files(self, with_file_info:bool=False) -> [list, dict]:
         """

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/rapidpro_api/datasets/RawDataset.py

@@ -1,5 +1,6 @@
 import json
 import gzip
+import ijson
 from fsspec.implementations.local import LocalFileSystem
 from fsspec.core import url_to_fs
 import pandas as pd
@@ -610,7 +611,61 @@ class RawDataset:
             with self.fs.open(file_path, "r") as f:
                 data = f.read()
                 return json.loads(data)
-        
+
+    def _stream_file_records(self, file_path):
+        """
+        Lazily parse the top-level JSON array of a file, one record at a time.
+        Args:
+            file_path (str): Path to the file.
+        Yields:
+            dict: Each record (array item) found in the file.
+        Raises:
+            FileNotFoundError: If the file does not exist.
+        """
+        if not self.fs.exists(file_path):
+            raise FileNotFoundError(f"File does not exist. The file {file_path} does not exist")
+
+        if file_path.endswith(".gzip"):
+            with self.fs.open(file_path, "rb") as f:
+                with gzip.GzipFile(fileobj=f, mode='rb') as gz:
+                    yield from ijson.items(gz, "item")
+        else:
+            with self.fs.open(file_path, "rb") as f:
+                yield from ijson.items(f, "item")
+
+    def iter_file_records(self, file_path, chunk_size:int=None):
+        """
+        Iterate over the records of a file (top-level JSON array) without loading the whole file into memory.
+        Args:
+            file_path (str): Path to the file.
+            chunk_size (int, optional): If set, records are grouped into lists of at most this size. If not set (default),
+                each record is yielded individually.
+        Yields:
+            dict: A single record, if chunk_size is None.
+            list: A list of up to chunk_size records, if chunk_size is set.
+        Raises:
+            FileNotFoundError: If the file does not exist.
+        Example:
+            >>> ds = RawDataset("contacts", base_folder="raw/")
+            >>> for record in ds.iter_file_records("raw/contacts/batch-000000001.gzip"):
+            >>>     print(record)  # one record (dict) at a time
+            >>> for chunk in ds.iter_file_records("raw/contacts/batch-000000001.gzip", chunk_size=10000):
+            >>>     print(len(chunk))  # a list of up to 10000 records at a time
+        """
+        records = self._stream_file_records(file_path)
+        if chunk_size is None:
+            yield from records
+            return
+
+        chunk = []
+        for record in records:
+            chunk.append(record)
+            if len(chunk) >= chunk_size:
+                yield chunk
+                chunk = []
+        if chunk:
+            yield chunk
+
     def download(self, local_base_folder: str, partition: list = None,):
         """
         Download the files of a dataset or a partition to a local folder.

rapidpro_api-0.2.1/rapidpro_api/datasets/StatisticsDataset.py

@@ -0,0 +1,281 @@
+import logging
+from datetime import datetime, timedelta
+
+import pandas as pd
+import requests
+
+from .BaseDataset import BaseDataset
+from ..time_utils import get_date_range, yesterday_end_of_day
+from ..workspaces import Workspace
+
+
+class StatisticsDataset(BaseDataset):
+    """
+    Dataset for RapidPro daily per-channel statistics.
+
+    This endpoint is not exposed by the temba client used elsewhere in the package,
+    so downloads are performed with direct HTTP requests against the RapidPro API.
+    """
+
+    REQUEST_TIMEOUT = 60
+
+    def __init__(self, workspace: Workspace = None,
+                 base_folder: str = "./datasets",
+                 storage_options: dict = None,
+                 raw_dataset_batch_size: int = None):
+        """
+        Initialize a statistics dataset handler.
+
+        Args:
+            workspace (Workspace, optional): Workspace containing host/token metadata used
+                to authenticate against RapidPro.
+            base_folder (str, optional): Root folder where dataset files are stored.
+                The statistics raw dataset is stored under ``<base_folder>/statistics/raw``.
+            storage_options (dict, optional): Optional fsspec storage options for remote
+                filesystems (for example Azure Blob or S3 credentials).
+            raw_dataset_batch_size (int, optional): Included for consistency with the
+                base dataset API. Statistics downloads are saved per period as a single
+                final batch.
+
+        Returns:
+            None
+
+        Raises:
+            ValueError: Propagated from base classes if workspace/partition operations
+                receive invalid values.
+
+        Example:
+            >>> ws = Workspace(code="rw", host="https://app.rapidpro.io", token="token")
+            >>> ds = StatisticsDataset(workspace=ws, base_folder="./datasets")
+            >>> ds.name
+            'statistics'
+        """
+        super().__init__(name="statistics",
+                         workspace=workspace,
+                         base_folder=base_folder,
+                         storage_options=storage_options,
+                         raw_dataset_batch_size=raw_dataset_batch_size)
+
+    def _statistics_url(self) -> str:
+        """
+        Build the RapidPro statistics endpoint URL for the current workspace.
+
+        Args:
+            None
+
+        Returns:
+            str: Absolute URL to ``/api/v2/statistics.json`` for the workspace host.
+
+        Raises:
+            AttributeError: If ``self.workspace`` is missing required ``host`` field.
+
+        Example:
+            >>> ds._statistics_url()
+            'https://app.rapidpro.io/api/v2/statistics.json'
+        """
+        host = self.workspace.host.rstrip("/")
+        if not host.startswith("http://") and not host.startswith("https://"):
+            host = f"https://{host}"
+        return f"{host}/api/v2/statistics.json"
+
+    def _download_statistics(self, since_date: datetime, until_date: datetime) -> list:
+        """
+        Download statistics rows from RapidPro for a specific date range.
+
+        The endpoint is called with ``since`` as an inclusive date and ``until`` as an
+        exclusive date in ``YYYY-MM-DD`` format.
+
+        Args:
+            since_date (datetime): Inclusive start date for the request.
+            until_date (datetime): Exclusive end date for the request.
+
+        Returns:
+            list: Value of the ``results`` array from the API response, or an empty list
+            when the response does not include results.
+
+        Raises:
+            requests.HTTPError: If the API returns a non-success status code.
+            requests.RequestException: For transport-level request failures.
+            ValueError: If JSON parsing fails in the underlying response parser.
+
+        Example:
+            >>> from datetime import datetime
+            >>> rows = ds._download_statistics(datetime(2026, 3, 1), datetime(2026, 4, 1))
+            >>> isinstance(rows, list)
+            True
+        """
+        response = requests.get(
+            self._statistics_url(),
+            headers={"Authorization": f"Token {self.workspace.token}"},
+            params={
+                "since": since_date.strftime("%Y-%m-%d"),
+                "until": until_date.strftime("%Y-%m-%d"),
+            },
+            timeout=self.REQUEST_TIMEOUT,
+        )
+        response.raise_for_status()
+        payload = response.json() or {}
+        return payload.get("results") or []
+
+    def _download_period(self,
+                         ws_partition: list,
+                         since_date: datetime,
+                         until_date: datetime,
+                         clear: bool = False,
+                         continue_if_exists: bool = True) -> bool:
+        """
+        Download and store one statistics period into a workspace partition.
+
+        This method enforces partition state rules:
+        - clear existing partition when ``clear=True``
+        - skip completed partitions when ``continue_if_exists=True``
+        - redownload incomplete partitions when ``continue_if_exists=True``
+        - raise when partition is incomplete and continuation is not allowed
+
+        Args:
+            ws_partition (list): Workspace-relative partition, usually
+                ``["YYYY", "YYYY-MM-DD"]``.
+            since_date (datetime): Inclusive start date for the API request.
+            until_date (datetime): Exclusive end date for the API request.
+            clear (bool, optional): Whether to clear the partition before download.
+            continue_if_exists (bool, optional): Whether to continue processing when
+                partition data already exists.
+
+        Returns:
+            bool: ``True`` when the period is either successfully saved or intentionally
+            skipped because it is already complete.
+
+        Raises:
+            ValueError: If partition exists and is incomplete while
+                ``continue_if_exists=False``.
+            requests.HTTPError: If the API request fails with a non-success code.
+            requests.RequestException: For transport-level request failures.
+
+        Example:
+            >>> ds._download_period(
+            ...     ws_partition=["2026", "2026-03-01"],
+            ...     since_date=datetime(2026, 3, 1),
+            ...     until_date=datetime(2026, 4, 1),
+            ...     clear=False,
+            ...     continue_if_exists=True,
+            ... )
+            True
+        """
+        partition = self._add_ws_to_partition(ws_partition=ws_partition)
+
+        if not self.raw_dataset.is_empty(partition=partition):
+            if clear:
+                logging.info("%s: %s [%s] Clearing partition %s before statistics download.", self.name, self.workspace.code, self.workspace.vendor, partition)
+                self.raw_dataset.clear(partition=partition)
+            elif self.raw_dataset.has_final_batch(partition=partition):
+                logging.info("%s: %s [%s] Partition %s is already marked as completed. Skipping.", self.name, self.workspace.code, self.workspace.vendor, partition)
+                return True
+            elif continue_if_exists:
+                logging.info("%s: %s [%s] Partition %s is incomplete. Clearing and redownloading period.", self.name, self.workspace.code, self.workspace.vendor, partition)
+                self.raw_dataset.clear(partition=partition)
+            else:
+                raise ValueError(
+                    f"Partition {partition} in {self.workspace.code} already exists and is not empty. "
+                    "Set clear=True to clear the partition or continue_if_exists=True to continue adding batches."
+                )
+
+        results = self._download_statistics(since_date=since_date, until_date=until_date)
+        # Only mark the partition as final when the full period has already elapsed.
+        # Partial periods (e.g. the current month) are saved without the final-batch
+        # flag so subsequent weekly runs will redownload and update them.
+        is_final_batch = until_date <= datetime.now()
+        logging.info("%s: %s [%s] Saving statistics period %s -> %s (%d records, final=%s).",
+                     self.name,
+                     self.workspace.code,
+                     self.workspace.vendor,
+                     since_date.strftime("%Y-%m-%d"),
+                     until_date.strftime("%Y-%m-%d"),
+                     len(results),
+                     is_final_batch)
+        self.raw_dataset.append_save(results, partition=partition, is_final_batch=is_final_batch)
+        return True
+
+    def download_dated_dataset(self,
+                               clear: bool = False,
+                               continue_if_exists: bool = True,
+                               after_date: datetime = None,
+                               before_date: datetime = None,
+                               frequency: str = "ME") -> bool:
+        """
+        Download statistics in dated partitions (monthly by default).
+
+        This overload replaces the base class downloader because the underlying
+        RapidPro client does not provide a ``get_statistics`` iterator. It performs
+        one HTTP request per period and stores each period as a final raw batch in the
+        partition ``[workspace_code, YYYY, YYYY-MM-DD]``.
+
+        Args:
+            clear (bool, optional): Whether to clear each target partition before
+                downloading the corresponding period.
+            continue_if_exists (bool, optional): If ``True``, already completed
+                partitions are skipped and incomplete partitions are redownloaded.
+                If ``False``, incomplete existing partitions raise ``ValueError``.
+            after_date (datetime, optional): Start date for iteration. If not provided,
+                defaults to ``BaseDataset.INIT_DATE`` or the last downloaded date when
+                ``continue_if_exists=True``.
+            before_date (datetime, optional): End boundary for iteration. If not
+                provided, defaults to yesterday end-of-day.
+            frequency (str, optional): One of ``D``, ``W``, ``ME``, or ``Y``.
+                Defaults to ``ME`` (month-end periods).
+
+        Returns:
+            bool: ``True`` when all periods are processed successfully.
+
+        Raises:
+            ValueError: If ``after_date`` is greater than ``before_date``.
+            ValueError: If ``frequency`` is not one of ``D``, ``W``, ``ME``, ``Y``.
+            ValueError: If ``continue_if_exists=True`` and the last downloaded date is
+                later than ``before_date``.
+            requests.HTTPError: If an API request fails with a non-success status code.
+            requests.RequestException: For transport-level request failures.
+
+        Example:
+            >>> ds.download_dated_dataset(
+            ...     after_date=datetime(2026, 1, 1),
+            ...     before_date=datetime(2026, 3, 31, 23, 59, 59),
+            ...     frequency="ME",
+            ...     continue_if_exists=True,
+            ... )
+            True
+        """
+        if after_date and before_date and after_date > before_date:
+            raise ValueError("after_date cannot be greater than before_date")
+
+        if frequency not in ["D", "W", "ME", "Y"]:
+            raise ValueError("frequency must be one of the following: D, W, ME, Y")
+
+        if continue_if_exists and after_date is None:
+            last_date = self.get_last_downloaded_date()
+            if last_date:
+                after_date = last_date
+            if before_date and last_date and last_date > before_date:
+                raise ValueError(f"last_date downloaded ({last_date}) cannot be greater than before_date ({before_date})")
+
+        after_date = after_date or self.INIT_DATE
+        end_loop = before_date or yesterday_end_of_day()
+
+        # Use period-start frequencies so that the current incomplete period
+        # (e.g. the month in progress) is always included in the iteration.
+        # With "ME" the last generated date would be the previous month-end,
+        # leaving the current partial month out of the loop entirely.
+        _PERIOD_START_FREQ = {"ME": "MS", "Y": "YS", "W": "W-MON", "D": "D"}
+        iter_freq = _PERIOD_START_FREQ.get(frequency, frequency)
+
+        for i_date in pd.date_range(start=after_date, end=end_loop, freq=iter_freq):
+            i_after_date, i_before_date = get_date_range(a_date=i_date, frequency=frequency)
+            ws_partition = [i_after_date.strftime("%Y"), i_after_date.strftime("%Y-%m-%d")]
+            until_exclusive = i_before_date + timedelta(microseconds=1)
+            self._download_period(
+                ws_partition=ws_partition,
+                since_date=i_after_date,
+                until_date=until_exclusive,
+                clear=clear,
+                continue_if_exists=continue_if_exists,
+            )
+
+        return True

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/rapidpro_api/datasets/__init__.py

@@ -5,4 +5,7 @@ from .MessagesDataset import MessagesDataset
 from .FlowsDataset import FlowsDataset
 from .RunsDataset import RunsDataset
 from .GroupsDataset import GroupsDataset
-from .FieldsDataset import FieldsDataset
+from .FieldsDataset import FieldsDataset
+from . import StatisticsDataset as _StatisticsDatasetModule
+
+StatisticsDataset = _StatisticsDatasetModule.StatisticsDataset

rapidpro_api-0.2.1/rapidpro_api/statistics_processors.py

@@ -0,0 +1,75 @@
+import copy
+
+
+STATISTICS_COUNT_FIELDS = (
+    "text:in",
+    "text:out",
+    "voice:in",
+    "voice:out",
+)
+
+
+def _normalize_count_key(key):
+    return key.replace(":", "_")
+
+
+def process_statistic(statistic, metadata=None):
+    """
+    Process a single RapidPro statistics day entry into flat per-channel rows.
+
+    Args:
+        statistic (dict): Daily statistics entry returned by the RapidPro API.
+        metadata (dict, optional): Additional metadata merged into each output row.
+
+    Returns:
+        list: One flattened row per channel for the given date.
+    """
+    if not isinstance(statistic, dict):
+        return []
+
+    date = statistic.get("date")
+    channels = statistic.get("channels") or {}
+    if not isinstance(channels, dict):
+        return []
+
+    processed = []
+    for channel_uuid, channel_stats in channels.items():
+        if not isinstance(channel_stats, dict):
+            continue
+
+        row = {
+            "date": date,
+            "date_month": date[:7] if date else None,   # "YYYY-MM"
+            "date_year": date[:4] if date else None,    # "YYYY"
+            "channel_uuid": channel_uuid,
+            "channel_type": channel_stats.get("type"),
+        }
+        for field in STATISTICS_COUNT_FIELDS:
+            row[_normalize_count_key(field)] = channel_stats.get(field, 0)
+
+        if metadata:
+            row.update(copy.deepcopy(metadata))
+
+        processed.append(row)
+
+    return processed
+
+
+def process_statistics(stats: list, metadata: dict = None):
+    """
+    Process a RapidPro statistics ``results`` array into flat per-channel rows.
+
+    Args:
+        stats (list): List of daily statistics entries from the RapidPro API response.
+        metadata (dict, optional): Additional metadata merged into each output row.
+
+    Returns:
+        list: Flattened statistics rows.
+    """
+    if not isinstance(stats, list):
+        return []
+
+    processed = []
+    for statistic in stats:
+        processed.extend(process_statistic(statistic, metadata))
+    return processed

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/rapidpro_api/ureport_processors_pl.py

@@ -176,7 +176,7 @@ def process_ureport_contacts_pl(contacts, fields: list = None, groups:list = Non
         #   "metadata_col1": "value1",
         #   "field1": "value1",
         #   "field2": "value2",
-        #   "age_group": "0-14",
+        #   "age_group": "0-17",
         #.  "registration_date": "2023-01-01",
         #   "country": "Country A",
         #.  "state": "State A",
@@ -205,7 +205,7 @@ def process_ureport_contacts_pl(contacts, fields: list = None, groups:list = Non
 
     # merge fields with the ureport specific fields
     fields = fields or []
-    fields.extend([registration_field, state_field, district_field, ward_field, born_field, gender_field])
+    fields.extend([registration_field, state_field, district_field, ward_field, born_field, gender_field, country_field])
     logging.debug("Fields after merging with ureport specific fields: %s", fields)
 
     # merge groups with the ureport specific fields
@@ -233,7 +233,7 @@ def process_ureport_contacts_pl(contacts, fields: list = None, groups:list = Non
 
     # cast any of the columns that are null to utf8
     cast_operations = []
-    utf8_cols = ['registration_date', 'state', 'district', 'ward', 'gender', 'born']
+    utf8_cols = ['registration_date', 'country', 'state', 'district', 'ward', 'gender', 'born']
     bool_cols = ['is_ureporter']
 
     for col in utf8_cols:

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/rapidpro_api/version.py

@@ -2,7 +2,7 @@
 """
 Version
 """
-__version__ = "0.1.1"
+__version__ = "0.2.1"
 
 def get_version():
     return __version__

{rapidpro_api-0.1.1 → rapidpro_api-0.2.1}/rapidpro_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rapidpro-api
-Version: 0.1.1
+Version: 0.2.1
 Summary: A library for extracting and transforming data from RapidPro
 Home-page: https://github.com/unicef/magasin-primero-paquet
 Author: merlos
@@ -20,6 +20,7 @@ Requires-Dist: fsspec==2025.3.2
 Requires-Dist: pyarrow==18.1.0
 Requires-Dist: adlfs==2024.12.0
 Requires-Dist: deltalake==0.25.5
+Requires-Dist: ijson==3.3.0
 Provides-Extra: dev
 Requires-Dist: pytest==8.3.5; extra == "dev"
 Requires-Dist: pytest-cov==6.1.1; extra == "dev"