openforis-whisp 3.0.0a3__py3-none-any.whl → 3.0.0a5__py3-none-any.whl

openforis_whisp/data_checks.py

@@ -750,23 +750,43 @@ def validate_geojson_constraints(
     return results
 
 
-def suggest_method(polygon_count, mean_area_ha, mean_vertices=None, verbose=True):
+def suggest_processing_mode(
+    feature_count,
+    mean_area_ha=None,
+    mean_vertices=None,
+    feature_type="polygon",
+    verbose=True,
+):
     """
-    Suggest processing method based on polygon characteristics.
+    Suggest processing mode based on feature characteristics.
+
+    Decision thresholds from comprehensive benchmark data (Nov 2025):
 
-    Decision thresholds from benchmark data (area per polygon × polygon count):
-    - Small polygons (10 ha): need 250+ polygons for concurrent
-    - Medium polygons (100 ha): breakeven at ~100 polygons
-    - Large polygons (500 ha): concurrent wins at 50+ polygons
+    POINTS:
+    - Break-even: 750-1000 features
+    - Sequential faster: < 750 features
+    - Concurrent faster: >= 750 features
+
+    POLYGONS (area-based thresholds):
+    - Tiny (< 1 ha): break-even ~500 features
+    - Small (1-5 ha, simple): break-even ~500 features
+    - Small (1-5 ha, complex 20-50v): break-even ~500 features
+    - Medium (5-20 ha): break-even ~250 features
+    - Large (20-100 ha): break-even ~250 features
+    - Very large (50-200 ha): break-even ~250 features
+
+    Vertex complexity adjustment: High vertex counts (>50) favor concurrent at lower thresholds
 
     Parameters:
     -----------
-    polygon_count : int
-        Number of polygons
-    mean_area_ha : float
-        Mean area per polygon in hectares
+    feature_count : int
+        Number of features (polygons or points)
+    mean_area_ha : float, optional
+        Mean area per polygon in hectares (required for polygons, ignored for points)
     mean_vertices : float, optional
-        Mean number of vertices per polygon (can influence decision for complex geometries)
+        Mean number of vertices per polygon (influences decision for complex geometries)
+    feature_type : str
+        'polygon', 'multipolygon', or 'point' (default: 'polygon')
     verbose : bool
         Print recommendation explanation
 
@@ -775,31 +795,63 @@ def suggest_method(polygon_count, mean_area_ha, mean_vertices=None, verbose=True
     str: 'concurrent' or 'sequential'
     """
 
-    # Primary decision based on area
-    if mean_area_ha >= 300:  # Large polygons
-        breakeven = 50
-        method = "concurrent" if polygon_count >= breakeven else "sequential"
-    elif mean_area_ha >= 50:  # Medium polygons
-        breakeven = 100
-        method = "concurrent" if polygon_count >= breakeven else "sequential"
-    else:  # Small polygons
+    # Points: simple threshold-based decision
+    if feature_type == "point":
+        breakeven = 750
+        method = "concurrent" if feature_count >= breakeven else "sequential"
+
+        if verbose:
+            print(f"\nMETHOD RECOMMENDATION (Points)")
+            print(f"   Features: {feature_count} points")
+            print(f"   Break-even: {breakeven} features | Method: {method.upper()}")
+
+        return method
+
+    # Polygons and MultiPolygons: area and complexity-based decision
+    # MultiPolygons use same breakpoints as Polygons
+    if mean_area_ha is None:
+        # Default to conservative threshold if area unknown
+        breakeven = 500
+        method = "concurrent" if feature_count >= breakeven else "sequential"
+
+        if verbose:
+            print(f"\nMETHOD RECOMMENDATION (Polygons - area unknown)")
+            print(f"   Features: {feature_count} polygons")
+            print(
+                f"   Break-even: {breakeven} (conservative) | Method: {method.upper()}"
+            )
+
+        return method
+
+    # Area-based thresholds from benchmark data
+    if mean_area_ha >= 20:  # Large to very large polygons
+        breakeven = 250
+    elif mean_area_ha >= 5:  # Medium polygons
         breakeven = 250
-        method = "concurrent" if polygon_count >= breakeven else "sequential"
+    elif mean_area_ha >= 1:  # Small polygons
+        # Vertex complexity matters more for small polygons
+        if mean_vertices is not None and mean_vertices >= 30:
+            breakeven = 500  # Complex small polygons
+        else:
+            breakeven = 500  # Simple small polygons
+    else:  # Tiny polygons (< 1 ha)
+        breakeven = 500
+
+    # Vertex complexity adjustment for high-complexity geometries
+    if mean_vertices is not None and mean_vertices >= 50:
+        # High complexity: reduce breakeven by 20% (concurrent beneficial sooner)
+        breakeven = int(breakeven * 0.8)
 
-    # Optional adjustment based on vertex complexity (very high complexity favors concurrent)
-    if mean_vertices is not None and mean_vertices > 500:
-        # Reduce breakeven by 25% for very complex geometries
-        adjusted_breakeven = int(breakeven * 0.75)
-        method = "concurrent" if polygon_count >= adjusted_breakeven else "sequential"
+    method = "concurrent" if feature_count >= breakeven else "sequential"
 
     if verbose:
-        print(f"\nMETHOD RECOMMENDATION")
+        print(f"\nMETHOD RECOMMENDATION (Polygons)")
         print(
-            f"   Polygons: {polygon_count} | Mean Area: {mean_area_ha:.1f} ha", end=""
+            f"   Features: {feature_count} | Mean Area: {mean_area_ha:.1f} ha", end=""
         )
         if mean_vertices is not None:
             print(f" | Mean Vertices: {mean_vertices:.1f}", end="")
         print()
-        print(f"   Breakeven: {breakeven} polygons | Method: {method.upper()}")
+        print(f"   Break-even: {breakeven} features | Method: {method.upper()}")
 
     return method

openforis_whisp/datasets.py

@@ -1160,6 +1160,20 @@ def nci_ocs2020_prep():
     ).selfMask()  # cocoa from national land cover map for Côte d'Ivoire
 
 
+# nCM - Cameroon
+# data from Aurelie Shapiro (FAO) working directly with country experts - info on methods and accuracy assessment to follow
+
+
+def ncm_treecover_2020_prep():
+    return (
+        ee.Image("projects/ee-cocoacmr/assets/land_cover/CMR_TNTMMU_2020")
+        .select("FNF_2020")
+        .eq(1)
+        .rename("nCM_Treecover_2020")
+        .selfMask()
+    )
+
+
 # ============================================================================
 # CONTEXT BANDS (Administrative boundaries and water mask)
 # ============================================================================

openforis_whisp/logger.py

@@ -8,9 +8,21 @@ BASE_MSG_FORMAT = (
 
 class StdoutLogger:
     def __init__(self, name: str, msg_format: str = BASE_MSG_FORMAT) -> None:
-        self.handler = logging.StreamHandler(sys.stdout)
-        self.handler.setFormatter(logging.Formatter(msg_format))
-        self.handler.setLevel(logging.DEBUG)
+        # Create handler that auto-flushes for Colab/notebook visibility
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(logging.Formatter(msg_format))
+        handler.setLevel(logging.DEBUG)
+
+        # Override emit to force flush after each message
+        original_emit = handler.emit
+
+        def emit_with_flush(record):
+            original_emit(record)
+            sys.stdout.flush()
+
+        handler.emit = emit_with_flush
+
+        self.handler = handler
         self.logger = logging.getLogger(name)
         self.logger.addHandler(self.handler)
         self.logger.propagate = False

openforis_whisp/parameters/lookup_context_and_metadata.csv

@@ -1,5 +1,5 @@
 name,order,ISO2_code,theme,theme_timber,use_for_risk,use_for_risk_timber,exclude_from_output,col_type,is_nullable,is_required,corresponding_variable
-plotId,-10,,context_and_metadata,context_and_metadata,NA,NA,0,string,1,0,plot_id_column
+plotId,-10,,context_and_metadata,context_and_metadata,NA,NA,0,int64,1,0,plot_id_column
 external_id,-9,,context_and_metadata,context_and_metadata,NA,NA,0,string,1,0,external_id_column
 Area,-8,,context_and_metadata,context_and_metadata,NA,NA,0,float32,1,1,geometry_area_column
 Geometry_type,-7,,context_and_metadata,context_and_metadata,NA,NA,0,string,1,1,geometry_type_column

openforis_whisp/parameters/lookup_gee_datasets.csv

@@ -2,9 +2,9 @@ name,order,ISO2_code,theme,theme_timber,use_for_risk,use_for_risk_timber,exclude
 EUFO_2020,10,,treecover,naturally_reg_2020,1,1,0,float32,1,0,g_jrc_gfc_2020_prep
 GLAD_Primary,20,,treecover,primary,1,1,0,float32,1,0,g_glad_pht_prep
 TMF_undist,30,,treecover,primary,1,1,0,float32,1,0,g_jrc_tmf_undisturbed_prep
-GFC_TC_2020,50,,treecover,naturally_reg_2020,1,1,0,float32,1,0,g_glad_gfc_10pc_prep
+GFC_TC_2020,50,,treecover,naturally_reg_2020,0,0,0,float32,1,0,g_glad_gfc_10pc_prep
 Forest_FDaP,60,,treecover,naturally_reg_2020,1,1,0,float32,1,0,g_glad_gfc_10pc_prep
-ESA_TC_2020,70,,treecover,naturally_reg_2020,1,1,0,float32,1,0,g_esa_worldcover_trees_prep
+ESA_TC_2020,70,,treecover,naturally_reg_2020,0,0,0,float32,1,0,g_esa_worldcover_trees_prep
 TMF_plant,80,,commodities,NA,1,1,0,float32,1,0,g_jrc_tmf_plantation_prep
 Oil_palm_Descals,90,,commodities,NA,1,1,0,float32,1,0,g_creaf_descals_palm_prep
 Oil_palm_FDaP,100,,commodities,NA,1,1,0,float32,1,0,g_fdap_palm_prep
@@ -197,3 +197,4 @@ nBR_INPE_TCamz_pasture_2020,2422,BR,commodities,NA,1,1,0,float32,1,0,nbr_terracl
 nBR_INPE_TCcer_pasture_2020,2423,BR,commodities,NA,1,1,0,float32,1,0,nbr_terraclass_cer20_ac_prep
 nBR_MapBiomas_col9_pasture_2020,2424,BR,commodities,NA,1,1,0,float32,1,0,nbr_mapbiomasc9_pasture_prep
 nCI_Cocoa_bnetd,3000,CI,commodities,NA,1,1,0,float32,1,0,nci_ocs2020_prep
+nCM_Treecover_2020,3100,CM,treecover,NA,1,0,0,float32,1,0,ncm_treecover_2020_prep

openforis_whisp/pd_schemas.py

@@ -1,5 +1,10 @@
-import pandera as pa
-from pandera.typing import DataFrame, Series
+# Support both old and new pandera import paths
+try:
+    import pandera.pandas as pa
+    from pandera.typing.pandas import DataFrame, Series
+except (ImportError, ModuleNotFoundError):
+    import pandera as pa
+    from pandera.typing import DataFrame, Series
 
 # Define a schema for validating a DataFrame related to GEE (Google Earth Engine) datasets.
 class DataLookupSchema(pa.DataFrameModel):

openforis_whisp/reformat.py

@@ -1,5 +1,10 @@
 # !pip install pandera[io] # special version used
-import pandera as pa
+# Support both old and new pandera import paths
+try:
+    import pandera.pandas as pa
+except (ImportError, ModuleNotFoundError):
+    import pandera as pa
+
 import pandas as pd
 import os
 import logging

openforis_whisp/stats.py

@@ -88,7 +88,6 @@ def get_admin_boundaries_fc():
 def whisp_formatted_stats_geojson_to_df_legacy(
     input_geojson_filepath: Path | str,
     external_id_column=None,
-    remove_geom=False,
     national_codes=None,
     unit_type="ha",
     whisp_image=None,
@@ -147,7 +146,6 @@ def whisp_formatted_stats_geojson_to_df_legacy(
     return whisp_formatted_stats_ee_to_df(
         feature_collection,
         external_id_column,
-        remove_geom,
         national_codes=national_codes,
         unit_type=unit_type,
         whisp_image=whisp_image,
@@ -167,6 +165,7 @@ def whisp_formatted_stats_geojson_to_df(
     batch_size: int = 10,
     max_concurrent: int = 20,
     geometry_audit_trail: bool = False,
+    status_file: str = None,
 ) -> pd.DataFrame:
     """
     Main entry point for converting GeoJSON to Whisp statistics.
@@ -188,11 +187,7 @@ def whisp_formatted_stats_geojson_to_df(
         The column in the GeoJSON containing external IDs to be preserved in the output DataFrame.
         This column must exist as a property in ALL features of the GeoJSON file.
         Use debug_feature_collection_properties() to inspect available properties if you encounter errors.
-    remove_geom : bool, default=False
-        If True, the geometry of the GeoJSON is removed from the output DataFrame.
     national_codes : list, optional
-        List of ISO2 country codes to include national datasets.
-    unit_type: str, optional
         Whether to use hectares ("ha") or percentage ("percent"), by default "ha".
     whisp_image : ee.Image, optional
         Pre-combined multiband Earth Engine Image containing all Whisp datasets.
@@ -224,6 +219,13 @@ def whisp_formatted_stats_geojson_to_df(
 
         Processing metadata stored in df.attrs['processing_metadata'].
         These columns enable full transparency for geometry modifications during processing.
+    status_file : str, optional
+        Path to JSON status file or directory for real-time progress tracking.
+        If a directory is provided, creates 'whisp_processing_status.json' in that directory.
+        Updates every 3 minutes and at progress milestones (5%, 10%, etc.).
+        Format: {"status": "processing", "progress": "450/1000", "percent": 45.0,
+                 "elapsed_sec": 120, "eta_sec": 145, "updated_at": "2025-11-13T14:23:45"}
+        Most useful for large concurrent jobs. Works in both concurrent and sequential modes.
 
     Returns
     -------
@@ -283,7 +285,6 @@ def whisp_formatted_stats_geojson_to_df(
         return whisp_formatted_stats_geojson_to_df_legacy(
             input_geojson_filepath=input_geojson_filepath,
             external_id_column=external_id_column,
-            remove_geom=remove_geom,
             national_codes=national_codes,
             unit_type=unit_type,
             whisp_image=whisp_image,
@@ -306,7 +307,6 @@ def whisp_formatted_stats_geojson_to_df(
         return whisp_formatted_stats_geojson_to_df_fast(
             input_geojson_filepath=input_geojson_filepath,
             external_id_column=external_id_column,
-            remove_geom=remove_geom,
             national_codes=national_codes,
             unit_type=unit_type,
             whisp_image=whisp_image,
@@ -315,6 +315,7 @@ def whisp_formatted_stats_geojson_to_df(
             batch_size=batch_size,
             max_concurrent=max_concurrent,
             geometry_audit_trail=geometry_audit_trail,
+            status_file=status_file,
         )
     else:
         raise ValueError(
@@ -473,7 +474,6 @@ def whisp_formatted_stats_ee_to_df(
 def whisp_stats_geojson_to_df(
     input_geojson_filepath: Path | str,
     external_id_column=None,
-    remove_geom=False,
     national_codes=None,
     unit_type="ha",
     whisp_image=None,  # New parameter
@@ -506,7 +506,6 @@ def whisp_stats_geojson_to_df(
     return whisp_stats_ee_to_df(
         feature_collection,
         external_id_column,
-        remove_geom,
         national_codes=national_codes,
         unit_type=unit_type,
         whisp_image=whisp_image,  # Pass through
@@ -990,7 +989,7 @@ def whisp_stats_ee_to_drive(
         )
         task.start()
         print(
-            "Exporting to Google Drive: 'whisp_results/whisp_output_table.csv'. To track progress: https://code.earthengine.google.com/tasks"
+            "Exporting to Google Drive: 'whisp_output_table.csv'. To track progress: https://code.earthengine.google.com/tasks"
         )
     except Exception as e:
         print(f"An error occurred during the export: {e}")

openforis_whisp/utils.py

@@ -5,6 +5,8 @@ import os
 import pandas as pd
 import random
 import numpy as np
+import logging
+import sys
 
 import urllib.request
 import os
@@ -19,6 +21,23 @@ from shapely.validation import make_valid
 
 from .logger import StdoutLogger
 
+# Configure the "whisp" logger with auto-flush handler for Colab visibility
+_whisp_logger = logging.getLogger("whisp")
+if not _whisp_logger.handlers:
+    _handler = logging.StreamHandler(sys.stdout)
+    _handler.setLevel(logging.DEBUG)
+    _handler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+    # Override emit to force flush after each message for Colab
+    _original_emit = _handler.emit
+
+    def _emit_with_flush(record):
+        _original_emit(record)
+        sys.stdout.flush()
+
+    _handler.emit = _emit_with_flush
+    _whisp_logger.addHandler(_handler)
+    _whisp_logger.setLevel(logging.INFO)
+    _whisp_logger.propagate = False  # Don't propagate to root to avoid duplicates
 
 logger = StdoutLogger(__name__)
 

{openforis_whisp-3.0.0a3.dist-info → openforis_whisp-3.0.0a5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: openforis-whisp
-Version: 3.0.0a3
+Version: 3.0.0a5
 Summary: Whisp (What is in that plot) is an open-source solution which helps to produce relevant forest monitoring information and support compliance with deforestation-related regulations.
 License: MIT
 Keywords: whisp,geospatial,data-processing

openforis_whisp-3.0.0a5.dist-info/RECORD

@@ -0,0 +1,20 @@
+openforis_whisp/__init__.py,sha256=5zJK84LYnlslxSajdCz6ZIYxRS4xgN3sGxSD6_GXEHs,3547
+openforis_whisp/advanced_stats.py,sha256=qg5Xsx9B3rAIDOsVTeHL-viRrY2KjS5f7CRlLK9WQt8,105329
+openforis_whisp/data_checks.py,sha256=ErIKGbCa3R8eYP0sVoAl-ZUl607W1QrG0Jr2SIVgm2I,34056
+openforis_whisp/data_conversion.py,sha256=L2IsiUyQUt3aHgSYGbIhgPGwM7eyS3nLVEoNO9YqQeM,21888
+openforis_whisp/datasets.py,sha256=F1WxXc93mxxmN-WHa0bf-XX-FloSQyEBJKmnrQEHYn8,53855
+openforis_whisp/logger.py,sha256=gFkRTwJDJKIBWcHDOK74Uln3JM7fAybURo7pQpGL790,3395
+openforis_whisp/parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+openforis_whisp/parameters/config_runtime.py,sha256=NOo39MAi60XCwEx5pwkS0EHKJBh0XY1q06y4j0HAABg,1421
+openforis_whisp/parameters/lookup_context_and_metadata.csv,sha256=57Nz7t8GPMYpx81IY5RWiJGVMbIT7_0jIgZWkyih7no,1297
+openforis_whisp/parameters/lookup_gaul1_admin.py,sha256=cQr5liRdXi85QieTxrz4VAkn0COvRCp82ZV0dYFWOio,474980
+openforis_whisp/parameters/lookup_gee_datasets.csv,sha256=7KdnFocEgbZO5m8JmWQchzZTurg9rJ96y17z8UyLtI0,17537
+openforis_whisp/pd_schemas.py,sha256=0z-oPmYIDUIn7mNY41W_uUpmTwjoR7e254mOCoHVsOg,2878
+openforis_whisp/reformat.py,sha256=gvhIa-_kTT5BSO8LuVmJ1TQcf_NwheskXboFM9e0KJY,32758
+openforis_whisp/risk.py,sha256=d_Di5XB8BnHdVXG56xdHTcpB4-CIF5vo2ZRMQRG7Pek,34420
+openforis_whisp/stats.py,sha256=pTSYs77ISRBOIglRpq4SUx3lKRkrUZOKROLRX5IP9IY,63941
+openforis_whisp/utils.py,sha256=AISWF-MpfFdYkhd6bei4BViw2Iag20mmq61ykrF9YTk,31287
+openforis_whisp-3.0.0a5.dist-info/LICENSE,sha256=nqyqICO95iw_iwzP1t_IIAf7ZX3DPbL_M9WyQfh2q1k,1085
+openforis_whisp-3.0.0a5.dist-info/METADATA,sha256=mYugV2a00b1X9D88x5MF7S6CEq86FlW6EjaCY1L1piA,16684
+openforis_whisp-3.0.0a5.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+openforis_whisp-3.0.0a5.dist-info/RECORD,,

openforis_whisp-3.0.0a3.dist-info/RECORD

@@ -1,20 +0,0 @@
-openforis_whisp/__init__.py,sha256=s42Q0VJdzm8mgnxfYg1hUEJPM2VLWIva2h-mdKyr444,3538
-openforis_whisp/advanced_stats.py,sha256=tvhgNTCGlT3aYecUPP6QCTO0FRrjk0qjs95NoVZvIt4,90935
-openforis_whisp/data_checks.py,sha256=KwgD72FA_n7joiJadGRpzntd2sLo0aqGNbOjRkB8iQI,32293
-openforis_whisp/data_conversion.py,sha256=L2IsiUyQUt3aHgSYGbIhgPGwM7eyS3nLVEoNO9YqQeM,21888
-openforis_whisp/datasets.py,sha256=aGJy0OYN4d0nsH3_IOYlHl-WCB7KFwZwMJ-dBi5Hc5Y,53470
-openforis_whisp/logger.py,sha256=9M6_3mdpoiWfC-pDwM9vKmB2l5Gul6Rb5rNTNh-_nzs,3054
-openforis_whisp/parameters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-openforis_whisp/parameters/config_runtime.py,sha256=NOo39MAi60XCwEx5pwkS0EHKJBh0XY1q06y4j0HAABg,1421
-openforis_whisp/parameters/lookup_context_and_metadata.csv,sha256=KgK0ik_Gd4t_Nq5cUkGPT4ZFZVO93HWSG82jRrOukt4,1298
-openforis_whisp/parameters/lookup_gaul1_admin.py,sha256=cQr5liRdXi85QieTxrz4VAkn0COvRCp82ZV0dYFWOio,474980
-openforis_whisp/parameters/lookup_gee_datasets.csv,sha256=UDvZrQsL5rXJn6CW6P3wofUrPLRmUFZWt6ETbXaxBMs,17454
-openforis_whisp/pd_schemas.py,sha256=W_ocS773LHfc05dJqvWRa-bRdX0wKFoNp0lMxgFx94Y,2681
-openforis_whisp/reformat.py,sha256=MPjP5lb218GTcTpd_Qvbj5ER_8EY4JjLDteQaS5OZCQ,32620
-openforis_whisp/risk.py,sha256=d_Di5XB8BnHdVXG56xdHTcpB4-CIF5vo2ZRMQRG7Pek,34420
-openforis_whisp/stats.py,sha256=nVzQpSu7BoSb2S6HheLeoK_pmguZ9Lyw0ZfbTTMVq4Q,63720
-openforis_whisp/utils.py,sha256=Q-EwhUaohk63WCx7Rr5VuR3X-oGtgILZDc8JsjbWhgg,30538
-openforis_whisp-3.0.0a3.dist-info/LICENSE,sha256=nqyqICO95iw_iwzP1t_IIAf7ZX3DPbL_M9WyQfh2q1k,1085
-openforis_whisp-3.0.0a3.dist-info/METADATA,sha256=6xuNhUpQWyzKU3m13FnJ7SX39jAVry1YEKNAdH0D2to,16684
-openforis_whisp-3.0.0a3.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-openforis_whisp-3.0.0a3.dist-info/RECORD,,