posthoganalytics 7.0.1__py3-none-any.whl → 7.4.1__py3-none-any.whl

posthoganalytics/client.py

@@ -2,6 +2,7 @@ import atexit
 import logging
 import os
 import sys
+import warnings
 from datetime import datetime, timedelta
 from typing import Any, Dict, Optional, Union
 from typing_extensions import Unpack
@@ -28,10 +29,17 @@ from posthoganalytics.feature_flags import (
     RequiresServerEvaluation,
     match_feature_flag_properties,
 )
+from posthoganalytics.flag_definition_cache import (
+    FlagDefinitionCacheData,
+    FlagDefinitionCacheProvider,
+)
 from posthoganalytics.poller import Poller
 from posthoganalytics.request import (
     DEFAULT_HOST,
     APIError,
+    QuotaLimitError,
+    RequestsConnectionError,
+    RequestsTimeout,
     batch_post,
     determine_server_host,
     flags,
@@ -49,6 +57,7 @@ from posthoganalytics.contexts import (
 )
 from posthoganalytics.types import (
     FeatureFlag,
+    FeatureFlagError,
     FeatureFlagResult,
     FlagMetadata,
     FlagsAndPayloads,
@@ -184,6 +193,7 @@ class Client(object):
         before_send=None,
         flag_fallback_cache_url=None,
         enable_local_evaluation=True,
+        flag_definition_cache_provider: Optional[FlagDefinitionCacheProvider] = None,
         capture_exception_code_variables=False,
         code_variables_mask_patterns=None,
         code_variables_ignore_patterns=None,
@@ -222,8 +232,8 @@ class Client(object):
         self.timeout = timeout
         self._feature_flags = None  # private variable to store flags
         self.feature_flags_by_key = None
-        self.group_type_mapping = None
-        self.cohorts = None
+        self.group_type_mapping: Optional[dict[str, str]] = None
+        self.cohorts: Optional[dict[str, Any]] = None
         self.poll_interval = poll_interval
         self.feature_flags_request_timeout_seconds = (
             feature_flags_request_timeout_seconds
@@ -232,6 +242,8 @@ class Client(object):
         self.distinct_ids_feature_flags_reported = SizeLimitedDict(MAX_DICT_SIZE, set)
         self.flag_cache = self._initialize_flag_cache(flag_fallback_cache_url)
         self.flag_definition_version = 0
+        self._flags_etag: Optional[str] = None
+        self._flag_definition_cache_provider = flag_definition_cache_provider
         self.disabled = disabled
         self.disable_geoip = disable_geoip
         self.historical_migration = historical_migration
@@ -622,7 +634,28 @@ class Client(object):
         if flag_options["should_send"]:
             try:
                 if flag_options["only_evaluate_locally"] is True:
-                    # Only use local evaluation
+                    # Local evaluation explicitly requested
+                    feature_variants = self.get_all_flags(
+                        distinct_id,
+                        groups=(groups or {}),
+                        person_properties=flag_options["person_properties"],
+                        group_properties=flag_options["group_properties"],
+                        disable_geoip=disable_geoip,
+                        only_evaluate_locally=True,
+                        flag_keys_to_evaluate=flag_options["flag_keys_filter"],
+                    )
+                elif flag_options["only_evaluate_locally"] is False:
+                    # Remote evaluation explicitly requested
+                    feature_variants = self.get_feature_variants(
+                        distinct_id,
+                        groups,
+                        person_properties=flag_options["person_properties"],
+                        group_properties=flag_options["group_properties"],
+                        disable_geoip=disable_geoip,
+                        flag_keys_to_evaluate=flag_options["flag_keys_filter"],
+                    )
+                elif self.feature_flags:
+                    # Local flags available, prefer local evaluation
                     feature_variants = self.get_all_flags(
                         distinct_id,
                         groups=(groups or {}),
@@ -633,7 +666,7 @@ class Client(object):
                         flag_keys_to_evaluate=flag_options["flag_keys_filter"],
                     )
                 else:
-                    # Default behavior - use remote evaluation
+                    # Fall back to remote evaluation
                     feature_variants = self.get_feature_variants(
                         distinct_id,
                         groups,
@@ -647,15 +680,6 @@ class Client(object):
                     f"[FEATURE FLAGS] Unable to get feature variants: {e}"
                 )
 
-        elif self.feature_flags and event != "$feature_flag_called":
-            # Local evaluation is enabled, flags are loaded, so try and get all flags we can without going to the server
-            feature_variants = self.get_all_flags(
-                distinct_id,
-                groups=(groups or {}),
-                disable_geoip=disable_geoip,
-                only_evaluate_locally=True,
-            )
-
         for feature, variant in (feature_variants or {}).items():
             extra_properties[f"$feature/{feature}"] = variant
 
@@ -979,10 +1003,6 @@ class Client(object):
             all_exceptions_with_trace_and_in_app = event["exception"]["values"]
 
             properties = {
-                "$exception_type": all_exceptions_with_trace_and_in_app[0].get("type"),
-                "$exception_message": all_exceptions_with_trace_and_in_app[0].get(
-                    "value"
-                ),
                 "$exception_list": all_exceptions_with_trace_and_in_app,
                 **properties,
             }
@@ -1147,17 +1167,25 @@ class Client(object):
             posthog.join()
             ```
         """
-        for consumer in self.consumers:
-            consumer.pause()
-            try:
-                consumer.join()
-            except RuntimeError:
-                # consumer thread has not started
-                pass
+        if self.consumers:
+            for consumer in self.consumers:
+                consumer.pause()
+                try:
+                    consumer.join()
+                except RuntimeError:
+                    # consumer thread has not started
+                    pass
 
         if self.poller:
             self.poller.stop()
 
+        # Shutdown the cache provider (release locks, cleanup)
+        if self._flag_definition_cache_provider:
+            try:
+                self._flag_definition_cache_provider.shutdown()
+            except Exception as e:
+                self.log.error(f"[FEATURE FLAGS] Cache provider shutdown error: {e}")
+
     def shutdown(self):
         """
         Flush all messages and cleanly shutdown the client. Call this before the process ends in serverless environments to avoid data loss.
@@ -1173,7 +1201,71 @@ class Client(object):
         if self.exception_capture:
             self.exception_capture.close()
 
+    def _update_flag_state(
+        self, data: FlagDefinitionCacheData, old_flags_by_key: Optional[dict] = None
+    ) -> None:
+        """Update internal flag state from cache data and invalidate evaluation cache if changed."""
+        self.feature_flags = data["flags"]
+        self.group_type_mapping = data["group_type_mapping"]
+        self.cohorts = data["cohorts"]
+
+        # Invalidate evaluation cache if flag definitions changed
+        if (
+            self.flag_cache
+            and old_flags_by_key is not None
+            and old_flags_by_key != (self.feature_flags_by_key or {})
+        ):
+            old_version = self.flag_definition_version
+            self.flag_definition_version += 1
+            self.flag_cache.invalidate_version(old_version)
+
     def _load_feature_flags(self):
+        should_fetch = True
+        if self._flag_definition_cache_provider:
+            try:
+                should_fetch = (
+                    self._flag_definition_cache_provider.should_fetch_flag_definitions()
+                )
+            except Exception as e:
+                self.log.error(
+                    f"[FEATURE FLAGS] Cache provider should_fetch error: {e}"
+                )
+                # Fail-safe: fetch from API if cache provider errors
+                should_fetch = True
+
+        # If not fetching, try to get from cache
+        if not should_fetch and self._flag_definition_cache_provider:
+            try:
+                cached_data = (
+                    self._flag_definition_cache_provider.get_flag_definitions()
+                )
+                if cached_data:
+                    self.log.debug(
+                        "[FEATURE FLAGS] Using cached flag definitions from external cache"
+                    )
+                    self._update_flag_state(
+                        cached_data, old_flags_by_key=self.feature_flags_by_key or {}
+                    )
+                    self._last_feature_flag_poll = datetime.now(tz=tzutc())
+                    return
+                else:
+                    # Emergency fallback: if cache is empty and we have no flags, fetch anyway.
+                    # There's really no other way of recovering in this case.
+                    if not self.feature_flags:
+                        self.log.debug(
+                            "[FEATURE FLAGS] Cache empty and no flags loaded, falling back to API fetch"
+                        )
+                        should_fetch = True
+            except Exception as e:
+                self.log.error(f"[FEATURE FLAGS] Cache provider get error: {e}")
+                # Fail-safe: fetch from API if cache provider errors
+                should_fetch = True
+
+        if should_fetch:
+            self._fetch_feature_flags_from_api()
+
+    def _fetch_feature_flags_from_api(self):
+        """Fetch feature flags from the PostHog API."""
         try:
             # Store old flags to detect changes
             old_flags_by_key: dict[str, dict] = self.feature_flags_by_key or {}
@@ -1183,19 +1275,41 @@ class Client(object):
                 f"/api/feature_flag/local_evaluation/?token={self.api_key}&send_cohorts",
                 self.host,
                 timeout=10,
+                etag=self._flags_etag,
             )
 
-            self.feature_flags = response["flags"] or []
-            self.group_type_mapping = response["group_type_mapping"] or {}
-            self.cohorts = response["cohorts"] or {}
+            # Update stored ETag (clear if server stops sending one)
+            self._flags_etag = response.etag
 
-            # Check if flag definitions changed and update version
-            if self.flag_cache and old_flags_by_key != (
-                self.feature_flags_by_key or {}
-            ):
-                old_version = self.flag_definition_version
-                self.flag_definition_version += 1
-                self.flag_cache.invalidate_version(old_version)
+            # If 304 Not Modified, flags haven't changed - skip processing
+            if response.not_modified:
+                self.log.debug(
+                    "[FEATURE FLAGS] Flags not modified (304), using cached data"
+                )
+                self._last_feature_flag_poll = datetime.now(tz=tzutc())
+                return
+
+            if response.data is None:
+                self.log.error(
+                    "[FEATURE FLAGS] Unexpected empty response data in non-304 response"
+                )
+                return
+
+            self._update_flag_state(response.data, old_flags_by_key=old_flags_by_key)
+
+            # Store in external cache if provider is configured
+            if self._flag_definition_cache_provider:
+                try:
+                    self._flag_definition_cache_provider.on_flag_definitions_received(
+                        {
+                            "flags": self.feature_flags or [],
+                            "group_type_mapping": self.group_type_mapping or {},
+                            "cohorts": self.cohorts or {},
+                        }
+                    )
+                except Exception as e:
+                    self.log.error(f"[FEATURE FLAGS] Cache provider store error: {e}")
+                    # Flags are already in memory, so continue normally
 
         except APIError as e:
             if e.status == 401:
@@ -1295,7 +1409,8 @@ class Client(object):
         flag_filters = feature_flag.get("filters") or {}
         aggregation_group_type_index = flag_filters.get("aggregation_group_type_index")
         if aggregation_group_type_index is not None:
-            group_name = self.group_type_mapping.get(str(aggregation_group_type_index))
+            group_type_mapping = self.group_type_mapping or {}
+            group_name = group_type_mapping.get(str(aggregation_group_type_index))
 
             if not group_name:
                 self.log.warning(
@@ -1387,6 +1502,19 @@ class Client(object):
             return None
         return bool(response)
 
+    def _get_stale_flag_fallback(
+        self, distinct_id: ID_TYPES, key: str
+    ) -> Optional[FeatureFlagResult]:
+        """Returns a stale cached flag value if available, otherwise None."""
+        if self.flag_cache:
+            stale_result = self.flag_cache.get_stale_cached_flag(distinct_id, key)
+            if stale_result:
+                self.log.info(
+                    f"[FEATURE FLAGS] Using stale cached value for flag {key}"
+                )
+                return stale_result
+        return None
+
     def _get_feature_flag_result(
         self,
         key: str,
@@ -1419,6 +1547,8 @@ class Client(object):
         flag_result = None
         flag_details = None
         request_id = None
+        evaluated_at = None
+        feature_flag_error: Optional[str] = None
 
         flag_value = self._locally_evaluate_flag(
             key, distinct_id, groups, person_properties, group_properties
@@ -1443,14 +1573,24 @@ class Client(object):
                 )
         elif not only_evaluate_locally:
             try:
-                flag_details, request_id = self._get_feature_flag_details_from_server(
-                    key,
-                    distinct_id,
-                    groups,
-                    person_properties,
-                    group_properties,
-                    disable_geoip,
+                flag_details, request_id, evaluated_at, errors_while_computing = (
+                    self._get_feature_flag_details_from_server(
+                        key,
+                        distinct_id,
+                        groups,
+                        person_properties,
+                        group_properties,
+                        disable_geoip,
+                    )
                 )
+                errors = []
+                if errors_while_computing:
+                    errors.append(FeatureFlagError.ERRORS_WHILE_COMPUTING)
+                if flag_details is None:
+                    errors.append(FeatureFlagError.FLAG_MISSING)
+                if errors:
+                    feature_flag_error = ",".join(errors)
+
                 flag_result = FeatureFlagResult.from_flag_details(
                     flag_details, override_match_value
                 )
@@ -1464,19 +1604,26 @@ class Client(object):
                 self.log.debug(
                     f"Successfully computed flag remotely: #{key} -> #{flag_result}"
                 )
+            except QuotaLimitError as e:
+                self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}")
+                feature_flag_error = FeatureFlagError.QUOTA_LIMITED
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
+            except RequestsTimeout as e:
+                self.log.warning(f"[FEATURE FLAGS] Request timed out: {e}")
+                feature_flag_error = FeatureFlagError.TIMEOUT
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
+            except RequestsConnectionError as e:
+                self.log.warning(f"[FEATURE FLAGS] Connection error: {e}")
+                feature_flag_error = FeatureFlagError.CONNECTION_ERROR
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
+            except APIError as e:
+                self.log.warning(f"[FEATURE FLAGS] API error: {e}")
+                feature_flag_error = FeatureFlagError.api_error(e.status)
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
             except Exception as e:
                 self.log.exception(f"[FEATURE FLAGS] Unable to get flag remotely: {e}")
-
-                # Fallback to cached value if remote evaluation fails
-                if self.flag_cache:
-                    stale_result = self.flag_cache.get_stale_cached_flag(
-                        distinct_id, key
-                    )
-                    if stale_result:
-                        self.log.info(
-                            f"[FEATURE FLAGS] Using stale cached value for flag {key}"
-                        )
-                        flag_result = stale_result
+                feature_flag_error = FeatureFlagError.UNKNOWN_ERROR
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
 
         if send_feature_flag_events:
             self._capture_feature_flag_called(
@@ -1488,7 +1635,9 @@ class Client(object):
                 groups,
                 disable_geoip,
                 request_id,
+                evaluated_at,
                 flag_details,
+                feature_flag_error,
             )
 
         return flag_result
@@ -1640,7 +1789,7 @@ class Client(object):
         person_properties=None,
         group_properties=None,
         only_evaluate_locally=False,
-        send_feature_flag_events=True,
+        send_feature_flag_events=False,
         disable_geoip=None,
     ):
         """
@@ -1654,7 +1803,7 @@ class Client(object):
             person_properties: A dictionary of person properties.
             group_properties: A dictionary of group properties.
             only_evaluate_locally: Whether to only evaluate locally.
-            send_feature_flag_events: Whether to send feature flag events.
+            send_feature_flag_events: Deprecated. Use get_feature_flag() instead if you need events.
             disable_geoip: Whether to disable GeoIP for this request.
 
         Examples:
@@ -1670,6 +1819,14 @@ class Client(object):
         Category:
             Feature flags
         """
+        if send_feature_flag_events:
+            warnings.warn(
+                "send_feature_flag_events is deprecated in get_feature_flag_payload() and will be removed "
+                "in a future version. Use get_feature_flag() if you want to send $feature_flag_called events.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         feature_flag_result = self._get_feature_flag_result(
             key,
             distinct_id,
@@ -1691,9 +1848,10 @@ class Client(object):
         person_properties: dict[str, str],
         group_properties: dict[str, str],
         disable_geoip: Optional[bool],
-    ) -> tuple[Optional[FeatureFlag], Optional[str]]:
+    ) -> tuple[Optional[FeatureFlag], Optional[str], Optional[int], bool]:
         """
-        Calls /flags and returns the flag details and request id
+        Calls /flags and returns the flag details, request id, evaluated at timestamp,
+        and whether there were errors while computing flags.
         """
         resp_data = self.get_flags_decision(
             distinct_id,
@@ -1704,9 +1862,11 @@ class Client(object):
             flag_keys_to_evaluate=[key],
         )
         request_id = resp_data.get("requestId")
+        evaluated_at = resp_data.get("evaluatedAt")
+        errors_while_computing = resp_data.get("errorsWhileComputingFlags", False)
         flags = resp_data.get("flags")
         flag_details = flags.get(key) if flags else None
-        return flag_details, request_id
+        return flag_details, request_id, evaluated_at, errors_while_computing
 
     def _capture_feature_flag_called(
         self,
@@ -1718,7 +1878,9 @@ class Client(object):
         groups: Dict[str, str],
         disable_geoip: Optional[bool],
         request_id: Optional[str],
+        evaluated_at: Optional[int],
         flag_details: Optional[FeatureFlag],
+        feature_flag_error: Optional[str] = None,
     ):
         feature_flag_reported_key = (
             f"{key}_{'::null::' if response is None else str(response)}"
@@ -1741,6 +1903,8 @@ class Client(object):
 
             if request_id:
                 properties["$feature_flag_request_id"] = request_id
+            if evaluated_at:
+                properties["$feature_flag_evaluated_at"] = evaluated_at
             if isinstance(flag_details, FeatureFlag):
                 if flag_details.reason and flag_details.reason.description:
                     properties["$feature_flag_reason"] = flag_details.reason.description
@@ -1751,6 +1915,8 @@ class Client(object):
                         )
                     if flag_details.metadata.id:
                         properties["$feature_flag_id"] = flag_details.metadata.id
+            if feature_flag_error:
+                properties["$feature_flag_error"] = feature_flag_error
 
             self.capture(
                 "$feature_flag_called",

posthoganalytics/exception_utils.py

@@ -54,6 +54,10 @@ DEFAULT_CODE_VARIABLES_MASK_PATTERNS = [
     r"(?i).*privatekey.*",
     r"(?i).*private_key.*",
     r"(?i).*token.*",
+    r"(?i).*aws_access_key_id.*",
+    r"(?i).*_pass",
+    r"(?i)sk_.*",
+    r"(?i).*jwt.*",
 ]
 
 DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS = [r"^__.*"]
@@ -941,7 +945,31 @@ def _pattern_matches(name, patterns):
     return False
 
 
-def _serialize_variable_value(value, limiter, max_length=1024):
+def _mask_sensitive_data(value, compiled_mask):
+    if not compiled_mask:
+        return value
+
+    if isinstance(value, dict):
+        result = {}
+        for k, v in value.items():
+            key_str = str(k) if not isinstance(k, str) else k
+            if _pattern_matches(key_str, compiled_mask):
+                result[k] = CODE_VARIABLES_REDACTED_VALUE
+            else:
+                result[k] = _mask_sensitive_data(v, compiled_mask)
+        return result
+    elif isinstance(value, (list, tuple)):
+        masked_items = [_mask_sensitive_data(item, compiled_mask) for item in value]
+        return type(value)(masked_items)
+    elif isinstance(value, str):
+        if _pattern_matches(value, compiled_mask):
+            return CODE_VARIABLES_REDACTED_VALUE
+        return value
+    else:
+        return value
+
+
+def _serialize_variable_value(value, limiter, max_length=1024, compiled_mask=None):
     try:
         if value is None:
             result = "None"
@@ -954,9 +982,13 @@ def _serialize_variable_value(value, limiter, max_length=1024):
             limiter.add(result_size)
             return value
         elif isinstance(value, str):
-            result = value
+            if compiled_mask and _pattern_matches(value, compiled_mask):
+                result = CODE_VARIABLES_REDACTED_VALUE
+            else:
+                result = value
         else:
-            result = json.dumps(value)
+            masked_value = _mask_sensitive_data(value, compiled_mask)
+            result = json.dumps(masked_value)
 
         if len(result) > max_length:
             result = result[: max_length - 3] + "..."
@@ -1043,7 +1075,9 @@ def serialize_code_variables(
             limiter.add(redacted_size)
             result[name] = redacted_value
         else:
-            serialized = _serialize_variable_value(value, limiter, max_length)
+            serialized = _serialize_variable_value(
+                value, limiter, max_length, compiled_mask
+            )
             if serialized is None:
                 break
             result[name] = serialized
@@ -1053,6 +1087,17 @@ def serialize_code_variables(
 
 def try_attach_code_variables_to_frames(
     all_exceptions, exc_info, mask_patterns, ignore_patterns
+):
+    try:
+        attach_code_variables_to_frames(
+            all_exceptions, exc_info, mask_patterns, ignore_patterns
+        )
+    except Exception:
+        pass
+
+
+def attach_code_variables_to_frames(
+    all_exceptions, exc_info, mask_patterns, ignore_patterns
 ):
     exc_type, exc_value, traceback = exc_info