user-analytics-tracker 2.2.0 → 3.0.0

package/CHANGELOG.md

@@ -1,3 +1,41 @@
+# [3.0.0](https://github.com/switch-org/analytics-tracker/compare/v2.2.0...v3.0.0) (2025-12-30)
+
+
+### Features
+
+* optimize essential mode with deduplication and reduced deviceInfo fields ([b64eefa](https://github.com/switch-org/analytics-tracker/commit/b64eefa4992e0570b1011fa029d0db8de89c321b))
+
+
+### BREAKING CHANGES
+
+* Essential mode deviceInfo now stores only 8 fields instead of 15
+
+## Changes
+
+### Core Optimizations
+- Reduce deviceInfo essential fields from 15 to 8
+- Implement automatic deduplication between location and customData.ipLocation
+- Remove duplicate fields from location object
+- Add null/undefined value removal from filtered objects
+- Improve nested object handling
+
+### Field Storage Improvements
+- Add generic field storage transformer for all data types
+- Support essential/all/custom modes for all data types
+- Implement recursive null value cleaning
+- Add mode-aware deduplication logic
+
+### Documentation
+- Merge and consolidate essential mode documentation
+- Add comprehensive essential-mode-guide.md
+- Update documentation index
+- Add example payload JSON
+
+## Impact
+- Significantly smaller payloads (~30-40% reduction)
+- No duplicate data storage
+- All crucial information preserved
+
 # [2.2.0](https://github.com/switch-org/analytics-tracker/compare/v2.1.0...v2.2.0) (2025-12-30)
 
 

package/README.md

@@ -93,6 +93,21 @@ const analytics = useAnalytics({
     
     // Metrics configuration
     enableMetrics: false,       // Enable metrics collection (default: false)
+    
+    // Field storage configuration (optional) - control which fields are stored
+    fieldStorage: {
+      ipLocation: { mode: 'essential' },    // IP location fields
+      deviceInfo: { mode: 'essential' },   // Device info fields
+      networkInfo: { mode: 'essential' },  // Network info fields
+      location: { mode: 'essential' },     // Location fields
+      attribution: { mode: 'essential' },   // Attribution fields
+      // Each can be: 'essential' (default) | 'all' | 'custom'
+      // For 'custom': specify fields array
+      // For 'all': specify exclude array
+    },
+    
+    // Legacy: IP Location storage (backward compatible)
+    ipLocationFields: { mode: 'essential' },
   },
 });
 ```

package/dist/index.cjs.js

@@ -4,6 +4,97 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 var react = require('react');
 
+/**
+ * Core types for the analytics tracker package
+ */
+/**
+ * Default essential fields for IP location storage
+ * These fields are stored when mode is 'essential' (default)
+ */
+const DEFAULT_ESSENTIAL_IP_FIELDS = [
+    // Core identification
+    'ip',
+    'country',
+    'countryCode',
+    'region',
+    'city',
+    // Geographic coordinates (stored here, not duplicated in location)
+    'lat',
+    'lon',
+    // Additional geographic info
+    'continent',
+    'continentCode',
+    // Network info
+    'type',
+    'isEu',
+    'isp',
+    'connection',
+    'connection.asn',
+    'connection.org',
+    'connection.isp',
+    'connection.domain',
+    // Timezone (stored here, not duplicated in location)
+    'timezone',
+    'timezoneDetails',
+    'timezoneDetails.id',
+    'timezoneDetails.abbr',
+    'timezoneDetails.utc',
+    // Flag (only emoji in essential mode)
+    'flag.emoji',
+];
+/**
+ * Default essential fields for Device Info storage
+ */
+const DEFAULT_ESSENTIAL_DEVICE_FIELDS = [
+    'type',
+    'os',
+    'osVersion',
+    'browser',
+    'browserVersion',
+    'deviceModel',
+    'deviceBrand',
+    'userAgent',
+];
+/**
+ * Default essential fields for Network Info storage
+ */
+const DEFAULT_ESSENTIAL_NETWORK_FIELDS = [
+    'type',
+    'effectiveType',
+    'downlink',
+    'rtt',
+    'saveData',
+];
+/**
+ * Default essential fields for Location Info storage
+ */
+const DEFAULT_ESSENTIAL_LOCATION_FIELDS = [
+    // Minimal location fields - only coordinates and source
+    // All IP-related data (ip, country, city, etc.) is stored in customData.ipLocation to avoid duplication
+    'lat',
+    'lon',
+    'source',
+    'ts',
+];
+/**
+ * Default essential fields for Attribution Info storage
+ */
+const DEFAULT_ESSENTIAL_ATTRIBUTION_FIELDS = [
+    'landingUrl',
+    'path',
+    'hostname',
+    'referrerUrl',
+    'referrerDomain',
+    'navigationType',
+    'isReload',
+    'isBackForward',
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_term',
+    'utm_content',
+];
+
 /**
  * Network Type Detector
  * Detects WiFi, Mobile Data (Cellular), Hotspot, Ethernet, or Unknown
@@ -2010,18 +2101,206 @@ class MetricsCollector {
 // Global metrics collector instance
 const metricsCollector = new MetricsCollector();
 
+/**
+ * Generic field storage transformer
+ * Filters object fields based on storage configuration
+ */
+/**
+ * Filter object fields based on storage configuration
+ *
+ * @param data - The data object to filter
+ * @param config - Storage configuration
+ * @param defaultEssentialFields - Default essential fields for this data type
+ * @returns Filtered data object with only configured fields
+ */
+function filterFieldsByConfig(data, config, defaultEssentialFields) {
+    if (!data) {
+        return null;
+    }
+    const mode = config?.mode || 'essential';
+    let fieldsToInclude = [];
+    if (mode === 'essential') {
+        // Use default essential fields
+        fieldsToInclude = [...defaultEssentialFields];
+    }
+    else if (mode === 'all') {
+        // Include all fields, then exclude specified ones
+        fieldsToInclude = ['*']; // Special marker for "all fields"
+    }
+    else if (mode === 'custom' && config) {
+        // Use custom field list
+        fieldsToInclude = config.fields || [];
+    }
+    // If mode is 'all', just exclude specified fields
+    if (mode === 'all') {
+        const filtered = { ...data };
+        if (config?.exclude && config.exclude.length > 0) {
+            const excludeSet = new Set(config.exclude);
+            Object.keys(filtered).forEach(key => {
+                if (excludeSet.has(key)) {
+                    delete filtered[key];
+                }
+            });
+            // Handle nested exclusions
+            if (filtered.connection && excludeSet.has('connection')) {
+                delete filtered.connection;
+            }
+            if (filtered.timezoneDetails && excludeSet.has('timezoneDetails')) {
+                delete filtered.timezoneDetails;
+            }
+            if (filtered.flag && excludeSet.has('flag')) {
+                delete filtered.flag;
+            }
+            if (filtered.firstTouch && excludeSet.has('firstTouch')) {
+                delete filtered.firstTouch;
+            }
+            if (filtered.lastTouch && excludeSet.has('lastTouch')) {
+                delete filtered.lastTouch;
+            }
+        }
+        return filtered;
+    }
+    // For 'essential' or 'custom' mode, only include specified fields
+    const filtered = {};
+    const includeSet = new Set(fieldsToInclude);
+    // Helper to check if a field path should be included
+    const shouldInclude = (fieldPath) => {
+        // Direct match - most specific
+        if (includeSet.has(fieldPath))
+            return true;
+        // For nested fields (e.g., 'flag.emoji'), only include if explicitly listed
+        // Don't auto-include all children just because parent is included
+        const parts = fieldPath.split('.');
+        if (parts.length > 1) {
+            // For nested fields, require explicit inclusion
+            // This prevents 'flag' from including all 'flag.*' fields
+            return includeSet.has(fieldPath);
+        }
+        // For top-level fields only, check if parent path is included
+        // This allows 'connection' to work when all connection.* fields are listed
+        return false;
+    };
+    // Helper to check if a parent object should be created (for nested objects)
+    const shouldIncludeParent = (parentPath) => {
+        // Check if any child of this parent is included
+        for (const field of fieldsToInclude) {
+            if (field.startsWith(parentPath + '.')) {
+                return true;
+            }
+        }
+        // Also check if parent itself is explicitly included
+        return includeSet.has(parentPath);
+    };
+    // Filter top-level fields
+    Object.keys(data).forEach(key => {
+        if (shouldInclude(key)) {
+            filtered[key] = data[key];
+        }
+    });
+    // Handle nested objects - only create if at least one child field is included
+    if (data.connection && shouldIncludeParent('connection')) {
+        filtered.connection = {};
+        if (shouldInclude('connection.asn'))
+            filtered.connection.asn = data.connection.asn;
+        if (shouldInclude('connection.org'))
+            filtered.connection.org = data.connection.org;
+        if (shouldInclude('connection.isp'))
+            filtered.connection.isp = data.connection.isp;
+        if (shouldInclude('connection.domain'))
+            filtered.connection.domain = data.connection.domain;
+        // If no connection fields were included, remove the object
+        if (Object.keys(filtered.connection).length === 0) {
+            delete filtered.connection;
+        }
+    }
+    if (data.timezoneDetails && shouldIncludeParent('timezoneDetails')) {
+        filtered.timezoneDetails = {};
+        if (shouldInclude('timezoneDetails.id'))
+            filtered.timezoneDetails.id = data.timezoneDetails.id;
+        if (shouldInclude('timezoneDetails.abbr'))
+            filtered.timezoneDetails.abbr = data.timezoneDetails.abbr;
+        if (shouldInclude('timezoneDetails.utc'))
+            filtered.timezoneDetails.utc = data.timezoneDetails.utc;
+        if (shouldInclude('timezoneDetails.isDst'))
+            filtered.timezoneDetails.isDst = data.timezoneDetails.isDst;
+        if (shouldInclude('timezoneDetails.offset'))
+            filtered.timezoneDetails.offset = data.timezoneDetails.offset;
+        if (shouldInclude('timezoneDetails.currentTime'))
+            filtered.timezoneDetails.currentTime = data.timezoneDetails.currentTime;
+        // If no timezoneDetails fields were included, remove the object
+        if (Object.keys(filtered.timezoneDetails).length === 0) {
+            delete filtered.timezoneDetails;
+        }
+    }
+    if (data.flag && shouldIncludeParent('flag')) {
+        filtered.flag = {};
+        // Only include specific flag fields if they're explicitly in the include list
+        if (shouldInclude('flag.emoji'))
+            filtered.flag.emoji = data.flag.emoji;
+        if (shouldInclude('flag.img'))
+            filtered.flag.img = data.flag.img;
+        if (shouldInclude('flag.emojiUnicode'))
+            filtered.flag.emojiUnicode = data.flag.emojiUnicode;
+        // If no specific flag fields are included, don't add the flag object
+        if (Object.keys(filtered.flag).length === 0) {
+            delete filtered.flag;
+        }
+    }
+    if (data.firstTouch && shouldInclude('firstTouch')) {
+        filtered.firstTouch = data.firstTouch;
+    }
+    if (data.lastTouch && shouldInclude('lastTouch')) {
+        filtered.lastTouch = data.lastTouch;
+    }
+    // Remove null and undefined values to reduce payload size
+    const cleanValue = (val) => {
+        if (val === null || val === undefined) {
+            return undefined; // Will be filtered out
+        }
+        // For objects, recursively clean nested null/undefined values
+        if (typeof val === 'object' && !Array.isArray(val) && val !== null) {
+            const cleaned = {};
+            let hasAnyValue = false;
+            Object.keys(val).forEach(key => {
+                const cleanedChild = cleanValue(val[key]);
+                if (cleanedChild !== undefined) {
+                    cleaned[key] = cleanedChild;
+                    hasAnyValue = true;
+                }
+            });
+            return hasAnyValue ? cleaned : undefined;
+        }
+        // For arrays, clean each element
+        if (Array.isArray(val)) {
+            const cleaned = val.map(cleanValue).filter(item => item !== undefined);
+            return cleaned.length > 0 ? cleaned : undefined;
+        }
+        return val;
+    };
+    const cleaned = {};
+    Object.keys(filtered).forEach(key => {
+        const cleanedValue = cleanValue(filtered[key]);
+        if (cleanedValue !== undefined) {
+            cleaned[key] = cleanedValue;
+        }
+    });
+    return cleaned;
+}
+
 /**
  * Transform IP location data from API format (snake_case) to backend-expected format (camelCase)
- * This ensures compatibility with the analytics backend integration
+ * Supports configurable field storage to optimize storage capacity
  *
  * @param ipLocation - Raw IP location data from ipwho.is API
- * @returns Transformed IP location data matching backend schema
+ * @param config - Optional configuration for which fields to store
+ * @returns Transformed IP location data matching backend schema (only includes configured fields)
  */
-function transformIPLocationForBackend(ipLocation) {
+function transformIPLocationForBackend(ipLocation, config) {
     if (!ipLocation) {
         return null;
     }
     // Transform to match backend expected format (camelCase)
+    // Build complete object first, then filter based on configuration
     const transformed = {
         // Basic fields
         ip: ipLocation.ip,
@@ -2056,9 +2335,11 @@ function transformIPLocationForBackend(ipLocation) {
         timezoneDetails: ipLocation.timezone && typeof ipLocation.timezone === 'object' ? {
             id: ipLocation.timezone.id,
             abbr: ipLocation.timezone.abbr,
+            utc: ipLocation.timezone.utc,
+            // Exclude these in essential mode: isDst, offset, currentTime
+            // They will be filtered out by filterFieldsByConfig if not in essential fields
             isDst: ipLocation.timezone.is_dst,
             offset: ipLocation.timezone.offset,
-            utc: ipLocation.timezone.utc,
             currentTime: ipLocation.timezone.current_time,
         } : undefined,
         // Flag - transform to camelCase
@@ -2074,7 +2355,8 @@ function transformIPLocationForBackend(ipLocation) {
             delete transformed[key];
         }
     });
-    return transformed;
+    // Filter fields based on configuration using generic filter
+    return filterFieldsByConfig(transformed, config, DEFAULT_ESSENTIAL_IP_FIELDS);
 }
 
 /**
@@ -2340,20 +2622,49 @@ class AnalyticsService {
      * Track user journey with full context
      */
     static async trackUserJourney({ sessionId, pageUrl, networkInfo, deviceInfo, location, attribution, ipLocation, userId, customData, pageVisits = 1, interactions = 0, }) {
-        // Transform IP location data to match backend expected format (camelCase)
-        const transformedIPLocation = transformIPLocationForBackend(ipLocation);
+        // Get field storage config (support both new and legacy format)
+        const fieldStorage = this.config.fieldStorage || {};
+        const ipLocationConfig = fieldStorage.ipLocation || this.config.ipLocationFields;
+        // Transform and filter all data types based on configuration
+        const transformedIPLocation = transformIPLocationForBackend(ipLocation, ipLocationConfig);
+        const filteredDeviceInfo = filterFieldsByConfig(deviceInfo, fieldStorage.deviceInfo, DEFAULT_ESSENTIAL_DEVICE_FIELDS);
+        const filteredNetworkInfo = filterFieldsByConfig(networkInfo, fieldStorage.networkInfo, DEFAULT_ESSENTIAL_NETWORK_FIELDS);
+        // For location: In essential mode, remove duplicate fields that are already in customData.ipLocation
+        // This prevents storing the same data twice (e.g., ip, country, city, region, timezone)
+        const locationConfig = fieldStorage.location;
+        const locationMode = locationConfig?.mode || 'essential';
+        let filteredLocation = filterFieldsByConfig(location, locationConfig, DEFAULT_ESSENTIAL_LOCATION_FIELDS);
+        // In essential mode, if we have IP location data, remove duplicate fields from location
+        // to avoid storing the same data twice
+        if (locationMode === 'essential' && transformedIPLocation && filteredLocation) {
+            // Remove fields that are duplicated in customData.ipLocation
+            const duplicateFields = ['ip', 'country', 'countryCode', 'city', 'region', 'timezone'];
+            const minimalLocation = { ...filteredLocation };
+            duplicateFields.forEach(field => {
+                delete minimalLocation[field];
+            });
+            // Only keep essential location fields: lat, lon, source, ts
+            filteredLocation = {
+                lat: minimalLocation.lat,
+                lon: minimalLocation.lon,
+                source: minimalLocation.source,
+                ts: minimalLocation.ts,
+            };
+        }
+        const filteredAttribution = filterFieldsByConfig(attribution, fieldStorage.attribution, DEFAULT_ESSENTIAL_ATTRIBUTION_FIELDS);
         await this.trackEvent({
             sessionId,
             pageUrl,
-            networkInfo,
-            deviceInfo,
-            location,
-            attribution,
-            ipLocation,
+            networkInfo: filteredNetworkInfo || undefined,
+            deviceInfo: filteredDeviceInfo || undefined,
+            location: filteredLocation || undefined,
+            attribution: filteredAttribution || undefined,
+            // Don't include raw ipLocation - we have the filtered/transformed version in customData
+            ipLocation: undefined,
             userId: userId ?? sessionId,
             customData: {
                 ...customData,
-                // Store transformed IP location in customData for backend integration
+                // Store transformed and filtered IP location in customData for backend integration
                 ...(transformedIPLocation && { ipLocation: transformedIPLocation }),
             },
             eventName: 'page_view', // Auto-tracked as page view
@@ -2430,15 +2741,41 @@ class AnalyticsService {
         const ipLocationData = locationData && typeof locationData === 'object'
             ? locationData?.ipLocationData
             : undefined;
-        // Transform IP location data to match backend expected format
-        const transformedIPLocation = transformIPLocationForBackend(ipLocationData);
+        // Get field storage config (support both new and legacy format)
+        const fieldStorage = this.config.fieldStorage || {};
+        const ipLocationConfig = fieldStorage.ipLocation || this.config.ipLocationFields;
+        // Transform and filter all data types based on configuration
+        const transformedIPLocation = transformIPLocationForBackend(ipLocationData, ipLocationConfig);
+        const filteredDeviceInfo = filterFieldsByConfig(context?.deviceInfo || autoContext?.deviceInfo, fieldStorage.deviceInfo, DEFAULT_ESSENTIAL_DEVICE_FIELDS);
+        const filteredNetworkInfo = filterFieldsByConfig(context?.networkInfo || autoContext?.networkInfo, fieldStorage.networkInfo, DEFAULT_ESSENTIAL_NETWORK_FIELDS);
+        // For location: In essential mode, remove duplicate fields that are already in customData.ipLocation
+        const locationConfig = fieldStorage.location;
+        const locationMode = locationConfig?.mode || 'essential';
+        let filteredLocation = filterFieldsByConfig((context?.location || autoContext?.location), locationConfig, DEFAULT_ESSENTIAL_LOCATION_FIELDS);
+        // In essential mode, if we have IP location data, remove duplicate fields from location
+        if (locationMode === 'essential' && transformedIPLocation && filteredLocation) {
+            // Remove fields that are duplicated in customData.ipLocation
+            const duplicateFields = ['ip', 'country', 'countryCode', 'city', 'region', 'timezone'];
+            const minimalLocation = { ...filteredLocation };
+            duplicateFields.forEach(field => {
+                delete minimalLocation[field];
+            });
+            // Only keep essential location fields: lat, lon, source, ts
+            filteredLocation = {
+                lat: minimalLocation.lat,
+                lon: minimalLocation.lon,
+                source: minimalLocation.source,
+                ts: minimalLocation.ts,
+            };
+        }
+        const filteredAttribution = filterFieldsByConfig(context?.attribution || autoContext?.attribution, fieldStorage.attribution, DEFAULT_ESSENTIAL_ATTRIBUTION_FIELDS);
         await this.trackEvent({
             sessionId: finalSessionId,
             pageUrl: finalPageUrl,
-            networkInfo: context?.networkInfo || autoContext?.networkInfo,
-            deviceInfo: context?.deviceInfo || autoContext?.deviceInfo,
-            location: context?.location || autoContext?.location,
-            attribution: context?.attribution || autoContext?.attribution,
+            networkInfo: filteredNetworkInfo || undefined,
+            deviceInfo: filteredDeviceInfo || undefined,
+            location: filteredLocation || undefined,
+            attribution: filteredAttribution || undefined,
             userId: context?.userId || finalSessionId,
             eventName,
             eventParameters: parameters || {},
@@ -2624,6 +2961,8 @@ function useAnalytics(options = {}) {
                 logLevel: config.logLevel,
                 enableMetrics: config.enableMetrics,
                 sessionTimeout: config.sessionTimeout,
+                fieldStorage: config.fieldStorage,
+                ipLocationFields: config.ipLocationFields, // Legacy support
             });
         }
     }, [
@@ -2911,6 +3250,11 @@ function useAnalytics(options = {}) {
 
 exports.AnalyticsService = AnalyticsService;
 exports.AttributionDetector = AttributionDetector;
+exports.DEFAULT_ESSENTIAL_ATTRIBUTION_FIELDS = DEFAULT_ESSENTIAL_ATTRIBUTION_FIELDS;
+exports.DEFAULT_ESSENTIAL_DEVICE_FIELDS = DEFAULT_ESSENTIAL_DEVICE_FIELDS;
+exports.DEFAULT_ESSENTIAL_IP_FIELDS = DEFAULT_ESSENTIAL_IP_FIELDS;
+exports.DEFAULT_ESSENTIAL_LOCATION_FIELDS = DEFAULT_ESSENTIAL_LOCATION_FIELDS;
+exports.DEFAULT_ESSENTIAL_NETWORK_FIELDS = DEFAULT_ESSENTIAL_NETWORK_FIELDS;
 exports.DeviceDetector = DeviceDetector;
 exports.LocationDetector = LocationDetector;
 exports.NetworkDetector = NetworkDetector;
@@ -2919,6 +3263,7 @@ exports.checkAndSetLocationConsent = checkAndSetLocationConsent;
 exports.clearLocationConsent = clearLocationConsent;
 exports.clearSession = clearSession;
 exports.default = useAnalytics;
+exports.filterFieldsByConfig = filterFieldsByConfig;
 exports.getCompleteIPLocation = getCompleteIPLocation;
 exports.getIPFromRequest = getIPFromRequest;
 exports.getIPLocation = getIPLocation;