pidatametrics1 0.3.7__py2.py3-none-any.whl → 0.4.0__py2.py3-none-any.whl

pidatametrics/exporter.py

@@ -14,6 +14,7 @@ except ImportError:
 try:
     import gspread
     import google.auth
+    from gspread.exceptions import APIError
 except ImportError:
     gspread = None
 
@@ -24,7 +25,6 @@ class PiExporter:
         """Removes characters that cause Excel to crash/corrupt."""
         if not isinstance(text, str):
             return text
-        # Remove illegal control characters (null bytes, etc.)
         if ILLEGAL_CHARACTERS_RE:
             return ILLEGAL_CHARACTERS_RE.sub('', text)
         return text
@@ -47,9 +47,7 @@ class PiExporter:
 
     @staticmethod
     def to_excel(data, filename):
-        """
-        Exports to Excel with sanitization to prevent 'Corrupted File' errors.
-        """
+        """Exports to Excel with sanitization."""
         if pd is None:
             print("Error: Pandas/Openpyxl not installed.")
             return
@@ -63,15 +61,12 @@ class PiExporter:
         print(f"Preparing Excel file: {filename} ({len(data)} rows)...")
 
         try:
-            # 1. Create DataFrame
             df = pd.DataFrame(data)
 
-            # 2. Sanitize Data (Fixes "Excel found a problem with content")
-            # Apply cleaning to all string columns
+            # Sanitize Data (Fixes "Excel found a problem with content")
             for col in df.select_dtypes(include=['object']).columns:
                 df[col] = df[col].apply(PiExporter._clean_for_excel)
 
-            # 3. Export
             df.to_excel(filename, index=False, engine='openpyxl')
             print(f"Successfully saved {len(data)} rows to {filename}")
             
@@ -90,11 +85,6 @@ class PiExporter:
         row_count = len(data)
         print(f"Preparing Google Sheet upload: {row_count} rows...")
 
-        # --- WARNING FOR LARGE DATASETS ---
-        if row_count > 50000:
-            print(f"⚠️  WARNING: You are uploading {row_count} rows.")
-            print("   Google Sheets may become slow. Uploading in chunks...")
-
         try:
             # 1. Auth
             creds, _ = google.auth.default()
@@ -118,24 +108,30 @@ class PiExporter:
             headers = list(data[0].keys())
             rows = [[row.get(col, '') for col in headers] for row in data]
             
-            # 5. Upload Headers first
+            # 5. Upload Headers
             worksheet.update([headers], 'A1')
 
-            # 6. CHUNKED UPLOAD (To prevent timeouts on large data)
-            chunk_size = 5000  # Safe limit for gspread
+            # 6. CHUNKED UPLOAD (Optimized for Speed)
+            # 30,000 is the sweet spot. 
+            # 90k often causes "Payload Too Large" or "Timeout" errors.
+            chunk_size = 30000 
             total_chunks = math.ceil(len(rows) / chunk_size)
 
-            print(f"Starting upload in {total_chunks} chunks...")
+            print(f"Starting upload in {total_chunks} chunks (Size: {chunk_size})...")
 
             for i in range(total_chunks):
                 start = i * chunk_size
                 end = start + chunk_size
                 chunk = rows[start:end]
                 
-                # Append rows is safer for large datasets than update range
-                worksheet.append_rows(chunk, value_input_option='RAW')
-                
-                print(f"   - Uploaded chunk {i+1}/{total_chunks} ({len(chunk)} rows)")
+                try:
+                    # value_input_option='RAW' is faster than 'USER_ENTERED'
+                    worksheet.append_rows(chunk, value_input_option='RAW')
+                    print(f"   - Uploaded chunk {i+1}/{total_chunks} ({len(chunk)} rows)")
+                except APIError as e:
+                    print(f"   ❌ Error on chunk {i+1}: {e}")
+                    print("      (If this is a timeout, try reducing chunk_size in the code)")
+                    raise e
 
             print(f"✅ Successfully uploaded {row_count} rows to '{spreadsheet_name}' (Tab: {tab_name})")
 

{pidatametrics1-0.3.7.dist-info → pidatametrics1-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pidatametrics1
-Version: 0.3.7
+Version: 0.4.0
 Summary: A test wrapper for Pi Datametrics API with CSV and BigQuery support.
 Requires-Dist: google-auth
 Requires-Dist: google-cloud-bigquery

{pidatametrics1-0.3.7.dist-info → pidatametrics1-0.4.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
 pidatametrics/__init__.py,sha256=cmNSHvjvMsYO1tMv0Nf-7LGjIJ8AFXmUIwiv8jQ34BI,137
 pidatametrics/client.py,sha256=tHH0GV0rk2SizVqRdKepjdDQevkfdWlHOJHwsPR2PCk,4399
-pidatametrics/exporter.py,sha256=yLN40kqwibHWs45gXKe_I1j9td9lJt-8LLFSS-Pk9-U,5813
+pidatametrics/exporter.py,sha256=PffQ0f515du1JnvUmh4xDM8YQ7ww8WgeFKSss233Y-I,5723
 pidatametrics/manager.py,sha256=tUeeJ-wKAlhpWsaZEAjxtZCtA2EbQcTBB1JkXPEVV50,9101
 pidatametrics/parsers.py,sha256=fiLx3080wNubT1VqSIeDvlrKT85KdqlKhY6FaB2XuC8,5989
-pidatametrics1-0.3.7.dist-info/METADATA,sha256=3rqys7d0-SNpsghd25f3eS99WSl-1QYzTY0HjbAYNak,293
-pidatametrics1-0.3.7.dist-info/WHEEL,sha256=aha0VrrYvgDJ3Xxl3db_g_MDIW-ZexDdrc_m-Hk8YY4,105
-pidatametrics1-0.3.7.dist-info/RECORD,,
+pidatametrics1-0.4.0.dist-info/METADATA,sha256=g4rCEmRmzUuuokAKK7v5EaAVpnredVEnPIE_ukOV0ao,293
+pidatametrics1-0.4.0.dist-info/WHEEL,sha256=aha0VrrYvgDJ3Xxl3db_g_MDIW-ZexDdrc_m-Hk8YY4,105
+pidatametrics1-0.4.0.dist-info/RECORD,,