pidatametrics1 0.3.6__tar.gz → 0.4.0__tar.gz

{pidatametrics1-0.3.6 → pidatametrics1-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pidatametrics1
-Version: 0.3.6
+Version: 0.4.0
 Summary: A test wrapper for Pi Datametrics API with CSV and BigQuery support.
 Requires-Dist: google-auth
 Requires-Dist: google-cloud-bigquery

{pidatametrics1-0.3.6 → pidatametrics1-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pidatametrics1"
-version = "0.3.6"
+version = "0.4.0"
 description = "A test wrapper for Pi Datametrics API with CSV and BigQuery support."
 dependencies = [
     "requests",

{pidatametrics1-0.3.6 → pidatametrics1-0.4.0}/src/pidatametrics/exporter.py

@@ -1,20 +1,34 @@
 import csv
-import json
+import re
+import math
 from google.cloud import bigquery
 
-# Optional imports with error handling
+# Optional imports
 try:
     import pandas as pd
+    from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
 except ImportError:
     pd = None
+    ILLEGAL_CHARACTERS_RE = None
 
 try:
     import gspread
     import google.auth
+    from gspread.exceptions import APIError
 except ImportError:
     gspread = None
 
 class PiExporter:
+    
+    @staticmethod
+    def _clean_for_excel(text):
+        """Removes characters that cause Excel to crash/corrupt."""
+        if not isinstance(text, str):
+            return text
+        if ILLEGAL_CHARACTERS_RE:
+            return ILLEGAL_CHARACTERS_RE.sub('', text)
+        return text
+
     @staticmethod
     def to_csv(data, filename):
         if not data:
@@ -33,9 +47,9 @@ class PiExporter:
 
     @staticmethod
     def to_excel(data, filename):
-        """Exports data to Excel. Requires pandas and openpyxl."""
+        """Exports to Excel with sanitization."""
         if pd is None:
-            print("Error: Pandas not installed. Run: pip install pandas openpyxl")
+            print("Error: Pandas/Openpyxl not installed.")
             return
         if not data:
             print("No data to export.")
@@ -44,58 +58,85 @@ class PiExporter:
         if not filename.endswith('.xlsx'):
             filename += '.xlsx'
 
+        print(f"Preparing Excel file: {filename} ({len(data)} rows)...")
+
         try:
             df = pd.DataFrame(data)
-            df.to_excel(filename, index=False)
+
+            # Sanitize Data (Fixes "Excel found a problem with content")
+            for col in df.select_dtypes(include=['object']).columns:
+                df[col] = df[col].apply(PiExporter._clean_for_excel)
+
+            df.to_excel(filename, index=False, engine='openpyxl')
             print(f"Successfully saved {len(data)} rows to {filename}")
+            
         except Exception as e:
             print(f"Excel Export Failed: {e}")
 
     @staticmethod
     def to_google_sheet(data, spreadsheet_name, tab_name="Sheet1"):
-        """
-        Exports to Google Sheet using the Filename (not ID).
-        Uses the active Colab authentication.
-        """
         if gspread is None:
-            print("Error: gspread not installed. Run: pip install gspread google-auth")
+            print("Error: gspread not installed.")
             return
         if not data:
             print("No data to upload.")
             return
 
-        print(f"Connecting to Google Sheet: '{spreadsheet_name}'...")
+        row_count = len(data)
+        print(f"Preparing Google Sheet upload: {row_count} rows...")
 
         try:
-            # 1. Get Default Credentials (works with Colab auth.authenticate_user)
+            # 1. Auth
             creds, _ = google.auth.default()
             client = gspread.authorize(creds)
 
-            # 2. Open by Name (Title)
+            # 2. Open Sheet
             try:
                 sh = client.open(spreadsheet_name)
             except gspread.SpreadsheetNotFound:
                 print(f"Sheet '{spreadsheet_name}' not found. Creating it...")
                 sh = client.create(spreadsheet_name)
 
-            # 3. Select or Create Worksheet (Tab)
+            # 3. Setup Tab
             try:
                 worksheet = sh.worksheet(tab_name)
-                worksheet.clear() # Clear old data
+                worksheet.clear()
             except gspread.WorksheetNotFound:
-                worksheet = sh.add_worksheet(title=tab_name, rows=len(data)+100, cols=20)
+                worksheet = sh.add_worksheet(title=tab_name, rows=row_count+100, cols=20)
 
             # 4. Prepare Data
             headers = list(data[0].keys())
             rows = [[row.get(col, '') for col in headers] for row in data]
-            all_values = [headers] + rows
-
-            # 5. Update
-            worksheet.update(all_values)
-            print(f"Successfully uploaded {len(data)} rows to '{spreadsheet_name}' (Tab: {tab_name})")
+            
+            # 5. Upload Headers
+            worksheet.update([headers], 'A1')
+
+            # 6. CHUNKED UPLOAD (Optimized for Speed)
+            # 30,000 is the sweet spot. 
+            # 90k often causes "Payload Too Large" or "Timeout" errors.
+            chunk_size = 30000 
+            total_chunks = math.ceil(len(rows) / chunk_size)
+
+            print(f"Starting upload in {total_chunks} chunks (Size: {chunk_size})...")
+
+            for i in range(total_chunks):
+                start = i * chunk_size
+                end = start + chunk_size
+                chunk = rows[start:end]
+                
+                try:
+                    # value_input_option='RAW' is faster than 'USER_ENTERED'
+                    worksheet.append_rows(chunk, value_input_option='RAW')
+                    print(f"   - Uploaded chunk {i+1}/{total_chunks} ({len(chunk)} rows)")
+                except APIError as e:
+                    print(f"   ❌ Error on chunk {i+1}: {e}")
+                    print("      (If this is a timeout, try reducing chunk_size in the code)")
+                    raise e
+
+            print(f"✅ Successfully uploaded {row_count} rows to '{spreadsheet_name}' (Tab: {tab_name})")
 
         except Exception as e:
-            print(f"Google Sheet Upload Failed: {e}")
+            print(f"❌ Google Sheet Upload Failed: {e}")
 
     @staticmethod
     def to_bigquery(data, project_id, dataset_id, table_id):