pidatametrics1 0.3.4__py2.py3-none-any.whl

pidatametrics/__init__.py

@@ -0,0 +1,4 @@
+from .client import PiDataMetrics
+from .parsers import PiParsers
+from .exporter import PiExporter
+from .manager import PiReportManager

pidatametrics/client.py

@@ -0,0 +1,104 @@
+# --- START OF FILE client.py ---
+
+import requests
+from requests.auth import HTTPBasicAuth
+from urllib.parse import urlencode
+
+class PiDataMetrics:
+    def __init__(self, client_id, client_secret, account_id=1377):
+        self.account_id = account_id
+        self.auth_url = "https://app.pi-datametrics.com/api/auth"
+        self.base_url = f"https://app.pi-datametrics.com/api/accounts/{account_id}"
+        self.access_token = self._get_access_token(client_id, client_secret)
+        self.headers = {
+            'Authorization': f'Bearer {self.access_token}',
+            'Content-Type': 'application/json'
+        }
+
+    def _get_access_token(self, client_id, client_secret):
+        data = {"grant_type": "client_credentials"}
+        auth = HTTPBasicAuth(client_id, client_secret)
+        try:
+            response = requests.post(self.auth_url, data=data, auth=auth)
+            response.raise_for_status()
+            return response.json()['access_token']
+        except requests.exceptions.RequestException as e:
+            raise SystemExit(f"Authentication Failed: {e}")
+
+    def fetch_endpoint(self, endpoint_path, params=None):
+        url = f"{self.base_url}/{endpoint_path}"
+        
+        # --- FIX: Manually encode params to preserve brackets [] and Order ---
+        if params:
+            # safe='[]' prevents encoding brackets to %5B%5D
+            # doseq=True handles lists correctly (e.g. key[]=val1&key[]=val2)
+            query_string = urlencode(params, doseq=True, safe='[]')
+            url = f"{url}?{query_string}"
+            params = None  # Clear params so requests doesn't append them again
+        # -----------------------------------------------------------
+
+        response = requests.get(url, headers=self.headers, params=params)
+        
+        try:
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            print(f"API Error: {e}")
+            print(f"Requested URL: {response.url}")
+            print(f"Response Body: {response.text}")
+            raise e
+            
+        return response.json().get('data', [])
+
+    def get_workspaces(self):
+        return self.fetch_endpoint("workspaces")
+
+    def get_stgs(self, workspace_id):
+        return self.fetch_endpoint(f"workspaces/{workspace_id}/search-term-groups")
+
+    def get_search_terms(self, workspace_id, stg_id):
+        return self.fetch_endpoint(f"workspaces/{workspace_id}/search-term-groups/{stg_id}/search-terms")
+
+    def get_bulk_serp_data(self, workspace_id, search_engine_id, period, **kwargs):
+        # The order here matters. 
+        # 1. search-engine-id
+        # 2. period
+        # 3. kwargs (which now has number-of-results FIRST, then serp-feature[])
+        params = {"search-engine-id": search_engine_id, "period": period}
+        params.update(kwargs)
+        return self.fetch_endpoint(f"workspaces/{workspace_id}/search-data/bulk-search-results", params=params)
+
+    def get_bulk_volume(self, workspace_id, start_date=None, end_date=None):
+        params = {}
+        if start_date and end_date:
+            params = {'start-period': start_date, 'end-period': end_date}
+        return self.fetch_endpoint(f"workspaces/{workspace_id}/volume-data/bulk-search-volume", params=params)
+
+    def get_llm_mentions(self, workspace_id, search_engine_id, start_period, end_period, stg_ids=None):
+        url = "https://app.pi-datametrics.com/api/data/llm/mentions"
+        
+        params = {
+            "account-id": self.account_id,
+            "workspace-id": workspace_id,
+            "start-period": start_period,
+            "end-period": end_period,
+            "search-engine-id": search_engine_id
+        }
+
+        if stg_ids:
+            params["search-term-group-id[]"] = stg_ids
+
+        # Apply the same fix here for consistency
+        query_string = urlencode(params, doseq=True, safe='[]')
+        url = f"{url}?{query_string}"
+
+        response = requests.get(url, headers=self.headers)
+        
+        try:
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            print(f"API Error: {e}")
+            print(f"Requested URL: {response.url}")
+            print(f"Response Body: {response.text}")
+            raise e
+
+        return response.json().get('data', [])

pidatametrics/exporter.py

@@ -0,0 +1,124 @@
+import csv
+import json
+from google.cloud import bigquery
+
+# Optional imports with error handling
+try:
+    import pandas as pd
+except ImportError:
+    pd = None
+
+try:
+    import gspread
+    import google.auth
+except ImportError:
+    gspread = None
+
+class PiExporter:
+    @staticmethod
+    def to_csv(data, filename):
+        if not data:
+            print("No data to export.")
+            return
+        
+        if not filename.endswith('.csv'):
+            filename += '.csv'
+            
+        keys = data[0].keys()
+        with open(filename, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=keys)
+            writer.writeheader()
+            writer.writerows(data)
+        print(f"Successfully saved {len(data)} rows to {filename}")
+
+    @staticmethod
+    def to_excel(data, filename):
+        """Exports data to Excel. Requires pandas and openpyxl."""
+        if pd is None:
+            print("Error: Pandas not installed. Run: pip install pandas openpyxl")
+            return
+        if not data:
+            print("No data to export.")
+            return
+
+        if not filename.endswith('.xlsx'):
+            filename += '.xlsx'
+
+        try:
+            df = pd.DataFrame(data)
+            df.to_excel(filename, index=False)
+            print(f"Successfully saved {len(data)} rows to {filename}")
+        except Exception as e:
+            print(f"Excel Export Failed: {e}")
+
+    @staticmethod
+    def to_google_sheet(data, spreadsheet_name, tab_name="Sheet1"):
+        """
+        Exports to Google Sheet using the Filename (not ID).
+        Uses the active Colab authentication.
+        """
+        if gspread is None:
+            print("Error: gspread not installed. Run: pip install gspread google-auth")
+            return
+        if not data:
+            print("No data to upload.")
+            return
+
+        print(f"Connecting to Google Sheet: '{spreadsheet_name}'...")
+
+        try:
+            # 1. Get Default Credentials (works with Colab auth.authenticate_user)
+            creds, _ = google.auth.default()
+            client = gspread.authorize(creds)
+
+            # 2. Open by Name (Title)
+            try:
+                sh = client.open(spreadsheet_name)
+            except gspread.SpreadsheetNotFound:
+                print(f"Sheet '{spreadsheet_name}' not found. Creating it...")
+                sh = client.create(spreadsheet_name)
+
+            # 3. Select or Create Worksheet (Tab)
+            try:
+                worksheet = sh.worksheet(tab_name)
+                worksheet.clear() # Clear old data
+            except gspread.WorksheetNotFound:
+                worksheet = sh.add_worksheet(title=tab_name, rows=len(data)+100, cols=20)
+
+            # 4. Prepare Data
+            headers = list(data[0].keys())
+            rows = [[row.get(col, '') for col in headers] for row in data]
+            all_values = [headers] + rows
+
+            # 5. Update
+            worksheet.update(all_values)
+            print(f"Successfully uploaded {len(data)} rows to '{spreadsheet_name}' (Tab: {tab_name})")
+
+        except Exception as e:
+            print(f"Google Sheet Upload Failed: {e}")
+
+    @staticmethod
+    def to_bigquery(data, project_id, dataset_id, table_id):
+        if not data:
+            print("No data to upload.")
+            return
+
+        client = bigquery.Client(project=project_id)
+        table_ref = f"{project_id}.{dataset_id}.{table_id}"
+        
+        print(f"Uploading {len(data)} rows to BigQuery table {table_ref}...")
+        
+        job_config = bigquery.LoadJobConfig(
+            autodetect=True,
+            source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
+            write_disposition=bigquery.WriteDisposition.WRITE_APPEND
+        )
+
+        try:
+            job = client.load_table_from_json(data, table_ref, job_config=job_config)
+            job.result() 
+            print(f"Upload successful. Loaded {job.output_rows} rows.")
+        except Exception as e:
+            print(f"BigQuery Upload Failed: {e}")
+            if hasattr(e, 'errors'):
+                print(e.errors)

pidatametrics/manager.py

@@ -0,0 +1,156 @@
+from .client import PiDataMetrics
+from .parsers import PiParsers
+from .exporter import PiExporter
+import datetime
+from dateutil.relativedelta import relativedelta
+
+class PiReportManager(PiDataMetrics):
+    def _resolve_workspaces(self, ids_str=None, name_pattern=None):
+        all_ws = self.get_workspaces()
+        targets = {}
+        if ids_str and ids_str.strip():
+            target_ids = [int(x.strip()) for x in ids_str.split(',') if x.strip().isdigit()]
+            for ws in all_ws:
+                if ws['id'] in target_ids:
+                    targets[ws['id']] = ws['name']
+        elif name_pattern:
+            for ws in all_ws:
+                if ws.get('tracked') and name_pattern.lower() in ws['name'].lower():
+                    targets[ws['id']] = ws['name']
+        return targets
+
+    def _generate_historical_dates(self, start_date_str, duration, frequency):
+        dates = []
+        try:
+            current_date = datetime.datetime.strptime(start_date_str, "%Y-%m-%d")
+        except ValueError:
+            print(f"Error: Invalid date format {start_date_str}. Using yesterday.")
+            current_date = datetime.datetime.now() - datetime.timedelta(days=1)
+        if current_date > datetime.datetime.now():
+            print(f"WARNING: Start date {current_date.strftime('%Y-%m-%d')} is in the future!")
+        if frequency == 'weekly':
+            days_since_sunday = (current_date.weekday() + 1) % 7
+            if days_since_sunday > 0:
+                current_date -= datetime.timedelta(days=days_since_sunday)
+                print(f"Note: Adjusted start date to previous Sunday: {current_date.strftime('%Y-%m-%d')}")
+        for _ in range(int(duration)):
+            dates.append(current_date.strftime("%Y-%m-%d"))
+            if frequency == 'daily':
+                current_date -= datetime.timedelta(days=1)
+            elif frequency == 'weekly':
+                current_date -= datetime.timedelta(weeks=1)
+            elif frequency == 'monthly':
+                current_date -= relativedelta(months=1)
+        return dates
+
+    # --- HELPER METHOD FOR EXPORTING ---
+    def _export_data(self, data, output_mode, filename, bq_config, spreadsheet_name, tab_name="Sheet1"):
+        if not data:
+            print("No data to export.")
+            return
+
+        if output_mode == 'bigquery' and bq_config:
+            PiExporter.to_bigquery(data, bq_config['project'], bq_config['dataset'], bq_config['table'])
+        elif output_mode == 'excel':
+            PiExporter.to_excel(data, filename)
+        elif output_mode == 'gsheet' and spreadsheet_name:
+            PiExporter.to_google_sheet(data, spreadsheet_name, tab_name)
+        else:
+            # Default to CSV
+            PiExporter.to_csv(data, filename)
+
+    def run_volume_report(self, filename, workspace_ids=None, workspace_name=None, output_mode='csv', bq_config=None, spreadsheet_name=None):
+        targets = self._resolve_workspaces(workspace_ids, workspace_name)
+        if not targets: return
+        all_rows = []
+        for ws_id, ws_name in targets.items():
+            vol_data = self.get_bulk_volume(ws_id)
+            stgs = self.get_stgs(ws_id)
+            for stg in stgs:
+                terms = self.get_search_terms(ws_id, stg['id'])
+                rows = PiParsers.parse_volume_data(vol_data, stg['name'], terms, ws_name)
+                all_rows.extend(rows)
+        
+        self._export_data(all_rows, output_mode, filename, bq_config, spreadsheet_name, tab_name="Volume")
+
+    def run_serp_report(self, data_sources, output_mode='csv', bq_config=None, filename=None, manual_duplication=None, spreadsheet_name=None):
+        yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
+        all_rows = []
+        for source in data_sources:
+            market, w_id, w_name, se_id, se_name = source
+            raw_data = self.get_bulk_serp_data(w_id, se_id, yesterday)
+            cat_map = PiParsers.build_category_map(self, w_id)
+            rows = PiParsers.parse_serp_response(raw_data, market, w_name, se_name, yesterday, cat_map, manual_duplication)
+            all_rows.extend(rows)
+        
+        self._export_data(all_rows, output_mode, filename or "serp_output", bq_config, spreadsheet_name, tab_name="SERP")
+
+    def run_historical_serp_report(self, data_sources, duration, frequency, start_date=None, features=None, num_results=25, output_mode='csv', bq_config=None, filename="historical_data", spreadsheet_name=None):
+        if features is None:
+            features = ['classicLink', 'popularProducts']
+
+        if not start_date:
+            start_date = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
+
+        target_dates = self._generate_historical_dates(start_date, duration, frequency)
+        
+        print(f"Starting Historical Report ({frequency}) for last {duration} periods...")
+        
+        all_file_rows = [] # Used for CSV, Excel, and GSheet
+
+        for i, date in enumerate(target_dates):
+            print(f"[{i+1}/{len(target_dates)}] Processing Date: {date}...")
+            
+            daily_rows = [] # Reset container for this specific date
+
+            for source in data_sources:
+                market, w_id, w_name, se_id, se_name = source
+                try:
+                    params = {
+                        'number-of-results': num_results,
+                        'serp-feature[]': features
+                    }
+                    raw_data = self.get_bulk_serp_data(w_id, se_id, date, **params)
+                    
+                    rows = PiParsers.parse_serp_response(
+                        raw_data, market, w_name, se_name, date, category_map=None
+                    )
+                    
+                    daily_rows.extend(rows)
+
+                except Exception as e:
+                    print(f"Failed to fetch {w_name} on {date}: {e}")
+
+            # --- UPLOAD LOGIC: PER DATE (BigQuery Only) ---
+            if output_mode == 'bigquery' and bq_config:
+                if daily_rows:
+                    print(f"Uploading {len(daily_rows)} rows for {date} to BigQuery...")
+                    PiExporter.to_bigquery(daily_rows, bq_config['project'], bq_config['dataset'], bq_config['table'])
+                else:
+                    print(f"No data found for {date}, skipping upload.")
+            
+            # --- FILE LOGIC: ACCUMULATE (CSV, Excel, GSheet) ---
+            elif output_mode in ['csv', 'excel', 'gsheet']:
+                all_file_rows.extend(daily_rows)
+
+        # Final Save for File-based outputs
+        if output_mode in ['csv', 'excel', 'gsheet']:
+            self._export_data(all_file_rows, output_mode, filename, bq_config, spreadsheet_name, tab_name="Historical")
+
+    def run_llm_report(self, data_sources, start_period, end_period, stg_ids=None, output_mode='csv', bq_config=None, filename="llm_output", spreadsheet_name=None):
+        all_rows = []
+        print(f"Starting LLM Report from {start_period} to {end_period}...")
+
+        for source in data_sources:
+            market, w_id, w_name, se_id, se_name = source
+            try:
+                print(f"Fetching LLM data for {w_name} ({se_name})...")
+                raw_data = self.get_llm_mentions(w_id, se_id, start_period, end_period, stg_ids)
+                
+                rows = PiParsers.parse_llm_response(raw_data, market, w_name, se_name)
+                all_rows.extend(rows)
+                print(f"Found {len(rows)} mentions/queries.")
+            except Exception as e:
+                print(f"Failed to fetch LLM data for {w_name}: {e}")
+
+        self._export_data(all_rows, output_mode, filename, bq_config, spreadsheet_name, tab_name="LLM_Mentions")

pidatametrics/parsers.py

@@ -0,0 +1,156 @@
+import json
+
+class PiParsers:
+    @staticmethod
+    def parse_serp_response(raw_data, market, workspace_name, search_engine_name, date, category_map=None, manual_duplication=None):
+        """
+        Transforms the nested 'Bulk Search Results' JSON into a flat list of dicts.
+        """
+        flat_rows = []
+        
+        for item in raw_data:
+            search_term = item.get('searchTerm')
+            
+            category = ""
+            if category_map:
+                category = category_map.get(search_term, "")
+
+            results = item.get('results', [])
+            previous_pos = None
+
+            if results:
+                for res in results:
+                    feature_type = res.get('feature')
+                    title = res.get('title')
+
+                    # If feature is popularProducts AND title is null/empty, SKIP this row.
+                    if feature_type == 'popularProducts' and not title:
+                        continue 
+
+                    # --- Logic for Position Fill-Down ---
+                    pos = res.get('position')
+                    if pos is None:
+                        pos = previous_pos
+                    
+                    # --- Logic for Attributes (Popular Products) ---
+                    attributes = res.get('attributes', {})
+                    price = None
+                    site_name = None
+                    
+                    if isinstance(attributes, dict):
+                        price = attributes.get('price')
+                        site_name = attributes.get('site')
+                        attr_str = json.dumps(attributes)
+                    else:
+                        attr_str = None
+
+                    # Create Row
+                    row = {
+                        'Date': date,
+                        'Market': market,
+                        'SearchTerm': search_term,
+                        'URL': res.get('url'),
+                        'Position': pos,
+                        'SERPFeature': feature_type,
+                        'PageTitle': title,
+                        'Price': price,
+                        'SiteName': site_name,
+                        'SearchEngine': search_engine_name,
+                        'Attributes': attr_str,
+                        'Category': category,
+                        'Workspace': workspace_name
+                    }
+                    
+                    flat_rows.append(row)
+                    previous_pos = pos
+            else:
+                pass
+                
+        return flat_rows
+
+    @staticmethod
+    def parse_volume_data(volume_data, stg_name, stg_terms, workspace_name):
+        rows = []
+        volume_lookup = {item.get('search-term'): item for item in volume_data}
+
+        for term in stg_terms:
+            term_text = term if isinstance(term, str) else term.get('term', '')
+            
+            if term_text in volume_lookup:
+                item = volume_lookup[term_text]
+                cpc = item.get('cpc', '')
+                monthly_volume = item.get('monthly-volume', {})
+                
+                for month, vol in monthly_volume.items():
+                    rows.append({
+                        "Workspace": workspace_name,
+                        "STG": stg_name,
+                        "Search Term": term_text,
+                        "Month": month,
+                        "Search Volume": vol,
+                        "CPC": cpc
+                    })
+        return rows
+
+    @staticmethod
+    def build_category_map(pi_client, workspace_id):
+        mapping = {}
+        stgs = pi_client.get_stgs(workspace_id)
+        for stg in stgs:
+            terms = pi_client.get_search_terms(workspace_id, stg['id'])
+            for term in terms:
+                t_text = term if isinstance(term, str) else term.get('term')
+                mapping[t_text] = stg['name']
+        return mapping
+
+    # --- NEW: LLM Parser ---
+    @staticmethod
+    def parse_llm_response(raw_data, market, workspace_name, search_engine_name):
+        """
+        Flattens LLM mentions data.
+        Each mention becomes a row. If no mentions exist for a query, the query is still recorded.
+        """
+        rows = []
+        for item in raw_data:
+            period = item.get('period')
+            query = item.get('query')
+            response_text = item.get('response')
+            mentions = item.get('mentions', [])
+
+            base_row = {
+                'Period': period,
+                'Market': market,
+                'Workspace': workspace_name,
+                'SearchEngine': search_engine_name,
+                'Query': query,
+                'LLM_Response': response_text
+            }
+
+            if mentions:
+                for m in mentions:
+                    row = base_row.copy()
+                    row.update({
+                        'Entity': m.get('entity'),
+                        'Citation': m.get('citation'),
+                        'Source_Site': m.get('source_site'),
+                        'Source_Domain': m.get('source_domain'),
+                        'Source_URL': m.get('source_url'),
+                        'Sentiment': m.get('sentiment'),
+                        'Sentiment_Category': m.get('sentiment_category')
+                    })
+                    rows.append(row)
+            else:
+                # Record the query even if no citations found
+                row = base_row.copy()
+                row.update({
+                    'Entity': None,
+                    'Citation': None,
+                    'Source_Site': None,
+                    'Source_Domain': None,
+                    'Source_URL': None,
+                    'Sentiment': None,
+                    'Sentiment_Category': None
+                })
+                rows.append(row)
+        
+        return rows

pidatametrics1-0.3.4.dist-info/METADATA

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: pidatametrics1
+Version: 0.3.4
+Summary: A wrapper for Pi Datametrics API with CSV and BigQuery support.
+Requires-Dist: google-auth
+Requires-Dist: google-cloud-bigquery
+Requires-Dist: gspread
+Requires-Dist: openpyxl
+Requires-Dist: pandas
+Requires-Dist: requests

pidatametrics1-0.3.4.dist-info/RECORD

@@ -0,0 +1,8 @@
+pidatametrics/__init__.py,sha256=cmNSHvjvMsYO1tMv0Nf-7LGjIJ8AFXmUIwiv8jQ34BI,137
+pidatametrics/client.py,sha256=tHH0GV0rk2SizVqRdKepjdDQevkfdWlHOJHwsPR2PCk,4399
+pidatametrics/exporter.py,sha256=CcsdVhxI6rXi0zlQaYzFEGX0GL3ZaNV94Pj5r_WrZc4,4226
+pidatametrics/manager.py,sha256=WbOQ-iZMhabkE49h3Rha0AyloqQgqdgiWBMc5S7v9jk,7888
+pidatametrics/parsers.py,sha256=fiLx3080wNubT1VqSIeDvlrKT85KdqlKhY6FaB2XuC8,5989
+pidatametrics1-0.3.4.dist-info/METADATA,sha256=PVNEkux2CMTwXSE0mVI-tmZu9LWzNTBzoke2WwsC9bk,288
+pidatametrics1-0.3.4.dist-info/WHEEL,sha256=aha0VrrYvgDJ3Xxl3db_g_MDIW-ZexDdrc_m-Hk8YY4,105
+pidatametrics1-0.3.4.dist-info/RECORD,,

pidatametrics1-0.3.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any