piholelongtermstats 0.2.2__tar.gz → 0.2.3__tar.gz

{piholelongtermstats-0.2.2 → piholelongtermstats-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: piholelongtermstats
-Version: 0.2.2
+Version: 0.2.3
 Summary: A dashboard to explore long-term DNS query data from Pi-hole FTL database files.
 Author-email: Davis Thomas Daniel <davisthomasdaniel@gmail.com>
 Maintainer-email: Davis Thomas Daniel <davisthomasdaniel@gmail.com>
@@ -26,11 +26,21 @@ Requires-Dist: twine==6.2.0; extra == "develop"
 Requires-Dist: pytest-cov==7.0.0; extra == "develop"
 Dynamic: license-file
 
-# Pi Hole Long Term Statistics v.0.2.2
+# Pi Hole Long Term Statistics v.0.2.3
+
+<div align="center">
+
+![PyPI - Version](https://img.shields.io/pypi/v/piholelongtermstats)
+![GitHub last commit](https://img.shields.io/github/last-commit/davistdaniel/piholelongtermstats)
+![GitHub License](https://img.shields.io/github/license/davistdaniel/piholelongtermstats)
+![phlts_tests](https://img.shields.io/github/actions/workflow/status/davistdaniel/piholelongtermstats/run-tests.yml?label=tests)
 
-A dashboard built with **Dash** and **Plotly** to explore long-term DNS query data from **Pi-hole v.6** FTL database files. Visualize allowed vs blocked domains, top clients, and query trends over time. I originally made this for my own use, but if you find it useful, a ⭐ would be awesome!
 
 
+</div>
+
+A dashboard built with **Dash** and **Plotly** to explore long-term DNS query data from **Pi-hole v.6** FTL database files. Visualize allowed vs blocked domains, top clients, and query trends over time. I originally made this for my own use, but if you find it useful, a ⭐ would be awesome!
+
 **Disclaimer : This is an unofficial, third-party project. The Pi Hole team and the development of [PiHole](https://pi-hole.net/) software is not related to this project.**
 
 

{piholelongtermstats-0.2.2 → piholelongtermstats-0.2.3}/README.md

@@ -1,8 +1,18 @@
-# Pi Hole Long Term Statistics v.0.2.2
+# Pi Hole Long Term Statistics v.0.2.3
+
+<div align="center">
+
+![PyPI - Version](https://img.shields.io/pypi/v/piholelongtermstats)
+![GitHub last commit](https://img.shields.io/github/last-commit/davistdaniel/piholelongtermstats)
+![GitHub License](https://img.shields.io/github/license/davistdaniel/piholelongtermstats)
+![phlts_tests](https://img.shields.io/github/actions/workflow/status/davistdaniel/piholelongtermstats/run-tests.yml?label=tests)
 
-A dashboard built with **Dash** and **Plotly** to explore long-term DNS query data from **Pi-hole v.6** FTL database files. Visualize allowed vs blocked domains, top clients, and query trends over time. I originally made this for my own use, but if you find it useful, a ⭐ would be awesome!
 
 
+</div>
+
+A dashboard built with **Dash** and **Plotly** to explore long-term DNS query data from **Pi-hole v.6** FTL database files. Visualize allowed vs blocked domains, top clients, and query trends over time. I originally made this for my own use, but if you find it useful, a ⭐ would be awesome!
+
 **Disclaimer : This is an unofficial, third-party project. The Pi Hole team and the development of [PiHole](https://pi-hole.net/) software is not related to this project.**
 
 

piholelongtermstats-0.2.3/piholelongtermstats/__init__.py

@@ -0,0 +1,3 @@
+## Author :  Davis T. Daniel
+## PiHoleLongTermStats v.0.2.3
+## License :  MIT

{piholelongtermstats-0.2.2 → piholelongtermstats-0.2.3}/piholelongtermstats/app.py

@@ -1,5 +1,5 @@
 ## Author :  Davis T. Daniel
-## PiHoleLongTermStats v.0.2.2
+## PiHoleLongTermStats v.0.2.3
 ## License :  MIT
 
 import os
@@ -25,7 +25,7 @@ from piholelongtermstats.plot import (
     generate_queries_over_time,
 )
 
-__version__ = "0.2.1"
+__version__ = "0.2.3"
 
 # logging setup
 logging.basicConfig(
@@ -629,9 +629,15 @@ def serve_layout(
                             html.Div(
                                 [
                                     html.H3("Longest Idle Period"),
-                                    html.P(f"{stats['max_idle_ms']:,.0f} s"),
                                     html.P(
-                                        f"Between {stats['before_gap']} and {stats['after_gap']}",
+                                        f"{stats['max_idle_ms']:,.0f} s"
+                                        if stats['max_idle_ms'] is not None
+                                        else "N/A"
+                                    ),
+                                    html.P(
+                                        f"Between {stats['before_gap']} and {stats['after_gap']}"
+                                        if stats['before_gap'] and stats['after_gap']
+                                        else "N/A",
                                         style={"fontSize": "14px", "color": "#777"},
                                     ),
                                 ],
@@ -652,7 +658,9 @@ def serve_layout(
                                         },
                                     ),
                                     html.P(
-                                        f"Avg reply time: {stats['slowest_avg_reply_time'] * 1000:.2f} ms",
+                                        f"Avg reply time: {stats['slowest_avg_reply_time'] * 1000:.2f} ms"
+                                        if stats['slowest_avg_reply_time']
+                                        else "N/A",
                                         style={"fontSize": "14px", "color": "#777"},
                                     ),
                                 ],
@@ -993,11 +1001,13 @@ chunksize_list, latest_ts_list, oldest_ts_list = (
 
 for db in db_paths:
     conn = connect_to_sql(db)
-    chunksize, latest_ts, oldest_ts = probe_sample_df(conn)
-    chunksize_list.append(chunksize)
-    latest_ts_list.append(latest_ts.tz_convert(ZoneInfo(args.timezone)))
-    oldest_ts_list.append(oldest_ts.tz_convert(ZoneInfo(args.timezone)))
-    conn.close()
+    try:
+        chunksize, latest_ts, oldest_ts = probe_sample_df(conn)
+        chunksize_list.append(chunksize)
+        latest_ts_list.append(latest_ts.tz_convert(ZoneInfo(args.timezone)))
+        oldest_ts_list.append(oldest_ts.tz_convert(ZoneInfo(args.timezone)))
+    finally:
+        conn.close()
 
 logging.info(
     f"Latest date-time from all databases : {max(latest_ts_list)} (TZ: {args.timezone})"
@@ -1063,11 +1073,13 @@ def reload_page(n_clicks, start_date, end_date):
 
     for db in db_paths:
         conn = connect_to_sql(db)
-        chunksize, latest_ts, oldest_ts = probe_sample_df(conn)
-        chunksize_list.append(chunksize)
-        latest_ts_list.append(latest_ts.tz_convert(ZoneInfo(args.timezone)))
-        oldest_ts_list.append(oldest_ts.tz_convert(ZoneInfo(args.timezone)))
-        conn.close()
+        try:
+            chunksize, latest_ts, oldest_ts = probe_sample_df(conn)
+            chunksize_list.append(chunksize)
+            latest_ts_list.append(latest_ts.tz_convert(ZoneInfo(args.timezone)))
+            oldest_ts_list.append(oldest_ts.tz_convert(ZoneInfo(args.timezone)))
+        finally:
+            conn.close()
 
     logging.info(
         f"Latest date-time from all databases : {max(latest_ts_list)} (TZ: {args.timezone})"

{piholelongtermstats-0.2.2 → piholelongtermstats-0.2.3}/piholelongtermstats/db.py

@@ -1,5 +1,5 @@
 ## Author :  Davis T. Daniel
-## PiHoleLongTermStats v.0.2.2
+## PiHoleLongTermStats v.0.2.3
 ## License :  MIT
 
 import sqlite3
@@ -8,7 +8,7 @@ from pathlib import Path
 import psutil
 import pandas as pd
 import logging
-from zoneinfo import ZoneInfo
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 import gc
 
 
@@ -31,14 +31,20 @@ def connect_to_sql(db_path):
 
 
 def probe_sample_df(conn):
-    """compute basic stats from a subset of the databases"""
-
-    # calculate safe chunksize to not overload system memory
+    """Calculate safe chunksize based on available memory and retrieve timestamp range.
+    
+    Analyzes a sample of the database to estimate memory requirements per row
+    and retrieves the oldest and latest timestamps in the database.
+    """
     sample_query = """SELECT id, timestamp, type, status, domain, client, reply_time
-    FROM queries LIMIT 5"""
+    FROM queries LIMIT 100"""
     sample_df = pd.read_sql_query(sample_query, conn)
+    if sample_df.empty:
+        raise ValueError("No data from database for the selected time frame.")
+    
     sample_df["timestamp"] = pd.to_datetime(sample_df["timestamp"], unit="s")
 
+
     available_memory = psutil.virtual_memory().available
     memory_per_row = sample_df.memory_usage(deep=True).sum() / len(sample_df)
     safe_memory = available_memory * 0.5
@@ -63,7 +69,7 @@ def probe_sample_df(conn):
 def get_timestamp_range(days, start_date, end_date, timezone):
     try:
         tz = ZoneInfo(timezone)
-    except Exception:
+    except ZoneInfoNotFoundError:
         logging.warning(f"Invalid timezone '{timezone}', using UTC")
         tz = ZoneInfo("UTC")
 
@@ -120,11 +126,13 @@ def read_pihole_ftl_db(
         f"Reading data from PiHole-FTL database(s) for timestamps ranging from {start_timestamp} to {end_timestamp} (TZ: UTC)..."
     )
 
-    query = f"""
+    # no sql injection
+    query = """
     SELECT id, timestamp, type, status, domain, client, reply_time	 
     FROM queries
-    WHERE timestamp >= {start_timestamp} AND timestamp < {end_timestamp};
+    WHERE timestamp >= ? AND timestamp < ?;
     """
+    params = [start_timestamp, end_timestamp]
 
     for db_idx, db_path in enumerate(db_paths):
         logging.info(
@@ -132,12 +140,13 @@ def read_pihole_ftl_db(
         )
         conn = connect_to_sql(db_path)
 
-        chunk_num = 0
-        for chunk in pd.read_sql_query(query, conn, chunksize=chunksize[db_idx]):
-            chunk_num += 1
-            logging.info(
-                f"Processing dataframe chunk {chunk_num} from database {db_idx + 1} at {db_path}..."
-            )
-            yield chunk
-
-        conn.close()
+        try:
+            chunk_num = 0
+            for chunk in pd.read_sql_query(query, conn, params=params, chunksize=chunksize[db_idx]):
+                chunk_num += 1
+                logging.info(
+                    f"Processing dataframe chunk {chunk_num} from database {db_idx + 1} at {db_path}..."
+                )
+                yield chunk
+        finally:
+            conn.close()

{piholelongtermstats-0.2.2 → piholelongtermstats-0.2.3}/piholelongtermstats/plot.py

@@ -1,5 +1,5 @@
 ## Author :  Davis T. Daniel
-## PiHoleLongTermStats v.0.2.2
+## PiHoleLongTermStats v.0.2.3
 ## License :  MIT
 
 import logging
@@ -10,7 +10,16 @@ import itertools
 
 
 def generate_plot_data(df, n_clients, n_domains):
-    """Generate plot data"""
+    """Generate plot data for dashboard visualizations.
+    
+    Args:
+        df: Pre-processed DataFrame with query data.
+        n_clients: Number of top clients to include in plots.
+        n_domains: Number of top domains to include in plots.
+        
+    Returns:
+        dict: Dictionary containing DataFrames and data for various plots.
+    """
 
     logging.info("Generating plot data...")
 
@@ -24,17 +33,19 @@ def generate_plot_data(df, n_clients, n_domains):
         .size()
         .reset_index(name="count")
     )
-    top_clients_stacked["client"] = pd.Categorical(
-        top_clients_stacked["client"],
-        categories=top_clients_stacked.groupby("client")["count"]
-        .sum()
-        .sort_values(ascending=False)
-        .index,
-        ordered=True,
-    )
-    top_clients_stacked = top_clients_stacked.sort_values(
-        ["client", "count"], ascending=[True, False]
-    )
+    
+    if not top_clients_stacked.empty:
+        top_clients_stacked["client"] = pd.Categorical(
+            top_clients_stacked["client"],
+            categories=top_clients_stacked.groupby("client")["count"]
+            .sum()
+            .sort_values(ascending=False)
+            .index,
+            ordered=True,
+        )
+        top_clients_stacked = top_clients_stacked.sort_values(
+            ["client", "count"], ascending=[True, False]
+        )
     logging.info("Generated plot data for top clients.")
 
     # plot data for allowed and blocked domains
@@ -46,7 +57,7 @@ def generate_plot_data(df, n_clients, n_domains):
         .value_counts()
         .nlargest(n_domains)
         .reset_index()
-        .rename(columns={"index": "Count", "domain": "Domain"})
+        .rename(columns={"domain": "Domain", "count": "count"})
     )
 
     tmp_allowed = df[df["status_type"] == "Allowed"].copy()
@@ -57,7 +68,7 @@ def generate_plot_data(df, n_clients, n_domains):
         .value_counts()
         .nlargest(n_domains)
         .reset_index()
-        .rename(columns={"index": "Count", "domain": "Domain"})
+        .rename(columns={"domain": "Domain", "count": "count"})
     )
 
     logging.info("Generated plot data for allowed and blocked domains.")
@@ -131,6 +142,11 @@ def generate_plot_data(df, n_clients, n_domains):
     gc.collect()
 
     logging.info("Plot data generation complete")
+    
+    if not df.empty:
+        data_span_days = (df["timestamp"].max() - df["timestamp"].min()).days
+    else:
+        data_span_days = 0
 
     return {
         "top_clients_stacked": top_clients_stacked,
@@ -138,7 +154,7 @@ def generate_plot_data(df, n_clients, n_domains):
         "allowed_df": allowed_df,
         "reply_time_df": reply_time_df,
         "client_list": client_list,
-        "data_span_days": (df["timestamp"].max() - df["timestamp"].min()).days,
+        "data_span_days": data_span_days,
         "client_domain_scatter_df": client_domain_scatter_df,
         "day_hour_heatmap": day_hour_heatmap,
         "blocked_day_hour_heatmap": blocked_day_hour_heatmap,
@@ -147,6 +163,15 @@ def generate_plot_data(df, n_clients, n_domains):
 
 
 def generate_queries_over_time(callback_data, client=None):
+    """Generate an area chart showing DNS queries over time.
+    
+    Args:
+        callback_data: Dictionary containing hourly aggregation data.
+        client: Optional client filter to show queries for a specific client.
+        
+    Returns:
+        plotly.graph_objects.Figure: Area chart of queries over time.
+    """
     dff_grouped = callback_data["hourly_agg"]
 
     if client is not None:
@@ -240,6 +265,16 @@ def generate_queries_over_time(callback_data, client=None):
 
 
 def generate_client_activity_over_time(callback_data, n_clients, client=None):
+    """Generate an area chart showing client activity over time.
+    
+    Args:
+        callback_data: Dictionary containing hourly aggregation data and top clients list.
+        n_clients: Number of top clients to display when no specific client is selected.
+        client: Optional client filter to show activity for a specific client.
+        
+    Returns:
+        plotly.graph_objects.Figure: Area chart of client activity over time.
+    """
     dff_grouped = callback_data["hourly_agg"]
     top_clients = callback_data["top_clients"]
 
@@ -295,7 +330,8 @@ def generate_client_activity_over_time(callback_data, n_clients, client=None):
     )
 
     default_colors = px.colors.qualitative.Plotly
-    client_color_map = dict(zip(top_clients, itertools.cycle(default_colors)))
+    # build color map for all clients that will be shown (not just top_clients)
+    client_color_map = dict(zip(clients_to_show, itertools.cycle(default_colors)))
 
     fig = px.area(
         pivot_df,

{piholelongtermstats-0.2.2 → piholelongtermstats-0.2.3}/piholelongtermstats/process.py

@@ -1,14 +1,22 @@
 ## Author :  Davis T. Daniel
-## PiHoleLongTermStats v.0.2.2
+## PiHoleLongTermStats v.0.2.3
 ## License :  MIT
 
 import re
 import logging
-from zoneinfo import ZoneInfo
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 import pandas as pd
 
 
 def _is_valid_regex(pattern):
+    """Check if a string is a valid regular expression pattern.
+    
+    Args:
+        pattern: String to validate as regex pattern.
+        
+    Returns:
+        bool: True if pattern is valid regex, False otherwise.
+    """
     try:
         re.compile(pattern)
         return True
@@ -17,6 +25,15 @@ def _is_valid_regex(pattern):
 
 
 def regex_ignore_domains(df, pattern):
+    """Filter out rows where domain matches the given regex pattern.
+    
+    Args:
+        df: DataFrame containing a 'domain' column.
+        pattern: Regex pattern to match domains for exclusion.
+        
+    Returns:
+        DataFrame with matching domains removed, or original DataFrame if pattern is invalid.
+    """
     if _is_valid_regex(pattern):
         mask = df["domain"].str.contains(pattern, regex=True, na=False)
         return df[~mask].reset_index(drop=True)
@@ -33,9 +50,9 @@ def preprocess_df(df, timezone="UTC"):
     logging.info("Pre-processing dataframe...")
 
     try:
-        tz = ZoneInfo(timezone)  # noqa: F841
-    except Exception as e:
-        logging.warning(f"Invalid timezone '{timezone}', falling back to UTC: {e}")
+        ZoneInfo(timezone)
+    except ZoneInfoNotFoundError:
+        logging.warning(f"Invalid timezone '{timezone}', falling back to UTC")
         timezone = "UTC"
 
     logging.info(f"Selected timezone : {timezone}")
@@ -51,13 +68,13 @@ def preprocess_df(df, timezone="UTC"):
 
     # status ids for pihole ftl db, see pi-hole FTL docs
     logging.info("Processing allowed and blocked status codes...")
-    allowed_statuses = [2, 3, 12, 13, 14, 17]
-    blocked_statuses = [1, 4, 5, 6, 7, 8, 9, 10, 11, 15, 16, 18]
-    df["status_type"] = df["status"].apply(
-        lambda x: "Allowed"
-        if x in allowed_statuses
-        else ("Blocked" if x in blocked_statuses else "Other")
-    )
+    allowed_statuses = {2, 3, 12, 13, 14, 17}
+    blocked_statuses = {1, 4, 5, 6, 7, 8, 9, 10, 11, 15, 16, 18}
+    
+    # Use vectorized operations for better performance
+    df["status_type"] = "Other"
+    df.loc[df["status"].isin(allowed_statuses), "status_type"] = "Allowed"
+    df.loc[df["status"].isin(blocked_statuses), "status_type"] = "Blocked"
 
     df["day_name"] = df["timestamp"].dt.day_name()
     df["reply_time"] = pd.to_numeric(df["reply_time"], errors="coerce")
@@ -67,7 +84,16 @@ def preprocess_df(df, timezone="UTC"):
 
 
 def prepare_hourly_aggregated_data(df, n_clients):
-    """Pre-aggregate data by hour"""
+    """Pre-aggregate data by hour for callback processing.
+    
+    Args:
+        df: Pre-processed DataFrame with timestamp, status_type, and client columns.
+        n_clients: Number of top clients to identify for client activity views.
+        
+    Returns:
+        dict: Contains 'hourly_agg' DataFrame with hourly aggregations and 
+              'top_clients' list of top n_clients client identifiers.
+    """
     logging.info("Pre-aggregating data by hour for callbacks...")
 
     # aggregate by hour, status_type, and client