py-backend-analytics 0.1.0__py3-none-any.whl

py_backend_analytics/__init__.py

@@ -0,0 +1,10 @@
+from py_backend_analytics.enums import PyBackendAnalyticsDB as PyBackendAnalyticsDB
+from py_backend_analytics.input_data import (
+    PyBackendAnalyticsInputData as PyBackendAnalyticsInputData,
+)
+from py_backend_analytics.middlewares.fastapi_middleware import (
+    PyBackendAnalyticsFastAPIMiddleware as PyBackendAnalyticsFastAPIMiddleware,
+)
+from py_backend_analytics.visualization.fastapi_visualizer import (
+    py_backend_analytics_fastapi_visualization as py_backend_analytics_fastapi_visualization,
+)

py_backend_analytics/constants.py

@@ -0,0 +1 @@
+PACKAGE_NAME = "py_backend_analytics"

py_backend_analytics/db/clients/abstract_db_client.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+from py_backend_analytics.models import RequestInfo
+
+
+@dataclass
+class AbstractDBClient(ABC):
+    """Client for interaction with the DB. Methods names are very self-descriptive"""
+
+    connection_string: str
+
+    @classmethod
+    async def create(cls, connection_string: str):
+        self = cls(connection_string)
+        await self.db_setup()
+        return self
+
+    async def db_setup(self):
+        if not await self._db_table_exists():
+            await self._create_db_table()
+
+    @abstractmethod
+    async def insert_request_info(self, model: RequestInfo): ...
+
+    @abstractmethod
+    async def get_analytics_summary(self) -> dict: ...
+
+    @abstractmethod
+    async def _create_db_table(self): ...
+
+    @abstractmethod
+    async def _db_table_exists(self) -> bool: ...

py_backend_analytics/db/clients/sqlite_db_client.py

@@ -0,0 +1,163 @@
+from datetime import datetime, timedelta, timezone
+from contextlib import asynccontextmanager
+
+import aiosqlite
+
+from py_backend_analytics.db.clients.abstract_db_client import AbstractDBClient
+from py_backend_analytics.db.constants import (
+    DB_TABLE_NAME,
+    DBColumns,
+    DB_INDEX_SUFFIX,
+    TOP_AGGREGATION_LIMIT,
+    UNKNOWN,
+    DEFAULT_DB_TIMEOUT,
+)
+from py_backend_analytics.db.models import AnalyticsSummaryFields as F
+from py_backend_analytics.models import RequestInfo
+
+
+class SQLiteDBClient(AbstractDBClient):
+    async def insert_request_info(self, model: RequestInfo):
+        async with self._get_connection() as conn:
+            cur = await conn.cursor()
+            await cur.execute(
+                f"""INSERT INTO {DB_TABLE_NAME}
+                    ({DBColumns.location}, {DBColumns.page}, {DBColumns.source}, {DBColumns.datestamp})
+                    VALUES (?, ?, ?, ?)""",
+                (model.location, model.page, model.source, model.datestamp),
+            )
+            await conn.commit()
+
+    async def get_analytics_summary(self) -> dict:
+        async with self._get_connection() as conn:
+            now = datetime.now(timezone.utc)
+
+            last_month = (now - timedelta(days=30)).isoformat()
+            last_year = (now - timedelta(days=365)).isoformat()
+
+            all_time = {
+                F.TOP_COUNTRIES: await self._get_top(conn, DBColumns.location),
+                F.TOP_PAGES: await self._get_top(conn, DBColumns.page),
+                F.TOP_SOURCES: await self._get_top(conn, DBColumns.source),
+            }
+
+            month_stats = {
+                F.TOP_COUNTRIES: await self._get_top(
+                    conn, DBColumns.location, last_month
+                ),
+                F.TOP_PAGES: await self._get_top(conn, DBColumns.page, last_month),
+                F.TOP_SOURCES: await self._get_top(conn, DBColumns.source, last_month),
+            }
+
+            year_stats = {
+                F.TOP_COUNTRIES: await self._get_top(
+                    conn, DBColumns.location, last_year
+                ),
+                F.TOP_PAGES: await self._get_top(conn, DBColumns.page, last_year),
+                F.TOP_SOURCES: await self._get_top(conn, DBColumns.source, last_year),
+            }
+
+            recent_query = f"""
+                SELECT
+                    {DBColumns.location},
+                    {DBColumns.page},
+                    {DBColumns.source},
+                    {DBColumns.datestamp}
+                FROM {DB_TABLE_NAME}
+                ORDER BY {DBColumns.datestamp} DESC
+                LIMIT 100
+            """
+
+            cur = await conn.execute(recent_query)
+            recent_rows = await cur.fetchall()
+
+            recent_requests = [
+                {
+                    F.LOCATION: row[0],
+                    F.PAGE: row[1],
+                    F.SOURCE: row[2],
+                    F.DATESTAMP: row[3],
+                }
+                for row in recent_rows
+            ]
+
+            return {
+                F.ALL_TIME: all_time,
+                F.LAST_MONTH: month_stats,
+                F.LAST_YEAR: year_stats,
+                F.RECENT_REQUESTS: recent_requests,
+            }
+
+    async def _create_db_table(self):
+        async with self._get_connection() as conn:
+            cur = await conn.cursor()
+            await cur.execute(
+                f"""CREATE TABLE {DB_TABLE_NAME}(
+                        {DBColumns.id} INTEGER PRIMARY KEY,
+                        {DBColumns.location} TEXT NULL,
+                        {DBColumns.page} TEXT NOT NULL,
+                        {DBColumns.source} TEXT NULL,
+                        {DBColumns.datestamp} TEXT NOT NULL
+                ) STRICT"""
+            )
+            await cur.execute(
+                f"""CREATE INDEX {DBColumns.datestamp}{DB_INDEX_SUFFIX}
+                    ON {DB_TABLE_NAME}({DBColumns.datestamp})"""
+            )
+
+    async def _db_table_exists(self) -> bool:
+        async with self._get_connection() as conn:
+            cur = await conn.cursor()
+            result = await cur.execute(
+                f"""
+                SELECT name
+                FROM sqlite_schema
+                WHERE type = 'table' AND name = '{DB_TABLE_NAME}'
+                """
+            )
+            name = await result.fetchone()
+            return name is not None
+
+    @staticmethod
+    async def _get_top(
+        conn: aiosqlite.Connection,
+        column: str,
+        min_date: str | None = None,
+        limit: int = TOP_AGGREGATION_LIMIT,
+    ) -> list[dict]:
+        query = f"""
+            SELECT {column}, COUNT(*) as count
+            FROM {DB_TABLE_NAME}
+        """
+        params = []
+
+        if min_date:
+            query += f" WHERE {DBColumns.datestamp} >= ?"
+            params.append(min_date)
+
+        query += f"""
+            GROUP BY {column}
+            ORDER BY count DESC
+            LIMIT {limit}
+        """
+
+        cur = await conn.execute(query, params)
+        rows = await cur.fetchall()
+
+        return [
+            {
+                F.VALUE: row[0] or UNKNOWN,
+                F.COUNT: row[1],
+            }
+            for row in rows
+        ]
+
+    @asynccontextmanager
+    async def _get_connection(self):
+        connection = await aiosqlite.connect(
+            self.connection_string, timeout=DEFAULT_DB_TIMEOUT
+        )
+        try:
+            yield connection
+        finally:
+            await connection.close()

py_backend_analytics/db/constants.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+
+from py_backend_analytics.constants import PACKAGE_NAME
+
+DB_TABLE_NAME = f"{PACKAGE_NAME}_requests_info"
+DB_INDEX_SUFFIX = "_index"
+
+
+TOP_AGGREGATION_LIMIT = 10
+UNKNOWN = "UNKNOWN"
+DEFAULT_DB_TIMEOUT = 60
+
+
+@dataclass(frozen=True)
+class DBColumns:
+    id = "id"
+    location = "location"
+    page = "page"
+    source = "source"
+    datestamp = "datestamp"

py_backend_analytics/db/models.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class AnalyticsSummaryFields:
+    # root
+    ALL_TIME: str = "all_time"
+    LAST_MONTH: str = "last_month"
+    LAST_YEAR: str = "last_year"
+    RECENT_REQUESTS: str = "recent_requests"
+
+    # TimeRangeStats
+    TOP_COUNTRIES: str = "top_countries"
+    TOP_PAGES: str = "top_pages"
+    TOP_SOURCES: str = "top_sources"
+
+    # TopEntry
+    VALUE: str = "value"
+    COUNT: str = "count"
+
+    # RequestInfo
+    LOCATION: str = "location"
+    PAGE: str = "page"
+    SOURCE: str = "source"
+    DATESTAMP: str = "datestamp"

py_backend_analytics/db/registry.py

@@ -0,0 +1,15 @@
+from py_backend_analytics.db.clients.abstract_db_client import AbstractDBClient
+from py_backend_analytics.db.clients.sqlite_db_client import SQLiteDBClient
+from py_backend_analytics.enums import PyBackendAnalyticsDB
+
+DB_CLIENTS = {PyBackendAnalyticsDB.SQLITE: SQLiteDBClient}
+
+
+async def get_db_client(
+    connection_string: str, db_type: PyBackendAnalyticsDB
+) -> AbstractDBClient:
+    """Unified interface for getting the correct type of DB Client"""
+    db_client_class = DB_CLIENTS.get(db_type)
+    if not db_client_class:
+        raise ValueError(f"Provided wrong DB type: {db_type}")
+    return await db_client_class.create(connection_string)

py_backend_analytics/enums.py

@@ -0,0 +1,5 @@
+from enum import Enum
+
+
+class PyBackendAnalyticsDB(str, Enum):
+    SQLITE = "sqlite"

py_backend_analytics/extraction/constants.py

@@ -0,0 +1,9 @@
+TEMP_DIR_NAME = "dbip_geo"
+MMDB_FILE_NAME = "dbip-country.mmdb"
+GZ_FILE_NAME = "dbip-country.mmdb.gz"
+
+DBIP_URL = "https://download.db-ip.com/free/dbip-country-lite-{month}.mmdb.gz"
+
+COUNTRY = "country"
+NAMES = "names"
+EN = "en"

py_backend_analytics/extraction/extractors/abstract_extractor.py

@@ -0,0 +1,17 @@
+from abc import ABC, abstractmethod
+from typing import TypeVar, Generic
+
+from py_backend_analytics.extraction.geo_lookup import IpCountryLookup
+from py_backend_analytics.models import RequestInfo
+
+T = TypeVar("T")
+
+
+class AbstractExtractor(ABC, Generic[T]):
+    geo_lookup: IpCountryLookup
+
+    def __init__(self):
+        self.geo_lookup = IpCountryLookup()
+
+    @abstractmethod
+    def extract(self, request: T) -> RequestInfo: ...

py_backend_analytics/extraction/extractors/fastapi_extractor.py

@@ -0,0 +1,28 @@
+from datetime import datetime, timezone
+from starlette.routing import Match
+
+from py_backend_analytics.extraction.extractors.abstract_extractor import (
+    AbstractExtractor,
+    T,
+)
+from py_backend_analytics.models import RequestInfo
+
+
+class FastAPIExtractor(AbstractExtractor):
+    def extract(self, request: T) -> RequestInfo:
+        """Extracts info from a FastAPI request. No FastAPI typing, so FastAPI is not needed at import."""
+        page = request.url.path  # /users/12313 - backup
+        # best effort to get /users/{id} instead of /users/12313
+        for route in request.app.router.routes:
+            match, scope = route.matches(request.scope)
+
+            if match == Match.FULL:
+                page = scope["route"].path
+                break
+        source = request.headers.get("referer", "direct")
+        datestamp = datetime.now(timezone.utc)
+        ip = request.client.host
+        location = self.geo_lookup.get_country(ip)
+        return RequestInfo(
+            location=location, page=page, source=source, datestamp=datestamp
+        )

py_backend_analytics/extraction/geo_lookup.py

@@ -0,0 +1,57 @@
+import gzip
+import shutil
+import tempfile
+from pathlib import Path
+from datetime import date
+from urllib.request import urlopen
+
+import maxminddb
+
+from py_backend_analytics.extraction.constants import (
+    TEMP_DIR_NAME,
+    MMDB_FILE_NAME,
+    GZ_FILE_NAME,
+    DBIP_URL,
+    COUNTRY,
+    NAMES,
+    EN,
+)
+
+
+class IpCountryLookup:
+    """
+    Installs IP DB, if it's not present, in the TEMP folder.
+
+    The DB is Taken from db-ip.com and is free. Then we use maxminddb to extract data from it.
+    """
+
+    def __init__(self):
+        self._dir = Path(tempfile.gettempdir()) / TEMP_DIR_NAME
+        self._dir.mkdir(exist_ok=True)
+        self._mmdb = self._dir / MMDB_FILE_NAME
+        if not self._mmdb.exists():
+            self._download()
+        self._db = maxminddb.open_database(str(self._mmdb))
+
+    def get_country(self, ip: str) -> str | None:
+        data = self._db.get(ip)
+        if not data:
+            return None
+        return data.get(COUNTRY, {}).get(NAMES, {}).get(EN)
+
+    def _download(self):
+        url = self._url()
+        gz = self._dir / GZ_FILE_NAME
+
+        with urlopen(url) as r:
+            with open(gz, "wb") as f:
+                f.write(r.read())
+
+        with gzip.open(gz, "rb") as f_in:
+            with open(self._mmdb, "wb") as f_out:
+                shutil.copyfileobj(f_in, f_out)
+
+    def _url(self) -> str:
+        month = date.today().strftime("%Y-%m")
+
+        return DBIP_URL.format(month=month)

py_backend_analytics/input_data.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass, field
+from typing import List, Any, Set
+
+from py_backend_analytics.constants import PACKAGE_NAME
+from py_backend_analytics.enums import PyBackendAnalyticsDB
+
+
+@dataclass
+class PyBackendAnalyticsInputData:
+    db_connection_string: str
+    db_type: PyBackendAnalyticsDB = PyBackendAnalyticsDB.SQLITE
+    excluded_endpoints: List[str] = field(default_factory=list)
+    excluded_path_fragments: List[str] = field(default_factory=list)
+    excluded_path_prefixes: Set[str] = field(default_factory=set)
+    logger: Any | None = None
+
+    def __post_init__(self):
+        if not self.excluded_endpoints:
+            self.excluded_endpoints = ["/favicon.ico", "/style.css"]
+        if not self.excluded_path_fragments:
+            self.excluded_path_fragments = ["static", PACKAGE_NAME]

py_backend_analytics/middlewares/fastapi_middleware.py

@@ -0,0 +1,71 @@
+from starlette.middleware.base import BaseHTTPMiddleware
+from fastapi import Request
+
+from py_backend_analytics.db.registry import get_db_client
+from py_backend_analytics.extraction.extractors.fastapi_extractor import (
+    FastAPIExtractor,
+)
+from py_backend_analytics.input_data import PyBackendAnalyticsInputData
+from py_backend_analytics.models import RequestInfo
+
+
+class PyBackendAnalyticsFastAPIMiddleware(BaseHTTPMiddleware):
+    """
+    DB is initted in the dispatch because there is no reliable way to call async method from __init__, since
+    there may already be an event loop running.
+
+    The flow:
+    1) Init DB if needed
+    2) Extract info from the request
+    3) Save the info to the DB if the request page is not excluded
+    4) On error: best effort to log the error message
+    """
+
+    def __init__(self, app, input_data: PyBackendAnalyticsInputData):
+        super().__init__(app)
+        self._initialized = False
+        self._data = input_data
+        self._logger = input_data.logger
+        self._db_client = None
+
+    async def dispatch(self, request: Request, call_next):
+        try:
+            if not self._initialized:
+                await self._setup_db()
+            request_info = FastAPIExtractor().extract(request)
+            if self._should_save_request(request_info):
+                await self._db_client.insert_request_info(request_info)
+        except Exception as e:
+            self._debug(
+                f"Got an error when trying to extract info from the request: {e}"
+            )
+        return await call_next(request)
+
+    async def _setup_db(self):
+        self._db_client = await get_db_client(
+            self._data.db_connection_string, self._data.db_type
+        )
+        self._initialized = True
+
+    def _debug(self, message: str):
+        """Best effort to log"""
+        if self._logger is not None:
+            try:
+                self._logger.debug(message)
+            except Exception:
+                pass
+
+    def _should_save_request(self, request_info: RequestInfo):
+        page = request_info.page
+        return (
+            page not in self._data.excluded_endpoints
+            and not any(
+                page.startswith(fragment)
+                for fragment in self._data.excluded_path_prefixes
+            )
+            and not any(
+                fragment
+                for fragment in self._data.excluded_path_fragments
+                if fragment in page
+            )
+        )

py_backend_analytics/models.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass
+class RequestInfo:
+    page: str
+    source: str
+    datestamp: datetime
+    location: str | None = None

py_backend_analytics/visualization/fastapi_visualizer.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+from starlette.requests import Request
+from starlette.staticfiles import StaticFiles
+from starlette.templating import Jinja2Templates, _TemplateResponse
+
+from py_backend_analytics.constants import PACKAGE_NAME
+from py_backend_analytics.db.registry import get_db_client
+from py_backend_analytics.input_data import PyBackendAnalyticsInputData
+
+
+_TEMPLATES_DIR = Path(__file__).parent / "templates"
+_STATIC_DIR = Path(__file__).parent / "static"
+_TEMPLATES = Jinja2Templates(directory=_TEMPLATES_DIR)
+ANALYTICS_STATIC = PACKAGE_NAME
+
+
+async def py_backend_analytics_fastapi_visualization(
+    app: FastAPI, input_data: PyBackendAnalyticsInputData, request: Request
+) -> _TemplateResponse:
+    """Visualizes the DB data as an HTML file, it also uses CSS."""
+    app.mount(
+        f"/{ANALYTICS_STATIC}",
+        StaticFiles(directory=_STATIC_DIR),
+        name=ANALYTICS_STATIC,
+    )
+
+    db_client = await get_db_client(input_data.db_connection_string, input_data.db_type)
+
+    results = await db_client.get_analytics_summary()
+
+    return _TEMPLATES.TemplateResponse(
+        request=request, name="visualization.html", context=results
+    )

py_backend_analytics-0.1.0.dist-info/METADATA

@@ -0,0 +1,118 @@
+Metadata-Version: 2.4
+Name: py-backend-analytics
+Version: 0.1.0
+Summary: easy-to-setup network analytics for python backends
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiosqlite>=0.22.1
+Requires-Dist: maxminddb>=3.1.1
+Dynamic: license-file
+
+# py-backend-analytics
+
+Easy-to-set-up python backend traffic analytics package.
+
+![CI](https://img.shields.io/github/actions/workflow/status/KrzysztofCzapla/py-backend-analytics/.github/workflows/pr.yaml)
+![PyPI](https://img.shields.io/pypi/v/py-backend-analytics)
+
+![img.png](docs/images/img.png)
+
+---
+
+py-backend-analytics is a package that saves data about your traffic in a DB and then lets
+you easily visualize your traffic for small to medium apps.
+
+Key features:
+- Set-up-once middleware that handles all the logic
+- Saving data about your traffic: location, path/page, source, datetime
+- Async DB client
+- out-of-the-box visualization
+
+Currently supported frameworks:
+- FastAPI
+
+Currently supported databases:
+- SQLite
+
+## Overview flow
+
+You specify your data using `PyBackendAnalyticsInputData` object.
+Then you add the middleware. The middleware will save information about each request
+in the chosen database.
+
+Then, you can add an endpoint that will visualize this data over time.
+
+## Future Plans
+
+Support for Django and Postgres is planned for the future releases.
+
+Also, improvements for the visualization layers.
+
+## Installation
+
+```bash
+$ pip install py_backend_analytics
+```
+
+## Quickstart
+
+```python
+import uvicorn
+from fastapi import FastAPI, Request, APIRouter
+
+from py_backend_analytics import PyBackendAnalyticsInputData, PyBackendAnalyticsFastAPIMiddleware, py_backend_analytics_fastapi_visualization
+
+app = FastAPI()
+my_router = APIRouter()
+
+# Setup py_backend_analytics
+db_string = "test1.db"
+input_data = PyBackendAnalyticsInputData(db_string)
+app.add_middleware(PyBackendAnalyticsFastAPIMiddleware, input_data)
+
+# Create endpoint with visualization
+@my_router.get("/")
+async def get(request: Request):
+    # This will return HTML page that you can see at the top of this doc
+    return await py_backend_analytics_fastapi_visualization(app, input_data, request)
+
+# run the ap
+app.include_router(my_router, tags=["my_router"])
+uvicorn.run(app, port=8080)
+```
+
+## Usage
+
+### Input Data
+
+You must create a `PyBackendAnalyticsInputData` object and specify its attributes.
+
+The only required attribute is the connection string to the DB, rest is optional.
+
+Attributes are:
+- `db_connection_string: str`
+- `db_type: PyBackendAnalyticsDB` - Enum that chooses the database type, defaults to `PyBackendAnalyticsDB.SQLITE`
+- `excluded_endpoints: List[str]` - list of excluded endpoints. defaults are: `["/favicon.ico", "/style.css"]`
+- `excluded_path_fragments: List[str]` - excluded path fragments. defaults are: `["static", "py_backend_analytics"]`
+- `excluded_path_prefixes: Set[str]` - excluded path prefixes. They will be excluded only if the path starts by them.
+- `logger: Any | None` - Optional logger, used if there are any errors. Defaults to None.
+
+DB is currently only for SQLite. It is recommended to provide an additional SQLite DB instead of the main you are using
+for safety. The DB client will create a new table if it doesn't exist and one manual index.
+
+### Middleware
+
+Added middleware takes in the input data and no further setup is needed.
+
+If there are any errors it will fail without raising exceptions; If logger is provided, it will try to log the error
+
+### Visualization
+
+You must provide your app, input_data and a request from inside the endpoint, and you will get an HTML template with the
+visualization of the data in the DB.
+
+## IP geo-location data
+
+The geo-location data is taken from:
+`db-ip.com` and uses `maxminddb` library to access it