ao3statscraper 0.0.1__py3-none-any.whl

ao3statscraper/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+from .__version__ import __version__
+
+from .configuration import Config
+from .scrape import UserSession
+from .statsdata import StatsData, WorkStatsData, TotStatsData
+from .utils import clear_terminal
+from .plotting import plot_total_stats, plot_work_stats
+from .pw import store_secrets, read_secrets
+
+from .scripts.ao3plot import ao3plot
+from .scripts.ao3scrape import ao3scrape
+from .scripts.ao3_purge_stats_data import ao3_purge_stats_data
+from .scripts.ao3_hits_to_kudos import ao3_hits_to_kudos

ao3statscraper/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

ao3statscraper/ao3requester.py

@@ -0,0 +1,70 @@
+import requests
+
+
+class Requester:
+    """Requester object"""
+
+    def __init__(self, rqtw=-1, timew=60):
+        """Limits the request rate to prevent HTTP 429 (rate limiting) responses.
+        12 request per minute seems to be the limit.
+
+        Args:
+            rqm (int, optional): Maximum requests per time window (-1 -> no limit). Defaults to -1.
+            timew (int, optional): Time window (seconds). Defaults to 60.
+        """
+
+        self._requests = []
+        self._rqtw = rqtw
+        self._timew = timew
+        self.total = 0
+
+    def setRQTW(self, value):
+        self._rqtw = value
+
+    def setTimeW(self, value):
+        self._timew = value
+
+    def request(self, *args, **kwargs):
+        """Requests a web page once enough time has passed since the last request
+
+        Mostly lifted from https://github.com/ArmindoFlores/ao3_api
+
+        Args:
+            session(requests.Session, optional): Session object to request with
+
+        Returns:
+            requests.Response: Response object
+        """
+
+        # We've made a bunch of requests, time to rate limit?
+        if self._rqtw != -1:
+            if len(self._requests) >= self._rqtw:
+                t = time.time()
+                # Reduce list to only requests made within the current time window
+                while len(self._requests):
+                    if t - self._requests[0] >= self._timew:
+                        self._requests.pop(0)  # Older than window, forget about it
+                    else:
+                        break  # Inside window, the rest of them must be too
+                # Have we used up all available requests within our window?
+                if len(self._requests) >= self._rqtw:  # Yes
+                    # Wait until the oldest request exits the window, giving us a slot for the new one
+                    time.sleep(self._requests[0] + self._timew - t)
+                    # Now outside window, drop it
+                    self._requests.pop(0)
+
+            if self._rqtw != -1:
+                self._requests.append(time.time())
+            self.total += 1
+
+        if "session" in kwargs:
+            sess = kwargs["session"]
+            del kwargs["session"]
+            req = sess.request(*args, **kwargs)
+        else:
+            req = requests.request(*args, **kwargs)
+
+        return req
+
+
+requester = Requester()

ao3statscraper/configuration.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+
+
+import os
+import platform
+import yaml
+from getpass import getpass
+
+from .utils import clear_terminal
+from .pw import get_credentials, read_secrets, store_secrets
+
+
+class PlottingConfig(object):
+    """
+    Configuration for the plotting script.
+    """
+
+    def __init__(self):
+        self.prettify = True
+        return
+
+
+class Config(object):
+    store_login = None
+    username = "Your AO3 Username"
+    password = "Don't type your password in plain text"
+
+    ao3dir = None
+    datadir = None
+    conffile = None
+    pklfile = None
+
+    plotting = None
+
+    def __init__(self, reset_conffile=False):
+        """
+        Initialize configuration.
+        If `reset_conffile` is True, then write a new config file from scratch
+        and exit.
+        """
+
+        if platform.system() == "Windows":
+            raise NotImplementedError("Setup for windows is ToDo")
+
+        # Get config directory
+        userdir = os.path.expanduser("~")
+        ao3dir = os.path.join(userdir, ".ao3statscraper")
+        self.ao3dir = ao3dir
+
+        # Make sure it exists.
+        try:
+            os.mkdir(ao3dir)
+        except FileExistsError:
+            pass
+
+        self.plotting = PlottingConfig()
+
+        # Set full paths for config files.
+        conffilename = "ao3statscraper.conf.yml"
+        conffile = os.path.join(ao3dir, conffilename)
+        self.conffile = conffile
+        pklfilename = "ao3statscraper.conf.pkl"
+        pklfile = os.path.join(ao3dir, pklfilename)
+        self.pklfile = pklfile
+
+        if not os.path.exists(conffile):
+            clear_terminal()
+            input(
+                "Didn't find a configuration file. We need to set one up. Hit any key to coninue.\n"
+            )
+            self.setup_config_file()
+        else:
+            if reset_conffile:
+                self.setup_config_file()
+                quit()
+
+        # get contents of config file.
+        self._read_config_file()
+
+        # Make sure everything is set.
+        self._check_config()
+
+        return
+
+    def _check_config(self):
+        """
+        Check that no attributes are None. Mainly intended as a dev tool to make
+        sure everything's intialized properly.
+        """
+
+        d = self.__dir__()
+
+        for key in d:
+            if key.startswith("__"):
+                continue
+            if getattr(self, key) is None:
+                raise ValueError(f"Attribute {key} not initialized.")
+        return
+
+    def set_credentials(self, username, password):
+        """
+        Store username and password.
+        """
+
+        self.username = username
+        self.password = password
+
+        return
+
+    def setup_config_file(self):
+        """
+        Set up permanent configuration and dump them in the configure file.
+        """
+
+        clear_terminal()
+        print("Welcome to the configuration setup for the AO3 Stat Scraper.")
+        print(
+            f"After this setup, a configuration file will be written in `{self.conffile}`"
+        )
+        print("You can freely modify it on your own without calling this dialogue.")
+        print(
+            "Select a directory to store downloaded statistics. Hit Enter to use default."
+        )
+        datadir = input(f"(default: '{self.ao3dir}')\n")
+
+        clear_terminal()
+        print("Optionally, you can store your AO3 login on your local machine.")
+        print("Your username and password will be encrypted and locked with a ")
+        print("master password that you'll be asked to set next.")
+        print("(If you forget the master password, you'll need to re-run this")
+        print("configuration dialogue. You can do that by running `getAO3stats.py -c`)")
+        store_login_str = "b"
+        a = 0
+        while not store_login_str.startswith(("n", "N", "y", "Y")):
+            if a > 0:
+                print("Invalid input.")
+            store_login_str = input(
+                "Do you want to store your AO3 login on your local machine? [y/n]\n"
+            )
+            a += 1
+            if a > 10:
+                print("Too many attempts. Exiting.")
+                quit()
+
+        store_login = True
+        if store_login_str.startswith(("n", "N")):
+            store_login = False
+
+        if store_login:
+            # Grab master password.
+            clear_terminal()
+            master_pwd = getpass(
+                "Enter a master password (not your AO3 login password):\n"
+            )
+            clear_terminal()
+            master_pwd2 = getpass("Enter the password again:\n")
+
+            if master_pwd != master_pwd2:
+                print("Error: Passwords do not match.")
+                quit()
+
+            # Grab AO3 credentials.
+            username, password = get_credentials()
+            # And store them.
+            self.set_credentials(username, password)
+            store_secrets(master_pwd, username, password, self.pklfile)
+
+        if datadir == "":
+            datadir = self.ao3dir
+
+        try:
+            os.mkdir(datadir)
+        except FileExistsError:
+            pass
+
+        # Store the config
+        self.datadir = datadir
+        self.store_login = store_login
+
+        # Now write the config file.
+        confdict = {}
+        global_conf = {
+            "data_directory": self.datadir,
+            "store_login": self.store_login,
+        }
+
+        confdict = {
+            "Globals": global_conf,
+        }
+
+        ymlfp = open(self.conffile, "w")
+        yaml.dump(confdict, ymlfp)
+        ymlfp.close()
+
+        return
+
+    def _read_config_file(self):
+        ymlfp = open(self.conffile, "r")
+        confdata = yaml.load(ymlfp, Loader=yaml.Loader)
+        ymlfp.close()
+
+        try:
+            self.datadir = confdata["Globals"]["data_directory"]
+            self.store_login = confdata["Globals"]["store_login"]
+        except KeyError:
+            print("ERROR: Some fields are missing from your config file.")
+            print("It might be out of date.")
+            print("Re-generate it using `getAO3stats.py -c`")
+            quit(1)
+
+        return
+
+    def prep_scrape(self):
+        """
+        Ensure everything is set for a scrape.
+        """
+
+        if self.store_login:
+            master_pwd = getpass(
+                "Enter master password (not your AO3 login password):\n"
+            )
+            username, password = read_secrets(master_pwd, self.pklfile, retry=True)
+        else:
+            username, password = get_credentials()
+
+        self.set_credentials(username, password)
+
+        return

ao3statscraper/plotting.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+
+import matplotlib
+from matplotlib import pyplot as plt
+import pandas as pd
+from typing import Union
+
+from .configuration import Config
+from .statsdata import WorkStatsData, TotStatsData
+
+
+_plotkwargs = {
+    "linewidth": 2,
+    "linestyle": "-",
+    "alpha": 0.75,
+}
+
+_scatterkwargs = {
+    "s": 10,
+    "alpha": 0.75,
+}
+
+
+def _set_matplotlib_style():
+    styles = [
+        #  "seaborn-v0_8-darkgrid",
+        "ggplot"
+    ]
+
+    for style in styles:
+        if style in matplotlib.style.available:
+            plt.style.use(style)
+            return
+
+
+def _plot_on_axis(
+    ax: matplotlib.axes.Axes,
+    abscissa: Union[pd.DataFrame, pd.Series],
+    ordinate: Union[pd.DataFrame, pd.Series],
+    prettify: bool = True,
+):
+    """
+    Plot abscissa data on the x-axis and ordinate data on the y-axis.
+    Set x_label and y_label according to data names.
+    """
+
+    ax.plot(abscissa, ordinate, **_plotkwargs)
+    ax.scatter(abscissa, ordinate, **_scatterkwargs)
+
+    #  ax.set_xlabel(abscissa.name)
+    ax.set_ylabel(ordinate.name)
+    if prettify:
+        ax.grid(True)
+        ax.tick_params(axis="x", labelrotation=45)
+
+    return
+
+
+def _plot_all_normalized_on_axis(
+    ax: matplotlib.axes.Axes,
+    data: pd.DataFrame,
+    legendax: matplotlib.axes.Axes = None,
+    prettify: bool = True,
+):
+    """
+    plot all columns of the dataframe and normalize data by setting the last
+    entry as 1.
+
+    If `legendax` is not none, that axis will be used to display the legend.
+    """
+
+    time = data["Timestamp"]
+    handles = []
+    labels = []
+
+    skip_columns = ["Timestamp", "index", "Title", "Fandom", "ID"]
+
+    for col in data.columns:
+        if col in skip_columns:
+            continue
+
+        d = data[col]
+        val = d.iat[-1]
+        d_norm = d / val
+
+        (h,) = ax.plot(time, d_norm, label=col, **_plotkwargs)
+        ax.scatter(time, d_norm, **_scatterkwargs)
+
+        handles.append(h)
+        labels.append(col)
+
+    #  ax.set_xlabel("Timestamp")
+    ax.set_ylabel("All data (normalized)")
+
+    if legendax is None:
+        ax.legend(loc="upper left", ncols=2, frameon=True)
+    else:
+        legendax.grid(False)
+        legendax.axis("off")
+        legendax.legend(handles=handles, labels=labels, loc="center left")
+
+    if prettify:
+        ax.tick_params(axis="x", labelrotation=45)
+        ax.grid(True)
+
+    return
+
+
+def plot_total_stats(tsfiles: list, conf: Config):
+    """
+    Plots the total user statistics.
+
+    tsfiles: list
+        list of file names of total user statistics to read in
+
+    conf:
+        AO3Stats configuration object
+    """
+
+    prettify = conf.plotting.prettify
+
+    # Grab data
+    ts_data = []
+
+    for f in tsfiles:
+        ts = TotStatsData(conf, source=f)
+        ts.data["Timestamp"] = ts.timestamp
+
+        ts_data.append(ts.data)
+
+    # Combine it into a single dataframe
+    alldata = pd.concat(ts_data)
+    alldata.reset_index(inplace=True)
+    alldata.sort_values(by="Timestamp")
+
+    # get shorthands
+    time = alldata["Timestamp"]
+    user_subscriptions = alldata["User Subscriptions"]
+    kudos = alldata["Kudos"]
+    comments = alldata["Comment Threads"]
+    bookmarks = alldata["Bookmarks"]
+    subscriptions = alldata["Subscriptions"]
+    words = alldata["Word Count"]
+    hits = alldata["Hits"]
+
+    if prettify:
+        _set_matplotlib_style()
+
+    fig = plt.figure()
+    fig.suptitle("Total User Statistics")
+
+    ax1 = fig.add_subplot(3, 3, 1)
+    ax2 = fig.add_subplot(3, 3, 2)
+    ax3 = fig.add_subplot(3, 3, 3)
+    ax4 = fig.add_subplot(3, 3, 4)
+    ax5 = fig.add_subplot(3, 3, 5)
+    ax6 = fig.add_subplot(3, 3, 6)
+    ax7 = fig.add_subplot(3, 3, 7)
+    ax8 = fig.add_subplot(3, 3, 8)
+    ax9 = fig.add_subplot(3, 3, 9)
+
+    _plot_on_axis(ax1, time, user_subscriptions, prettify)
+    _plot_on_axis(ax2, time, kudos, prettify)
+    _plot_on_axis(ax3, time, comments, prettify)
+    _plot_on_axis(ax4, time, bookmarks, prettify)
+    _plot_on_axis(ax5, time, subscriptions, prettify)
+    _plot_on_axis(ax6, time, words, prettify)
+    _plot_on_axis(ax7, time, hits, prettify)
+    _plot_all_normalized_on_axis(ax8, alldata, ax9, prettify)
+
+    if prettify:
+        plt.subplots_adjust(
+            top=0.952, bottom=0.078, left=0.044, right=0.992, hspace=0.337, wspace=0.169
+        )
+
+    plt.show()
+
+    return
+
+
+def plot_work_stats(wsfiles: list, ID: int, conf: Config):
+    """
+    Plots the statistics of a single work specified via its AO3 ID.
+
+    wsfiles: list
+        list of file names of work statistics to read in
+
+    ID: int
+        AO3 ID of work to plot
+
+    conf:
+        AO3Stats configuration object
+    """
+
+    prettify = conf.plotting.prettify
+
+    # Grab data
+    ws_data = []
+
+    for f in wsfiles:
+        ws = WorkStatsData(conf, source=f)
+        ws.data["Timestamp"] = ws.timestamp
+
+        work = ws.data[ws.data["ID"] == ID]
+        if work.empty:
+            continue
+        else:
+            ws_data.append(work)
+
+    if len(ws_data) == 0:
+        raise ValueError(f"Didn't find any data on work with ID {ID}.")
+
+    # Combine it into a single dataframe
+    alldata = pd.concat(ws_data)
+    alldata.reset_index(inplace=True)
+    alldata.sort_values(by="Timestamp")
+
+    # get shorthands
+    time = alldata["Timestamp"]
+    words = alldata["Words"]
+    hits = alldata["Hits"]
+    kudos = alldata["Kudos"]
+    comments = alldata["Comment Threads"]
+    bookmarks = alldata["Bookmarks"]
+    subscriptions = alldata["Subscriptions"]
+
+    title = alldata["Title"].at[0]
+
+    if prettify:
+        _set_matplotlib_style()
+
+    fig1 = plt.figure()
+    fig1.suptitle(f"Work Statistics for '{title}'")
+
+    ax1 = fig1.add_subplot(2, 3, 1)
+    ax2 = fig1.add_subplot(2, 3, 2)
+    ax3 = fig1.add_subplot(2, 3, 3)
+    ax4 = fig1.add_subplot(2, 3, 4)
+    ax5 = fig1.add_subplot(2, 3, 5)
+    ax6 = fig1.add_subplot(2, 3, 6)
+
+    _plot_on_axis(ax1, time, words, prettify)
+    _plot_on_axis(ax2, time, hits, prettify)
+    _plot_on_axis(ax3, time, kudos, prettify)
+    _plot_on_axis(ax4, time, comments, prettify)
+    _plot_on_axis(ax5, time, bookmarks, prettify)
+    _plot_on_axis(ax6, time, subscriptions, prettify)
+
+    if prettify:
+        plt.subplots_adjust(
+            top=0.952, bottom=0.078, left=0.044, right=0.992, hspace=0.337, wspace=0.169
+        )
+
+    fig2, axes2 = plt.subplots(1, 2, width_ratios=[3, 1])
+    fig2.suptitle(f"Work Statistics for '{title}'")
+    ax21 = axes2[0]
+    ax22 = axes2[1]
+
+    _plot_all_normalized_on_axis(ax21, alldata, ax22, prettify)
+
+    plt.show()
+
+    return