promnesia 1.1.20230129__tar.gz → 1.2.20240810__tar.gz

promnesia-1.2.20240810/.ci/end2end/.dockerignore

@@ -0,0 +1 @@
+build_and_run.sh

promnesia-1.2.20240810/.ci/end2end/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY scripts /scripts
+
+RUN /scripts/setup_firefox.sh \
+ && /scripts/setup_chrome.sh  \
+ && /scripts/setup_node.sh    \
+ && apt install --yes tox git \
+# using python docs as a source of some html test data
+# need to prevent dpkg from excluding doc files...
+ && sed -i '/usr.share.doc/d' /etc/dpkg/dpkg.cfg.d/excludes && apt install --yes python3-doc \
+ && apt clean                     \
+ && mkdir /promnesia
+
+WORKDIR /promnesia
+
+ENTRYPOINT ["/scripts/build_and_run_tests.sh"]

promnesia-1.2.20240810/.ci/end2end/build_and_run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eux
+
+# TODO assert we're running under github ci?
+# since this setup is kinda elaborate and can be somewhat unsafe to run blindly
+
+# supposed to be called from promnesia repository root
+[ -e src/promnesia ]
+[ -e extension/src ]
+
+PROMNESIA_SRC="$(pwd)"
+
+cd .ci/end2end
+
+IMAGE='promnesia_end2end_tests'
+
+docker build -t "$IMAGE" .
+
+# NOTE: dev/shm mount to prevent crashes during headless chrome
+docker run -v /dev/shm:/dev/shm --mount "type=bind,src=$PROMNESIA_SRC,dst=/promnesia_source,readonly=true" -e CI "$IMAGE" "$@"

promnesia-1.2.20240810/.ci/end2end/scripts/build_and_run_tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -eux
+
+# Seems wrong to keep the whole repository in docker build context.
+# So instead, we mount the repository inside the container (into /promnesia_source)
+# (as read only to avoid messing up host files and crapping with caches etc.)
+# However to actually run tests we do need a writable directory..
+# So we copy the repo to the actual working dir here
+
+# ugh, kinda annoying -- not sure how to update source files when we change them on the host system...
+cp -R -T /promnesia_source /promnesia
+extension/.ci/build
+
+git init  # todo ??? otherwise setuptools-scm fails to detect the version...
+python3 -m tox -e end2end -- "$@"

promnesia-1.2.20240810/.ci/end2end/scripts/setup_chrome.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -eux -o pipefail
+
+apt update --yes
+
+apt install --yes wget
+
+install -d -m 0755 /etc/apt/keyrings
+wget -q https://dl.google.com/linux/linux_signing_key.pub -O- | tee /etc/apt/keyrings/linux_signing_key.pub.asc > /dev/null
+echo "deb [signed-by=/etc/apt/keyrings/linux_signing_key.pub.asc] https://dl.google.com/linux/chrome/deb/ stable main" | tee -a /etc/apt/sources.list.d/google-chrome.list > /dev/null
+
+apt update
+
+apt install --yes google-chrome-stable
+
+# sadly latest version of chrome/chromedriver isn't working due to some bugs with iframes (see install_custom_chrome)
+
+# remove the actual chrome to get it out of the way (we do want dependencies though)
+apt remove --yes google-chrome-stable
+! which google-chrome  # check there is no binary (in case of virtual packages or whatever)
+
+function install_custom_chrome() {
+    ## this installs last revision that was actually working (1110897) or 113.0.5623.0
+    ## see https://bugs.chromium.org/p/chromedriver/issues/detail?id=4440
+    apt install --yes unzip
+
+    mkdir /tmp/chrome
+
+    wget -q 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchrome-linux.zip?generation=1677589092014487&alt=media'         \
+       -O /tmp/chrome/chrome-linux.zip
+    unzip /tmp/chrome/chrome-linux.zip         -d /tmp/chrome
+    ln -sf /tmp/chrome/chrome-linux/chrome /usr/bin/google-chrome
+
+    wget -q 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchromedriver_linux64.zip?generation=1677589097630198&alt=media' \
+       -O /tmp/chrome/chromedriver_linux64.zip
+    unzip /tmp/chrome/chromedriver_linux64.zip -d /tmp/chrome
+    ln -sf /tmp/chrome/chromedriver_linux64/chromedriver /usr/bin/chromedriver
+}
+
+install_custom_chrome

promnesia-1.2.20240810/.ci/end2end/scripts/setup_firefox.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -eux -o pipefail
+
+apt update --yes
+
+apt install --yes wget
+
+# NOTE: these days ubuntu provisions firefox via snap, and it's a nightmare to make it work with webdriver
+# so we force it to use a regular package (following these instructions https://askubuntu.com/a/1510872/427470)
+install -d -m 0755 /etc/apt/keyrings
+wget -q https://packages.mozilla.org/apt/repo-signing-key.gpg -O- | tee /etc/apt/keyrings/packages.mozilla.org.asc > /dev/null
+echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" | tee -a /etc/apt/sources.list.d/mozilla.list > /dev/null
+
+# prevent snap version from overriding:
+echo '
+Package: *
+Pin: origin packages.mozilla.org
+Pin-Priority: 1000
+' | tee /etc/apt/preferences.d/mozilla
+# to check: -- should not show anything mentioning snap
+# apt install --verbose-versions --dry-run firefox
+
+apt update
+
+apt install --yes firefox
+# NOTE: selenium should download the corresponding geckodriver itself via selenium_manager

promnesia-1.2.20240810/.ci/end2end/scripts/setup_node.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eux -o pipefail
+
+apt update --yes
+apt install --yes curl
+curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
+apt install --yes nodejs

{promnesia-1.1.20230129 → promnesia-1.2.20240810}/.ci/release

@@ -19,10 +19,8 @@ from pathlib import Path
 from subprocess import check_call
 import shutil
 
-
 is_ci = os.environ.get('CI') is not None
 
-
 def main() -> None:
     import argparse
     p = argparse.ArgumentParser()
@@ -31,7 +29,7 @@ def main() -> None:
 
     extra = []
     if args.test:
-        extra.extend(['--repository-url', 'https://test.pypi.org/legacy/'])
+        extra.extend(['--repository', 'testpypi'])
 
     root = Path(__file__).absolute().parent.parent
     os.chdir(root) # just in case
@@ -44,7 +42,7 @@ def main() -> None:
     if dist.exists():
         shutil.rmtree(dist)
 
-    check_call('python3 setup.py sdist bdist_wheel', shell=True)
+    check_call(['python3', '-m', 'build'])
 
     TP = 'TWINE_PASSWORD'
     password = os.environ.get(TP)

{promnesia-1.1.20230129 → promnesia-1.2.20240810}/.ci/run

@@ -45,4 +45,4 @@ if ! command -v python3 &> /dev/null; then
 fi
 
 "$PY_BIN" -m pip install --user tox
-"$PY_BIN" -m tox
+"$PY_BIN" -m tox --parallel --parallel-live "$@"

{promnesia-1.1.20230129 → promnesia-1.2.20240810}/.github/workflows/main.yml

@@ -9,28 +9,32 @@ on:
   pull_request: # needed to trigger on others' PRs
   # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
   workflow_dispatch: # needed to trigger workflows manually
+    # todo cron?
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
 
 
 jobs:
   build:
     strategy:
-      # useful to quickly debug all versions, otherwise a bit wasteful
-      # fail-fast: false
+      fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         exclude: [
-            # windows runners are pretty scarce, so let's only run one of them..
-            # also 3.7 on windows is a bit broken due to the WindowsPath bug
-            {platform: windows-latest, python-version: '3.7'},
-            {platform: windows-latest, python-version: '3.9'},
-            # 3.11 on windows has this bug, lxml setup fails
-            #https://bugs.launchpad.net/lxml/+bug/1977998
+            # windows runners are pretty scarce, so let's only run lowest and highest python version
+            {platform: windows-latest, python-version: '3.9' },
+            {platform: windows-latest, python-version: '3.10'},
             {platform: windows-latest, python-version: '3.11'},
 
-            # perhaps no need to run all macos versions -- it's a bit too slow and ubuntu covers python quirks well
-            {platform: macos-latest  , python-version: '3.8' },
+            # same, macos is a bit too slow and ubuntu covers python quirks well
             {platform: macos-latest  , python-version: '3.9' },
+            {platform: macos-latest  , python-version: '3.10'},
+            {platform: macos-latest  , python-version: '3.11'},
         ]
 
     runs-on: ${{ matrix.platform }}
@@ -40,31 +44,29 @@ jobs:
     steps:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-    - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }}
-      # hmm somehow only seems necessary for 3.11 on osx??
-      run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: recursive
-   
-    # uncomment for SSH debugging
-    # - uses: mxschmitt/action-tmate@v3
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
+
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
     # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
     - run: bash .ci/run
 
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy-core/
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy-misc/
@@ -73,24 +75,22 @@ jobs:
   end2end_tests_chrome:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: recursive
-    # NOTE: dev/shm mount to prevent crashes during headless chrome
-    - run: |
-        docker build -f .ci/end2end_tests.Dockerfile . -t promnesia_end2end_tests
-        docker run -v /dev/shm:/dev/shm -e CI promnesia_end2end_tests -- -k chrome
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+    - run: .ci/end2end/build_and_run.sh -k chrome
 
   end2end_tests_firefox:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: recursive
-    # NOTE: dev/shm mount to prevent crashes during headless chrome
-    - run: |
-        docker build -f .ci/end2end_tests.Dockerfile . -t promnesia_end2end_tests
-        docker run -v /dev/shm:/dev/shm -e CI promnesia_end2end_tests -- -k firefox
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+    - run: .ci/end2end/build_and_run.sh -k firefox
 
   install_and_run_test:
     # todo use setup-python thing?
@@ -98,7 +98,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: recursive
 
@@ -119,11 +119,11 @@ jobs:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.10'
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: recursive
 
@@ -132,8 +132,7 @@ jobs:
       if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
-      run: pip3 install --user wheel twine && .ci/release --test
-      # TODO run pip install just to test?
+      run: pip3 install --user --upgrade build twine && .ci/release --test
 
     - name: 'release to pypi'
       # always deploy tags to release pypi
@@ -141,7 +140,7 @@ jobs:
       if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
       env:
         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-      run: pip3 install --user wheel twine && .ci/release
+      run: pip3 install --user --upgrade build twine && .ci/release
 
   ###
   build_extension:
@@ -149,31 +148,33 @@ jobs:
       name: 'promnesia'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: recursive
-    - uses: actions/setup-node@v3
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
+
+    - uses: actions/setup-node@v4
       with:
-        node-version: '18'
+        node-version: '20'
 
     - run: extension/.ci/build --lint # debug version
     - run: extension/.ci/build --lint --release
 
     # TODO ugh. can't share github actions artifacts publicly...
     # TODO for fuck's sake... why does it end up named as .zip.zip ????
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: '${{ env.name }}-chrome-debug-latest.zip'
         path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip'
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: '${{ env.name }}-chrome-release-latest.zip'
         path: 'extension/dist/artifacts/chrome/${{ env.name }}-*.zip'
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: '${{ env.name }}-firefox-debug-latest.zip'
         path: 'extension/dist/artifacts/firefox/${{ env.name }}_dev_-*.zip'
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: '${{ env.name }}-firefox-release-latest.zip'
         path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip'

promnesia-1.2.20240810/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.1
+Name: promnesia
+Version: 1.2.20240810
+Summary: Enhancement of your browsing history
+Home-page: https://github.com/karlicoss/promnesia
+Author: Dmitrii Gerasimov
+Author-email: karlicoss@gmail.com
+Requires-Python: >=3.8
+License-File: LICENSE
+Requires-Dist: appdirs
+Requires-Dist: tzlocal
+Requires-Dist: more_itertools
+Requires-Dist: pytz
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: urlextract
+Requires-Dist: fastapi
+Requires-Dist: uvicorn[standard]
+Provides-Extra: testing
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-timeout; extra == "testing"
+Requires-Dist: pytest-xdist; extra == "testing"
+Requires-Dist: hypothesis; extra == "testing"
+Requires-Dist: psutil; extra == "testing"
+Requires-Dist: requests; extra == "testing"
+Requires-Dist: selenium; extra == "testing"
+Requires-Dist: click; extra == "testing"
+Requires-Dist: ruff; extra == "testing"
+Requires-Dist: mypy; extra == "testing"
+Requires-Dist: lxml; extra == "testing"
+Requires-Dist: loguru; extra == "testing"
+Provides-Extra: testing-gui
+Requires-Dist: pyautogui; extra == "testing-gui"
+Provides-Extra: optional
+Requires-Dist: logzero; extra == "optional"
+Requires-Dist: python-magic; extra == "optional"
+Provides-Extra: hpi
+Requires-Dist: HPI; extra == "hpi"
+Provides-Extra: html
+Requires-Dist: beautifulsoup4; extra == "html"
+Requires-Dist: lxml; extra == "html"
+Provides-Extra: markdown
+Requires-Dist: mistletoe; extra == "markdown"
+Provides-Extra: org
+Requires-Dist: orgparse>=0.3.0; extra == "org"
+Provides-Extra: telegram
+Provides-Extra: all
+Requires-Dist: logzero; extra == "all"
+Requires-Dist: python-magic; extra == "all"
+Requires-Dist: HPI; extra == "all"
+Requires-Dist: beautifulsoup4; extra == "all"
+Requires-Dist: lxml; extra == "all"
+Requires-Dist: mistletoe; extra == "all"
+Requires-Dist: orgparse>=0.3.0; extra == "all"

{promnesia-1.1.20230129 → promnesia-1.2.20240810}/README.org

@@ -406,7 +406,7 @@ The best support for me would be if you contribute to this or my other projects.
 
 I don't need money, but I understand it's often easier to give away than time, so here are some of projects that I donate to:
 
-- [[https://orgmode.org/worg/donate.html][org-mode]]
+- [[https://liberapay.com/org-mode][org-mode]]
 - [[https://archive.org/donate][Internet Archive]]
 - [[https://web.hypothes.is/donate][Hypothes.is]]
 - [[https://github.com/hlissner/doom-emacs#contribute][Doom Emacs]]

promnesia-1.2.20240810/benchmarks/20231115.org

@@ -0,0 +1,17 @@
+Running on @karlicoss desktop PC, =python3.10= (under docker)
+
+- dumping via cachew NTBinder, using regular sqlalchemy insert statement
+
+#+begin_example
+$ python3 -m pytest --import-mode=importlib --pyargs promnesia.tests -s -k 'gc_off and benchmark and 100000'
+12.90s call     src/promnesia/tests/test_db_dump.py::test_benchmark_visits_dumping[gc_off-100000]
+PASSED src/promnesia/tests/test_db_dump.py::test_benchmark_visits_dumping[gc_off-100000]
+#+end_example
+
+
+- dumping via cachew NTBinder, using dbengine to insert directly -- massive speedup (added a test with 1M visits too)
+#+begin_example
+$ python3 -m pytest --import-mode=importlib --pyargs promnesia.tests -s -k 'gc_off and benchmark and 100000'
+0.85s call     src/promnesia/tests/test_db_dump.py::test_benchmark_visits_dumping[gc_off-100000]
+8.23s call     src/promnesia/tests/test_db_dump.py::test_benchmark_visits_dumping[gc_off-1000000]
+#+end_example

{promnesia-1.1.20230129 → promnesia-1.2.20240810}/doc/DEVELOPMENT.org

@@ -52,7 +52,7 @@ I'm not against it, but with limited time resources full-scale tests give better
    : ./build --firefox # you can also use --chrome
    :         --lint    # [optional], run webext linter
    :         --release # [optional], build in the release mode (with optimizations)
-   :         --publish # [optional], release to the Chrome Web Store/Mozilla addons
+   :         --publish listed # [optional], release to the Chrome Web Store/Mozilla addons
 
    You'll find the result in =dist/=. After that, you can load it in your browser and develop.
 
@@ -61,7 +61,7 @@ I'm not against it, but with limited time resources full-scale tests give better
 
 * Run end-to-end tests
 
-End-to-end test go thought the full cycle from indexing & backend to automated extension tests, and are very helpful to make sure you don't break things.
+End-to-end tests go through the full cycle from indexing & backend to automated extension tests, and are very helpful to make sure you don't break things.
 
 You need:
 
@@ -74,18 +74,21 @@ Example of running a test:
 
 You can also run them via tox:
 
+# TODO https://github.com/tox-dev/tox/discussions/2681
 : TOX_TESTENV_PASSENV='DISPLAY HOME' tox -e end2end -- -s -k 'test_blacklist_custom[firefox]'
 
 - it's useful to pass =--pdb= to pytest, so you don't have to restart webdriver all over
 - after initial run, it's best pass =--skip-pkg-install= to tox to speedup repetitive test runs
 - when you're in development/debugging process, perhaps best to run the firefox version of tests -- Chrome has a 5-ish seconds lag after installing the extension
+- since these tests can be quite slow, you can benefit from installing =pytest-xdist= plugin and using =-n= flag to parallelize
+- to debug flaky tests, it's useful to install =pytest-repeat= in addition, e.g. =--repeat=5 -n 5=
 
 * Releasing
 ** AMO (addons.mozilla.org)
 
 : cd extension
 : npm install # (if necessary)
-: ./build --firefox --lint --release --publish
+: ./build --firefox --lint --release --publish listed
 
 Note that this will only upload the extension (see [[https://extensionworkshop.com/documentation/develop/web-ext-command-reference/#channel][web-ext sign command reference]] and [[https://github.com/fregante/web-ext-submit#readme][web-ext-submit]] for more info).
 You need go to https://addons.mozilla.org/en-GB/developers/addon/promnesia/versions, there you should see your extension as 'pending approval'. Usually it's approved within minutes.
@@ -97,7 +100,7 @@ For build instructions, you can point the reviewers at [[file:addons-mozilla-org
 ** CWS (chrome web store)
 : cd extension
 : npm install # (if necessary)
-: ./build --chrome  --lint --release --publish
+: ./build --chrome  --lint --release --publish listed
 
 This command should work in theory and upload automatically... however getting the api keys is super annoying (see [[https://github.com/DrewML/chrome-webstore-upload/blob/master/How%20to%20generate%20Google%20API%20keys.md][this]]).