promnesia 1.2.20240810__tar.gz → 1.4.20250909__tar.gz

promnesia-1.4.20250909/.ci/end2end/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY scripts /scripts
+
+RUN /scripts/setup_firefox.sh \
+ && /scripts/setup_chrome.sh  \
+ && /scripts/setup_node.sh    \
+ && apt install --yes pipx git \
+# using python docs as a source of some html test data
+# need to prevent dpkg from excluding doc files...
+ && sed -i '/usr.share.doc/d' /etc/dpkg/dpkg.cfg.d/excludes && apt install --yes python3-doc \
+ && apt clean                     \
+ && mkdir /promnesia
+
+WORKDIR /promnesia
+
+ENTRYPOINT ["/scripts/build_and_run_tests.sh"]

promnesia-1.4.20250909/.ci/end2end/scripts/build_and_run_tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -eux
+
+# Seems wrong to keep the whole repository in docker build context.
+# So instead, we mount the repository inside the container (into /promnesia_source)
+# (as read only to avoid messing up host files and crapping with caches etc.)
+# However to actually run tests we do need a writable directory..
+# So we copy the repo to the actual working dir here
+
+# ugh, kinda annoying -- not sure how to update source files when we change them on the host system...
+cp -R -T /promnesia_source /promnesia
+extension/.ci/build
+
+git init  # todo ??? otherwise setuptools-scm fails to detect the version...
+
+# eh. kinda annoying to jump over so many venv layer here...
+# but docker runs as root and it doesn't like pip install uv now
+# even if you pass --break-system-packages, then subsequent uv invocation also fails
+pipx run uv tool run --with=tox-uv tox -e end2end -- "$@"

promnesia-1.4.20250909/.ci/github-ci-compat

@@ -0,0 +1,20 @@
+#!/bin/bash -eux
+
+# install sudo if it's missing
+# probably means that we're running under local docker..
+if ! which sudo; then
+    apt update
+    apt -y install sudo
+fi
+
+sudo apt update  # github actions might fail if we don't update it?
+
+# make up for differences between ubuntu:focal and github action image...
+sudo apt -y install python3.12 python3.12-dev
+sudo apt -y install python3-pip python3-setuptools
+
+# otherwise setuptools don't work..
+sudo apt -y install git
+
+# jq wants it??
+sudo apt -y install dh-autoreconf

promnesia-1.4.20250909/.ci/release-uv

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+'''
+Deploys Python package onto [[https://pypi.org][PyPi]] or [[https://test.pypi.org][test PyPi]].
+
+- running manually
+
+  You'll need =UV_PUBLISH_TOKEN= env variable
+
+- running on Github Actions
+
+  Instead of env variable, relies on configuring github as Trusted publisher (https://docs.pypi.org/trusted-publishers/) -- both for test and regular pypi
+
+  It's running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]].
+  Packages are deployed on:
+  - every master commit, onto test pypi
+  - every new tag, onto production pypi
+'''
+
+UV_PUBLISH_TOKEN = 'UV_PUBLISH_TOKEN'
+
+import argparse
+import os
+import shutil
+from pathlib import Path
+from subprocess import check_call
+
+is_ci = os.environ.get('CI') is not None
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument('--use-test-pypi', action='store_true')
+    args = p.parse_args()
+
+    publish_url = ['--publish-url', 'https://test.pypi.org/legacy/'] if args.use_test_pypi else []
+
+    root = Path(__file__).absolute().parent.parent
+    os.chdir(root)  # just in case
+
+    # TODO ok, for now uv won't remove dist dir if it already exists
+    #  https://github.com/astral-sh/uv/issues/10293
+    dist = root / 'dist'
+    if dist.exists():
+        shutil.rmtree(dist)
+
+    check_call(['uv', 'build'])
+
+    if not is_ci:
+        # CI relies on trusted publishers so doesn't need env variable
+        assert UV_PUBLISH_TOKEN in os.environ, f'no {UV_PUBLISH_TOKEN} passed'
+
+    check_call(['uv', 'publish', *publish_url])
+
+
+if __name__ == '__main__':
+    main()

promnesia-1.4.20250909/.ci/run

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -eu
+
+cd "$(dirname "$0")"
+cd .. # git root
+
+if ! command -v sudo; then
+    # CI or Docker sometimes doesn't have it, so useful to have a dummy
+    function sudo {
+        "$@"
+    }
+fi
+
+# --parallel-live to show outputs while it's running
+tox_cmd='run-parallel --parallel-live'
+if [ -n "${CI-}" ]; then
+    # install OS specific stuff here
+    # TODO: pyjq is not necessary anymore? will keep CI deps just in case I guess
+    PYJQ_DEPS=('autoconf' 'automake' 'libtool') # see https://github.com/mwilliamson/jq.py#installation
+    case "$OSTYPE" in
+    darwin*) 
+        # macos
+        brew install "${PYJQ_DEPS[@]}"
+
+        # TODO hmm. this should be in setup.py?
+        brew install libmagic # for python-magic
+        ;;
+    cygwin* | msys* | win*)
+        # windows
+        # ugh. parallel stuff seems super flaky under windows, some random failures, "file used by other process" and crap like that
+        tox_cmd='run'
+        ;;
+    *)
+        # must be linux?
+        sudo apt update
+
+        # TODO also need to warn from readme??
+        sudo apt install "${PYJQ_DEPS[@]}" python3-dev
+        ;;
+    esac
+fi
+
+# NOTE: expects uv installed
+uv tool run --with tox-uv tox $tox_cmd "$@"

promnesia-1.4.20250909/.github/workflows/main.yml

@@ -0,0 +1,201 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+
+name: CI
+on:
+  push:
+    branches: '*'
+    tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi
+    # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug:
+
+  # Needed to trigger on others' PRs.
+  # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
+  pull_request:
+
+  # Needed to trigger workflows manually.
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+  schedule:
+    - cron: '31 18 * * 5'  # run every Friday
+
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+        exclude: [
+            # windows runners are pretty scarce, so let's only run lowest and highest python version
+            {platform: windows-latest, python-version: '3.11'},
+            {platform: windows-latest, python-version: '3.12'},
+
+            # same, macos is a bit too slow and ubuntu covers python quirks well
+            {platform: macos-latest  , python-version: '3.11'},
+            {platform: macos-latest  , python-version: '3.12'},
+        ]
+
+    runs-on: ${{ matrix.platform }}
+
+    # useful for 'optional' pipelines
+    # continue-on-error: ${{ matrix.platform == 'windows-latest' }}
+
+    steps:
+    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
+    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+      
+    - uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: false  # we don't have lock files, so can't use them as cache key
+
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+
+    # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
+    - run: bash .ci/run
+      env:
+        # only compute lxml coverage on ubuntu; it crashes on windows
+        CI_MYPY_COVERAGE: ${{ matrix.platform == 'ubuntu-latest' && '--cobertura-xml-report .coverage.mypy-all' || '' }}
+
+    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
+      uses: codecov/codecov-action@v5
+      with:
+        fail_ci_if_error: true  # default false
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: mypy-${{ matrix.python-version }}
+        files: .coverage.mypy-all/cobertura.xml
+
+
+  pypi:
+    # Do not run it for PRs/cron schedule etc.
+    # NOTE: release tags are guarded by on: push: tags on the top.
+    if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/tags/') || (github.event.ref == format('refs/heads/{0}', github.event.repository.master_branch)))
+    # Ugh, I tried using matrix or something to explicitly generate only test pypi or prod pypi pipelines.
+    # But github actions is so shit, it's impossible to do any logic at all, e.g. doesn't support conditional matrix, if/else statements for variables etc.
+
+    needs: [build, end2end_tests_chrome, end2end_tests_firefox, install_and_run_test] # add all other jobs here
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      # necessary for Trusted Publishing
+      id-token: write
+
+    steps:
+    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
+    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        fetch-depth: 0  # pull all commits to correctly infer vcs version
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: false  # we don't have lock files, so can't use them as cache key
+
+    - name: 'release to test pypi'
+      # always deploy merged master to test pypi
+      if: github.event.ref == format('refs/heads/{0}', github.event.repository.master_branch)
+      run: .ci/release-uv --use-test-pypi
+
+    - name: 'release to prod pypi'
+      # always deploy tags to release pypi
+      if: startsWith(github.event.ref, 'refs/tags/')
+      run: .ci/release-uv
+
+  ###
+  build_extension:
+    env:
+      name: 'promnesia'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
+
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+
+    - run: extension/.ci/build --lint # debug version
+    - run: extension/.ci/build --lint --release
+
+    # TODO ugh. can't share github actions artifacts publicly...
+    # TODO for fuck's sake... why does it end up named as .zip.zip ????
+    - uses: actions/upload-artifact@v4
+      with:
+        name: '${{ env.name }}-chrome-debug-latest.zip'
+        path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip'
+    - uses: actions/upload-artifact@v4
+      with:
+        name: '${{ env.name }}-chrome-release-latest.zip'
+        path: 'extension/dist/artifacts/chrome/${{ env.name }}-*.zip'
+    - uses: actions/upload-artifact@v4
+      with:
+        name: '${{ env.name }}-firefox-debug-latest.zip'
+        path: 'extension/dist/artifacts/firefox/${{ env.name }}_dev_-*.zip'
+    - uses: actions/upload-artifact@v4
+      with:
+        name: '${{ env.name }}-firefox-release-latest.zip'
+        path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip'
+
+  # split in two pipelines to speedup running
+  end2end_tests_chrome:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+    - run: .ci/end2end/build_and_run.sh -k chrome
+
+  end2end_tests_firefox:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+    - run: .ci/end2end/build_and_run.sh -k firefox
+
+  install_and_run_test:
+    # todo use setup-python thing?
+    # todo run on macos too?
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+
+    - run: .ci/github-ci-compat
+
+    - run: |
+        python3 -m pip install .
+        export PATH=.ci/fake-systemd:$PATH
+        tests/install_and_run
+

promnesia-1.4.20250909/CHANGELOG.org

@@ -0,0 +1,38 @@
+* =v1.0.20210415=
+
+Thanks @ankostis, @purarue, @gms8994, @Cobertos and others for changes!
+
+** general
+- *deprecate*: if you have =import promnesia= in the config you should switch it to =import promnesia.common=
+   
+    see https://github.com/karlicoss/promnesia/pull/225
+    This brings us closer towards making promnesia a namespaced package to allow for better extensibility.
+- better Windows support https://github.com/karlicoss/promnesia/pull/197
+
+** indexer
+- *new*: 'update' style indexing is now the default https://github.com/karlicoss/promnesia/pull/211
+
+  It means that database won't be emptied before reindexing, so if you only index a single datasource, the data for other datasources will be untouched.
+  If you want the previous behaviour, you can use =--overwrite=
+- fixes for race conditions during 'update' style indexing https://github.com/karlicoss/promnesia/pull/220
+- minor cannon enhancements
+
+** server
+- fix deprecation in sqlalchemy API https://github.com/karlicoss/promnesia/pull/221
+
+** sources
+
+- *new*: viber data source (local desktop database)
+  - https://github.com/karlicoss/promnesia/pull/204
+  - https://github.com/karlicoss/promnesia/pull/208
+  - https://github.com/karlicoss/promnesia/pull/224
+- *new*: safari browser data https://github.com/karlicoss/promnesia/pull/207
+- *new*: stackexchange source https://github.com/karlicoss/promnesia/pull/189
+- auto indexer: better directory pruning https://github.com/karlicoss/promnesia/pull/209
+- telegram: enhancements to opt out of non-http link extraction
+- firefox: handle Fenix databases properly https://github.com/karlicoss/promnesia/pull/227
+- hypothesis: 
+  - extract tags https://github.com/karlicoss/promnesia/pull/199
+  - extract URLs from annotation text https://github.com/karlicoss/promnesia/pull/222
+
+* for older versions, see https://github.com/karlicoss/promnesia/releases

promnesia-1.4.20250909/PKG-INFO

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: promnesia
+Version: 1.4.20250909
+Summary: Enhancement of your browsing history
+Project-URL: Homepage, https://github.com/karlicoss/promnesia
+Author-email: "Dima Gerasimov (@karlicoss)" <karlicoss@gmail.com>
+Maintainer-email: "Dima Gerasimov (@karlicoss)" <karlicoss@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2018 Dmitrii Gerasimov
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: more-itertools
+Requires-Dist: platformdirs
+Requires-Dist: promnesia[indexer]
+Requires-Dist: promnesia[server]
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: typing-extensions
+Requires-Dist: tzlocal
+Provides-Extra: all
+Requires-Dist: beautifulsoup4; extra == 'all'
+Requires-Dist: fastapi; extra == 'all'
+Requires-Dist: hpi; extra == 'all'
+Requires-Dist: logzero; extra == 'all'
+Requires-Dist: lxml; extra == 'all'
+Requires-Dist: mistletoe; extra == 'all'
+Requires-Dist: orgparse>=0.3.0; extra == 'all'
+Requires-Dist: python-magic; extra == 'all'
+Requires-Dist: urlextract; extra == 'all'
+Requires-Dist: uvicorn[standard]; extra == 'all'
+Provides-Extra: hpi
+Requires-Dist: hpi; extra == 'hpi'
+Provides-Extra: html
+Requires-Dist: beautifulsoup4; extra == 'html'
+Requires-Dist: lxml; extra == 'html'
+Provides-Extra: indexer
+Requires-Dist: urlextract; extra == 'indexer'
+Provides-Extra: markdown
+Requires-Dist: mistletoe; extra == 'markdown'
+Provides-Extra: optional
+Requires-Dist: logzero; extra == 'optional'
+Requires-Dist: python-magic; extra == 'optional'
+Provides-Extra: org
+Requires-Dist: orgparse>=0.3.0; extra == 'org'
+Provides-Extra: server
+Requires-Dist: fastapi; extra == 'server'
+Requires-Dist: uvicorn[standard]; extra == 'server'
+Provides-Extra: telegram