promnesia 1.2.20230515__py3-none-any.whl → 1.2.20240810__py3-none-any.whl

promnesia/tests/test_db_dump.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+from concurrent.futures import ProcessPoolExecutor
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Iterable
+
+
+from hypothesis import settings, given
+from hypothesis.strategies import from_type
+# NOTE: pytest ... -s --hypothesis-verbosity=debug is useful for seeing what hypothesis is doing
+import pytest
+import pytz
+
+
+from ..common import Loc
+from ..database.common import DbVisit
+from ..database.dump import visits_to_sqlite
+from ..database.load import get_all_db_visits
+from ..sqlite import sqlite_connection
+
+from .common import gc_control, running_on_ci
+
+
+HSETTINGS: dict[str, Any] = dict(
+    derandomize=True,
+    deadline=timedelta(seconds=2),  # sometimes slow on ci
+)
+
+
+def test_no_visits(tmp_path: Path) -> None:
+    visits: list[DbVisit] = []
+
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(
+        vit=visits,
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert db.exists()
+    [err] = [errors]
+    assert 'No visits were indexed' in str(err)
+
+
+def test_one_visit(tmp_path: Path) -> None:
+    dt = datetime.fromisoformat('2023-11-14T23:11:01')
+    dt = pytz.timezone('Europe/Warsaw').localize(dt)
+    visit = DbVisit(
+        norm_url='google.com',
+        orig_url='https://google.com',
+        dt=dt,
+        locator=Loc.make(title='title', href='https://whatever.com'),
+        duration=123,
+        src='whatever',
+    )
+
+    visits = [visit]
+
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(
+        vit=visits,
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert len(errors) == 0
+    assert db.exists()
+
+    with sqlite_connection(db, row_factory='dict') as conn:
+        [sqlite_visit] = conn.execute('SELECT * FROM visits')
+
+    assert sqlite_visit == {
+        'context': None,
+        'dt': '2023-11-14T23:11:01+01:00',
+        'duration': 123,
+        'locator_href': 'https://whatever.com',
+        'locator_title': 'title',
+        'norm_url': 'google.com',
+        'orig_url': 'https://google.com',
+        'src': 'whatever',
+    }
+
+    visits_in_db = get_all_db_visits(db)
+    assert visits_in_db == [visit]
+
+
+def test_read_db_visits(tmp_path: Path) -> None:
+    """
+    Deliberately test against "hardcoded" database to check for backwards compatibility
+    """
+    db = tmp_path / 'db.sqlite'
+    with sqlite_connection(db) as conn:
+        conn.execute(
+            '''
+CREATE TABLE visits (
+    norm_url VARCHAR,
+    orig_url VARCHAR,
+    dt VARCHAR,
+    locator_title VARCHAR,
+    locator_href VARCHAR,
+    src VARCHAR,
+    context VARCHAR,
+    duration INTEGER
+);
+'''
+        )
+        # this dt format (zone name after iso timestap) might occur in legacy databases
+        # (that were created when promnesia was using cachew NTBinder)
+        conn.execute(
+            '''
+INSERT INTO visits VALUES(
+    'i.redd.it/alala.jpg',
+    'https://i.redd.it/alala.jpg',
+    '2019-04-13T11:55:09-04:00 America/New_York',
+    'Reddit save',
+    'https://reddit.com/r/whatever',
+    'reddit',
+    '',
+    NULL
+);
+'''
+        )
+    [visit_in_db] = get_all_db_visits(db)
+    assert visit_in_db == DbVisit(
+        norm_url='i.redd.it/alala.jpg',
+        orig_url='https://i.redd.it/alala.jpg',
+        dt=datetime(2019, 4, 13, 11, 55, 9, tzinfo=timezone(timedelta(hours=-4))),
+        locator=Loc.make(title='Reddit save', href='https://reddit.com/r/whatever'),
+        src='reddit',
+        context='',
+    )
+
+
+def _test_random_visit_aux(visit: DbVisit, tmp_path: Path) -> None:
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(
+        vit=[visit],
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert db.exists()
+    assert len(errors) == 0, errors
+    # TODO query the db?
+
+
+@given(
+    visit=from_type(DbVisit).filter(
+        # if duration is too big it fails to insert in sqlite
+        lambda v: (v.duration is None or 0 <= v.duration <= 10**5)
+    )
+)
+@settings(**HSETTINGS, max_examples=100)
+def test_random_visit(visit: DbVisit) -> None:
+    with TemporaryDirectory() as tdir:
+        tmp_path = Path(tdir)
+        _test_random_visit_aux(visit=visit, tmp_path=tmp_path)
+
+
+_dt_naive = datetime.fromisoformat('2023-11-14T23:11:01')
+_dt_aware = pytz.timezone('America/New_York').localize(_dt_naive)
+
+def make_testvisit(i: int) -> DbVisit:
+    return DbVisit(
+        norm_url=f'google.com/{i}',
+        orig_url=f'https://google.com/{i}',
+        dt=(_dt_naive if i % 2 == 0 else _dt_aware) + timedelta(seconds=i),
+        locator=Loc.make(title=f'title{i}', href=f'https://whatever.com/{i}'),
+        duration=i,
+        src='whatever',
+    )
+
+
+@pytest.mark.parametrize('count', [99, 100_000, 1_000_000])
+@pytest.mark.parametrize('gc_on', [True, False], ids=['gc_on', 'gc_off'])
+def test_benchmark_visits_dumping(count: int, gc_control, tmp_path: Path) -> None:
+    # [20231212] testing differernt CHUNK_BY values with 1_000_000 visits on @karlicoss desktop pc
+    # 1: 25s (perhaps most overhead is from temporary lists?)
+    # 10 (current default): 8s
+    # 100: 6s
+    # 1000: 6s
+    # TODO maybe consider changing default to 100?
+    if count > 99 and running_on_ci:
+        pytest.skip("test would be too slow on CI, only meant to run manually")
+
+    visits = (make_testvisit(i) for i in range(count))
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(  # TODO maybe this method should return db stats? would make testing easier
+        vit=visits,
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert db.exists()
+    assert len(errors) == 0, errors
+
+
+def _populate_db(db_path: Path, *, overwrite_db: bool, count: int) -> None:
+    visits = [make_testvisit(i) for i in range(count)]
+    errors = visits_to_sqlite(visits, _db_path=db_path, overwrite_db=overwrite_db)
+    assert len(errors) == 0
+
+
+@pytest.mark.parametrize('mode', ['update', 'overwrite'])
+def test_concurrent(tmp_path: Path, mode: str) -> None:
+    overwrite_db = {'overwrite': True, 'update': False}[mode]
+
+    db_path = tmp_path / 'db.sqlite'
+    # do initial indexing to initialize the db
+    _populate_db(db_path, overwrite_db=True, count=1)
+    assert db_path.exists()  # just in case
+
+    # this simply tests correctness by running many concurrent indexers
+    parallel = 100  # 100 indexers
+    with ProcessPoolExecutor(max_workers=8) as pool:
+        futures = []
+        for _ in range(parallel):
+            futures.append(pool.submit(_populate_db, db_path, overwrite_db=overwrite_db, count=1_000))
+        for f in futures:
+            f.result()
+    assert db_path.exists()  # just in case
+
+
+# TODO test to make sure db is readable while we're indexing?
+# kinda nicer version of test_query_while_indexing

promnesia/tests/test_extract.py

@@ -0,0 +1,61 @@
+from datetime import datetime, timezone
+
+from ..common import Visit, DbVisit, Loc, Source
+from ..extract import extract_visits
+
+from .common import get_testdata, unwrap, running_on_ci, gc_control
+
+from more_itertools import ilen
+import pytest
+
+
+def test_with_error() -> None:
+    class ExtractionError(Exception):
+        pass
+
+    def indexer():
+        yield Visit(url='http://test1', dt=datetime.fromtimestamp(0, tz=timezone.utc), locator=Loc.make('whatever'))
+        yield ExtractionError()
+        yield Visit(url='http://test2', dt=datetime.fromtimestamp(0, tz=timezone.utc), locator=Loc.make('whatever'))
+
+    [v1, e, v2] = extract_visits(source=Source(indexer), src='whatever')
+    assert isinstance(v1, DbVisit)
+    assert isinstance(e, Exception)
+    assert isinstance(v2, DbVisit)
+
+
+def test_urls_are_normalised() -> None:
+    # generally this stuff is covered by cannon tests, but good to check it's actually inserted in the db
+    # TODO maybe this should be a separate test which takes DbVisit.make separately?
+    # especially to decouple from shellcmd source
+    from ..sources import shellcmd
+    from ..sources.plaintext import extract_from_path
+
+    visits = list(extract_visits(
+        source=Source(shellcmd.index, extract_from_path(get_testdata('normalise'))),
+        src='whatever',
+    ))
+    assert len(visits) == 7
+
+    assert {unwrap(v).norm_url for v in visits} == {
+        'hi.com',
+        'reddit.com/post',
+        'argos.co.uk/webapp/wcs/stores/servlet/OrderItemDisplay',
+        'youtube.com/watch?v=XXlZfc1TrD0',
+        'youtube.com/watch?v=XXlZfc1Tr11',
+    }
+
+
+@pytest.mark.parametrize('count', [99, 100_000, 1_000_000])
+@pytest.mark.parametrize('gc_on', [True, False], ids=['gc_on', 'gc_off'])
+def test_benchmark(count: int, gc_control) -> None:
+    # NOTE: at the moment most time is spent canonifying urls, so not much point optimizing this in isolation
+    # TODO maybe could specify custom cannonifying strategy that doesn't do anything to isolate benchmark
+    if count > 99 and running_on_ci:
+        pytest.skip("test would be too slow on CI, only meant to run manually")
+
+    from ..sources import demo
+    source = Source(demo.index, count=count)
+
+    total = ilen(extract_visits(source=source, src='whatever'))
+    assert total == count  # sanity check

promnesia/tests/test_extract_urls.py

@@ -0,0 +1,43 @@
+from ..common import extract_urls
+
+
+def test_extract_simple() -> None:
+    lines = """
+ I've enjoyed [Chandler Carruth's _There Are No Zero-cost Abstractions_](
+ https://www.youtube.com/watch?v=rHIkrotSwcc) very much.
+""".strip()
+    assert set(extract_urls(lines)) == {'https://www.youtube.com/watch?v=rHIkrotSwcc'}
+
+
+def test_extract_2() -> None:
+    text = '''‍♂️ Чтобы снизить вероятность ошибиться, важно знать про когнитивные искажения.
+    Если для вас это новое словосочетание, начните с книжки
+    "Гарри Поттер и Методы рационального мышления" - http://hpmor.ru/, если вы знакомы с понятием - читайте цепочки на сайтах
+    lesswrong.ru и lesswrong.com, книжку Даниэля Канемана "Thinking, fast and slow" и канал Пион https://t.me/ontologics
+    '''
+    assert set(extract_urls(text)) == {'http://hpmor.ru/', 'lesswrong.ru', 'lesswrong.com', 'https://t.me/ontologics'}
+
+
+def test_extract_md() -> None:
+    lines = '''
+Hey, I recently implemented a new extension for that [addons.mozilla.org](https://addons.mozilla.org/en-US/firefox/addon/org-grasp-for-org-capture/), [github](https://github.com/karlicoss/grasp), perhaps it could be useful for you!
+    '''
+    assert set(extract_urls(lines)) == {
+        'addons.mozilla.org',
+        'https://addons.mozilla.org/en-US/firefox/addon/org-grasp-for-org-capture/',
+        'https://github.com/karlicoss/grasp',
+    }
+
+
+# just random links to test multiline/whitespace behaviour
+def test_extract_3() -> None:
+    lines = '''
+python.org/one.html ?? https://python.org/two.html some extra text
+
+    whatever.org
+    '''
+    assert set(extract_urls(lines, syntax='org')) == {
+        'python.org/one.html',
+        'https://python.org/two.html',
+        'whatever.org',
+    }

promnesia/tests/test_indexer.py

@@ -0,0 +1,245 @@
+from collections import Counter
+from pathlib import Path
+from subprocess import check_call, Popen
+
+from ..__main__ import do_index, read_example_config
+from ..common import DbVisit, _is_windows
+from ..database.load import get_all_db_visits
+
+import pytest
+
+from .common import get_testdata, promnesia_bin, reset_filters, write_config
+
+
+def get_stats(tmp_path: Path) -> Counter:
+    visits = get_all_db_visits(tmp_path / 'promnesia.sqlite')
+    return Counter(v.src for v in visits)
+
+
+@pytest.mark.parametrize('mode', ['update', 'overwrite'])
+def test_indexing_mode(tmp_path: Path, mode: str) -> None:
+    # ugh. we modify the config very fast during tests
+    # and pycache distinguishes identical filenames based on int mtime in seconds
+    # so best to use different names to prevent undesired caching
+    # https://github.com/python/cpython/blob/fb202af4470d6051a69bb9d2f44d7e8a1c99eb4f/Lib/importlib/_bootstrap_external.py#L714-L739
+    # TODO could probably relax that if we switch from importlib config loading to exec()?
+
+    def cfg1() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [
+            Source(demo.index, count=10, base_dt='2000-01-01', delta=30, name='demo1'),
+            Source(demo.index, count=20, base_dt='2001-01-01', delta=30, name='demo2'),
+        ]
+
+    cfg_path = tmp_path / 'config1.py'
+    write_config(cfg_path, cfg1)
+    do_index(cfg_path)
+
+    stats = get_stats(tmp_path)
+    assert stats == {'demo1': 10, 'demo2': 20}
+
+    def cfg2() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [
+            Source(demo.index, count=30, base_dt='2005-01-01', delta=30, name='demo2'),
+            Source(demo.index, count=40, base_dt='2010-01-01', delta=30, name='demo3'),
+        ]
+
+    cfg_path = tmp_path / 'config2.py'
+    write_config(cfg_path, cfg2)
+    do_index(cfg_path, overwrite_db={'overwrite': True, 'update': False}[mode])
+    # TODO use some sort of test helper?
+    stats = get_stats(tmp_path)
+
+    if mode == 'update':
+        # should keep the original visits too!
+        assert stats == {'demo1': 10, 'demo2': 30, 'demo3': 40}
+    else:
+        # should overwrite with newly indexed visits
+        assert stats == {'demo2': 30, 'demo3': 40}
+
+
+# TODO check both modes?
+def test_concurrent_indexing(tmp_path: Path) -> None:
+    def cfg_fast() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, count=10)]
+
+    cfg_fast_path = tmp_path / 'cfg_fast.py'
+    write_config(cfg_fast_path, cfg_fast)
+
+    def cfg_slow() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, count=100_000)]
+
+    cfg_slow_path = tmp_path / 'cfg_slow.py'
+    write_config(cfg_slow_path, cfg_slow)
+
+    # init it first, to create the database
+    # TODO ideally this shouldn't be necessary but it's reasonable that people would already have the index
+    # otherwise it would fail at db creation point.. which is kinda annoying to work around
+    # todo in principle can work around same way as in cachew, by having a loop around PRAGMA WAL command?
+    check_call(promnesia_bin('index', '--config', cfg_fast_path, '--overwrite'))
+
+    total_runs = 0
+    # run in the background
+    with Popen(promnesia_bin('index', '--config', cfg_slow_path, '--overwrite')) as slow_indexer:
+        while slow_indexer.poll() is None:
+            # create a bunch of 'smaller' indexers running in parallel
+            fasts = [
+                Popen(promnesia_bin('index', '--config', cfg_fast_path, '--overwrite'))
+                for _ in range(10)
+            ]
+            for fast in fasts:
+                assert fast.wait() == 0, fast  # should succeed
+                total_runs += 1
+        assert slow_indexer.poll() == 0, slow_indexer
+
+    # FIXME ok, need to uncomment this once proper concurrent indexing is supported
+    # if not, slow indexer is too fast, so crank up the count in it
+    # assert total_runs > 20
+
+
+def test_filter(tmp_path: Path, reset_filters) -> None:
+    domain_to_filter = 'some-weird-domain.xyz'
+    testdata = get_testdata('custom')
+    assert any(domain_to_filter in p.read_text() for p in testdata.glob('*.txt'))  # precondition
+
+    def cfg(testdata, domain_to_filter) -> None:
+        from promnesia.common import Source
+        from promnesia.sources import shellcmd
+        from promnesia.sources.plaintext import extract_from_path
+
+        FILTERS = [
+            domain_to_filter,
+        ]
+
+        SOURCES = [Source(shellcmd.index, extract_from_path(testdata))]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg, testdata=testdata, domain_to_filter=domain_to_filter)
+    do_index(cfg_path)
+
+    visits = get_all_db_visits(tmp_path / 'promnesia.sqlite')
+    urls = {v.orig_url for v in visits}
+    assert not any(domain_to_filter in u for u in urls), urls
+    assert len(visits) == 4  # just in case
+
+
+def test_weird_urls(tmp_path: Path) -> None:
+    # specifically test this here (rather than in cannon)
+    # to make sure it's not messed up when we insert/extract from sqlite
+
+    def cfg(testdata: str) -> None:
+        from promnesia.common import Source
+        from promnesia.sources import shellcmd
+        from promnesia.sources.plaintext import extract_from_path
+
+        SOURCES = [Source(shellcmd.index, extract_from_path(testdata))]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg, testdata=get_testdata('weird.txt'))
+    do_index(cfg_path)
+
+    [v1, v2] = get_all_db_visits(tmp_path / 'promnesia.sqlite')
+
+    assert v1.norm_url == "urbandictionary.com/define.php?term=Belgian%20Whistle"
+
+    assert v2.norm_url == "en.wikipedia.org/wiki/Dinic%27s_algorithm"
+    assert v2.locator.title.endswith('weird.txt:2')
+    assert v2.context == 'right, so https://en.wikipedia.org/wiki/Dinic%27s_algorithm can be used for max flow'
+
+
+def test_errors_during_indexing(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        def indexer1():
+            visits = list(demo.index(count=10))
+            yield from visits[:5]
+            yield RuntimeError("some error during visits extraction")
+            yield from visits[5:]
+
+        def indexer2():
+            raise RuntimeError("in this case indexer itself crashed")
+
+        SOURCES = [Source(indexer1), Source(indexer2)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    stats = get_stats(tmp_path)
+    assert stats == {
+        'error': 2,
+        'config': 10,
+    }
+
+
+def test_hook(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, count=7, name='somename')]
+
+        from typing import cast, Iterator
+        from promnesia.common import DbVisit, Loc, Res
+        from promnesia.sources import demo
+
+        def HOOK(visit: Res[DbVisit]) -> Iterator[Res[DbVisit]]:
+            visit = cast(DbVisit, visit)
+
+            # NOTE: might be a good idea to check that the visit is an exception first and yield it intact?
+            nurl = visit.norm_url
+            if 'page1' in nurl:
+                yield visit._replace(norm_url='patched.com')
+            elif 'page2' in nurl:
+                raise Exception('boom')  # deliberately crash
+            elif 'page3' in nurl:
+                # just don't yield anything! it will be omitted
+                pass
+            elif 'page4' in nurl:
+                # can emit multiple!
+                yield visit
+                yield visit
+            elif 'page6' in nurl:
+                # patch locator
+                yield visit._replace(locator=Loc.make(title='some custom timte', href='/can/replace/original/path'))
+            else:
+                yield visit
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    [p0, p1, e2, p41, p42, p5, p6] = get_all_db_visits(tmp_path / 'promnesia.sqlite')
+    assert p0.norm_url == 'demo.com/page0.html'
+    assert p1.norm_url == 'patched.com'
+    assert e2.norm_url == '<error>'
+    assert p41 == p42
+    assert isinstance(p6, DbVisit)
+    assert p6.locator is not None
+
+
+def test_example_config(tmp_path: Path) -> None:
+    if _is_windows:
+        pytest.skip("doesn't work on Windows: example config references /usr/include paths")
+
+    config = read_example_config() + '\n' + f'OUTPUT_DIR = "{str(tmp_path)}"'
+    cfg_path = tmp_path / 'example_config.py'
+    cfg_path.write_text(config)
+
+    do_index(cfg_path)
+
+    visits = [v for v in get_all_db_visits(tmp_path / 'promnesia.sqlite') if v.src != 'error']
+    assert len(visits) > 50  # random sanity check