promnesia 1.1.20230129__py3-none-any.whl → 1.2.20240810__py3-none-any.whl

promnesia/tests/test_config.py

@@ -0,0 +1,290 @@
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Union, List
+
+from ..common import Source
+from ..config import import_config, Config
+
+
+from more_itertools import ilen
+import pytest
+
+from .common import throw
+
+
+def make(body: str) -> Config:
+    with TemporaryDirectory() as td:
+        tdir = Path(td)
+        cp = tdir / 'cfg.py'
+        cp.write_text(body)
+        return import_config(cp)
+
+
+@contextmanager
+def with_config(cfg: Union[str, Config]):
+    from .. import config as C
+
+    assert not C.has()
+    cfg2: Config = make(cfg) if isinstance(cfg, str) else cfg
+    try:
+        C.instance = cfg2
+        assert C.has()
+        yield
+    finally:
+        C.reset()
+
+
+def index(cfg: Union[str, Config], check=True) -> List[Exception]:
+    from ..__main__ import _do_index
+
+    with with_config(cfg):
+        errors = list(_do_index())
+        if check:
+            assert len(errors) == 0, errors
+        # visits = cfg.output_dir / 'promnesia.sqlite'
+        # TODO query visit count too
+        return errors
+
+
+def test_minimal() -> None:
+    '''
+    Example of a smallest possible config, using a 'demo' source
+    '''
+    # import directly from promnesia, not promnesia.common
+    cfg = make(
+        '''
+from promnesia import Source
+from promnesia.sources import demo
+
+SOURCES = [
+    Source(demo.index),
+]
+'''
+    )
+    assert ilen(cfg.sources) == 1
+    assert all(isinstance(s, Source) for s in cfg.sources)
+    # todo output dirs?
+    index(cfg)
+
+
+def test_sources_style_1() -> None:
+    '''
+    Testing 'styles' of specifying sources
+    '''
+    cfg = make(
+        '''
+from promnesia.common import Source
+from promnesia.sources import demo
+
+SOURCES = [
+    # you can pass arguments to index functions
+    Source(demo.index, count=10, name='explicit name'),
+
+    # or rely on the default argument!
+    Source(demo.index, name='another name'),
+
+    # or rely on default source name name (will be guessed as 'demo')
+    Source(demo.index),
+
+    # rely on default index function
+    Source(demo),
+
+    # no need for Source() either!
+    demo.index,
+    demo,
+
+    # I guess this is as simple as it possibly gets...
+    'promnesia.sources.demo',
+
+    # just in case, test lambdas
+    # with list
+    lambda: list(demo.index()),
+
+    # with generator
+    lambda: iter(list(demo.index())),
+
+    # example of lazy source
+    # useful when arguments are somehow computed dynamically in config
+    Source(lambda: demo.index(count=10), name='lazy'),
+]
+    '''
+    )
+
+    srcs = [s if isinstance(s, Source) else throw(s) for s in cfg.sources]
+
+    [s1, s2, s3, s4, s5, s55, s6, s7, s77, s777] = srcs
+
+    # just a quick check to make sure tests import promnesia package correctly
+    # (depends on conftests settings)
+    assert type(srcs[0]).__module__ == 'promnesia.common', srcs
+    assert s1.name == 'explicit name'
+    assert s2.name == 'another name'
+    assert s3.name == 'demo'
+    assert s4.name == 'demo'
+    assert s5.name == 'demo'
+    assert s55.name == 'demo'
+    assert s6.name == 'demo'
+
+    # can't say 'cfg' as name is intended here but anyway
+    assert s7.name == 'cfg'
+    assert s77.name == 'cfg'
+    assert s777.name == 'lazy'
+
+    index(cfg)
+    # TODO assert on results count?
+
+
+# TODO ugh. allow not to have locator
+# ideally you can construct a visit with a link and that's it
+def test_sources_style_2() -> None:
+    '''
+    Now, sources are not magic -- they are just functions emitting visits
+    '''
+    cfg = make(
+        '''
+from typing import Iterable
+from promnesia.common import Visit, Source, Loc
+
+def my_indexer() -> Iterable[Visit]:
+    from datetime import datetime
+    for link in ['reddit.com', 'beepb00p.xyz']:
+        yield Visit(
+            url=link,
+            dt=datetime.min,
+            locator=Loc.make('test'),
+        )
+
+SOURCES = [
+    # you can just pass the function name here
+    my_indexer,
+
+    # or give it an explicit name (instead of a guess)
+    Source(my_indexer, name='nice name'),
+]
+
+
+class MyIndexer:
+    def index():
+        from promnesia.sources import demo
+        return list(demo.index())
+
+SOURCES.append(
+    MyIndexer,
+)
+
+'''
+    )
+    [s1, s2, s3] = [s if isinstance(s, Source) else throw(s) for s in cfg.sources]
+
+    assert s1.name == 'cfg'  # TODO would be nice to guess 'my_indexer' instead...
+    assert s2.name == 'nice name'
+    assert s3.name == 'cfg'  # TODO fix it, make MyIndexer?
+
+    index(cfg)
+
+
+def test_sources_lazy():
+    '''
+    Demonstration of ways to return 'lazy' and generally more advanced sources
+
+    Lazy sources could be useful to do some conditional magic or make more defensive against imports, excra configuration. You'll know when you need it ;)
+    '''
+
+    cfg = make(
+        '''
+from promnesia.common import Source
+
+def lazy():
+    from promnesia.sources import demo
+    print("Hello, I'm so lazy...")
+    yield from demo.index()
+
+SOURCES = [
+    lazy,
+]
+    '''
+    )
+    srcs = [s if isinstance(s, Source) else throw(s) for s in cfg.sources]
+    [s] = srcs
+
+    assert s.name == 'cfg'  # TODO this should be fixed... but not a big deal
+
+    index(cfg)
+
+
+# TODO later
+# or like that:
+# (i for i in lazy()),
+
+# TODO later, support stuff that returns sources lazily? e.g. lambda: Source(...)
+# not sure if it's very useful
+
+
+def test_sources_errors() -> None:
+    '''
+    Testing defensiveness of config against various errors
+    '''
+    cfg = make(
+        '''
+SOURCES = [
+    'non.existing.module',
+
+    lambda: bad.attribute,
+
+    'promnesia.sources.demo',
+]
+    '''
+    )
+
+    # nothing fails so far! It's defensive!
+    srcs = list(cfg.sources)
+
+    [e1, s1, s2] = srcs
+
+    assert isinstance(e1, Exception)
+    assert isinstance(s1, Source)
+    assert isinstance(s2, Source)
+
+    errors = index(cfg, check=False)
+    assert len(errors) == 2  # errors simply propagate
+
+
+def test_no_sources() -> None:
+    cfg = make(
+        '''
+'''
+    )
+    # raises because no SOURCES
+    with pytest.raises(RuntimeError):
+        list(cfg.sources)
+
+
+def test_empty_sources() -> None:
+    cfg = make(
+        '''
+SOURCES = []
+    '''
+    )
+    # raises because empty SOURCES
+    with pytest.raises(RuntimeError):
+        list(cfg.sources)
+
+
+def test_legacy() -> None:
+    cfg = make(
+        '''
+from promnesia.common import Source
+from promnesia.sources import demo
+INDEXERS = [
+    Source(demo.index, src='legacy name'),
+]
+    '''
+    )
+
+    [s1] = cfg.sources
+    assert isinstance(s1, Source)
+
+    assert s1.name == 'legacy name'
+
+    index(cfg)

promnesia/tests/test_db_dump.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+from concurrent.futures import ProcessPoolExecutor
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, Iterable
+
+
+from hypothesis import settings, given
+from hypothesis.strategies import from_type
+# NOTE: pytest ... -s --hypothesis-verbosity=debug is useful for seeing what hypothesis is doing
+import pytest
+import pytz
+
+
+from ..common import Loc
+from ..database.common import DbVisit
+from ..database.dump import visits_to_sqlite
+from ..database.load import get_all_db_visits
+from ..sqlite import sqlite_connection
+
+from .common import gc_control, running_on_ci
+
+
+HSETTINGS: dict[str, Any] = dict(
+    derandomize=True,
+    deadline=timedelta(seconds=2),  # sometimes slow on ci
+)
+
+
+def test_no_visits(tmp_path: Path) -> None:
+    visits: list[DbVisit] = []
+
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(
+        vit=visits,
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert db.exists()
+    [err] = [errors]
+    assert 'No visits were indexed' in str(err)
+
+
+def test_one_visit(tmp_path: Path) -> None:
+    dt = datetime.fromisoformat('2023-11-14T23:11:01')
+    dt = pytz.timezone('Europe/Warsaw').localize(dt)
+    visit = DbVisit(
+        norm_url='google.com',
+        orig_url='https://google.com',
+        dt=dt,
+        locator=Loc.make(title='title', href='https://whatever.com'),
+        duration=123,
+        src='whatever',
+    )
+
+    visits = [visit]
+
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(
+        vit=visits,
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert len(errors) == 0
+    assert db.exists()
+
+    with sqlite_connection(db, row_factory='dict') as conn:
+        [sqlite_visit] = conn.execute('SELECT * FROM visits')
+
+    assert sqlite_visit == {
+        'context': None,
+        'dt': '2023-11-14T23:11:01+01:00',
+        'duration': 123,
+        'locator_href': 'https://whatever.com',
+        'locator_title': 'title',
+        'norm_url': 'google.com',
+        'orig_url': 'https://google.com',
+        'src': 'whatever',
+    }
+
+    visits_in_db = get_all_db_visits(db)
+    assert visits_in_db == [visit]
+
+
+def test_read_db_visits(tmp_path: Path) -> None:
+    """
+    Deliberately test against "hardcoded" database to check for backwards compatibility
+    """
+    db = tmp_path / 'db.sqlite'
+    with sqlite_connection(db) as conn:
+        conn.execute(
+            '''
+CREATE TABLE visits (
+    norm_url VARCHAR,
+    orig_url VARCHAR,
+    dt VARCHAR,
+    locator_title VARCHAR,
+    locator_href VARCHAR,
+    src VARCHAR,
+    context VARCHAR,
+    duration INTEGER
+);
+'''
+        )
+        # this dt format (zone name after iso timestap) might occur in legacy databases
+        # (that were created when promnesia was using cachew NTBinder)
+        conn.execute(
+            '''
+INSERT INTO visits VALUES(
+    'i.redd.it/alala.jpg',
+    'https://i.redd.it/alala.jpg',
+    '2019-04-13T11:55:09-04:00 America/New_York',
+    'Reddit save',
+    'https://reddit.com/r/whatever',
+    'reddit',
+    '',
+    NULL
+);
+'''
+        )
+    [visit_in_db] = get_all_db_visits(db)
+    assert visit_in_db == DbVisit(
+        norm_url='i.redd.it/alala.jpg',
+        orig_url='https://i.redd.it/alala.jpg',
+        dt=datetime(2019, 4, 13, 11, 55, 9, tzinfo=timezone(timedelta(hours=-4))),
+        locator=Loc.make(title='Reddit save', href='https://reddit.com/r/whatever'),
+        src='reddit',
+        context='',
+    )
+
+
+def _test_random_visit_aux(visit: DbVisit, tmp_path: Path) -> None:
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(
+        vit=[visit],
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert db.exists()
+    assert len(errors) == 0, errors
+    # TODO query the db?
+
+
+@given(
+    visit=from_type(DbVisit).filter(
+        # if duration is too big it fails to insert in sqlite
+        lambda v: (v.duration is None or 0 <= v.duration <= 10**5)
+    )
+)
+@settings(**HSETTINGS, max_examples=100)
+def test_random_visit(visit: DbVisit) -> None:
+    with TemporaryDirectory() as tdir:
+        tmp_path = Path(tdir)
+        _test_random_visit_aux(visit=visit, tmp_path=tmp_path)
+
+
+_dt_naive = datetime.fromisoformat('2023-11-14T23:11:01')
+_dt_aware = pytz.timezone('America/New_York').localize(_dt_naive)
+
+def make_testvisit(i: int) -> DbVisit:
+    return DbVisit(
+        norm_url=f'google.com/{i}',
+        orig_url=f'https://google.com/{i}',
+        dt=(_dt_naive if i % 2 == 0 else _dt_aware) + timedelta(seconds=i),
+        locator=Loc.make(title=f'title{i}', href=f'https://whatever.com/{i}'),
+        duration=i,
+        src='whatever',
+    )
+
+
+@pytest.mark.parametrize('count', [99, 100_000, 1_000_000])
+@pytest.mark.parametrize('gc_on', [True, False], ids=['gc_on', 'gc_off'])
+def test_benchmark_visits_dumping(count: int, gc_control, tmp_path: Path) -> None:
+    # [20231212] testing differernt CHUNK_BY values with 1_000_000 visits on @karlicoss desktop pc
+    # 1: 25s (perhaps most overhead is from temporary lists?)
+    # 10 (current default): 8s
+    # 100: 6s
+    # 1000: 6s
+    # TODO maybe consider changing default to 100?
+    if count > 99 and running_on_ci:
+        pytest.skip("test would be too slow on CI, only meant to run manually")
+
+    visits = (make_testvisit(i) for i in range(count))
+    db = tmp_path / 'db.sqlite'
+    errors = visits_to_sqlite(  # TODO maybe this method should return db stats? would make testing easier
+        vit=visits,
+        overwrite_db=True,
+        _db_path=db,
+    )
+    assert db.exists()
+    assert len(errors) == 0, errors
+
+
+def _populate_db(db_path: Path, *, overwrite_db: bool, count: int) -> None:
+    visits = [make_testvisit(i) for i in range(count)]
+    errors = visits_to_sqlite(visits, _db_path=db_path, overwrite_db=overwrite_db)
+    assert len(errors) == 0
+
+
+@pytest.mark.parametrize('mode', ['update', 'overwrite'])
+def test_concurrent(tmp_path: Path, mode: str) -> None:
+    overwrite_db = {'overwrite': True, 'update': False}[mode]
+
+    db_path = tmp_path / 'db.sqlite'
+    # do initial indexing to initialize the db
+    _populate_db(db_path, overwrite_db=True, count=1)
+    assert db_path.exists()  # just in case
+
+    # this simply tests correctness by running many concurrent indexers
+    parallel = 100  # 100 indexers
+    with ProcessPoolExecutor(max_workers=8) as pool:
+        futures = []
+        for _ in range(parallel):
+            futures.append(pool.submit(_populate_db, db_path, overwrite_db=overwrite_db, count=1_000))
+        for f in futures:
+            f.result()
+    assert db_path.exists()  # just in case
+
+
+# TODO test to make sure db is readable while we're indexing?
+# kinda nicer version of test_query_while_indexing

promnesia/tests/test_extract.py

@@ -0,0 +1,61 @@
+from datetime import datetime, timezone
+
+from ..common import Visit, DbVisit, Loc, Source
+from ..extract import extract_visits
+
+from .common import get_testdata, unwrap, running_on_ci, gc_control
+
+from more_itertools import ilen
+import pytest
+
+
+def test_with_error() -> None:
+    class ExtractionError(Exception):
+        pass
+
+    def indexer():
+        yield Visit(url='http://test1', dt=datetime.fromtimestamp(0, tz=timezone.utc), locator=Loc.make('whatever'))
+        yield ExtractionError()
+        yield Visit(url='http://test2', dt=datetime.fromtimestamp(0, tz=timezone.utc), locator=Loc.make('whatever'))
+
+    [v1, e, v2] = extract_visits(source=Source(indexer), src='whatever')
+    assert isinstance(v1, DbVisit)
+    assert isinstance(e, Exception)
+    assert isinstance(v2, DbVisit)
+
+
+def test_urls_are_normalised() -> None:
+    # generally this stuff is covered by cannon tests, but good to check it's actually inserted in the db
+    # TODO maybe this should be a separate test which takes DbVisit.make separately?
+    # especially to decouple from shellcmd source
+    from ..sources import shellcmd
+    from ..sources.plaintext import extract_from_path
+
+    visits = list(extract_visits(
+        source=Source(shellcmd.index, extract_from_path(get_testdata('normalise'))),
+        src='whatever',
+    ))
+    assert len(visits) == 7
+
+    assert {unwrap(v).norm_url for v in visits} == {
+        'hi.com',
+        'reddit.com/post',
+        'argos.co.uk/webapp/wcs/stores/servlet/OrderItemDisplay',
+        'youtube.com/watch?v=XXlZfc1TrD0',
+        'youtube.com/watch?v=XXlZfc1Tr11',
+    }
+
+
+@pytest.mark.parametrize('count', [99, 100_000, 1_000_000])
+@pytest.mark.parametrize('gc_on', [True, False], ids=['gc_on', 'gc_off'])
+def test_benchmark(count: int, gc_control) -> None:
+    # NOTE: at the moment most time is spent canonifying urls, so not much point optimizing this in isolation
+    # TODO maybe could specify custom cannonifying strategy that doesn't do anything to isolate benchmark
+    if count > 99 and running_on_ci:
+        pytest.skip("test would be too slow on CI, only meant to run manually")
+
+    from ..sources import demo
+    source = Source(demo.index, count=count)
+
+    total = ilen(extract_visits(source=source, src='whatever'))
+    assert total == count  # sanity check

promnesia/tests/test_extract_urls.py

@@ -0,0 +1,43 @@
+from ..common import extract_urls
+
+
+def test_extract_simple() -> None:
+    lines = """
+ I've enjoyed [Chandler Carruth's _There Are No Zero-cost Abstractions_](
+ https://www.youtube.com/watch?v=rHIkrotSwcc) very much.
+""".strip()
+    assert set(extract_urls(lines)) == {'https://www.youtube.com/watch?v=rHIkrotSwcc'}
+
+
+def test_extract_2() -> None:
+    text = '''‍♂️ Чтобы снизить вероятность ошибиться, важно знать про когнитивные искажения.
+    Если для вас это новое словосочетание, начните с книжки
+    "Гарри Поттер и Методы рационального мышления" - http://hpmor.ru/, если вы знакомы с понятием - читайте цепочки на сайтах
+    lesswrong.ru и lesswrong.com, книжку Даниэля Канемана "Thinking, fast and slow" и канал Пион https://t.me/ontologics
+    '''
+    assert set(extract_urls(text)) == {'http://hpmor.ru/', 'lesswrong.ru', 'lesswrong.com', 'https://t.me/ontologics'}
+
+
+def test_extract_md() -> None:
+    lines = '''
+Hey, I recently implemented a new extension for that [addons.mozilla.org](https://addons.mozilla.org/en-US/firefox/addon/org-grasp-for-org-capture/), [github](https://github.com/karlicoss/grasp), perhaps it could be useful for you!
+    '''
+    assert set(extract_urls(lines)) == {
+        'addons.mozilla.org',
+        'https://addons.mozilla.org/en-US/firefox/addon/org-grasp-for-org-capture/',
+        'https://github.com/karlicoss/grasp',
+    }
+
+
+# just random links to test multiline/whitespace behaviour
+def test_extract_3() -> None:
+    lines = '''
+python.org/one.html ?? https://python.org/two.html some extra text
+
+    whatever.org
+    '''
+    assert set(extract_urls(lines, syntax='org')) == {
+        'python.org/one.html',
+        'https://python.org/two.html',
+        'whatever.org',
+    }