promnesia 1.2.20230515__py3-none-any.whl → 1.2.20240810__py3-none-any.whl

promnesia/tests/test_server.py

@@ -0,0 +1,292 @@
+from datetime import datetime
+from pathlib import Path
+from subprocess import Popen
+
+import pytest
+
+from ..__main__ import do_index
+
+from .common import promnesia_bin, write_config
+from .server_helper import run_server
+
+
+def test_status_error() -> None:
+    """
+    If DB doesn't exist, server should handle it gracefully and respond with error
+    """
+    with run_server(db='/does/not/exist') as server:
+        response = server.post('/status')
+
+        # TODO ugh currently returns 200? maybe should return proper error, but need to handle in extension
+        # assert response.status_code == 404
+
+        body = response.json()
+
+        version = body['version']
+        assert version is not None
+        assert len(version.split('.')) >= 2  # random check..
+
+        assert 'ERROR' in body['db']  # defensive, it doesn't exist
+
+
+def test_status_ok(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, count=10)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    db_path = tmp_path / 'promnesia.sqlite'
+    with run_server(db=db_path, timezone='America/New_York') as server:
+        r = server.post('/status').json()
+        version = r['version']
+        assert version is not None
+        assert len(version.split('.')) >= 2  # random check..
+
+        assert r['db'] == str(db_path)
+
+        assert r['stats'] == {'total_visits': 10}
+
+
+def test_visits(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, base_dt='2000-01-01', delta=30 * 60)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    # force timezone here, otherwise dependeing on the test env response varies
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        r = server.post('/visits', json={'url': 'whatever'}).json()
+        assert r['visits'] == []
+
+        r = server.post('/visits', json={'url': 'https://demo.com/page0.html'})
+        rj = r.json()
+        assert rj['normalised_url'] == 'demo.com/page0.html'
+        [v] = rj['visits']
+        assert v['src'] == 'demo'
+        assert v['locator']['title'] == 'demo'
+
+        assert v['dt'] == '01 Jan 2000 00:00:00 -0500'
+
+
+def test_visits_hierarchy(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from datetime import datetime
+
+        from promnesia.common import Source, Visit, Loc
+        from promnesia.sources import demo
+
+        def indexer():
+            visits = list(demo.index(count=6))
+            yield Visit(
+                url='https://reddit.com/post1',
+                dt=datetime.fromisoformat('2023-12-04'),
+                locator=Loc.make('reddit'),
+            )
+            yield Visit(
+                url='https://reddit.com/post1/comment2',
+                dt=datetime.fromisoformat('2023-12-02'),
+                locator=Loc.make('reddit'),
+                context='I am comment 2',
+            )
+            yield from visits[:3]
+            yield Visit(
+                url='https://reddit.com/post2',
+                dt=datetime.fromisoformat('2023-12-05'),
+                locator=Loc.make('reddit'),
+            )
+            yield from visits[3:]
+            yield Visit(
+                url='https://reddit.com/post1/ihavenocontext',
+                dt=datetime.fromisoformat('2023-12-06'),
+                locator=Loc.make('reddit'),
+            )
+            yield Visit(
+                url='https://reddit.com/post1/comment1',
+                dt=datetime.fromisoformat('2023-12-06'),
+                locator=Loc.make('reddit'),
+                context='I am comment 1',
+            )
+
+        SOURCES = [Source(indexer)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    # force timezone here, otherwise dependeing on the test env response varies
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        r = server.post('/visits', json={'url': 'https://reddit.com/post1'}).json()
+        # retuns exact match + 'child' visits that are interesting (e.g. have context)
+        assert {v['original_url'] for v in r['visits']} == {
+            'https://reddit.com/post1',
+            'https://reddit.com/post1/comment1',
+            'https://reddit.com/post1/comment2',
+        }
+
+
+def test_visited(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, base_dt='2000-01-01', delta=30 * 60)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    test_url = 'https://demo.com/page5.html'
+
+    # force timezone here, otherwise dependeing on the test env response varies
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        r = server.post('/visited', json={'urls': []}).json()
+        assert r == []
+
+        r = server.post('/visited', json={'urls': [test_url, 'http://badurl.org']}).json()
+        [r1, r2] = r
+        assert r1['original_url'] == test_url
+        assert r2 is None
+
+
+def test_search(tmp_path: Path) -> None:
+    # TODO not sure if should index at all here or just insert DbVisits directly?
+    def cfg() -> None:
+        from datetime import datetime
+
+        from promnesia.common import Source, Visit, Loc
+        from promnesia.sources import demo
+
+        def indexer():
+            visits = list(demo.index(count=6))
+            yield Visit(
+                url='https://someone.org/something',
+                dt=datetime.fromisoformat('2023-12-04T11:12:13+03:00'),
+                locator=Loc.make('whatever'),
+            )
+            yield from visits[:3]
+            yield Visit(
+                url='https://wiki.termux.com/wiki/Termux-setup-storage',
+                locator=Loc.make(
+                    title='Reddit comment',
+                    href='https://reddit.com/r/termux/comments/m4qrxt/cant_open_storageshared_in_termux/gso0kak/',
+                ),
+                dt=datetime.fromisoformat('2023-12-02'),
+                context='perhaps it will help someone else https://wiki.termux.com/wiki/Termux-setup-storage',
+            )
+            yield from visits[3:]
+
+        SOURCES = [Source(indexer)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        # FIXME 'url' is actually kinda misleading -- it can be any text
+        rj = server.post('/search', json={'url': 'someone'}).json()
+        # TODO maybe return in chronological order or something? not sure
+        [v1, v2] = sorted(rj['visits'], key=lambda j: j['dt'])
+
+        assert v1['context'] == 'perhaps it will help someone else https://wiki.termux.com/wiki/Termux-setup-storage'
+        assert v1['dt'] == '02 Dec 2023 00:00:00 -0500'  # uses server timezone (original visit didn't have it)
+
+        assert v2['normalised_url'] == 'someone.org/something'
+        assert v2['dt'] == '04 Dec 2023 11:12:13 +0300'  # uses original visit timezone
+
+        rj = server.post('/search', json={'url': 'comment'}).json()
+        [v] = rj['visits']
+        assert v['context'] == 'perhaps it will help someone else https://wiki.termux.com/wiki/Termux-setup-storage'
+
+
+def test_search_around(tmp_path: Path) -> None:
+    # this should return visits up to 3 hours in the past
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        # generates 60 visits within 10 mins of each other -- so spanning over 10 hours
+        SOURCES = [Source(demo.index, count=60, base_dt='2000-01-01T00:00:00+03:00', delta=10 * 60)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    # TODO hmm. perhaps it makes more sense to run query in different process and server in main process for testing??
+    with run_server(db=tmp_path / 'promnesia.sqlite') as server:
+        rj = server.post(
+            '/search_around',
+            json={'timestamp': datetime.fromisoformat('2005-01-01T00:00:00+06:00').timestamp()},
+        ).json()
+        assert rj['visits'] == []
+
+        rj = server.post(
+            '/search_around',
+            json={'timestamp': datetime.fromisoformat('2000-01-01T07:55:00+06:00').timestamp()},
+        ).json()
+        visits = rj['visits']
+        assert len(visits) == 18  # 6 per hour * 3
+        assert visits[0 ]['dt'] == '01 Jan 2000 02:00:00 +0300'
+        assert visits[-1]['dt'] == '01 Jan 2000 04:50:00 +0300'
+
+
+@pytest.mark.parametrize('mode', ['update', 'overwrite'])
+def test_query_while_indexing(tmp_path: Path, mode: str) -> None:
+    overwrite = mode == 'overwrite'
+    moverwrite = ['--overwrite'] if overwrite else []
+
+    def _index(run_id: str) -> Popen:
+        def cfg(run_id: str) -> None:
+            from promnesia.common import Source
+            from promnesia.sources import demo
+
+            SOURCES = [Source(demo.index, count=1_000, name=run_id)]
+
+        cfg_path = tmp_path / f'config{run_id}.py'
+        write_config(cfg_path, cfg, run_id=run_id)
+
+        return Popen(promnesia_bin('index', '--config', cfg_path, *moverwrite))
+
+    # trigger initial indexing
+    with _index(run_id='0'):
+        pass
+
+    with run_server(db=tmp_path / 'promnesia.sqlite') as server:
+        rj = server.post(
+            '/search_around',
+            json={'timestamp': datetime.fromisoformat('2005-01-01T00:00:00+06:00').timestamp()},
+        ).json()
+        assert rj['visits'] == []
+
+        for run_id in range(1, 5):
+            with _index(run_id=str(run_id)) as indexer:
+                # hammer the backend to increase likelihood of race condition
+                while indexer.poll() is None:
+                    stats = server.post('/status').json()['stats']
+                    total_visits = stats['total_visits']
+                    if overwrite:
+                        assert total_visits >= 1_000
+                    else:
+                        assert total_visits >= 1_000 * run_id
+
+
+# TODO also could check server methods directly?
+# via something like this... but not sure if really makes much difference
+# import promnesia.server as S
+# S.EnvConfig.set(S.ServerConfig(
+#     # TODO populate with test db and benchmark properly...
+#     db=Path('/todo'),
+#     timezone=pytz.utc,
+# ))
+# links = [f'https://reddit.com/whatever{i}.html' for i in range(count)]
+# res = S.visited(links)
+# assert len(res) == len(links)

promnesia/tests/test_traverse.py

@@ -0,0 +1,41 @@
+from unittest.mock import patch
+
+from ..common import traverse
+
+from .common import get_testdata
+
+
+testDataPath = get_testdata('traverse')
+
+
+# Patch shutil.which so it always returns false (when trying to which fdfind, etc)
+# so that it falls back to find
+@patch('promnesia.common.shutil.which', return_value=False)
+def test_traverse_ignore_find(patched) -> None:
+    '''
+    traverse() with `find` but ignore some stuff
+    '''
+    paths = set(traverse(testDataPath, ignore=['ignoreme.txt', 'ignoreme2']))
+
+    assert paths == {testDataPath / 'imhere2/real.txt', testDataPath / 'imhere.txt'}
+
+
+def test_traverse_ignore_fdfind():
+    '''
+    traverse() with `fdfind` but ignore some stuff
+    '''
+    paths = set(traverse(testDataPath, ignore=['ignoreme.txt', 'ignoreme2']))
+
+    assert paths == {testDataPath / 'imhere.txt', testDataPath / 'imhere2/real.txt'}
+
+
+# TODO: It would be nice to test the implementation directly without having to do this
+# weird patching in the future
+@patch('promnesia.common._is_windows', new_callable=lambda: True)
+def test_traverse_ignore_windows(patched) -> None:
+    '''
+    traverse() with python when _is_windows is true but ignore some stuff
+    '''
+    paths = set(traverse(testDataPath, ignore=['ignoreme.txt', 'ignoreme2']))
+
+    assert paths == {testDataPath / 'imhere.txt', testDataPath / 'imhere2/real.txt'}

promnesia/tests/utils.py

@@ -0,0 +1,35 @@
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Mapping, Optional, Sequence, Tuple, Union
+
+from ..common import Source, Loc, Visit
+from ..database.dump import visits_to_sqlite
+from ..extract import extract_visits
+
+
+# TODO a bit shit... why did I make it dict at first??
+Urls = Union[
+           Mapping[str, Optional[str]],
+    Sequence[Tuple[str, Optional[str]]],
+]
+
+
+def index_urls(urls: Urls, *, source_name: str = 'test'):
+    uuu = list(urls.items()) if isinstance(urls, dict) else urls
+
+    def idx(tmp_path: Path) -> None:
+        def indexer():
+            for i, (url, ctx) in enumerate(uuu):
+                yield Visit(
+                    url=url,
+                    dt=datetime.min + timedelta(days=5000) + timedelta(hours=i),
+                    locator=Loc.make('test'),
+                    context=ctx,
+                )
+
+        db_visits = extract_visits(source=Source(indexer), src=source_name)
+        errors = visits_to_sqlite(vit=db_visits, overwrite_db=True, _db_path=tmp_path / 'promnesia.sqlite')
+
+        assert len(errors) == 0, errors
+
+    return idx

{promnesia-1.2.20230515.dist-info → promnesia-1.2.20240810.dist-info}/METADATA

@@ -1,19 +1,17 @@
 Metadata-Version: 2.1
 Name: promnesia
-Version: 1.2.20230515
+Version: 1.2.20240810
 Summary: Enhancement of your browsing history
 Home-page: https://github.com/karlicoss/promnesia
 Author: Dmitrii Gerasimov
 Author-email: karlicoss@gmail.com
-License: UNKNOWN
-Platform: UNKNOWN
 Requires-Python: >=3.8
+License-File: LICENSE
 Requires-Dist: appdirs
 Requires-Dist: tzlocal
 Requires-Dist: more-itertools
 Requires-Dist: pytz
-Requires-Dist: sqlalchemy
-Requires-Dist: cachew (>=0.8.0)
+Requires-Dist: sqlalchemy >=2.0
 Requires-Dist: urlextract
 Requires-Dist: fastapi
 Requires-Dist: uvicorn[standard]
@@ -26,33 +24,31 @@ Requires-Dist: HPI ; extra == 'all'
 Requires-Dist: beautifulsoup4 ; extra == 'all'
 Requires-Dist: lxml ; extra == 'all'
 Requires-Dist: mistletoe ; extra == 'all'
-Requires-Dist: orgparse (>=0.3.0) ; extra == 'all'
+Requires-Dist: orgparse >=0.3.0 ; extra == 'all'
 Provides-Extra: html
 Requires-Dist: beautifulsoup4 ; extra == 'html'
 Requires-Dist: lxml ; extra == 'html'
-Provides-Extra: linting
-Requires-Dist: pytest ; extra == 'linting'
-Requires-Dist: mypy ; extra == 'linting'
-Requires-Dist: lxml ; extra == 'linting'
 Provides-Extra: markdown
 Requires-Dist: mistletoe ; extra == 'markdown'
 Provides-Extra: optional
 Requires-Dist: logzero ; extra == 'optional'
 Requires-Dist: python-magic ; extra == 'optional'
 Provides-Extra: org
-Requires-Dist: orgparse (>=0.3.0) ; extra == 'org'
+Requires-Dist: orgparse >=0.3.0 ; extra == 'org'
 Provides-Extra: telegram
 Provides-Extra: testing
 Requires-Dist: pytest ; extra == 'testing'
 Requires-Dist: pytest-timeout ; extra == 'testing'
 Requires-Dist: pytest-xdist ; extra == 'testing'
+Requires-Dist: hypothesis ; extra == 'testing'
 Requires-Dist: psutil ; extra == 'testing'
-Requires-Dist: requests (<2.30.0) ; extra == 'testing'
-Requires-Dist: httpie ; extra == 'testing'
+Requires-Dist: requests ; extra == 'testing'
 Requires-Dist: selenium ; extra == 'testing'
 Requires-Dist: click ; extra == 'testing'
-Requires-Dist: pyautogui ; extra == 'testing'
-
-UNKNOWN
-
+Requires-Dist: ruff ; extra == 'testing'
+Requires-Dist: mypy ; extra == 'testing'
+Requires-Dist: lxml ; extra == 'testing'
+Requires-Dist: loguru ; extra == 'testing'
+Provides-Extra: testing-gui
+Requires-Dist: pyautogui ; extra == 'testing-gui'
 

promnesia-1.2.20240810.dist-info/RECORD

@@ -0,0 +1,83 @@
+promnesia/__init__.py,sha256=rxVINCLqxpZFknz9QeMA6McpnzR8Hi0zhZdoyYCGUlM,367
+promnesia/__main__.py,sha256=5HVE6tyBCzTKs19rDGZIkotrmaoeTRBVtTCgRnKluh8,15251
+promnesia/cannon.py,sha256=dRfRk4bnII5D0rxKnimZiN727imYuyR2Kfw3pWEOxLs,24596
+promnesia/common.py,sha256=iMBdyYHzjma1mGP02-ugDpkN-WoyBE6Uku4Il8fuWzc,19546
+promnesia/compare.py,sha256=XjhleEzl_hzSvuinDR_I9oin5U65HkAbNCJRJQ6BwM8,4639
+promnesia/compat.py,sha256=SP1Nj3jwFoKKYLnx5qVirHdsufehcb42DShqS3PPzR4,389
+promnesia/config.py,sha256=JFLY28C11tpvwBnPYT-VfdbsD1hivPvZMzojOh5tSJs,4743
+promnesia/extract.py,sha256=JIVCxCvkn02YnGoUvLL0pAGPHXpepq-C_2brYbtN4ts,2804
+promnesia/kjson.py,sha256=9hMw-sY-avA99aWV8cG03HL2AlfvlzE5YfWw1_5LwM4,3313
+promnesia/logging.py,sha256=s_dhobXuTl8tibcpKCxktGduECKWIiEkYe0atcAToIQ,5923
+promnesia/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/server.py,sha256=42sByMUialuZQoKRF_zgxky7ykc9d-cNMwKBJAEaH68,14636
+promnesia/sqlite.py,sha256=lB7YZ9jdALbo_gblmVn7z3Dn5M6FJg7Nz2y51Z0pl1c,1353
+promnesia/database/common.py,sha256=HRw47RhxqN6Wfb5LVfbc1eRzT8LSJus1colY_S9ijm8,1814
+promnesia/database/dump.py,sha256=JV3j8J5pJJmPocRZalJqq0zz1532p9lVxW7_lcXbvx4,6101
+promnesia/database/load.py,sha256=Q2TLyV0rzvfvq8kkAhIIvcCC4D-eEbCH5wmsZL6jYg0,1353
+promnesia/misc/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/misc/config_example.py,sha256=MXxJCVjm9MrEwPR8SHTwkJRDcqWcqJWtfJg_M6yhRMo,364
+promnesia/misc/install_server.py,sha256=-dPTJjM-jktVf25mcrXRL6Uj6Pji72lPnC3iyg_m8Gk,4820
+promnesia/sources/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/sources/auto.py,sha256=MhOesdn2YfPPepNkOBSihT_CDoGXfyTIA0y8wxcKqn0,11710
+promnesia/sources/auto_logseq.py,sha256=eXOHIOx9MladhyD1IyLd90lyqO6gG2F5uObnuHFQrZA,439
+promnesia/sources/auto_obsidian.py,sha256=PLWPHj9U3pnwLWw7-fzEk12F4Xdigq0VrvSAn_yS-yw,213
+promnesia/sources/browser.py,sha256=BzdVVMm6N-U0F8_sng51deara9maY1TzMF6e7s8VCXg,2992
+promnesia/sources/browser_legacy.py,sha256=3vARw78yXWvp3RQzfPbkXgtZB5iWSbXIKgadMIG4gGs,10400
+promnesia/sources/demo.py,sha256=-5vADo4S-fDNEEPHGrh9pYoIIW_tzApRl6hIy8K3b2Q,1041
+promnesia/sources/fbmessenger.py,sha256=_QdM3-vLutN-dh4vxfQlQkRDiorWA2EwixL49AH7lbM,969
+promnesia/sources/filetypes.py,sha256=W0OVmhtvJKbWk2dju8Uwbo0TfNHye--cKrCPfL_DfBw,3593
+promnesia/sources/github.py,sha256=hHTlJZvxe6TzzALX_z8R9FUQgVw39n22XYWPc62oPfo,2753
+promnesia/sources/guess.py,sha256=-km2_hsrznYrws1lA3ZN5L6ixJunEsgQBFDU7juqPpM,776
+promnesia/sources/hackernews.py,sha256=ReI362AKjZ79Lu99qJNPZtCoyHr6AycghvQrmDFRKgw,1228
+promnesia/sources/hpi.py,sha256=yQIOT-OQ4Rhs7xXbszkNWjj46jvLDgP-xt7bSQV9wRk,361
+promnesia/sources/html.py,sha256=FOLcXzV3J5VEHMkpuF4Q0rrht5zdGsDNQZJMu2WWYZ8,1108
+promnesia/sources/hypothesis.py,sha256=PNuYKV-G7o8trMtgEzK3qX2NcBOhlZuf5Yg71FpWoQY,1359
+promnesia/sources/instapaper.py,sha256=K2feGkgiFp6pNJoDYlyLs9m8MkDizoXy2QUg-Fjm-O0,969
+promnesia/sources/markdown.py,sha256=acI2FwkawcAZX21i4kK44ligKKbj7TZDB5c0a9ihZOk,3762
+promnesia/sources/org.py,sha256=oiZsNdW-oxq4a2D5_TYBUXPdPkPguSosaoyEp9dnyvc,5499
+promnesia/sources/plaintext.py,sha256=aiMwd7WWtE65iTWiLFIow-RBDqozbjGfPgTdkzuMjaI,3259
+promnesia/sources/pocket.py,sha256=4JCNrYs23Nq0c0hw_UxaxaIJbiqF2JVu5GAql-hfNZM,1105
+promnesia/sources/reddit.py,sha256=DIEWQoWGpuQXGOmcZ9lANkPYuLhB9rzq1u23HllqLF4,5623
+promnesia/sources/roamresearch.py,sha256=Su-2ykyjdylvDMogQjAgiGpLkh4Inw-K8yeAhGv678E,1068
+promnesia/sources/rss.py,sha256=5-DmUesDv_GCOF5Qo7e7P3QrS6WdryZpFidazfI24Wg,870
+promnesia/sources/shellcmd.py,sha256=m2tT8RSoOjXz7eC-OOXGGKleod-1A_XtWFlOucO3so0,2644
+promnesia/sources/signal.py,sha256=iTU9z3hFiWjuGr5noLl8Vy3TlKj8zIMFlhNj2741h00,14728
+promnesia/sources/smscalls.py,sha256=HHM8SGDx9qS5tZfpIn_iUcKYQIyryQMZvIkqUNZKHtk,755
+promnesia/sources/stackexchange.py,sha256=vlidNoqWNUtR8dyHya9D7ZlHX5a2OIfoZq951M_V8c8,630
+promnesia/sources/takeout.py,sha256=jDhpJkjtzC7OIlBASXHw0QNwwincBkEMgqyAhuboL7o,7918
+promnesia/sources/takeout_legacy.py,sha256=ZuaP9jOuxKrbm03IOudeneWHlgT95Qr86fcGqvplKeE,4040
+promnesia/sources/telegram.py,sha256=HCft7kWHdkBJiVI8OR6Wg34k4nW1vhb3tMAEJwGO1QA,2947
+promnesia/sources/telegram_legacy.py,sha256=E4izRfxJWy25R0Rm-E78Op2lxcl2w6Rq5XUeYfXw2wc,4468
+promnesia/sources/twitter.py,sha256=8vtLKyhtqioMmp_IKKVozLZC32TIQKjs_toB5cU2_6Q,1848
+promnesia/sources/vcs.py,sha256=K2rRopKlwiNQ2_2TMWSHrwpsd5nYtPMUjHvnpNBvzBg,1609
+promnesia/sources/viber.py,sha256=uFBtiJGWwuEE-gFIV0GV7eLExc0ddt3U3F_Wnkh1XKA,5948
+promnesia/sources/website.py,sha256=Ii0Xka3PZk2_KJHJy9l3EYvGgSKkUehhHSmWI9tXec4,1781
+promnesia/sources/zulip.py,sha256=s1bhYmx26VKC35CJF_2yiO4jdYHH0QwRYtAD3Ss6tt8,742
+promnesia/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/tests/common.py,sha256=eOFyfOLBUM8hK0gJ7JTXdjAMwRL6QN5DlRsZYAudMOM,4026
+promnesia/tests/server_helper.py,sha256=W76vmwozoqJEPrlxQB0azklR28gf2RzimTHAJWOgclg,2181
+promnesia/tests/test_cannon.py,sha256=bvfL5foAtDm6QzqE_nqtiSF6zAQnkWU0dlMoQp9kX0s,12702
+promnesia/tests/test_cli.py,sha256=YZAsqbJcka1rjlDkb925V4A97FkiQc63JF0eT4Zb5wE,1406
+promnesia/tests/test_compare.py,sha256=8pxY1DPZuGnrU1p0z4VzEawJVzzmL3sNVPce1CeiSNI,945
+promnesia/tests/test_config.py,sha256=zkZXQkADHhgFzoFjqTr3zv-o6hqS0eED3wuYIMvw6dw,6532
+promnesia/tests/test_db_dump.py,sha256=k6FPtcuSImMdealbgNr7mCW5NG1_jG8eESNwO1TSW1o,6751
+promnesia/tests/test_extract.py,sha256=Hn-4B7T_4YrUtcO6tLxYJUkXknU97i0nEGRxejRnozI,2306
+promnesia/tests/test_extract_urls.py,sha256=Ybez16VdYoveL9b_50Ca9vbMogWHOvFqDOJPGZrzQJg,1961
+promnesia/tests/test_indexer.py,sha256=ec5xR9whW6z6GwKjnCGrHDp9WgTebbq7qKOIXHPLGlU,8911
+promnesia/tests/test_server.py,sha256=pwlXT-qU6KXjE0ftlWB9CE1FxhpTrwJRdX8r4QDctYs,10787
+promnesia/tests/test_traverse.py,sha256=F2qd1vvX2u9jJ4GVksAh6Dum-XEVLg9iCyFU6H8CZhI,1352
+promnesia/tests/utils.py,sha256=etNWsXIeTMaYCTN9sIlgMKHyvY_Ek3l0Btu5rIpQEcI,1117
+promnesia/tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/tests/sources/test_auto.py,sha256=d9MwY8taALdT5C64u8G7uvk4iubZ3SHSCR50qmXzz3E,1855
+promnesia/tests/sources/test_filetypes.py,sha256=w5uPsfzHfl8hS3BbXxkA3TR8rKqniRGWSO8zeIKOiM4,1338
+promnesia/tests/sources/test_hypothesis.py,sha256=nUsfET_X0yGVdlpmvxaDGAPjznYsA0Jc04vXGl83QCU,1400
+promnesia/tests/sources/test_org.py,sha256=BYJqAkI7TfgOkWk1W33MMQUDSBGpvSgGbFvb2r9qqYo,2572
+promnesia/tests/sources/test_plaintext.py,sha256=O6lLrCvZpLaaWuyeKU3eZyf4Mx1rmarU689nifTw3hk,828
+promnesia/tests/sources/test_shellcmd.py,sha256=4wyoFB3UzdviNdQX_dOpIlrzZSMXtPqRgTkRb3pk3go,679
+promnesia/tests/sources/test_takeout.py,sha256=We0gSDpENuxoO9kInSmBExJrAqmfNZkA3cVQf876SsQ,1559
+promnesia-1.2.20240810.dist-info/LICENSE,sha256=rgO9acPmnw53ZBxiXBdp8kfxmRcekhg_Q7HN65BPihs,1074
+promnesia-1.2.20240810.dist-info/METADATA,sha256=7ilgQnAz53XkvjLFIZgf8cjh5qWwyyZ_i3G2dw5mfkI,1870
+promnesia-1.2.20240810.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+promnesia-1.2.20240810.dist-info/entry_points.txt,sha256=hz1qfzQSRh4kkVkJWk4hnYqE9A1nobEbKlLG_0nNxzE,54
+promnesia-1.2.20240810.dist-info/top_level.txt,sha256=7yvIpooFiuNLf9yLdu9MTADz57z0YTAqSu7aSG9ujSU,10
+promnesia-1.2.20240810.dist-info/RECORD,,

{promnesia-1.2.20230515.dist-info → promnesia-1.2.20240810.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{promnesia-1.2.20230515.dist-info → promnesia-1.2.20240810.dist-info}/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 promnesia = promnesia.__main__:main
-

promnesia/dump.py

@@ -1,105 +0,0 @@
-from pathlib import Path
-import shutil
-from typing import  List, Set, Iterable
-
-from more_itertools import chunked
-
-from sqlalchemy import create_engine, MetaData, Table, event, text
-
-from cachew import NTBinder
-
-from .common import get_logger, DbVisit, get_tmpdir, Res, now_tz, Loc
-from . import config
-
-
-# NOTE: I guess the main performance benefit from this is not creating too many tmp lists and avoiding overhead
-# since as far as sql is concerned it should all be in the same transaction. only a guess
-# not sure it's the proper way to handle it
-# see test_index_many
-_CHUNK_BY = 10
-
-# I guess 1 hour is definitely enough
-_CONNECTION_TIMEOUT_SECONDS = 3600
-
-# returns critical warnings
-def visits_to_sqlite(vit: Iterable[Res[DbVisit]], *, overwrite_db: bool) -> List[Exception]:
-    logger = get_logger()
-    db_path = config.get().db
-
-    now = now_tz()
-    ok = 0
-    errors = 0
-    def vit_ok() -> Iterable[DbVisit]:
-        nonlocal errors, ok
-        for v in vit:
-            if isinstance(v, DbVisit):
-                ok += 1
-                yield v
-            else:
-                errors += 1
-                # conform to the schema and dump. can't hurt anyway
-                ev = DbVisit(
-                    norm_url='<error>',
-                    orig_url='<error>',
-                    dt=now,
-                    locator=Loc.make('<errror>'),
-                    src='error',
-                    # todo attach backtrace?
-                    context=repr(v),
-                )
-                yield ev
-
-    tpath = Path(get_tmpdir().name) / 'promnesia.tmp.sqlite'
-    if overwrite_db:
-        # here we don't need timeout, since it's a brand new DB
-        engine = create_engine(f'sqlite:///{tpath}')
-    else:
-        # here we need a timeout, othewise concurrent indexing might not work
-        # (note that this also needs WAL mode)
-        # see test_concurrent_indexing
-        engine = create_engine(f'sqlite:///{db_path}', connect_args={'timeout': _CONNECTION_TIMEOUT_SECONDS})
-
-    # using WAL keeps database readable while we're writing in it
-    # this is tested by test_query_while_indexing
-    def enable_wal(dbapi_con, con_record) -> None:
-        dbapi_con.execute('PRAGMA journal_mode = WAL')
-    event.listen(engine, 'connect', enable_wal)
-
-    binder = NTBinder.make(DbVisit)
-    meta = MetaData()
-    table = Table('visits', meta, *binder.columns)
-
-    cleared: Set[str] = set()
-    ncleared = 0
-    with engine.begin() as conn:
-        table.create(conn, checkfirst=True)
-
-        for chunk in chunked(vit_ok(), n=_CHUNK_BY):
-            srcs = set(v.src or '' for v in chunk)
-            new = srcs.difference(cleared)
-
-            for src in new:
-                conn.execute(table.delete().where(table.c.src == src))
-                cursor = conn.execute(text("SELECT changes()")).fetchone()
-                assert cursor is not None
-                ncleared += cursor[0]
-                cleared.add(src)
-
-            bound = [binder.to_row(x) for x in chunk]
-            # pylint: disable=no-value-for-parameter
-            conn.execute(table.insert().values(bound))
-    engine.dispose()
-
-    if overwrite_db:
-        shutil.move(str(tpath), str(db_path))
-
-    errs = '' if errors == 0 else f', {errors} ERRORS'
-    total = ok + errors
-    what = 'overwritten' if overwrite_db else 'updated'
-    logger.info(
-        '%s database "%s". %d total (%d OK%s, %d cleared, +%d more)',
-        what, db_path, total, ok, errs, ncleared, ok - ncleared)
-    res: List[Exception] = []
-    if total == 0:
-        res.append(RuntimeError('No visits were indexed, something is probably wrong!'))
-    return res