promnesia 1.2.20230515__py3-none-any.whl → 1.3.20241021__py3-none-any.whl

promnesia/tests/test_server.py

@@ -0,0 +1,291 @@
+from datetime import datetime
+from pathlib import Path
+from subprocess import Popen
+
+import pytest
+
+from ..__main__ import do_index
+from .common import promnesia_bin, write_config
+from .server_helper import run_server
+
+
+def test_status_error() -> None:
+    """
+    If DB doesn't exist, server should handle it gracefully and respond with error
+    """
+    with run_server(db='/does/not/exist') as server:
+        response = server.post('/status')
+
+        # TODO ugh currently returns 200? maybe should return proper error, but need to handle in extension
+        # assert response.status_code == 404
+
+        body = response.json()
+
+        version = body['version']
+        assert version is not None
+        assert len(version.split('.')) >= 2  # random check..
+
+        assert 'ERROR' in body['db']  # defensive, it doesn't exist
+
+
+def test_status_ok(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, count=10)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    db_path = tmp_path / 'promnesia.sqlite'
+    with run_server(db=db_path, timezone='America/New_York') as server:
+        r = server.post('/status').json()
+        version = r['version']
+        assert version is not None
+        assert len(version.split('.')) >= 2  # random check..
+
+        assert r['db'] == str(db_path)
+
+        assert r['stats'] == {'total_visits': 10}
+
+
+def test_visits(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, base_dt='2000-01-01', delta=30 * 60)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    # force timezone here, otherwise dependeing on the test env response varies
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        r = server.post('/visits', json={'url': 'whatever'}).json()
+        assert r['visits'] == []
+
+        r = server.post('/visits', json={'url': 'https://demo.com/page0.html'})
+        rj = r.json()
+        assert rj['normalised_url'] == 'demo.com/page0.html'
+        [v] = rj['visits']
+        assert v['src'] == 'demo'
+        assert v['locator']['title'] == 'demo'
+
+        assert v['dt'] == '01 Jan 2000 00:00:00 -0500'
+
+
+def test_visits_hierarchy(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from datetime import datetime
+
+        from promnesia.common import Loc, Source, Visit
+        from promnesia.sources import demo
+
+        def indexer():
+            visits = list(demo.index(count=6))
+            yield Visit(
+                url='https://reddit.com/post1',
+                dt=datetime.fromisoformat('2023-12-04'),
+                locator=Loc.make('reddit'),
+            )
+            yield Visit(
+                url='https://reddit.com/post1/comment2',
+                dt=datetime.fromisoformat('2023-12-02'),
+                locator=Loc.make('reddit'),
+                context='I am comment 2',
+            )
+            yield from visits[:3]
+            yield Visit(
+                url='https://reddit.com/post2',
+                dt=datetime.fromisoformat('2023-12-05'),
+                locator=Loc.make('reddit'),
+            )
+            yield from visits[3:]
+            yield Visit(
+                url='https://reddit.com/post1/ihavenocontext',
+                dt=datetime.fromisoformat('2023-12-06'),
+                locator=Loc.make('reddit'),
+            )
+            yield Visit(
+                url='https://reddit.com/post1/comment1',
+                dt=datetime.fromisoformat('2023-12-06'),
+                locator=Loc.make('reddit'),
+                context='I am comment 1',
+            )
+
+        SOURCES = [Source(indexer)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    # force timezone here, otherwise dependeing on the test env response varies
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        r = server.post('/visits', json={'url': 'https://reddit.com/post1'}).json()
+        # retuns exact match + 'child' visits that are interesting (e.g. have context)
+        assert {v['original_url'] for v in r['visits']} == {
+            'https://reddit.com/post1',
+            'https://reddit.com/post1/comment1',
+            'https://reddit.com/post1/comment2',
+        }
+
+
+def test_visited(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        SOURCES = [Source(demo.index, base_dt='2000-01-01', delta=30 * 60)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    test_url = 'https://demo.com/page5.html'
+
+    # force timezone here, otherwise dependeing on the test env response varies
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        r = server.post('/visited', json={'urls': []}).json()
+        assert r == []
+
+        r = server.post('/visited', json={'urls': [test_url, 'http://badurl.org']}).json()
+        [r1, r2] = r
+        assert r1['original_url'] == test_url
+        assert r2 is None
+
+
+def test_search(tmp_path: Path) -> None:
+    # TODO not sure if should index at all here or just insert DbVisits directly?
+    def cfg() -> None:
+        from datetime import datetime
+
+        from promnesia.common import Loc, Source, Visit
+        from promnesia.sources import demo
+
+        def indexer():
+            visits = list(demo.index(count=6))
+            yield Visit(
+                url='https://someone.org/something',
+                dt=datetime.fromisoformat('2023-12-04T11:12:13+03:00'),
+                locator=Loc.make('whatever'),
+            )
+            yield from visits[:3]
+            yield Visit(
+                url='https://wiki.termux.com/wiki/Termux-setup-storage',
+                locator=Loc.make(
+                    title='Reddit comment',
+                    href='https://reddit.com/r/termux/comments/m4qrxt/cant_open_storageshared_in_termux/gso0kak/',
+                ),
+                dt=datetime.fromisoformat('2023-12-02'),
+                context='perhaps it will help someone else https://wiki.termux.com/wiki/Termux-setup-storage',
+            )
+            yield from visits[3:]
+
+        SOURCES = [Source(indexer)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    with run_server(db=tmp_path / 'promnesia.sqlite', timezone='America/New_York') as server:
+        # FIXME 'url' is actually kinda misleading -- it can be any text
+        rj = server.post('/search', json={'url': 'someone'}).json()
+        # TODO maybe return in chronological order or something? not sure
+        [v1, v2] = sorted(rj['visits'], key=lambda j: j['dt'])
+
+        assert v1['context'] == 'perhaps it will help someone else https://wiki.termux.com/wiki/Termux-setup-storage'
+        assert v1['dt'] == '02 Dec 2023 00:00:00 -0500'  # uses server timezone (original visit didn't have it)
+
+        assert v2['normalised_url'] == 'someone.org/something'
+        assert v2['dt'] == '04 Dec 2023 11:12:13 +0300'  # uses original visit timezone
+
+        rj = server.post('/search', json={'url': 'comment'}).json()
+        [v] = rj['visits']
+        assert v['context'] == 'perhaps it will help someone else https://wiki.termux.com/wiki/Termux-setup-storage'
+
+
+def test_search_around(tmp_path: Path) -> None:
+    # this should return visits up to 3 hours in the past
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import demo
+
+        # generates 60 visits within 10 mins of each other -- so spanning over 10 hours
+        SOURCES = [Source(demo.index, count=60, base_dt='2000-01-01T00:00:00+03:00', delta=10 * 60)]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+    do_index(cfg_path)
+
+    # TODO hmm. perhaps it makes more sense to run query in different process and server in main process for testing??
+    with run_server(db=tmp_path / 'promnesia.sqlite') as server:
+        rj = server.post(
+            '/search_around',
+            json={'timestamp': datetime.fromisoformat('2005-01-01T00:00:00+06:00').timestamp()},
+        ).json()
+        assert rj['visits'] == []
+
+        rj = server.post(
+            '/search_around',
+            json={'timestamp': datetime.fromisoformat('2000-01-01T07:55:00+06:00').timestamp()},
+        ).json()
+        visits = rj['visits']
+        assert len(visits) == 18  # 6 per hour * 3
+        assert visits[0 ]['dt'] == '01 Jan 2000 02:00:00 +0300'
+        assert visits[-1]['dt'] == '01 Jan 2000 04:50:00 +0300'
+
+
+@pytest.mark.parametrize('mode', ['update', 'overwrite'])
+def test_query_while_indexing(tmp_path: Path, mode: str) -> None:
+    overwrite = mode == 'overwrite'
+    moverwrite = ['--overwrite'] if overwrite else []
+
+    def _index(run_id: str) -> Popen:
+        def cfg(run_id: str) -> None:
+            from promnesia.common import Source
+            from promnesia.sources import demo
+
+            SOURCES = [Source(demo.index, count=1_000, name=run_id)]
+
+        cfg_path = tmp_path / f'config{run_id}.py'
+        write_config(cfg_path, cfg, run_id=run_id)
+
+        return Popen(promnesia_bin('index', '--config', cfg_path, *moverwrite))
+
+    # trigger initial indexing
+    with _index(run_id='0'):
+        pass
+
+    with run_server(db=tmp_path / 'promnesia.sqlite') as server:
+        rj = server.post(
+            '/search_around',
+            json={'timestamp': datetime.fromisoformat('2005-01-01T00:00:00+06:00').timestamp()},
+        ).json()
+        assert rj['visits'] == []
+
+        for run_id in range(1, 5):
+            with _index(run_id=str(run_id)) as indexer:
+                # hammer the backend to increase likelihood of race condition
+                while indexer.poll() is None:
+                    stats = server.post('/status').json()['stats']
+                    total_visits = stats['total_visits']
+                    if overwrite:
+                        assert total_visits >= 1_000
+                    else:
+                        assert total_visits >= 1_000 * run_id
+
+
+# TODO also could check server methods directly?
+# via something like this... but not sure if really makes much difference
+# import promnesia.server as S
+# S.EnvConfig.set(S.ServerConfig(
+#     # TODO populate with test db and benchmark properly...
+#     db=Path('/todo'),
+#     timezone=pytz.utc,
+# ))
+# links = [f'https://reddit.com/whatever{i}.html' for i in range(count)]
+# res = S.visited(links)
+# assert len(res) == len(links)

promnesia/tests/test_traverse.py

@@ -0,0 +1,39 @@
+from unittest.mock import patch
+
+from ..common import traverse
+from .common import get_testdata
+
+testDataPath = get_testdata('traverse')
+
+
+# Patch shutil.which so it always returns false (when trying to which fdfind, etc)
+# so that it falls back to find
+@patch('promnesia.common.shutil.which', return_value=False)
+def test_traverse_ignore_find(patched) -> None:
+    '''
+    traverse() with `find` but ignore some stuff
+    '''
+    paths = set(traverse(testDataPath, ignore=['ignoreme.txt', 'ignoreme2']))
+
+    assert paths == {testDataPath / 'imhere2/real.txt', testDataPath / 'imhere.txt'}
+
+
+def test_traverse_ignore_fdfind():
+    '''
+    traverse() with `fdfind` but ignore some stuff
+    '''
+    paths = set(traverse(testDataPath, ignore=['ignoreme.txt', 'ignoreme2']))
+
+    assert paths == {testDataPath / 'imhere.txt', testDataPath / 'imhere2/real.txt'}
+
+
+# TODO: It would be nice to test the implementation directly without having to do this
+# weird patching in the future
+@patch('promnesia.common._is_windows', new_callable=lambda: True)
+def test_traverse_ignore_windows(patched) -> None:
+    '''
+    traverse() with python when _is_windows is true but ignore some stuff
+    '''
+    paths = set(traverse(testDataPath, ignore=['ignoreme.txt', 'ignoreme2']))
+
+    assert paths == {testDataPath / 'imhere.txt', testDataPath / 'imhere2/real.txt'}

promnesia/tests/utils.py

@@ -0,0 +1,35 @@
+from collections.abc import Mapping, Sequence
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional, Union
+
+from ..common import Loc, Source, Visit
+from ..database.dump import visits_to_sqlite
+from ..extract import extract_visits
+
+# TODO a bit shit... why did I make it dict at first??
+Urls = Union[
+           Mapping[str, Optional[str]],
+    Sequence[tuple[str, Optional[str]]],
+]
+
+
+def index_urls(urls: Urls, *, source_name: str = 'test'):
+    uuu = list(urls.items()) if isinstance(urls, dict) else urls
+
+    def idx(tmp_path: Path) -> None:
+        def indexer():
+            for i, (url, ctx) in enumerate(uuu):
+                yield Visit(
+                    url=url,
+                    dt=datetime.min + timedelta(days=5000) + timedelta(hours=i),
+                    locator=Loc.make('test'),
+                    context=ctx,
+                )
+
+        db_visits = extract_visits(source=Source(indexer), src=source_name)
+        errors = visits_to_sqlite(vit=db_visits, overwrite_db=True, _db_path=tmp_path / 'promnesia.sqlite')
+
+        assert len(errors) == 0, errors
+
+    return idx

{promnesia-1.2.20230515.dist-info → promnesia-1.3.20241021.dist-info}/METADATA

@@ -1,19 +1,18 @@
 Metadata-Version: 2.1
 Name: promnesia
-Version: 1.2.20230515
+Version: 1.3.20241021
 Summary: Enhancement of your browsing history
 Home-page: https://github.com/karlicoss/promnesia
 Author: Dmitrii Gerasimov
 Author-email: karlicoss@gmail.com
-License: UNKNOWN
-Platform: UNKNOWN
-Requires-Python: >=3.8
+Requires-Python: >=3.9
+License-File: LICENSE
 Requires-Dist: appdirs
 Requires-Dist: tzlocal
 Requires-Dist: more-itertools
+Requires-Dist: typing-extensions
 Requires-Dist: pytz
-Requires-Dist: sqlalchemy
-Requires-Dist: cachew (>=0.8.0)
+Requires-Dist: sqlalchemy >=2.0
 Requires-Dist: urlextract
 Requires-Dist: fastapi
 Requires-Dist: uvicorn[standard]
@@ -26,33 +25,31 @@ Requires-Dist: HPI ; extra == 'all'
 Requires-Dist: beautifulsoup4 ; extra == 'all'
 Requires-Dist: lxml ; extra == 'all'
 Requires-Dist: mistletoe ; extra == 'all'
-Requires-Dist: orgparse (>=0.3.0) ; extra == 'all'
+Requires-Dist: orgparse >=0.3.0 ; extra == 'all'
 Provides-Extra: html
 Requires-Dist: beautifulsoup4 ; extra == 'html'
 Requires-Dist: lxml ; extra == 'html'
-Provides-Extra: linting
-Requires-Dist: pytest ; extra == 'linting'
-Requires-Dist: mypy ; extra == 'linting'
-Requires-Dist: lxml ; extra == 'linting'
 Provides-Extra: markdown
 Requires-Dist: mistletoe ; extra == 'markdown'
 Provides-Extra: optional
 Requires-Dist: logzero ; extra == 'optional'
 Requires-Dist: python-magic ; extra == 'optional'
 Provides-Extra: org
-Requires-Dist: orgparse (>=0.3.0) ; extra == 'org'
+Requires-Dist: orgparse >=0.3.0 ; extra == 'org'
 Provides-Extra: telegram
 Provides-Extra: testing
 Requires-Dist: pytest ; extra == 'testing'
 Requires-Dist: pytest-timeout ; extra == 'testing'
 Requires-Dist: pytest-xdist ; extra == 'testing'
+Requires-Dist: hypothesis ; extra == 'testing'
 Requires-Dist: psutil ; extra == 'testing'
-Requires-Dist: requests (<2.30.0) ; extra == 'testing'
-Requires-Dist: httpie ; extra == 'testing'
+Requires-Dist: requests ; extra == 'testing'
 Requires-Dist: selenium ; extra == 'testing'
 Requires-Dist: click ; extra == 'testing'
-Requires-Dist: pyautogui ; extra == 'testing'
-
-UNKNOWN
-
+Requires-Dist: ruff ; extra == 'testing'
+Requires-Dist: mypy ; extra == 'testing'
+Requires-Dist: lxml ; extra == 'testing'
+Requires-Dist: loguru ; extra == 'testing'
+Provides-Extra: testing-gui
+Requires-Dist: pyautogui ; extra == 'testing-gui'
 

promnesia-1.3.20241021.dist-info/RECORD

@@ -0,0 +1,83 @@
+promnesia/__init__.py,sha256=8ZrCJe2kJb0DuYIiNeiUm0XU0nsjTlctcjdFoy9DVYw,457
+promnesia/__main__.py,sha256=y7Jgcc1uSKbxKhFDrlBlYh75kZm_unS9vH_3HvWCpnk,15421
+promnesia/cannon.py,sha256=TZ4b5P0qZpo0CVKCQLthyPFtAW5DYZT5ylttr5TmikI,24522
+promnesia/common.py,sha256=iUQh3Z-XqWxOPqnOHL8sBHEEJcGTztITBXnYizA02zs,20098
+promnesia/compare.py,sha256=vREaDTOjrGG43qOZEq2E9jV7fihO0U-KBMuguUMNgnk,4572
+promnesia/compat.py,sha256=cxk8ZOv0LnSwLCab_UrsxQWcKiiZnWgf2xK13L-ql4w,456
+promnesia/config.py,sha256=hFHdFbcB2q35gU1zlmtYj6VkHR_1SSH4U2M5G4HQa_g,4721
+promnesia/extract.py,sha256=m6D-QVA54ldcffcAby8w2OIDg0GPmAgDSGVWhSTwA60,2810
+promnesia/kjson.py,sha256=GPpeIpvtXwaocBw1W1QzPWN8UJcF0USSlaMxRSEkH-U,3400
+promnesia/logging.py,sha256=z3Otc_JzZREnMG0QerqoOR6zFJ6Ls6ATGydczCADZSY,5923
+promnesia/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/server.py,sha256=LBka2jYuzISVAyNIQaf8IRGIbcpSFo1g-vn8TI5VOhE,14647
+promnesia/sqlite.py,sha256=4jZhqMGviSZK5wNqd7PbVC0z-D3UWyeM0RXrabAruZ8,1378
+promnesia/database/common.py,sha256=9iENKM0pRL1PKrSAhGzNreDFsQNQBIGZWKuV5E8aalw,1843
+promnesia/database/dump.py,sha256=v5m5mdnY7Tu7Yu-mlQHAFW46Kgt4dJRIOWOGOs88gEY,6116
+promnesia/database/load.py,sha256=kP-HdNL_3q94P_Cj5pnxarHqFgCVJFrsbr0vogDRtFU,1357
+promnesia/misc/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/misc/config_example.py,sha256=MXxJCVjm9MrEwPR8SHTwkJRDcqWcqJWtfJg_M6yhRMo,364
+promnesia/misc/install_server.py,sha256=NEEGvpun7_lMBGXKvRGrq5wcGT_xgMm7MvPcftR15pI,4757
+promnesia/sources/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/sources/auto.py,sha256=hiecNxttUGFskR5H2W35K-6daEhhiPpH2ZRpOZBBr50,11695
+promnesia/sources/auto_logseq.py,sha256=1npVDEXB8wAM6XwTku0nZb57yQ9mM7vWct9albgvGxw,445
+promnesia/sources/auto_obsidian.py,sha256=UoVWAkdCbh7Ij3JFIy6ylYOd6776c7-z2xaR6b8snYc,205
+promnesia/sources/browser.py,sha256=qmuDzu0iJjpQc-WX2LEZ3TnpHhb9H-b5H-hZV7lbVkY,3087
+promnesia/sources/browser_legacy.py,sha256=rNZydnWnnQ3sAzgRgFYnuqQKHUg8MxS5iiAmHAh9rV8,10429
+promnesia/sources/demo.py,sha256=TQVVl6089TlEntiGSO-9EhqXBf9U_ipUnhsVqDCOspg,1033
+promnesia/sources/fbmessenger.py,sha256=Dpvxh_5_ozaqGOSHxCApoIda58Z33r0_N6Uk2BIMde8,997
+promnesia/sources/filetypes.py,sha256=2d6taAuGT4jvfCEqF9US8jukT6sBuN0wBEPJToCKiww,3643
+promnesia/sources/github.py,sha256=arDUZShPi8Dzz9EhYfk4j1OK3ugsADsjAhfXy00ng0c,2777
+promnesia/sources/guess.py,sha256=DMC7pj7rAA5yGZ6hMp31y4QvSugVNA9osexBLd5oCmc,803
+promnesia/sources/hackernews.py,sha256=zocx4XrP2QY5UW-ErSMej7lrYZH0xmotqaIBVGyNH7M,1247
+promnesia/sources/hpi.py,sha256=cHQlEJAH1EeTiawB1kcjXPt4UYDL1ZTNIHadBvd5QH4,383
+promnesia/sources/html.py,sha256=ill3XtIObZoK1a8FY6OZGoOwmqjtSiiiwmqjzSbZhQQ,1153
+promnesia/sources/hypothesis.py,sha256=qbiP6xJN53A6YkSUgA7hUa3ZD4KmtUnhJfAzXUHdt0o,1387
+promnesia/sources/instapaper.py,sha256=zIq6AClPb8Zfkdft7U4VgEgODlEeQRowygGz06DaGUQ,988
+promnesia/sources/markdown.py,sha256=KHHwxezCQRpcJLoyt_qB44NQwZGL7BnPT-Ehud21eR4,3871
+promnesia/sources/org.py,sha256=BD1DpDcxXWWjirKnrovY2tDQpgj5YAt175z45Z_o0jI,5589
+promnesia/sources/plaintext.py,sha256=8aYkmBYmXKnxyshWqgwUgQ03oG-OMKRi7DXsWtcgpVA,3138
+promnesia/sources/pocket.py,sha256=PWmjAgg8nSDubBv_4WuWmyupB6NP8SAuuKiV8ZLE4xY,1133
+promnesia/sources/reddit.py,sha256=u-Ou0xTZO-s9SM_GKHfNawLV5Yv7PmH4lggy_yq1XnA,5630
+promnesia/sources/roamresearch.py,sha256=_WKurvGea9JHMKlapQyH44Kfg-AD40fb7xse26It0Fo,1077
+promnesia/sources/rss.py,sha256=6Ijy6omXGjzMLkmwijq6JBRIY7ruArb-hXppRYY4ln0,824
+promnesia/sources/shellcmd.py,sha256=bXWfHv6XFmVUQHUUWn2E_C96tIdmIxb1-yqAQRtTYIk,2739
+promnesia/sources/signal.py,sha256=4ZBUXQoab0OG1r8ieZhSHpMNrzsFDomVXBhqb8hRYPA,14793
+promnesia/sources/smscalls.py,sha256=N6jMHHr4bUlQORQcTVeGsdrmW4ep7jpCEFSi26sRpxY,774
+promnesia/sources/stackexchange.py,sha256=_s8HJfhblVNspvbLEnuse5J8BvFFrboIdv5jURTZ2eE,658
+promnesia/sources/takeout.py,sha256=mv6j8mkDb8lJsv41iBsJBzFpbGp7jXIBWYcAFMTvdls,8017
+promnesia/sources/takeout_legacy.py,sha256=iaSLu73RutEx1UW4m-n3hk7-ISlVl2d-W0ZKP__20XU,4130
+promnesia/sources/telegram.py,sha256=hSI6zxQh2zqdBiIAGJzY2Vi5A_233jDy0tGU0Q-Q-EU,2979
+promnesia/sources/telegram_legacy.py,sha256=gRe6Exw-svMQZhlussmBGg0EJq81XadSa5-mh2m8ztI,4524
+promnesia/sources/twitter.py,sha256=_tIU0rQ3b96WXKIPJQRyRAzG7AAnPp9ANytrcVJ5Z0U,1886
+promnesia/sources/vcs.py,sha256=wMcgXr1Nd7DxM5j405ip4SPxJIqhdY4SePpp9qPL9q8,1654
+promnesia/sources/viber.py,sha256=C7e9AxN5sgC-SLyCdUapzwC3eGuA0WQUGfO8Br5r53A,5990
+promnesia/sources/website.py,sha256=GbDZChKK1_KLWXiLUUcZLOvep6h2AWmqECudwmupSl8,1811
+promnesia/sources/zulip.py,sha256=GN8YiUmFa3jLXFmNst47YILu0U-kPawbxDKY-Mtobf0,770
+promnesia/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/tests/common.py,sha256=2D9YspEWhKplbG6ZrVACkBjp_NEJB7qaXzOMjWHitCY,4092
+promnesia/tests/server_helper.py,sha256=Z1Rc_88WmOwukCyVR5Cb4XaXg4xewHiy4vlFAxQkoBU,2199
+promnesia/tests/test_cannon.py,sha256=VZPWZAtcAdRzQwVJ01qUIgvxB3VJl72Gqg0GI9tr8To,12711
+promnesia/tests/test_cli.py,sha256=g-9mgs4_tu6bGNZz3_swP3oj1L2V_o9ycjlFzIR9WYE,1404
+promnesia/tests/test_compare.py,sha256=JDQzFWaIMkKCn6YYKJPrZjdbZ6LZn5ig793q62LwSFc,945
+promnesia/tests/test_config.py,sha256=VqX2R0UcXmxp8O-p6C5Unk9W2Iv3RkIhvKe3ga84GHc,6534
+promnesia/tests/test_db_dump.py,sha256=PMGiTnp_cfDxLUfzYrchKKewoF84q6TooEw445K-WiA,6766
+promnesia/tests/test_extract.py,sha256=kD2iNodCj2OHM7_sQ_3DTRHJMZItJ7FuE855GpxZ3jM,2340
+promnesia/tests/test_extract_urls.py,sha256=Ybez16VdYoveL9b_50Ca9vbMogWHOvFqDOJPGZrzQJg,1961
+promnesia/tests/test_indexer.py,sha256=Oo93qqPrP0xXY1XhcRQkoJuEGpREfs6qoH335f_5_dI,8981
+promnesia/tests/test_server.py,sha256=c1HxEKlqTxbBrKu7tmg58G8Z9E4jNcxq-YENxxfVqIQ,10786
+promnesia/tests/test_traverse.py,sha256=DUN-NbrZ8b6MKrtZaOavAnZelMPk_aqN2ylvRV7UqHo,1350
+promnesia/tests/utils.py,sha256=TsRaLYN9slHHoNJmPSKwgoP82ICHBvEjT2G4scAraIQ,1136
+promnesia/tests/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promnesia/tests/sources/test_auto.py,sha256=wWqf7x_BXeVSYTDIaD6ZXqqRvqk312biY4iOGcMa4PY,1848
+promnesia/tests/sources/test_filetypes.py,sha256=6xhpN9hQb2U6FF4lAEIXWupk-t_HxQjNfk9A6krEnx4,1359
+promnesia/tests/sources/test_hypothesis.py,sha256=4anFCfBqzJueiDhhg2WtvITMIofAWRMB2-Ja1Cb2beY,1400
+promnesia/tests/sources/test_org.py,sha256=cefCAc7JVtrzHblbLHYB28tsjzfV1b4h80_JRO1Zamk,2572
+promnesia/tests/sources/test_plaintext.py,sha256=Bn7v2HhL1FSLqCKPy-BX5OLy1PlWIyYjvRTLAWrXVWg,827
+promnesia/tests/sources/test_shellcmd.py,sha256=K4hDQl3yd4t-6JFeDtB1gtH2XFImSYdYe1pG0UOO-Uw,678
+promnesia/tests/sources/test_takeout.py,sha256=HVreW_4pZP8TjGNmrJva5JJfkexmOwwr7cRxwU1Qg_Q,1557
+promnesia-1.3.20241021.dist-info/LICENSE,sha256=rgO9acPmnw53ZBxiXBdp8kfxmRcekhg_Q7HN65BPihs,1074
+promnesia-1.3.20241021.dist-info/METADATA,sha256=YLJgGCkJ4LTXKid3Ieq-hBk5aJVc3c000Diutdk-RlI,1903
+promnesia-1.3.20241021.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+promnesia-1.3.20241021.dist-info/entry_points.txt,sha256=hz1qfzQSRh4kkVkJWk4hnYqE9A1nobEbKlLG_0nNxzE,54
+promnesia-1.3.20241021.dist-info/top_level.txt,sha256=7yvIpooFiuNLf9yLdu9MTADz57z0YTAqSu7aSG9ujSU,10
+promnesia-1.3.20241021.dist-info/RECORD,,

{promnesia-1.2.20230515.dist-info → promnesia-1.3.20241021.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{promnesia-1.2.20230515.dist-info → promnesia-1.3.20241021.dist-info}/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 promnesia = promnesia.__main__:main
-

promnesia/dump.py

@@ -1,105 +0,0 @@
-from pathlib import Path
-import shutil
-from typing import  List, Set, Iterable
-
-from more_itertools import chunked
-
-from sqlalchemy import create_engine, MetaData, Table, event, text
-
-from cachew import NTBinder
-
-from .common import get_logger, DbVisit, get_tmpdir, Res, now_tz, Loc
-from . import config
-
-
-# NOTE: I guess the main performance benefit from this is not creating too many tmp lists and avoiding overhead
-# since as far as sql is concerned it should all be in the same transaction. only a guess
-# not sure it's the proper way to handle it
-# see test_index_many
-_CHUNK_BY = 10
-
-# I guess 1 hour is definitely enough
-_CONNECTION_TIMEOUT_SECONDS = 3600
-
-# returns critical warnings
-def visits_to_sqlite(vit: Iterable[Res[DbVisit]], *, overwrite_db: bool) -> List[Exception]:
-    logger = get_logger()
-    db_path = config.get().db
-
-    now = now_tz()
-    ok = 0
-    errors = 0
-    def vit_ok() -> Iterable[DbVisit]:
-        nonlocal errors, ok
-        for v in vit:
-            if isinstance(v, DbVisit):
-                ok += 1
-                yield v
-            else:
-                errors += 1
-                # conform to the schema and dump. can't hurt anyway
-                ev = DbVisit(
-                    norm_url='<error>',
-                    orig_url='<error>',
-                    dt=now,
-                    locator=Loc.make('<errror>'),
-                    src='error',
-                    # todo attach backtrace?
-                    context=repr(v),
-                )
-                yield ev
-
-    tpath = Path(get_tmpdir().name) / 'promnesia.tmp.sqlite'
-    if overwrite_db:
-        # here we don't need timeout, since it's a brand new DB
-        engine = create_engine(f'sqlite:///{tpath}')
-    else:
-        # here we need a timeout, othewise concurrent indexing might not work
-        # (note that this also needs WAL mode)
-        # see test_concurrent_indexing
-        engine = create_engine(f'sqlite:///{db_path}', connect_args={'timeout': _CONNECTION_TIMEOUT_SECONDS})
-
-    # using WAL keeps database readable while we're writing in it
-    # this is tested by test_query_while_indexing
-    def enable_wal(dbapi_con, con_record) -> None:
-        dbapi_con.execute('PRAGMA journal_mode = WAL')
-    event.listen(engine, 'connect', enable_wal)
-
-    binder = NTBinder.make(DbVisit)
-    meta = MetaData()
-    table = Table('visits', meta, *binder.columns)
-
-    cleared: Set[str] = set()
-    ncleared = 0
-    with engine.begin() as conn:
-        table.create(conn, checkfirst=True)
-
-        for chunk in chunked(vit_ok(), n=_CHUNK_BY):
-            srcs = set(v.src or '' for v in chunk)
-            new = srcs.difference(cleared)
-
-            for src in new:
-                conn.execute(table.delete().where(table.c.src == src))
-                cursor = conn.execute(text("SELECT changes()")).fetchone()
-                assert cursor is not None
-                ncleared += cursor[0]
-                cleared.add(src)
-
-            bound = [binder.to_row(x) for x in chunk]
-            # pylint: disable=no-value-for-parameter
-            conn.execute(table.insert().values(bound))
-    engine.dispose()
-
-    if overwrite_db:
-        shutil.move(str(tpath), str(db_path))
-
-    errs = '' if errors == 0 else f', {errors} ERRORS'
-    total = ok + errors
-    what = 'overwritten' if overwrite_db else 'updated'
-    logger.info(
-        '%s database "%s". %d total (%d OK%s, %d cleared, +%d more)',
-        what, db_path, total, ok, errs, ncleared, ok - ncleared)
-    res: List[Exception] = []
-    if total == 0:
-        res.append(RuntimeError('No visits were indexed, something is probably wrong!'))
-    return res