promnesia 1.2.20230515__py3-none-any.whl → 1.3.20241021__py3-none-any.whl

promnesia/sources/viber.py

@@ -2,17 +2,18 @@
 Collects visits from Viber desktop app (e.g. `~/.ViberPC/XYZ123/viber.db`)
 """
 
+from __future__ import annotations
+
 import logging
+import sqlite3
 import textwrap
+from collections.abc import Iterable
 from os import PathLike
 from pathlib import Path
-import sqlite3
-from typing import Iterable, Optional
 
 from ..common import Loc, PathIsh, Results, Visit, extract_urls, from_epoch, join_tags
 from ..sqlite import sqlite_connection
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -34,12 +35,12 @@ def index(
 
     msgs_query = messages_query(http_only)
 
-    for db_path in _get_files(db_path):
-        assert db_path.is_file(), f"Is it a (Viber-desktop sqlite) file? {db_path}"
-        yield from _harvest_db(db_path, msgs_query, locator_schema)
+    for db in _get_files(db_path):
+        assert db.is_file(), f"Is it a (Viber-desktop sqlite) file? {db}"
+        yield from _harvest_db(db, msgs_query, locator_schema)
 
 
-def messages_query(http_only: Optional[bool]) -> str:
+def messages_query(http_only: bool | None) -> str:
     """
     An SQL-query returning 1 row for each message
 
@@ -123,7 +124,7 @@ def _handle_row(row: sqlite3.Row, db_path: PathLike, locator_schema: str) -> Res
     tags: str = row["tags"]
     url_title: str = row["url_title"]
 
-    assert (
+    assert (  # noqa: PT018
         text and mid and sender and chatname
     ), f"sql-query should eliminate messages without 'http' or missing ids: {row}"
 
@@ -154,7 +155,7 @@ def _get_files(path: PathIsh) -> Iterable[Path]:
     """
     path = Path(path).expanduser()
     parts = path.parts[1:] if path.is_absolute() else path.parts
-    return Path(path.root).glob(str(Path("").joinpath(*parts)))
+    return Path(path.root).glob(str(Path("").joinpath(*parts)))  # noqa: PTH201
 
 
 def _harvest_db(db_path: PathIsh, msgs_query: str, locator_schema: str) -> Results:

promnesia/sources/website.py

@@ -2,12 +2,12 @@
 Clones a website with wget and indexes via sources.auto
 '''
 
-from pathlib import Path
 import re
+from collections.abc import Iterable
+from pathlib import Path
 from subprocess import run
-from typing import Iterable
 
-from ..common import Extraction, PathIsh, get_tmpdir, slugify, get_logger
+from promnesia.common import Extraction, PathIsh, get_logger, get_tmpdir, slugify
 
 
 def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
@@ -30,7 +30,7 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
     ]
     # TODO follow sitemap? e.g. gwern
     logger.info(' '.join(cmd))
-    res = run(cmd)
+    res = run(cmd, check=False)
 
     if res.returncode == 8:
         # man wget: 8 means server error (e.g. broken link)

promnesia/sources/zulip.py

@@ -2,12 +2,13 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Zulip data.
 '''
 
-from ..common import Results, Visit, Loc, iter_urls
+from promnesia.common import Loc, Results, Visit, iter_urls
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.zulip.organization as Z
+
     for m in Z.messages():
         if isinstance(m, Exception):
             yield m

promnesia/sqlite.py

@@ -1,6 +1,9 @@
-from contextlib import contextmanager
+from __future__ import annotations
+
 import sqlite3
-from typing import Callable, Optional, Any, Iterator, Union, Literal
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any, Callable, Literal, Union
 
 from .common import PathIsh
 
@@ -10,13 +13,13 @@ SqliteRowFactory = Callable[[sqlite3.Cursor, sqlite3.Row], Any]
 
 def dict_factory(cursor, row):
     fields = [column[0] for column in cursor.description]
-    return {key: value for key, value in zip(fields, row)}
+    return dict(zip(fields, row))
 
 
 Factory = Union[SqliteRowFactory, Literal['row', 'dict']]
 
 @contextmanager
-def sqlite_connection(db: PathIsh, *, immutable: bool=False, row_factory: Optional[Factory]=None) -> Iterator[sqlite3.Connection]:
+def sqlite_connection(db: PathIsh, *, immutable: bool=False, row_factory: Factory | None=None) -> Iterator[sqlite3.Connection]:
     dbp = f'file:{db}'
     # https://www.sqlite.org/draft/uri.html#uriimmutable
     if immutable:

promnesia/tests/common.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import gc
+import inspect
+import os
+import socket
+import sys
+from collections.abc import Iterator
+from contextlib import closing, contextmanager
+from pathlib import Path
+from textwrap import dedent
+from typing import NoReturn, TypeVar
+
+import pytest
+
+from ..common import Res, _is_windows
+
+
+def under_ci() -> bool:
+    return 'CI' in os.environ
+
+
+def throw(x: Exception) -> NoReturn:
+    '''
+    like raise, but can be an expression...
+    '''
+    raise x
+
+
+@pytest.fixture
+def gc_control(*, gc_on: bool):
+    if gc_on:
+        # no need to do anything, should be on by default
+        yield
+        return
+
+    gc.disable()
+    try:
+        yield
+    finally:
+        gc.enable()
+
+
+running_on_ci = 'CI' in os.environ
+
+
+GIT_ROOT = Path(__file__).absolute().parent.parent.parent.parent
+TESTDATA = GIT_ROOT / 'tests/testdata'
+
+
+def get_testdata(path: str) -> Path:
+    assert TESTDATA.is_dir()
+    res = TESTDATA / path
+    if not res.exists():
+        raise RuntimeError(f"'{res}' not found! You propably need to run 'git submodule update --init --recursive'")
+    return TESTDATA / path
+
+
+@contextmanager
+def tmp_popen(*args, **kwargs):
+    import psutil
+    with psutil.Popen(*args, **kwargs) as p:
+        try:
+            yield p
+        finally:
+            for c in p.children(recursive=True):
+                c.kill()
+            p.kill()
+            p.wait()
+
+
+# meh
+def promnesia_bin(*args):
+    # not sure it's a good idea to diverge, but not sure if there's a better way either?
+    # ugh. on windows there is no bash so can't use the script
+    # whatever...
+    if under_ci() or _is_windows:
+        # should be able to use the installed version
+        return [sys.executable, '-m', 'promnesia', *args]
+    else:
+        # use version from the repository
+        root = Path(__file__).parent.parent.parent.parent
+        pm = root / 'scripts/promnesia'
+        return [pm, *args]
+
+
+# meh... not great
+@pytest.fixture
+def reset_filters():
+    from .. import extract
+
+    extract.filters.cache_clear()
+    try:
+        yield
+    finally:
+        extract.filters.cache_clear()
+
+
+# TODO could be a TypeGuard from 3.10
+V = TypeVar('V')
+
+def unwrap(r: Res[V]) -> V:
+    assert not isinstance(r, Exception), r
+    return r
+
+
+def write_config(path: Path, gen, **kwargs) -> None:
+    output_dir = path.parent
+    cfg_src = dedent('\n'.join(inspect.getsource(gen).splitlines()[1:])) + f"\nOUTPUT_DIR = r'{output_dir}'"
+    for k, v in kwargs.items():
+        assert k in cfg_src, k
+        cfg_src = cfg_src.replace(k, repr(str(v)))  # meh
+    path.write_text(cfg_src)
+
+
+@contextmanager
+def free_port() -> Iterator[int]:
+    # this is a generator to make sure there are no race conditions between the time we call this and launch program
+    #
+    # also some relevant articles about this 'technique'
+    # - https://eklitzke.org/binding-on-port-zero
+    # - https://idea.popcount.org/2014-04-03-bind-before-connect
+    # - https://blog.cloudflare.com/the-quantum-state-of-a-tcp-port
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        if sys.platform == 'linux':
+            # Ok, so from what I've been reading, SO_REUSEADDR should only be necessary in the program that reuses the port
+            # However, this answer (or man socket) claims we need it on both sites in Linux? see https://superuser.com/a/587955/300795
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        # also not sure where REUSEADDR is set in uvicorn (e.g. here reuse_address isn't passed?)
+        # https://github.com/encode/uvicorn/blob/6d666d99a285153bc4613e811543c39eca57054a/uvicorn/server.py#L162C37-L162C50
+        # but from strace looks like it is called somewhere :shrug:
+
+        # assign euphemeral port
+        # see table in
+        # https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ/14388707#14388707
+        # we rely on server binding to localhost later (or anything except 0.0.0.0 really)
+        s.bind(('', 0))
+
+        port = s.getsockname()[1]
+        yield port

promnesia/tests/server_helper.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import sys
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import psutil
+import requests
+
+from ..common import PathIsh
+from .common import free_port, promnesia_bin, tmp_popen
+
+
+@dataclass
+class Helper:
+    host: str
+    port: str
+    process: psutil.Popen
+
+    def get(self, path: str):
+        # check it's alive first so the error is cleaner
+        assert self.process.poll() is None, self.process
+        return requests.get(f'http://{self.host}:{self.port}' + path)
+
+    def post(self, path: str, *, json: dict[str, Any] | None = None):
+        assert self.process.poll() is None, self.process
+        return requests.post(f'http://{self.host}:{self.port}' + path, json=json)
+
+
+@contextmanager
+def run_server(db: PathIsh | None = None, *, timezone: str | None = None) -> Iterator[Helper]:
+    # TODO not sure, perhaps best to use a thread or something?
+    # but for some tests makes more sense to test in a separate process
+    with free_port() as pp:
+        # ugh. under docker 'localhost' tries to bind it to ipv6 (::1) for some reason???
+        host = '0.0.0.0' if Path('/.dockerenv').exists() else 'localhost'
+        port = str(pp)
+        args = [
+            'serve',
+            '--host', host,
+            '--quiet',
+            '--port', port,
+            *([] if timezone is None else ['--timezone', timezone]),
+            *([] if db is None else ['--db'  , str(db)]),
+        ]
+        with tmp_popen(promnesia_bin(*args)) as server_process:
+            server = Helper(host=host, port=port, process=server_process)
+
+            # wait till ready
+            for _ in range(50):
+                try:
+                    server.get('/status').json()
+                    break
+                except:
+                    time.sleep(0.1)
+            else:
+                raise RuntimeError("Cooldn't connect to '{st}' after 50 attempts")
+            print(f"Started server up, db: {db}", file=sys.stderr)
+
+            yield server
+
+            # TODO use logger!
+            print("Done with the server", file=sys.stderr)

promnesia/tests/sources/test_auto.py

@@ -0,0 +1,65 @@
+import os
+from itertools import groupby
+
+from ...sources import auto
+from ..common import get_testdata, throw
+
+sa2464 = 'https://www.scottaaronson.com/blog/?p=2464'
+
+_JSON_URLS = {
+    'https://johncarlosbaez.wordpress.com/2016/09/09/struggles-with-the-continuum-part-2/',
+    sa2464,
+}
+
+
+def makemap(visits):
+    key = lambda v: v.url
+
+    def it():
+        vit = (throw(v) if isinstance(v, Exception) else v for v in visits)
+        for k, g in groupby(sorted(vit, key=key), key=key):
+            yield k, sorted(g)
+
+    return dict(it())
+
+
+def test_json() -> None:
+    mm = makemap(auto.index(get_testdata('auto'), ignored='*/orgs/*'))
+    assert mm.keys() == _JSON_URLS
+
+    # TODO not sure if they deserve separate visits..
+    [v1, v2] = mm[sa2464]
+    assert v1.context == 'list::yyy::given_url'
+    # todo not sure if editor:// work on Windows
+    assert v1.locator.href.startswith('editor://')
+    assert v1.locator.href.endswith('pocket.json')
+    # TODO line number?
+
+
+def test_auto() -> None:
+    mm = makemap(auto.index(get_testdata('auto')))
+    org_link = 'https://www.youtube.com/watch?v=rHIkrotSwcc'
+    assert {
+        *_JSON_URLS,
+        org_link,
+    }.issubset(mm.keys())
+
+    [v] = mm[org_link]
+    assert v.locator.title == 'orgs' + os.sep + 'file.org:14'  # meh
+    assert v.locator.href.endswith('file.org:14')
+    assert "xxx /r/cpp" in v.context
+    assert "I've enjoyed [Chandler Carruth's" in v.context
+
+
+def test_obsidian() -> None:
+    mm = makemap(auto.index(get_testdata('obsidian-vault')))
+    example_url = 'https://example.com'
+    [v] = mm[example_url]
+    assert v.locator.href.startswith('obsidian://')
+
+
+def test_logseq() -> None:
+    mm = makemap(auto.index(get_testdata('logseq-graph')))
+    example_url = 'https://example.com'
+    [v] = mm[example_url]
+    assert v.locator.href.startswith('logseq://')

promnesia/tests/sources/test_filetypes.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+
+from ...common import PathIsh
+from ...common import _is_windows as windows
+from ...sources.auto import by_path
+
+
+def handled(p: PathIsh) -> bool:
+    idx, m = by_path(Path(p))
+    return idx is not None
+    # ideally these won't hit libmagic path (would try to open the file and cause FileNotFoundError)
+
+
+def test_filetypes() -> None:
+    # test media
+    for ext in 'avi mp4 mp3 webm'.split() + ([] if windows else 'mkv'.split()):
+        assert handled('file.' + ext)
+
+    # images
+    for ext in 'gif jpg png jpeg'.split():
+        assert handled('file.' + ext)
+
+    # TODO more granual checks that these are ignored?
+    # binaries
+    for ext in 'o sqlite'.split() + ([] if windows else 'class jar'.split()):
+        assert handled('file.' + ext)
+
+    # these might have potentially some links
+    for ext in [
+        'svg',
+        'pdf', 'epub', 'ps',
+        'doc', 'ppt', 'xsl',
+        # seriously, windows doesn't know about docx???
+        *([] if windows else 'docx pptx xlsx'.split()),
+        *([] if windows else 'ods odt rtf'.split()),
+    ] + ([] if windows else 'djvu'.split()):
+        assert handled('file.' + ext)
+
+    # source code
+    for ext in 'rs tex el js sh hs pl h py hpp c go css'.split() + ([] if windows else 'java cpp'.split()):
+        assert handled('file.' + ext)
+
+    assert handled('x.html')

promnesia/tests/sources/test_hypothesis.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from my.core.cfg import tmp_config
+
+from ...__main__ import do_index
+from ...database.load import get_all_db_visits
+from ..common import get_testdata, write_config
+
+
+def index_hypothesis(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import hypothesis
+
+        SOURCES = [Source(hypothesis.index, name='hyp')]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+
+    class hpi_config:
+        class hypothesis:
+            export_path = get_testdata('hypexport/testdata') / 'netrights-dashboard-mockup/data/*.json'
+
+    with tmp_config(modules='my.hypothesis', config=hpi_config):
+        do_index(cfg_path)
+
+
+def test_hypothesis(tmp_path: Path) -> None:
+    index_hypothesis(tmp_path)
+
+    visits = get_all_db_visits(tmp_path / 'promnesia.sqlite')
+    assert len(visits) > 100
+
+    [vis] = [x for x in visits if 'fundamental fact of evolution' in (x.context or '')]
+
+    assert vis.norm_url == 'wired.com/2017/04/the-myth-of-a-superhuman-ai'
+    assert vis.orig_url == 'https://www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
+    assert vis.locator.href == 'https://hyp.is/_Z9ccmVZEeexBOO7mToqdg/www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
+    assert 'misconception about evolution is fueling misconception about AI' in (vis.context or '')  # contains notes as well

promnesia/tests/sources/test_org.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from ...common import Visit
+from ...sources.org import extract_from_file
+from ..common import get_testdata, throw
+
+
+def delrf(s: str | None) -> str | None:
+    if s is None:
+        return None
+    # meh.. not sure how ot handle this properly, ideally should be via pytest?
+    # not sure if should just do it in the indexer? e.g. extension might not like it
+    return s.replace('\r', '')
+
+
+def test_org_indexer() -> None:
+    [_, cpp, cozy] = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file.org'))]
+
+    assert cpp.url == 'https://www.youtube.com/watch?v=rHIkrotSwcc'
+    # TODO not sure about filetags?
+    exp = '''
+xxx /r/cpp   :cpp:programming:
+ I've enjoyed [Chandler Carruth's _There Are No Zero-cost Abstractions_](
+ https://www.youtube.com/watch?v=rHIkrotSwcc) very much.
+
+'''.lstrip()
+    assert delrf(cpp.context) == exp
+
+    assert cozy.url == 'https://twitter.com/Mappletons/status/1255221220263563269'
+
+
+def test_org_indexer_2() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file3.org'))]
+
+    assert len(items) == 6
+    assert items[0].url == 'https://www.reddit.com/r/androidapps/comments/4i36z9/how_you_use_your_android_to_the_maximum/d2uq24i'
+    assert items[1].url == 'https://link.com'
+    assert items[-2].url == 'https://en.wikipedia.org/wiki/Resilio_Sync'
+    # TODO shit def need org specific url extractor (and then extract from everything remaining)
+    # assert results[-1].url == 'https://en.wikipedia.org/wiki/InterPlanetary_File_System'
+
+
+def test_heading() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file2.org'))]
+    assert {i.url for i in items} == {
+        'https://en.wikipedia.org/wiki/Computational_topology',
+        'http://graphics.stanford.edu/courses/cs468-09-fall/',
+        'https://en.wikipedia.org/wiki/Triangulation_(topology)',
+        'https://en.wikipedia.org/wiki/Digital_manifold',
+    }
+
+
+def test_url_in_properties() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file4.org'))]
+
+    assert len(items) == 2, items
+    assert items[0].url == 'https://example.org/ref_example'
+    assert items[1].url == 'http://example.org/a_test'
+
+
+def test_5() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file5.org'))]
+
+    assert len(items) == 0  # shouldn't crash at least

promnesia/tests/sources/test_plaintext.py

@@ -0,0 +1,25 @@
+from ...common import Source
+from ...extract import extract_visits
+from ...sources import plaintext, shellcmd
+from ..common import get_testdata, unwrap
+
+
+def test_plaintext_path_extractor() -> None:
+    visits = list(extract_visits(
+        Source(
+            shellcmd.index,
+            plaintext.extract_from_path(get_testdata('custom')),
+        ),
+        src='whatever',
+    ))
+    assert {unwrap(v).orig_url for v in visits} == {
+        'http://google.com',
+        'http://google.com/',
+        'http://some-weird-domain.xyz/whatever',
+        'https://google.com',
+        'http://what.about.this.link',
+    }
+
+    [wa] = [v for v in visits if unwrap(v).orig_url == 'http://what.about.this.link']
+    f2 = get_testdata('custom') / 'file2.txt'
+    assert unwrap(wa).locator.href == f'editor://{f2}:3'  # occurs line 3

promnesia/tests/sources/test_shellcmd.py

@@ -0,0 +1,21 @@
+import pytest
+
+from ...common import Source, _is_windows
+from ...extract import extract_visits
+from ...sources import shellcmd
+from ..common import get_testdata
+
+
+@pytest.mark.skipif(_is_windows, reason="no grep on windows")
+def test_via_grep() -> None:
+
+    visits = list(extract_visits(
+        Source(
+            shellcmd.index,
+            # meh. maybe should deprecate plain string here...
+            r"""grep -Eo -r --no-filename (http|https)://\S+ """ + str(get_testdata('custom')),
+        ),
+        src='whatever',
+    ))
+    # TODO I guess filtering of equivalent urls should rather be tested on something having context (e.g. org mode)
+    assert len(visits) == 5

promnesia/tests/sources/test_takeout.py

@@ -0,0 +1,56 @@
+from datetime import datetime, timezone
+
+import pytest
+from my.core.cfg import tmp_config
+
+from ...common import Source
+from ...extract import extract_visits
+from ...sources import takeout
+from ..common import get_testdata, unwrap
+
+
+# TODO apply in conftest so it's used in all tests?
+@pytest.fixture
+def no_cachew():
+    from my.core.cachew import disabled_cachew
+
+    with disabled_cachew():
+        yield
+
+
+# todo testing this logic probably belongs to hpi or google_takeout_export, but whatever
+def test_takeout_directory(no_cachew) -> None:
+    class config:
+        class google:
+            takeout_path = get_testdata('takeout')
+
+    with tmp_config(modules='my.google.takeout.*', config=config):
+        visits = list(extract_visits(Source(takeout.index), src='takeout'))
+
+    assert len(visits) == 3
+    assert all(unwrap(v).dt.tzinfo is not None for v in visits)
+
+
+def test_takeout_zip(no_cachew) -> None:
+    class config:
+        class google:
+            takeout_path = get_testdata('takeout-20150518T000000Z.zip')
+
+    with tmp_config(modules='my.google.takeout.*', config=config):
+        visits = list(extract_visits(Source(takeout.index), src='takeout'))
+
+    assert len(visits) == 3
+    assert all(unwrap(v).dt.tzinfo is not None for v in visits)
+
+    [vis] = [v for v in visits if unwrap(v).norm_url == 'takeout.google.com/settings/takeout']
+
+    edt = datetime(
+        year=2018,
+        month=9,
+        day=18,
+        hour=5,
+        minute=48,
+        second=23,
+        tzinfo=timezone.utc,
+    )
+    assert unwrap(vis).dt == edt