promnesia 1.1.20230129__py3-none-any.whl → 1.2.20240810__py3-none-any.whl

promnesia/sources/telegram_legacy.py

@@ -0,0 +1,117 @@
+'''
+Uses [[https://github.com/fabianonline/telegram_backup#readme][telegram_backup]] database for messages data
+'''
+
+from pathlib import Path
+import sqlite3
+from textwrap import dedent
+from typing import Union, TypeVar
+from urllib.parse import unquote # TODO mm, make it easier to rememember to use...
+
+from ..common import PathIsh, Visit, get_logger, Loc, extract_urls, from_epoch, Results, echain
+from ..sqlite import sqlite_connection
+
+T = TypeVar("T")
+
+
+def unwrap(res: Union[T, Exception]) -> T:
+    if isinstance(res, Exception):
+        raise res
+    else:
+        return res
+
+
+def index(database: PathIsh, *, http_only: bool=False) -> Results:
+    """
+    :param database:
+        the path of the sqlite generated by the _telegram_backup_ java program
+    :param http_only:
+        when true, do not collect IP-addresses and `python.py` strings
+    """
+    logger = get_logger()
+
+    path = Path(database)
+    assert path.is_file(), path
+
+    def make_query(text_query: str) -> str:
+        extra_criteria = "AND (M.has_media == 1 OR text LIKE '%http%')" if http_only else ""
+        return dedent(
+            f"""
+            WITH entities AS (
+            SELECT 'dialog' as type
+                , id
+                , coalesce(username, id) as handle
+                , coalesce(first_name || " " || last_name
+                    , username
+                    , id
+                ) as display_name FROM users
+            UNION
+            SELECT 'group' as type
+                , id
+                , id as handle
+                , coalesce(name, id) as display_name FROM chats
+            )
+            SELECT src.display_name AS chatname
+                , src.handle       AS chat
+                , snd.display_name AS sender
+                , M.time           AS time
+                , {text_query}     AS text
+                , M.message_id     AS mid
+            FROM messages AS M
+                                                                                /* chat types are 'dialog' (1-1), 'group' and 'supergroup' */
+                                                                                /* this is abit hacky way to handle all groups in one go */
+            LEFT JOIN entities AS src    ON M.source_id = src.id AND src.type = (CASE M.source_type WHEN 'supergroup' THEN 'group' ELSE M.source_type END)
+            LEFT JOIN entities AS snd    ON M.sender_id = snd.id AND snd.type = 'dialog'
+            WHERE
+                M.message_type NOT IN ('service_message', 'empty_message')
+                {extra_criteria}
+            ORDER BY time;
+            """)
+
+    with sqlite_connection(path, immutable=True, row_factory='row') as db:
+        # TODO yield error if chatname or chat or smth else is null?
+        for row in db.execute(make_query('M.text')):
+            try:
+                yield from _handle_row(row)
+            except Exception as ex:
+                yield echain(RuntimeError(f'While handling {row}'), ex)
+
+        # old (also 'stable') version doesn't have 'json' column yet...
+        messages_columns = [d[0] for d in db.execute('SELECT * FROM messages').description]
+        # todo hmm what is 'markup_json'??
+        if 'json' in messages_columns:
+            for row in db.execute(make_query("json_extract(json, '$.media.webpage.description')")):
+                try:
+                    yield from _handle_row(row)
+                except Exception as ex:
+                    yield echain(RuntimeError(f'While handling {row}'), ex)
+
+
+def _handle_row(row: sqlite3.Row) -> Results:
+    text = row['text']
+    if text is None:
+        return
+    urls = extract_urls(text)
+    if len(urls) == 0:
+        return
+    dt            = from_epoch(row['time'])
+    mid: str      = unwrap(row['mid'])
+
+    # TODO perhaps we could be defensive with null sender/chat etc and still emit the Visit
+    sender: str   = unwrap(row['sender'])
+    chatname: str = unwrap(row['chatname'])
+    chat: str     = unwrap(row['chat'])
+
+    in_context = f'https://t.me/{chat}/{mid}'
+    for u in urls:
+        # https://www.reddit.com/r/Telegram/comments/6ufwi3/link_to_a_specific_message_in_a_channel_possible/
+        # hmm, only seems to work on mobile app, but better than nothing...
+        yield Visit(
+            url=unquote(u),
+            dt=dt,
+            context=f"{sender}: {text}",
+            locator=Loc.make(
+                title=f"chat with {chatname}",
+                href=in_context,
+            ),
+        )

promnesia/sources/vcs.py

@@ -5,10 +5,10 @@ Clones & indexes Git repositories (via sources.auto)
 
 from pathlib import Path
 import re
+from subprocess import check_call
 from typing import Iterable
 
 from ..common import Extraction, PathIsh, get_tmpdir, slugify
-from ..compat import check_call
 
 
 def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:

promnesia/sources/viber.py

@@ -1,14 +1,16 @@
 """
-Adapted from `telegram.py` to read from `~/.ViberPC/XYZ123/viber.db`
+Collects visits from Viber desktop app (e.g. `~/.ViberPC/XYZ123/viber.db`)
 """
 
 import logging
 import textwrap
 from os import PathLike
 from pathlib import Path
+import sqlite3
 from typing import Iterable, Optional
 
 from ..common import Loc, PathIsh, Results, Visit, extract_urls, from_epoch, join_tags
+from ..sqlite import sqlite_connection
 
 
 logger = logging.getLogger(__name__)
@@ -37,17 +39,6 @@ def index(
         yield from _harvest_db(db_path, msgs_query, locator_schema)
 
 
-# TODO move to common?
-def _dataset_readonly(db: Path):
-    # see https://github.com/pudo/dataset/issues/136#issuecomment-128693122
-    import sqlite3
-
-    import dataset  # type: ignore
-
-    creator = lambda: sqlite3.connect(f"file:{db}?immutable=1", uri=True)
-    return dataset.connect("sqlite:///", engine_kwargs={"creator": creator})
-
-
 def messages_query(http_only: Optional[bool]) -> str:
     """
     An SQL-query returning 1 row for each message
@@ -118,7 +109,7 @@ def messages_query(http_only: Optional[bool]) -> str:
     )
 
 
-def _handle_row(row: dict, db_path: PathLike, locator_schema: str) -> Results:
+def _handle_row(row: sqlite3.Row, db_path: PathLike, locator_schema: str) -> Results:
     text = row["text"]
     urls = extract_urls(text)
     if not urls:
@@ -173,8 +164,8 @@ def _harvest_db(db_path: PathIsh, msgs_query: str, locator_schema: str) -> Resul
     # but it's safer for debugging resolved.
     db_path = Path(db_path).resolve()
 
-    with _dataset_readonly(db_path) as db:
-        for row in db.query(msgs_query):
+    with sqlite_connection(db_path, immutable=True, row_factory='row') as db:
+        for row in db.execute(msgs_query):
             try:
                 yield from _handle_row(row, db_path, locator_schema)
             except Exception as ex:

promnesia/sources/website.py

@@ -4,7 +4,7 @@ Clones a website with wget and indexes via sources.auto
 
 from pathlib import Path
 import re
-from ..compat import check_call, run
+from subprocess import run
 from typing import Iterable
 
 from ..common import Extraction, PathIsh, get_tmpdir, slugify, get_logger

promnesia/sqlite.py

@@ -0,0 +1,42 @@
+from contextlib import contextmanager
+import sqlite3
+from typing import Callable, Optional, Any, Iterator, Union, Literal
+
+from .common import PathIsh
+
+# NOTE: copy pasted from HPI
+
+SqliteRowFactory = Callable[[sqlite3.Cursor, sqlite3.Row], Any]
+
+def dict_factory(cursor, row):
+    fields = [column[0] for column in cursor.description]
+    return {key: value for key, value in zip(fields, row)}
+
+
+Factory = Union[SqliteRowFactory, Literal['row', 'dict']]
+
+@contextmanager
+def sqlite_connection(db: PathIsh, *, immutable: bool=False, row_factory: Optional[Factory]=None) -> Iterator[sqlite3.Connection]:
+    dbp = f'file:{db}'
+    # https://www.sqlite.org/draft/uri.html#uriimmutable
+    if immutable:
+        dbp = f'{dbp}?immutable=1'
+    row_factory_: Any = None
+    if row_factory is not None:
+        if callable(row_factory):
+            row_factory_ = row_factory
+        elif row_factory == 'row':
+            row_factory_ = sqlite3.Row
+        elif row_factory == 'dict':
+            row_factory_ = dict_factory
+        else:
+            raise RuntimeError("should not happen")
+
+    conn = sqlite3.connect(dbp, uri=True)
+    try:
+        conn.row_factory = row_factory_
+        with conn:
+            yield conn
+    finally:
+        # Connection context manager isn't actually closing the connection, only keeps transaction
+        conn.close()

promnesia/tests/common.py

@@ -0,0 +1,137 @@
+from contextlib import closing, contextmanager
+import gc
+import inspect
+import os
+from pathlib import Path
+import socket
+import sys
+from textwrap import dedent
+from typing import Iterator, NoReturn, TypeVar
+
+import pytest
+
+from ..common import _is_windows, Res
+
+
+def under_ci() -> bool:
+    return 'CI' in os.environ
+
+
+def throw(x: Exception) -> NoReturn:
+    '''
+    like raise, but can be an expression...
+    '''
+    raise x
+
+
+@pytest.fixture
+def gc_control(gc_on: bool):
+    if gc_on:
+        # no need to do anything, should be on by default
+        yield
+        return
+
+    gc.disable()
+    try:
+        yield
+    finally:
+        gc.enable()
+
+
+running_on_ci = 'CI' in os.environ
+
+
+GIT_ROOT = Path(__file__).absolute().parent.parent.parent.parent
+TESTDATA = GIT_ROOT / 'tests/testdata'
+
+
+def get_testdata(path: str) -> Path:
+    assert TESTDATA.is_dir()
+    res = TESTDATA / path
+    if not res.exists():
+        raise RuntimeError(f"'{res}' not found! You propably need to run 'git submodule update --init --recursive'")
+    return TESTDATA / path
+
+
+@contextmanager
+def tmp_popen(*args, **kwargs):
+    import psutil
+    with psutil.Popen(*args, **kwargs) as p:
+        try:
+            yield p
+        finally:
+            for c in p.children(recursive=True):
+                c.kill()
+            p.kill()
+            p.wait()
+
+
+# meh
+def promnesia_bin(*args):
+    # not sure it's a good idea to diverge, but not sure if there's a better way either?
+    # ugh. on windows there is no bash so can't use the script
+    # whatever...
+    if under_ci() or _is_windows:
+        # should be able to use the installed version
+        return [sys.executable, '-m', 'promnesia', *args]
+    else:
+        # use version from the repository
+        root = Path(__file__).parent.parent.parent.parent
+        pm = root / 'scripts/promnesia'
+        return [pm, *args]
+
+
+# meh... not great
+@pytest.fixture
+def reset_filters():
+    from .. import extract
+
+    extract.filters.cache_clear()
+    try:
+        yield
+    finally:
+        extract.filters.cache_clear()
+
+
+# TODO could be a TypeGuard from 3.10
+V = TypeVar('V')
+
+def unwrap(r: Res[V]) -> V:
+    assert not isinstance(r, Exception), r
+    return r
+
+
+def write_config(path: Path, gen, **kwargs) -> None:
+    output_dir = path.parent
+    cfg_src = dedent('\n'.join(inspect.getsource(gen).splitlines()[1:])) + f"\nOUTPUT_DIR = r'{output_dir}'"
+    for k, v in kwargs.items():
+        assert k in cfg_src, k
+        cfg_src = cfg_src.replace(k, repr(str(v)))  # meh
+    path.write_text(cfg_src)
+
+
+@contextmanager
+def free_port() -> Iterator[int]:
+    # this is a generator to make sure there are no race conditions between the time we call this and launch program
+    #
+    # also some relevant articles about this 'technique'
+    # - https://eklitzke.org/binding-on-port-zero
+    # - https://idea.popcount.org/2014-04-03-bind-before-connect
+    # - https://blog.cloudflare.com/the-quantum-state-of-a-tcp-port
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        if sys.platform == 'linux':
+            # Ok, so from what I've been reading, SO_REUSEADDR should only be necessary in the program that reuses the port
+            # However, this answer (or man socket) claims we need it on both sites in Linux? see https://superuser.com/a/587955/300795
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        # also not sure where REUSEADDR is set in uvicorn (e.g. here reuse_address isn't passed?)
+        # https://github.com/encode/uvicorn/blob/6d666d99a285153bc4613e811543c39eca57054a/uvicorn/server.py#L162C37-L162C50
+        # but from strace looks like it is called somewhere :shrug:
+
+        # assign euphemeral port
+        # see table in
+        # https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ/14388707#14388707
+        # we rely on server binding to localhost later (or anything except 0.0.0.0 really)
+        s.bind(('', 0))
+
+        port = s.getsockname()[1]
+        yield port

promnesia/tests/server_helper.py

@@ -0,0 +1,64 @@
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+import sys
+import time
+from typing import Any, Dict, Iterator, Optional
+
+import psutil
+import requests
+
+from ..common import PathIsh
+from .common import tmp_popen, promnesia_bin, free_port
+
+
+@dataclass
+class Helper:
+    host: str
+    port: str
+    process: psutil.Popen
+
+    def get(self, path: str, *args):
+        # check it's alive first so the error is cleaner
+        assert self.process.poll() is None, self.process
+        return requests.get(f'http://{self.host}:{self.port}' + path)
+
+    def post(self, path: str, *, json: Optional[Dict[str, Any]] = None):
+        assert self.process.poll() is None, self.process
+        return requests.post(f'http://{self.host}:{self.port}' + path, json=json)
+
+
+@contextmanager
+def run_server(db: Optional[PathIsh] = None, *, timezone: Optional[str] = None) -> Iterator[Helper]:
+    # TODO not sure, perhaps best to use a thread or something?
+    # but for some tests makes more sense to test in a separate process
+    with free_port() as pp:
+        # ugh. under docker 'localhost' tries to bind it to ipv6 (::1) for some reason???
+        host = '0.0.0.0' if Path('/.dockerenv').exists() else 'localhost'
+        port = str(pp)
+        args = [
+            'serve',
+            '--host', host,
+            '--quiet',
+            '--port', port,
+            *([] if timezone is None else ['--timezone', timezone]),
+            *([] if db is None else ['--db'  , str(db)]),
+        ]
+        with tmp_popen(promnesia_bin(*args)) as server_process:
+            server = Helper(host=host, port=port, process=server_process)
+
+            # wait till ready
+            for _ in range(50):
+                try:
+                    server.get('/status').json()
+                    break
+                except:
+                    time.sleep(0.1)
+            else:
+                raise RuntimeError("Cooldn't connect to '{st}' after 50 attempts")
+            print("Started server up, db: {db}".format(db=db), file=sys.stderr)
+
+            yield server
+
+            # TODO use logger!
+            print("Done with the server", file=sys.stderr)

promnesia/tests/sources/test_auto.py

@@ -0,0 +1,66 @@
+from itertools import groupby
+import os
+
+from ...sources import auto
+
+from ..common import get_testdata, throw
+
+sa2464 = 'https://www.scottaaronson.com/blog/?p=2464'
+
+_JSON_URLS = {
+    'https://johncarlosbaez.wordpress.com/2016/09/09/struggles-with-the-continuum-part-2/',
+    sa2464,
+}
+
+
+def makemap(visits):
+    key = lambda v: v.url
+
+    def it():
+        vit = (throw(v) if isinstance(v, Exception) else v for v in visits)
+        for k, g in groupby(sorted(vit, key=key), key=key):
+            yield k, list(sorted(g))
+
+    return dict(it())
+
+
+def test_json() -> None:
+    mm = makemap(auto.index(get_testdata('auto'), ignored='*/orgs/*'))
+    assert mm.keys() == _JSON_URLS
+
+    # TODO not sure if they deserve separate visits..
+    [v1, v2] = mm[sa2464]
+    assert v1.context == 'list::yyy::given_url'
+    # todo not sure if editor:// work on Windows
+    assert v1.locator.href.startswith('editor://')
+    assert v1.locator.href.endswith('pocket.json')
+    # TODO line number?
+
+
+def test_auto() -> None:
+    mm = makemap(auto.index(get_testdata('auto')))
+    org_link = 'https://www.youtube.com/watch?v=rHIkrotSwcc'
+    assert {
+        *_JSON_URLS,
+        org_link,
+    }.issubset(mm.keys())
+
+    [v] = mm[org_link]
+    assert v.locator.title == 'orgs' + os.sep + 'file.org:14'  # meh
+    assert v.locator.href.endswith('file.org:14')
+    assert "xxx /r/cpp" in v.context
+    assert "I've enjoyed [Chandler Carruth's" in v.context
+
+
+def test_obsidian() -> None:
+    mm = makemap(auto.index(get_testdata('obsidian-vault')))
+    example_url = 'https://example.com'
+    [v] = mm[example_url]
+    assert v.locator.href.startswith('obsidian://')
+
+
+def test_logseq() -> None:
+    mm = makemap(auto.index(get_testdata('logseq-graph')))
+    example_url = 'https://example.com'
+    [v] = mm[example_url]
+    assert v.locator.href.startswith('logseq://')

promnesia/tests/sources/test_filetypes.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from ...common import PathIsh, _is_windows as windows
+from ...sources.auto import by_path
+
+
+def handled(p: PathIsh) -> bool:
+    idx, m = by_path(Path(p))
+    return idx is not None
+    # ideally these won't hit libmagic path (would try to open the file and cause FileNotFoundError)
+
+
+def test_filetypes() -> None:
+    # test media
+    for ext in 'avi mp4 mp3 webm'.split() + ([] if windows else 'mkv'.split()):
+        assert handled('file.' + ext)
+
+    # images
+    for ext in 'gif jpg png jpeg'.split():
+        assert handled('file.' + ext)
+
+    # TODO more granual checks that these are ignored?
+    # binaries
+    for ext in 'o sqlite'.split() + ([] if windows else 'class jar'.split()):
+        assert handled('file.' + ext)
+
+    # these might have potentially some links
+    for ext in [
+        'svg',
+        'pdf', 'epub', 'ps',
+        'doc', 'ppt', 'xsl',
+        # seriously, windows doesn't know about docx???
+        *([] if windows else 'docx pptx xlsx'.split()),
+        *([] if windows else 'ods odt rtf'.split()),
+    ] + ([] if windows else 'djvu'.split()):
+        assert handled('file.' + ext)
+
+    # source code
+    for ext in 'rs tex el js sh hs pl h py hpp c go css'.split() + ([] if windows else 'java cpp'.split()):
+        assert handled('file.' + ext)
+
+    assert handled('x.html')

promnesia/tests/sources/test_hypothesis.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from ..common import write_config, get_testdata
+from ...__main__ import do_index
+from ...database.load import get_all_db_visits
+
+from my.core.cfg import tmp_config
+
+
+def index_hypothesis(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import hypothesis
+
+        SOURCES = [Source(hypothesis.index, name='hyp')]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+
+    class hpi_config:
+        class hypothesis:
+            export_path = get_testdata('hypexport/testdata') / 'netrights-dashboard-mockup/data/*.json'
+
+    with tmp_config(modules='my.hypothesis', config=hpi_config):
+        do_index(cfg_path)
+
+
+def test_hypothesis(tmp_path: Path) -> None:
+    index_hypothesis(tmp_path)
+
+    visits = get_all_db_visits(tmp_path / 'promnesia.sqlite')
+    assert len(visits) > 100
+
+    [vis] = [x for x in visits if 'fundamental fact of evolution' in (x.context or '')]
+
+    assert vis.norm_url == 'wired.com/2017/04/the-myth-of-a-superhuman-ai'
+    assert vis.orig_url == 'https://www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
+    assert vis.locator.href == 'https://hyp.is/_Z9ccmVZEeexBOO7mToqdg/www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
+    assert 'misconception about evolution is fueling misconception about AI' in (vis.context or '')  # contains notes as well

promnesia/tests/sources/test_org.py

@@ -0,0 +1,65 @@
+from typing import Optional
+
+from ...common import Visit
+from ...sources.org import extract_from_file
+
+from ..common import get_testdata, throw
+
+
+def delrf(s: Optional[str]) -> Optional[str]:
+    if s is None:
+        return None
+    # meh.. not sure how ot handle this properly, ideally should be via pytest?
+    # not sure if should just do it in the indexer? e.g. extension might not like it
+    return s.replace('\r', '')
+
+
+def test_org_indexer() -> None:
+    [_, cpp, cozy] = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file.org'))]
+
+    assert cpp.url == 'https://www.youtube.com/watch?v=rHIkrotSwcc'
+    # TODO not sure about filetags?
+    exp = '''
+xxx /r/cpp   :cpp:programming:
+ I've enjoyed [Chandler Carruth's _There Are No Zero-cost Abstractions_](
+ https://www.youtube.com/watch?v=rHIkrotSwcc) very much.
+
+'''.lstrip()
+    assert delrf(cpp.context) == exp
+
+    assert cozy.url == 'https://twitter.com/Mappletons/status/1255221220263563269'
+
+
+def test_org_indexer_2() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file3.org'))]
+
+    assert len(items) == 6
+    assert items[0].url == 'https://www.reddit.com/r/androidapps/comments/4i36z9/how_you_use_your_android_to_the_maximum/d2uq24i'
+    assert items[1].url == 'https://link.com'
+    assert items[-2].url == 'https://en.wikipedia.org/wiki/Resilio_Sync'
+    # TODO shit def need org specific url extractor (and then extract from everything remaining)
+    # assert results[-1].url == 'https://en.wikipedia.org/wiki/InterPlanetary_File_System'
+
+
+def test_heading() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file2.org'))]
+    assert {i.url for i in items} == {
+        'https://en.wikipedia.org/wiki/Computational_topology',
+        'http://graphics.stanford.edu/courses/cs468-09-fall/',
+        'https://en.wikipedia.org/wiki/Triangulation_(topology)',
+        'https://en.wikipedia.org/wiki/Digital_manifold',
+    }
+
+
+def test_url_in_properties() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file4.org'))]
+
+    assert len(items) == 2, items
+    assert items[0].url == 'https://example.org/ref_example'
+    assert items[1].url == 'http://example.org/a_test'
+
+
+def test_5() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file5.org'))]
+
+    assert len(items) == 0  # shouldn't crash at least

promnesia/tests/sources/test_plaintext.py

@@ -0,0 +1,26 @@
+from ...common import Source
+from ...extract import extract_visits
+from ...sources import plaintext, shellcmd
+
+from ..common import get_testdata, unwrap
+
+
+def test_plaintext_path_extractor() -> None:
+    visits = list(extract_visits(
+        Source(
+            shellcmd.index,
+            plaintext.extract_from_path(get_testdata('custom')),
+        ),
+        src='whatever',
+    ))
+    assert {unwrap(v).orig_url for v in visits} == {
+        'http://google.com',
+        'http://google.com/',
+        'http://some-weird-domain.xyz/whatever',
+        'https://google.com',
+        'http://what.about.this.link',
+    }
+
+    [wa] = [v for v in visits if unwrap(v).orig_url == 'http://what.about.this.link']
+    f2 = get_testdata('custom') / 'file2.txt'
+    assert unwrap(wa).locator.href == f'editor://{f2}:3'  # occurs line 3