promnesia 1.2.20230515__py3-none-any.whl → 1.2.20240810__py3-none-any.whl

promnesia/sources/takeout.py

@@ -1,19 +1,26 @@
 '''
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#mygoogletakeoutpaths][google.takeout]] module
 '''
-from typing import Iterable, Set, Any
+from typing import Iterable, Set, Any, NamedTuple
 import warnings
 
 from ..common import Visit, Loc, Results, logger
 from ..compat import removeprefix
 
 
+# incase user is using an old version of google_takeout_parser
+class YoutubeCSVStub(NamedTuple):
+    contentJSON: str
+
+
 def index() -> Results:
     from . import hpi
+    import json
 
     try:
         from my.google.takeout.parser import events
         from google_takeout_parser.models import Activity, YoutubeComment, LikedYoutubeVideo, ChromeHistory
+        from google_takeout_parser.parse_csv import reconstruct_comment_content, extract_comment_links
     except ModuleNotFoundError as ex:
         logger.exception(ex)
         yield ex
@@ -24,12 +31,24 @@ def index() -> Results:
         yield from takeout_legacy.index()
         return
 
+
     _seen: Set[str] = {
         # these are definitely not useful for promnesia
         'Location',
         'PlaceVisit',
         'PlayStoreAppInstall',
     }
+
+    imported_yt_csv_models = False
+    try:
+        from google_takeout_parser.models import CSVYoutubeComment, CSVYoutubeLiveChat
+        imported_yt_csv_models = True
+    except ImportError:
+        # warn user to upgrade google_takeout_parser
+        warnings.warn("Please upgrade google_takeout_parser (`pip install -U google_takeout_parser`) to support the new format for youtube comments")
+        CSVYoutubeComment = YoutubeCSVStub  # type: ignore[misc,assignment]
+        CSVYoutubeLiveChat = YoutubeCSVStub  # type: ignore[misc,assignment]
+
     def warn_once_if_not_seen(e: Any) -> Iterable[Exception]:
         et_name = type(e).__name__
         if et_name in _seen:
@@ -71,6 +90,8 @@ def index() -> Results:
                 elif e.products == ['Ads']:
                     # header contains some weird internal ad id in this case
                     context = None
+                else:
+                    context = None
                 # NOTE: at this point seems that context always ends up as None (at least for @karlicoss as of 20230131)
                 # so alternatively could just force it to be None instead of manual dispatching :shrug:
                 yield Visit(
@@ -109,6 +130,42 @@ def index() -> Results:
                 yield Visit(
                     url=url, dt=e.dt, context=e.content, locator=Loc(title=e.content, href=url)
                 )
+        elif imported_yt_csv_models and isinstance(e, CSVYoutubeComment):
+            contentJSON = json.loads(e.contentJSON)
+            content = reconstruct_comment_content(contentJSON, format='text')
+            if isinstance(content, Exception):
+                yield content
+                continue
+            links = extract_comment_links(contentJSON)
+            if isinstance(links, Exception):
+                yield links
+                continue
+            context = f"Commented on {e.video_url}"
+            for url in links:
+                yield Visit(
+                    url=url, dt=e.dt, context=content, locator=Loc(title=context, href=url)
+                )
+            yield Visit(
+                url=e.video_url, dt=e.dt, context=content, locator=Loc(title=context, href=e.video_url)
+            )
+        elif imported_yt_csv_models and isinstance(e, CSVYoutubeLiveChat):
+            contentJSON = json.loads(e.contentJSON)
+            content = reconstruct_comment_content(contentJSON, format='text')
+            if isinstance(content, Exception):
+                yield content
+                continue
+            links = extract_comment_links(contentJSON)
+            if isinstance(links, Exception):
+                yield links
+                continue
+            context = f"Commented on livestream {e.video_url}"
+            for url in links:
+                yield Visit(
+                    url=url, dt=e.dt, context=content, locator=Loc(title=context, href=url)
+                )
+            yield Visit(
+                url=e.video_url, dt=e.dt, context=content, locator=Loc(title=context, href=e.video_url)
+            )
         else:
             yield from warn_once_if_not_seen(e)
 

promnesia/sources/takeout_legacy.py

@@ -34,7 +34,15 @@ from .. import config
 
 
 from more_itertools import unique_everseen
-from cachew import cachew
+
+try:
+    from cachew import cachew
+except ModuleNotFoundError as me:
+    if me.name != 'cachew':
+        raise me
+    # this module is legacy anyway, so just make it defensive
+    def cachew(*args, **kwargs):  # type: ignore[no-redef]
+        return lambda f: f
 
 
 # TODO use CPath? Could encapsulate a path within an archive *or* within a directory
@@ -105,7 +113,7 @@ def read_browser_history_json(takeout: TakeoutPath) -> Iterable[Visit]:
     hist = j['Browser History']
     for item in hist:
         url = item['url']
-        time = datetime.utcfromtimestamp(item['time_usec'] / 10 ** 6).replace(tzinfo=pytz.utc)
+        time = datetime.fromtimestamp(item['time_usec'] / 10 ** 6, tz=pytz.utc)
         # TODO any more interesitng info?
         yield Visit(
             url=url,

promnesia/tests/common.py

@@ -0,0 +1,137 @@
+from contextlib import closing, contextmanager
+import gc
+import inspect
+import os
+from pathlib import Path
+import socket
+import sys
+from textwrap import dedent
+from typing import Iterator, NoReturn, TypeVar
+
+import pytest
+
+from ..common import _is_windows, Res
+
+
+def under_ci() -> bool:
+    return 'CI' in os.environ
+
+
+def throw(x: Exception) -> NoReturn:
+    '''
+    like raise, but can be an expression...
+    '''
+    raise x
+
+
+@pytest.fixture
+def gc_control(gc_on: bool):
+    if gc_on:
+        # no need to do anything, should be on by default
+        yield
+        return
+
+    gc.disable()
+    try:
+        yield
+    finally:
+        gc.enable()
+
+
+running_on_ci = 'CI' in os.environ
+
+
+GIT_ROOT = Path(__file__).absolute().parent.parent.parent.parent
+TESTDATA = GIT_ROOT / 'tests/testdata'
+
+
+def get_testdata(path: str) -> Path:
+    assert TESTDATA.is_dir()
+    res = TESTDATA / path
+    if not res.exists():
+        raise RuntimeError(f"'{res}' not found! You propably need to run 'git submodule update --init --recursive'")
+    return TESTDATA / path
+
+
+@contextmanager
+def tmp_popen(*args, **kwargs):
+    import psutil
+    with psutil.Popen(*args, **kwargs) as p:
+        try:
+            yield p
+        finally:
+            for c in p.children(recursive=True):
+                c.kill()
+            p.kill()
+            p.wait()
+
+
+# meh
+def promnesia_bin(*args):
+    # not sure it's a good idea to diverge, but not sure if there's a better way either?
+    # ugh. on windows there is no bash so can't use the script
+    # whatever...
+    if under_ci() or _is_windows:
+        # should be able to use the installed version
+        return [sys.executable, '-m', 'promnesia', *args]
+    else:
+        # use version from the repository
+        root = Path(__file__).parent.parent.parent.parent
+        pm = root / 'scripts/promnesia'
+        return [pm, *args]
+
+
+# meh... not great
+@pytest.fixture
+def reset_filters():
+    from .. import extract
+
+    extract.filters.cache_clear()
+    try:
+        yield
+    finally:
+        extract.filters.cache_clear()
+
+
+# TODO could be a TypeGuard from 3.10
+V = TypeVar('V')
+
+def unwrap(r: Res[V]) -> V:
+    assert not isinstance(r, Exception), r
+    return r
+
+
+def write_config(path: Path, gen, **kwargs) -> None:
+    output_dir = path.parent
+    cfg_src = dedent('\n'.join(inspect.getsource(gen).splitlines()[1:])) + f"\nOUTPUT_DIR = r'{output_dir}'"
+    for k, v in kwargs.items():
+        assert k in cfg_src, k
+        cfg_src = cfg_src.replace(k, repr(str(v)))  # meh
+    path.write_text(cfg_src)
+
+
+@contextmanager
+def free_port() -> Iterator[int]:
+    # this is a generator to make sure there are no race conditions between the time we call this and launch program
+    #
+    # also some relevant articles about this 'technique'
+    # - https://eklitzke.org/binding-on-port-zero
+    # - https://idea.popcount.org/2014-04-03-bind-before-connect
+    # - https://blog.cloudflare.com/the-quantum-state-of-a-tcp-port
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        if sys.platform == 'linux':
+            # Ok, so from what I've been reading, SO_REUSEADDR should only be necessary in the program that reuses the port
+            # However, this answer (or man socket) claims we need it on both sites in Linux? see https://superuser.com/a/587955/300795
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        # also not sure where REUSEADDR is set in uvicorn (e.g. here reuse_address isn't passed?)
+        # https://github.com/encode/uvicorn/blob/6d666d99a285153bc4613e811543c39eca57054a/uvicorn/server.py#L162C37-L162C50
+        # but from strace looks like it is called somewhere :shrug:
+
+        # assign euphemeral port
+        # see table in
+        # https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ/14388707#14388707
+        # we rely on server binding to localhost later (or anything except 0.0.0.0 really)
+        s.bind(('', 0))
+
+        port = s.getsockname()[1]
+        yield port

promnesia/tests/server_helper.py

@@ -0,0 +1,64 @@
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+import sys
+import time
+from typing import Any, Dict, Iterator, Optional
+
+import psutil
+import requests
+
+from ..common import PathIsh
+from .common import tmp_popen, promnesia_bin, free_port
+
+
+@dataclass
+class Helper:
+    host: str
+    port: str
+    process: psutil.Popen
+
+    def get(self, path: str, *args):
+        # check it's alive first so the error is cleaner
+        assert self.process.poll() is None, self.process
+        return requests.get(f'http://{self.host}:{self.port}' + path)
+
+    def post(self, path: str, *, json: Optional[Dict[str, Any]] = None):
+        assert self.process.poll() is None, self.process
+        return requests.post(f'http://{self.host}:{self.port}' + path, json=json)
+
+
+@contextmanager
+def run_server(db: Optional[PathIsh] = None, *, timezone: Optional[str] = None) -> Iterator[Helper]:
+    # TODO not sure, perhaps best to use a thread or something?
+    # but for some tests makes more sense to test in a separate process
+    with free_port() as pp:
+        # ugh. under docker 'localhost' tries to bind it to ipv6 (::1) for some reason???
+        host = '0.0.0.0' if Path('/.dockerenv').exists() else 'localhost'
+        port = str(pp)
+        args = [
+            'serve',
+            '--host', host,
+            '--quiet',
+            '--port', port,
+            *([] if timezone is None else ['--timezone', timezone]),
+            *([] if db is None else ['--db'  , str(db)]),
+        ]
+        with tmp_popen(promnesia_bin(*args)) as server_process:
+            server = Helper(host=host, port=port, process=server_process)
+
+            # wait till ready
+            for _ in range(50):
+                try:
+                    server.get('/status').json()
+                    break
+                except:
+                    time.sleep(0.1)
+            else:
+                raise RuntimeError("Cooldn't connect to '{st}' after 50 attempts")
+            print("Started server up, db: {db}".format(db=db), file=sys.stderr)
+
+            yield server
+
+            # TODO use logger!
+            print("Done with the server", file=sys.stderr)

promnesia/tests/sources/test_auto.py

@@ -0,0 +1,66 @@
+from itertools import groupby
+import os
+
+from ...sources import auto
+
+from ..common import get_testdata, throw
+
+sa2464 = 'https://www.scottaaronson.com/blog/?p=2464'
+
+_JSON_URLS = {
+    'https://johncarlosbaez.wordpress.com/2016/09/09/struggles-with-the-continuum-part-2/',
+    sa2464,
+}
+
+
+def makemap(visits):
+    key = lambda v: v.url
+
+    def it():
+        vit = (throw(v) if isinstance(v, Exception) else v for v in visits)
+        for k, g in groupby(sorted(vit, key=key), key=key):
+            yield k, list(sorted(g))
+
+    return dict(it())
+
+
+def test_json() -> None:
+    mm = makemap(auto.index(get_testdata('auto'), ignored='*/orgs/*'))
+    assert mm.keys() == _JSON_URLS
+
+    # TODO not sure if they deserve separate visits..
+    [v1, v2] = mm[sa2464]
+    assert v1.context == 'list::yyy::given_url'
+    # todo not sure if editor:// work on Windows
+    assert v1.locator.href.startswith('editor://')
+    assert v1.locator.href.endswith('pocket.json')
+    # TODO line number?
+
+
+def test_auto() -> None:
+    mm = makemap(auto.index(get_testdata('auto')))
+    org_link = 'https://www.youtube.com/watch?v=rHIkrotSwcc'
+    assert {
+        *_JSON_URLS,
+        org_link,
+    }.issubset(mm.keys())
+
+    [v] = mm[org_link]
+    assert v.locator.title == 'orgs' + os.sep + 'file.org:14'  # meh
+    assert v.locator.href.endswith('file.org:14')
+    assert "xxx /r/cpp" in v.context
+    assert "I've enjoyed [Chandler Carruth's" in v.context
+
+
+def test_obsidian() -> None:
+    mm = makemap(auto.index(get_testdata('obsidian-vault')))
+    example_url = 'https://example.com'
+    [v] = mm[example_url]
+    assert v.locator.href.startswith('obsidian://')
+
+
+def test_logseq() -> None:
+    mm = makemap(auto.index(get_testdata('logseq-graph')))
+    example_url = 'https://example.com'
+    [v] = mm[example_url]
+    assert v.locator.href.startswith('logseq://')

promnesia/tests/sources/test_filetypes.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from ...common import PathIsh, _is_windows as windows
+from ...sources.auto import by_path
+
+
+def handled(p: PathIsh) -> bool:
+    idx, m = by_path(Path(p))
+    return idx is not None
+    # ideally these won't hit libmagic path (would try to open the file and cause FileNotFoundError)
+
+
+def test_filetypes() -> None:
+    # test media
+    for ext in 'avi mp4 mp3 webm'.split() + ([] if windows else 'mkv'.split()):
+        assert handled('file.' + ext)
+
+    # images
+    for ext in 'gif jpg png jpeg'.split():
+        assert handled('file.' + ext)
+
+    # TODO more granual checks that these are ignored?
+    # binaries
+    for ext in 'o sqlite'.split() + ([] if windows else 'class jar'.split()):
+        assert handled('file.' + ext)
+
+    # these might have potentially some links
+    for ext in [
+        'svg',
+        'pdf', 'epub', 'ps',
+        'doc', 'ppt', 'xsl',
+        # seriously, windows doesn't know about docx???
+        *([] if windows else 'docx pptx xlsx'.split()),
+        *([] if windows else 'ods odt rtf'.split()),
+    ] + ([] if windows else 'djvu'.split()):
+        assert handled('file.' + ext)
+
+    # source code
+    for ext in 'rs tex el js sh hs pl h py hpp c go css'.split() + ([] if windows else 'java cpp'.split()):
+        assert handled('file.' + ext)
+
+    assert handled('x.html')

promnesia/tests/sources/test_hypothesis.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from ..common import write_config, get_testdata
+from ...__main__ import do_index
+from ...database.load import get_all_db_visits
+
+from my.core.cfg import tmp_config
+
+
+def index_hypothesis(tmp_path: Path) -> None:
+    def cfg() -> None:
+        from promnesia.common import Source
+        from promnesia.sources import hypothesis
+
+        SOURCES = [Source(hypothesis.index, name='hyp')]
+
+    cfg_path = tmp_path / 'config.py'
+    write_config(cfg_path, cfg)
+
+    class hpi_config:
+        class hypothesis:
+            export_path = get_testdata('hypexport/testdata') / 'netrights-dashboard-mockup/data/*.json'
+
+    with tmp_config(modules='my.hypothesis', config=hpi_config):
+        do_index(cfg_path)
+
+
+def test_hypothesis(tmp_path: Path) -> None:
+    index_hypothesis(tmp_path)
+
+    visits = get_all_db_visits(tmp_path / 'promnesia.sqlite')
+    assert len(visits) > 100
+
+    [vis] = [x for x in visits if 'fundamental fact of evolution' in (x.context or '')]
+
+    assert vis.norm_url == 'wired.com/2017/04/the-myth-of-a-superhuman-ai'
+    assert vis.orig_url == 'https://www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
+    assert vis.locator.href == 'https://hyp.is/_Z9ccmVZEeexBOO7mToqdg/www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
+    assert 'misconception about evolution is fueling misconception about AI' in (vis.context or '')  # contains notes as well

promnesia/tests/sources/test_org.py

@@ -0,0 +1,65 @@
+from typing import Optional
+
+from ...common import Visit
+from ...sources.org import extract_from_file
+
+from ..common import get_testdata, throw
+
+
+def delrf(s: Optional[str]) -> Optional[str]:
+    if s is None:
+        return None
+    # meh.. not sure how ot handle this properly, ideally should be via pytest?
+    # not sure if should just do it in the indexer? e.g. extension might not like it
+    return s.replace('\r', '')
+
+
+def test_org_indexer() -> None:
+    [_, cpp, cozy] = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file.org'))]
+
+    assert cpp.url == 'https://www.youtube.com/watch?v=rHIkrotSwcc'
+    # TODO not sure about filetags?
+    exp = '''
+xxx /r/cpp   :cpp:programming:
+ I've enjoyed [Chandler Carruth's _There Are No Zero-cost Abstractions_](
+ https://www.youtube.com/watch?v=rHIkrotSwcc) very much.
+
+'''.lstrip()
+    assert delrf(cpp.context) == exp
+
+    assert cozy.url == 'https://twitter.com/Mappletons/status/1255221220263563269'
+
+
+def test_org_indexer_2() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file3.org'))]
+
+    assert len(items) == 6
+    assert items[0].url == 'https://www.reddit.com/r/androidapps/comments/4i36z9/how_you_use_your_android_to_the_maximum/d2uq24i'
+    assert items[1].url == 'https://link.com'
+    assert items[-2].url == 'https://en.wikipedia.org/wiki/Resilio_Sync'
+    # TODO shit def need org specific url extractor (and then extract from everything remaining)
+    # assert results[-1].url == 'https://en.wikipedia.org/wiki/InterPlanetary_File_System'
+
+
+def test_heading() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file2.org'))]
+    assert {i.url for i in items} == {
+        'https://en.wikipedia.org/wiki/Computational_topology',
+        'http://graphics.stanford.edu/courses/cs468-09-fall/',
+        'https://en.wikipedia.org/wiki/Triangulation_(topology)',
+        'https://en.wikipedia.org/wiki/Digital_manifold',
+    }
+
+
+def test_url_in_properties() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file4.org'))]
+
+    assert len(items) == 2, items
+    assert items[0].url == 'https://example.org/ref_example'
+    assert items[1].url == 'http://example.org/a_test'
+
+
+def test_5() -> None:
+    items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file5.org'))]
+
+    assert len(items) == 0  # shouldn't crash at least

promnesia/tests/sources/test_plaintext.py

@@ -0,0 +1,26 @@
+from ...common import Source
+from ...extract import extract_visits
+from ...sources import plaintext, shellcmd
+
+from ..common import get_testdata, unwrap
+
+
+def test_plaintext_path_extractor() -> None:
+    visits = list(extract_visits(
+        Source(
+            shellcmd.index,
+            plaintext.extract_from_path(get_testdata('custom')),
+        ),
+        src='whatever',
+    ))
+    assert {unwrap(v).orig_url for v in visits} == {
+        'http://google.com',
+        'http://google.com/',
+        'http://some-weird-domain.xyz/whatever',
+        'https://google.com',
+        'http://what.about.this.link',
+    }
+
+    [wa] = [v for v in visits if unwrap(v).orig_url == 'http://what.about.this.link']
+    f2 = get_testdata('custom') / 'file2.txt'
+    assert unwrap(wa).locator.href == f'editor://{f2}:3'  # occurs line 3

promnesia/tests/sources/test_shellcmd.py

@@ -0,0 +1,22 @@
+from ...common import _is_windows, Source
+from ...extract import extract_visits
+from ...sources import shellcmd
+
+import pytest
+
+from ..common import get_testdata
+
+
+@pytest.mark.skipif(_is_windows, reason="no grep on windows")
+def test_via_grep() -> None:
+
+    visits = list(extract_visits(
+        Source(
+            shellcmd.index,
+            # meh. maybe should deprecate plain string here...
+            r"""grep -Eo -r --no-filename (http|https)://\S+ """ + str(get_testdata('custom')),
+        ),
+        src='whatever',
+    ))
+    # TODO I guess filtering of equivalent urls should rather be tested on something having context (e.g. org mode)
+    assert len(visits) == 5

promnesia/tests/sources/test_takeout.py

@@ -0,0 +1,58 @@
+from datetime import datetime, timezone
+
+from ...common import Source
+from ...extract import extract_visits
+from ...sources import takeout
+
+import pytest
+
+from ..common import get_testdata, unwrap
+
+from my.core.cfg import tmp_config
+
+
+# TODO apply in conftest so it's used in all tests?
+@pytest.fixture
+def no_cachew():
+    from my.core.cachew import disabled_cachew
+
+    with disabled_cachew():
+        yield
+
+
+# todo testing this logic probably belongs to hpi or google_takeout_export, but whatever
+def test_takeout_directory(no_cachew) -> None:
+    class config:
+        class google:
+            takeout_path = get_testdata('takeout')
+
+    with tmp_config(modules='my.google.takeout.*', config=config):
+        visits = list(extract_visits(Source(takeout.index), src='takeout'))
+
+    assert len(visits) == 3
+    assert all(unwrap(v).dt.tzinfo is not None for v in visits)
+
+
+def test_takeout_zip(no_cachew) -> None:
+    class config:
+        class google:
+            takeout_path = get_testdata('takeout-20150518T000000Z.zip')
+
+    with tmp_config(modules='my.google.takeout.*', config=config):
+        visits = list(extract_visits(Source(takeout.index), src='takeout'))
+
+    assert len(visits) == 3
+    assert all(unwrap(v).dt.tzinfo is not None for v in visits)
+
+    [vis] = [v for v in visits if unwrap(v).norm_url == 'takeout.google.com/settings/takeout']
+
+    edt = datetime(
+        year=2018,
+        month=9,
+        day=18,
+        hour=5,
+        minute=48,
+        second=23,
+        tzinfo=timezone.utc,
+    )
+    assert unwrap(vis).dt == edt