promnesia 1.2.20240810__py3-none-any.whl → 1.4.20250909__py3-none-any.whl

promnesia/sources/twitter.py

@@ -1,18 +1,19 @@
 '''
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Twitter data.
 '''
-from typing import Iterable
 
-from ..common import logger, Results, Visit, Loc, extract_urls, Res
+from collections.abc import Iterable
+
+from promnesia.common import Loc, Res, Results, Visit, extract_urls, logger
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.twitter.all as tw
+    from my.twitter.archive import Tweet  # todo extract to common or something?
+
     # TODO hmm. tweets themselves are sort of visits? not sure if they should contribute..
     processed = 0
-
-    from my.twitter.archive import Tweet # todo extract to common or something?
     tweets: Iterable[Res[Tweet]] = tw.tweets()
     for t in tweets:
         if isinstance(t, Exception):
@@ -22,7 +23,7 @@ def index() -> Results:
         processed += 1
         try:
             urls = t.urls
-        except Exception as e: # just in case..
+        except Exception as e:  # just in case..
             yield e
             urls = []
 

promnesia/sources/vcs.py

@@ -1,12 +1,15 @@
 '''
 Clones & indexes Git repositories (via sources.auto)
 '''
-# TODO not sure if worth exposing... could be just handled by auto or something?)
 
-from pathlib import Path
+from __future__ import annotations
+
 import re
+from collections.abc import Iterable
+
+# TODO not sure if worth exposing... could be just handled by auto or something?)
+from pathlib import Path
 from subprocess import check_call
-from typing import Iterable
 
 from ..common import Extraction, PathIsh, get_tmpdir, slugify
 
@@ -20,7 +23,7 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
     # note: https://bugs.python.org/issue33617 , it doesn't like Path here on Windows
     check_call(['git', 'clone', repo, str(tp)])
 
-    def replacer(p: PathIsh, prefix: str=str(tp), repo: str=repo) -> str:
+    def replacer(p: PathIsh, prefix: str = str(tp), repo: str = repo) -> str:
         ps = str(p)
         # TODO prefix is a bit misleading
         pos = ps.find(prefix)
@@ -29,13 +32,15 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
             return ps
         # TODO ugh. seems that blame view https://github.com/davidgasquez/handbook/blame/master/README.md#L25 is the most reliable
         # in raw mode can't jump onto line, when markdown is renderend can't jump either
-        rest = ps[pos + len(prefix):]
-        rest = re.sub(r':(\d+)$', r'#L\1', rest) # patch line number...
+        rest = ps[pos + len(prefix) :]
+        rest = re.sub(r':(\d+)$', r'#L\1', rest)  # patch line number...
         return repo + '/blame/master' + rest
 
         # TODO doesn't work for git:
         # TODO think about something more generic... this isn't too sustainable
+
     # TODO not sure if context should be local or github?...
 
     from . import auto
+
     yield from auto.index(tp, *args, replacer=replacer, **kwargs)

promnesia/sources/viber.py

@@ -2,23 +2,24 @@
 Collects visits from Viber desktop app (e.g. `~/.ViberPC/XYZ123/viber.db`)
 """
 
+from __future__ import annotations
+
 import logging
+import sqlite3
 import textwrap
+from collections.abc import Iterable
 from os import PathLike
 from pathlib import Path
-import sqlite3
-from typing import Iterable, Optional
 
 from ..common import Loc, PathIsh, Results, Visit, extract_urls, from_epoch, join_tags
 from ..sqlite import sqlite_connection
 
-
 logger = logging.getLogger(__name__)
 
 
 def index(
     db_path: PathIsh = "~/.ViberPC/*/viber.db",
-    locator_schema: str="editor",
+    locator_schema: str = "editor",
     *,
     http_only: bool = False,
 ) -> Results:
@@ -34,12 +35,12 @@ def index(
 
     msgs_query = messages_query(http_only)
 
-    for db_path in _get_files(db_path):
-        assert db_path.is_file(), f"Is it a (Viber-desktop sqlite) file? {db_path}"
-        yield from _harvest_db(db_path, msgs_query, locator_schema)
+    for db in _get_files(db_path):
+        assert db.is_file(), f"Is it a (Viber-desktop sqlite) file? {db}"
+        yield from _harvest_db(db, msgs_query, locator_schema)
 
 
-def messages_query(http_only: Optional[bool]) -> str:
+def messages_query(http_only: bool | None) -> str:  # noqa: FBT001
     """
     An SQL-query returning 1 row for each message
 
@@ -123,7 +124,7 @@ def _handle_row(row: sqlite3.Row, db_path: PathLike, locator_schema: str) -> Res
     tags: str = row["tags"]
     url_title: str = row["url_title"]
 
-    assert (
+    assert (  # noqa: PT018
         text and mid and sender and chatname
     ), f"sql-query should eliminate messages without 'http' or missing ids: {row}"
 
@@ -154,7 +155,7 @@ def _get_files(path: PathIsh) -> Iterable[Path]:
     """
     path = Path(path).expanduser()
     parts = path.parts[1:] if path.is_absolute() else path.parts
-    return Path(path.root).glob(str(Path("").joinpath(*parts)))
+    return Path(path.root).glob(str(Path("").joinpath(*parts)))  # noqa: PTH201
 
 
 def _harvest_db(db_path: PathIsh, msgs_query: str, locator_schema: str) -> Results:

promnesia/sources/website.py

@@ -2,12 +2,12 @@
 Clones a website with wget and indexes via sources.auto
 '''
 
-from pathlib import Path
 import re
+from collections.abc import Iterable
+from pathlib import Path
 from subprocess import run
-from typing import Iterable
 
-from ..common import Extraction, PathIsh, get_tmpdir, slugify, get_logger
+from promnesia.common import Extraction, PathIsh, get_logger, get_tmpdir, slugify
 
 
 def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
@@ -27,10 +27,10 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
         '-A', 'html,html,txt', # TODO eh, ideally would use mime type I guess...
         '--no-parent',
         url,
-    ]
+    ]  # fmt: skip
     # TODO follow sitemap? e.g. gwern
     logger.info(' '.join(cmd))
-    res = run(cmd)
+    res = run(cmd, check=False)
 
     if res.returncode == 8:
         # man wget: 8 means server error (e.g. broken link)
@@ -39,12 +39,12 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
         # rest of the errors are a bit more critical..
         res.check_returncode()
 
-    def replacer(p: PathIsh, prefix: str=str(tp), url: str=url) -> str:
+    def replacer(p: PathIsh, prefix: str = str(tp), url: str = url) -> str:
         ps = str(p)
         pos = ps.find(prefix)
         if pos == -1:
             return ps
-        rest = ps[pos + len(prefix):]
+        rest = ps[pos + len(prefix) :]
         # now this should look kinda like /domain.tld/rest (due to the way wget downloads stuff)
         rest = re.sub(r'/.*?/', '/', rest)
         return url + rest
@@ -54,4 +54,5 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
 
     # TODO smarter html handling
     from . import auto
+
     yield from auto.index(tp, *args, replacer=replacer, **kwargs)

promnesia/sources/zulip.py

@@ -2,12 +2,13 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Zulip data.
 '''
 
-from ..common import Results, Visit, Loc, iter_urls
+from promnesia.common import Loc, Results, Visit, iter_urls
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.zulip.organization as Z
+
     for m in Z.messages():
         if isinstance(m, Exception):
             yield m

promnesia/sqlite.py

@@ -1,22 +1,28 @@
-from contextlib import contextmanager
-import sqlite3
-from typing import Callable, Optional, Any, Iterator, Union, Literal
+from __future__ import annotations
 
-from .common import PathIsh
+import sqlite3
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Literal
 
 # NOTE: copy pasted from HPI
 
 SqliteRowFactory = Callable[[sqlite3.Cursor, sqlite3.Row], Any]
 
+
 def dict_factory(cursor, row):
     fields = [column[0] for column in cursor.description]
-    return {key: value for key, value in zip(fields, row)}
+    return dict(zip(fields, row, strict=True))
+
 
+Factory = SqliteRowFactory | Literal['row', 'dict']
 
-Factory = Union[SqliteRowFactory, Literal['row', 'dict']]
 
 @contextmanager
-def sqlite_connection(db: PathIsh, *, immutable: bool=False, row_factory: Optional[Factory]=None) -> Iterator[sqlite3.Connection]:
+def sqlite_connection(
+    db: Path | str, *, immutable: bool = False, row_factory: Factory | None = None
+) -> Iterator[sqlite3.Connection]:
     dbp = f'file:{db}'
     # https://www.sqlite.org/draft/uri.html#uriimmutable
     if immutable:

promnesia/tests/common.py

@@ -1,16 +1,19 @@
-from contextlib import closing, contextmanager
+from __future__ import annotations
+
 import gc
 import inspect
 import os
-from pathlib import Path
 import socket
 import sys
+from collections.abc import Iterator
+from contextlib import closing, contextmanager
+from pathlib import Path
 from textwrap import dedent
-from typing import Iterator, NoReturn, TypeVar
+from typing import NoReturn, TypeVar
 
 import pytest
 
-from ..common import _is_windows, Res
+from ..common import Res, _is_windows
 
 
 def under_ci() -> bool:
@@ -25,7 +28,7 @@ def throw(x: Exception) -> NoReturn:
 
 
 @pytest.fixture
-def gc_control(gc_on: bool):
+def gc_control(*, gc_on: bool):
     if gc_on:
         # no need to do anything, should be on by default
         yield
@@ -56,6 +59,7 @@ def get_testdata(path: str) -> Path:
 @contextmanager
 def tmp_popen(*args, **kwargs):
     import psutil
+
     with psutil.Popen(*args, **kwargs) as p:
         try:
             yield p
@@ -96,6 +100,7 @@ def reset_filters():
 # TODO could be a TypeGuard from 3.10
 V = TypeVar('V')
 
+
 def unwrap(r: Res[V]) -> V:
     assert not isinstance(r, Exception), r
     return r

promnesia/tests/server_helper.py

@@ -1,15 +1,18 @@
+from __future__ import annotations
+
+import sys
+import time
+from collections.abc import Iterator
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-import sys
-import time
-from typing import Any, Dict, Iterator, Optional
+from typing import Any
 
 import psutil
 import requests
 
 from ..common import PathIsh
-from .common import tmp_popen, promnesia_bin, free_port
+from .common import free_port, promnesia_bin, tmp_popen
 
 
 @dataclass
@@ -18,18 +21,18 @@ class Helper:
     port: str
     process: psutil.Popen
 
-    def get(self, path: str, *args):
+    def get(self, path: str):
         # check it's alive first so the error is cleaner
         assert self.process.poll() is None, self.process
         return requests.get(f'http://{self.host}:{self.port}' + path)
 
-    def post(self, path: str, *, json: Optional[Dict[str, Any]] = None):
+    def post(self, path: str, *, json: dict[str, Any] | None = None):
         assert self.process.poll() is None, self.process
         return requests.post(f'http://{self.host}:{self.port}' + path, json=json)
 
 
 @contextmanager
-def run_server(db: Optional[PathIsh] = None, *, timezone: Optional[str] = None) -> Iterator[Helper]:
+def run_server(db: PathIsh | None = None, *, timezone: str | None = None) -> Iterator[Helper]:
     # TODO not sure, perhaps best to use a thread or something?
     # but for some tests makes more sense to test in a separate process
     with free_port() as pp:
@@ -42,8 +45,8 @@ def run_server(db: Optional[PathIsh] = None, *, timezone: Optional[str] = None)
             '--quiet',
             '--port', port,
             *([] if timezone is None else ['--timezone', timezone]),
-            *([] if db is None else ['--db'  , str(db)]),
-        ]
+            *([] if db is None else ['--db', str(db)]),
+        ]  # fmt: skip
         with tmp_popen(promnesia_bin(*args)) as server_process:
             server = Helper(host=host, port=port, process=server_process)
 
@@ -56,7 +59,7 @@ def run_server(db: Optional[PathIsh] = None, *, timezone: Optional[str] = None)
                     time.sleep(0.1)
             else:
                 raise RuntimeError("Cooldn't connect to '{st}' after 50 attempts")
-            print("Started server up, db: {db}".format(db=db), file=sys.stderr)
+            print(f"Started server up, db: {db}", file=sys.stderr)
 
             yield server
 

promnesia/tests/sources/test_auto.py

@@ -1,8 +1,7 @@
-from itertools import groupby
 import os
+from itertools import groupby
 
 from ...sources import auto
-
 from ..common import get_testdata, throw
 
 sa2464 = 'https://www.scottaaronson.com/blog/?p=2464'
@@ -19,7 +18,7 @@ def makemap(visits):
     def it():
         vit = (throw(v) if isinstance(v, Exception) else v for v in visits)
         for k, g in groupby(sorted(vit, key=key), key=key):
-            yield k, list(sorted(g))
+            yield k, sorted(g)
 
     return dict(it())
 

promnesia/tests/sources/test_filetypes.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
-from ...common import PathIsh, _is_windows as windows
+from ...common import PathIsh
+from ...common import _is_windows as windows
 from ...sources.auto import by_path
 
 
@@ -12,16 +13,16 @@ def handled(p: PathIsh) -> bool:
 
 def test_filetypes() -> None:
     # test media
-    for ext in 'avi mp4 mp3 webm'.split() + ([] if windows else 'mkv'.split()):
+    for ext in ['avi', 'mp4', 'mp3', 'webm'] + ([] if windows else ['mkv']):
         assert handled('file.' + ext)
 
     # images
-    for ext in 'gif jpg png jpeg'.split():
+    for ext in ['gif', 'jpg', 'png', 'jpeg']:
         assert handled('file.' + ext)
 
     # TODO more granual checks that these are ignored?
     # binaries
-    for ext in 'o sqlite'.split() + ([] if windows else 'class jar'.split()):
+    for ext in ['o', 'sqlite'] + ([] if windows else ['class', 'jar']):
         assert handled('file.' + ext)
 
     # these might have potentially some links
@@ -30,13 +31,15 @@ def test_filetypes() -> None:
         'pdf', 'epub', 'ps',
         'doc', 'ppt', 'xsl',
         # seriously, windows doesn't know about docx???
-        *([] if windows else 'docx pptx xlsx'.split()),
-        *([] if windows else 'ods odt rtf'.split()),
-    ] + ([] if windows else 'djvu'.split()):
+        *([] if windows else ['docx', 'pptx', 'xlsx']),
+        *([] if windows else ['ods', 'odt', 'rtf']),
+    ] + ([] if windows else ['djvu']):  # fmt: skip
         assert handled('file.' + ext)
 
     # source code
-    for ext in 'rs tex el js sh hs pl h py hpp c go css'.split() + ([] if windows else 'java cpp'.split()):
+    for ext in ['rs', 'tex', 'el', 'js', 'sh', 'hs', 'pl', 'h', 'py', 'hpp', 'c', 'go', 'css'] + (
+        [] if windows else ['java', 'cpp']
+    ):
         assert handled('file.' + ext)
 
     assert handled('x.html')

promnesia/tests/sources/test_hypothesis.py

@@ -1,10 +1,10 @@
 from pathlib import Path
 
-from ..common import write_config, get_testdata
+from my.core.cfg import tmp_config
+
 from ...__main__ import do_index
 from ...database.load import get_all_db_visits
-
-from my.core.cfg import tmp_config
+from ..common import get_testdata, write_config
 
 
 def index_hypothesis(tmp_path: Path) -> None:
@@ -12,7 +12,7 @@ def index_hypothesis(tmp_path: Path) -> None:
         from promnesia.common import Source
         from promnesia.sources import hypothesis
 
-        SOURCES = [Source(hypothesis.index, name='hyp')]
+        SOURCES = [Source(hypothesis.index, name='hyp')]  # noqa: F841
 
     cfg_path = tmp_path / 'config.py'
     write_config(cfg_path, cfg)
@@ -35,5 +35,9 @@ def test_hypothesis(tmp_path: Path) -> None:
 
     assert vis.norm_url == 'wired.com/2017/04/the-myth-of-a-superhuman-ai'
     assert vis.orig_url == 'https://www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
-    assert vis.locator.href == 'https://hyp.is/_Z9ccmVZEeexBOO7mToqdg/www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
-    assert 'misconception about evolution is fueling misconception about AI' in (vis.context or '')  # contains notes as well
+    assert (
+        vis.locator.href == 'https://hyp.is/_Z9ccmVZEeexBOO7mToqdg/www.wired.com/2017/04/the-myth-of-a-superhuman-ai/'
+    )
+    assert 'misconception about evolution is fueling misconception about AI' in (
+        vis.context or ''
+    )  # contains notes as well

promnesia/tests/sources/test_org.py

@@ -1,12 +1,11 @@
-from typing import Optional
+from __future__ import annotations
 
 from ...common import Visit
 from ...sources.org import extract_from_file
-
 from ..common import get_testdata, throw
 
 
-def delrf(s: Optional[str]) -> Optional[str]:
+def delrf(s: str | None) -> str | None:
     if s is None:
         return None
     # meh.. not sure how ot handle this properly, ideally should be via pytest?
@@ -15,7 +14,9 @@ def delrf(s: Optional[str]) -> Optional[str]:
 
 
 def test_org_indexer() -> None:
-    [_, cpp, cozy] = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file.org'))]
+    [_, cpp, cozy] = [
+        v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file.org'))
+    ]
 
     assert cpp.url == 'https://www.youtube.com/watch?v=rHIkrotSwcc'
     # TODO not sure about filetags?
@@ -34,7 +35,10 @@ def test_org_indexer_2() -> None:
     items = [v if isinstance(v, Visit) else throw(v) for v in extract_from_file(get_testdata('auto/orgs/file3.org'))]
 
     assert len(items) == 6
-    assert items[0].url == 'https://www.reddit.com/r/androidapps/comments/4i36z9/how_you_use_your_android_to_the_maximum/d2uq24i'
+    assert (
+        items[0].url
+        == 'https://www.reddit.com/r/androidapps/comments/4i36z9/how_you_use_your_android_to_the_maximum/d2uq24i'
+    )
     assert items[1].url == 'https://link.com'
     assert items[-2].url == 'https://en.wikipedia.org/wiki/Resilio_Sync'
     # TODO shit def need org specific url extractor (and then extract from everything remaining)

promnesia/tests/sources/test_plaintext.py

@@ -1,18 +1,19 @@
 from ...common import Source
 from ...extract import extract_visits
 from ...sources import plaintext, shellcmd
-
 from ..common import get_testdata, unwrap
 
 
 def test_plaintext_path_extractor() -> None:
-    visits = list(extract_visits(
-        Source(
-            shellcmd.index,
-            plaintext.extract_from_path(get_testdata('custom')),
-        ),
-        src='whatever',
-    ))
+    visits = list(
+        extract_visits(
+            Source(
+                shellcmd.index,
+                plaintext.extract_from_path(get_testdata('custom')),
+            ),
+            src='whatever',
+        )
+    )
     assert {unwrap(v).orig_url for v in visits} == {
         'http://google.com',
         'http://google.com/',

promnesia/tests/sources/test_shellcmd.py

@@ -1,22 +1,22 @@
-from ...common import _is_windows, Source
-from ...extract import extract_visits
-from ...sources import shellcmd
-
 import pytest
 
+from ...common import Source, _is_windows
+from ...extract import extract_visits
+from ...sources import shellcmd
 from ..common import get_testdata
 
 
 @pytest.mark.skipif(_is_windows, reason="no grep on windows")
 def test_via_grep() -> None:
-
-    visits = list(extract_visits(
-        Source(
-            shellcmd.index,
-            # meh. maybe should deprecate plain string here...
-            r"""grep -Eo -r --no-filename (http|https)://\S+ """ + str(get_testdata('custom')),
-        ),
-        src='whatever',
-    ))
+    visits = list(
+        extract_visits(
+            Source(
+                shellcmd.index,
+                # meh. maybe should deprecate plain string here...
+                r"""grep -Eo -r --no-filename (http|https)://\S+ """ + str(get_testdata('custom')),
+            ),
+            src='whatever',
+        )
+    )
     # TODO I guess filtering of equivalent urls should rather be tested on something having context (e.g. org mode)
     assert len(visits) == 5

promnesia/tests/sources/test_takeout.py

@@ -1,15 +1,13 @@
 from datetime import datetime, timezone
 
+import pytest
+from my.core.cfg import tmp_config
+
 from ...common import Source
 from ...extract import extract_visits
 from ...sources import takeout
-
-import pytest
-
 from ..common import get_testdata, unwrap
 
-from my.core.cfg import tmp_config
-
 
 # TODO apply in conftest so it's used in all tests?
 @pytest.fixture