promnesia 1.2.20240810__py3-none-any.whl → 1.3.20241021__py3-none-any.whl

promnesia/sources/telegram.py

@@ -1,11 +1,12 @@
-from typing import Optional
-from urllib.parse import unquote # TODO mm, make it easier to rememember to use...
+from __future__ import annotations
+
 import warnings
+from urllib.parse import unquote  # TODO mm, make it easier to rememember to use...
 
-from promnesia.common import Results, logger, extract_urls, Visit, Loc, PathIsh
+from promnesia.common import Loc, PathIsh, Results, Visit, extract_urls, logger
 
 
-def index(database: Optional[PathIsh]=None, *, http_only: bool=False, with_extra_media_info: bool=False)  -> Results:
+def index(database: PathIsh | None=None, *, http_only: bool=False, with_extra_media_info: bool=False)  -> Results:
     if database is None:
         # fully relying on HPI
         yield from _index_new(http_only=http_only, with_extra_media_info=with_extra_media_info)
@@ -17,10 +18,11 @@ def index(database: Optional[PathIsh]=None, *, http_only: bool=False, with_extra
     )
     try:
         yield from _index_new_with_adhoc_config(database=database, http_only=http_only, with_extra_media_info=with_extra_media_info)
-        return
     except Exception as e:
         logger.exception(e)
         warnings.warn("Hacking my.config.telegram.telegram_backup didn't work. You probably need to update HPI.")
+    else:
+        return
 
     logger.warning("Falling back onto promnesia.sources.telegram_legacy module")
     yield from _index_legacy(database=database, http_only=http_only)
@@ -32,7 +34,7 @@ def _index_legacy(*, database: PathIsh, http_only: bool) -> Results:
 
 
 def _index_new_with_adhoc_config(*, database: PathIsh, http_only: bool, with_extra_media_info: bool) -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
 
     class config:
         class telegram:
@@ -45,14 +47,14 @@ def _index_new_with_adhoc_config(*, database: PathIsh, http_only: bool, with_ext
 
 
 def _index_new(*, http_only: bool, with_extra_media_info: bool) -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.telegram.telegram_backup import messages
 
     extra_where = "(has_media == 1 OR text LIKE '%http%')" if http_only else None
-    for i, m in enumerate(messages(
-            with_extra_media_info=with_extra_media_info,
-            extra_where=extra_where,
-    )):
+    for m in messages(
+        with_extra_media_info=with_extra_media_info,
+        extra_where=extra_where,
+    ):
         text = m.text
 
         urls = extract_urls(text)

promnesia/sources/telegram_legacy.py

@@ -2,23 +2,34 @@
 Uses [[https://github.com/fabianonline/telegram_backup#readme][telegram_backup]] database for messages data
 '''
 
-from pathlib import Path
+from __future__ import annotations
+
 import sqlite3
+from pathlib import Path
 from textwrap import dedent
-from typing import Union, TypeVar
-from urllib.parse import unquote # TODO mm, make it easier to rememember to use...
+from typing import TypeVar
+from urllib.parse import unquote  # TODO mm, make it easier to rememember to use...
+
+from promnesia.common import (
+    Loc,
+    PathIsh,
+    Results,
+    Visit,
+    echain,
+    extract_urls,
+    from_epoch,
+    get_logger,
+)
 
-from ..common import PathIsh, Visit, get_logger, Loc, extract_urls, from_epoch, Results, echain
 from ..sqlite import sqlite_connection
 
 T = TypeVar("T")
 
 
-def unwrap(res: Union[T, Exception]) -> T:
+def unwrap(res: T | Exception) -> T:
     if isinstance(res, Exception):
         raise res
-    else:
-        return res
+    return res
 
 
 def index(database: PathIsh, *, http_only: bool=False) -> Results:

promnesia/sources/twitter.py

@@ -1,18 +1,19 @@
 '''
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Twitter data.
 '''
-from typing import Iterable
 
-from ..common import logger, Results, Visit, Loc, extract_urls, Res
+from collections.abc import Iterable
+
+from promnesia.common import Loc, Res, Results, Visit, extract_urls, logger
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.twitter.all as tw
+    from my.twitter.archive import Tweet  # todo extract to common or something?
+
     # TODO hmm. tweets themselves are sort of visits? not sure if they should contribute..
     processed = 0
-
-    from my.twitter.archive import Tweet # todo extract to common or something?
     tweets: Iterable[Res[Tweet]] = tw.tweets()
     for t in tweets:
         if isinstance(t, Exception):

promnesia/sources/vcs.py

@@ -1,12 +1,14 @@
 '''
 Clones & indexes Git repositories (via sources.auto)
 '''
-# TODO not sure if worth exposing... could be just handled by auto or something?)
+from __future__ import annotations
 
-from pathlib import Path
 import re
+from collections.abc import Iterable
+
+# TODO not sure if worth exposing... could be just handled by auto or something?)
+from pathlib import Path
 from subprocess import check_call
-from typing import Iterable
 
 from ..common import Extraction, PathIsh, get_tmpdir, slugify
 

promnesia/sources/viber.py

@@ -2,17 +2,18 @@
 Collects visits from Viber desktop app (e.g. `~/.ViberPC/XYZ123/viber.db`)
 """
 
+from __future__ import annotations
+
 import logging
+import sqlite3
 import textwrap
+from collections.abc import Iterable
 from os import PathLike
 from pathlib import Path
-import sqlite3
-from typing import Iterable, Optional
 
 from ..common import Loc, PathIsh, Results, Visit, extract_urls, from_epoch, join_tags
 from ..sqlite import sqlite_connection
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -34,12 +35,12 @@ def index(
 
     msgs_query = messages_query(http_only)
 
-    for db_path in _get_files(db_path):
-        assert db_path.is_file(), f"Is it a (Viber-desktop sqlite) file? {db_path}"
-        yield from _harvest_db(db_path, msgs_query, locator_schema)
+    for db in _get_files(db_path):
+        assert db.is_file(), f"Is it a (Viber-desktop sqlite) file? {db}"
+        yield from _harvest_db(db, msgs_query, locator_schema)
 
 
-def messages_query(http_only: Optional[bool]) -> str:
+def messages_query(http_only: bool | None) -> str:
     """
     An SQL-query returning 1 row for each message
 
@@ -123,7 +124,7 @@ def _handle_row(row: sqlite3.Row, db_path: PathLike, locator_schema: str) -> Res
     tags: str = row["tags"]
     url_title: str = row["url_title"]
 
-    assert (
+    assert (  # noqa: PT018
         text and mid and sender and chatname
     ), f"sql-query should eliminate messages without 'http' or missing ids: {row}"
 
@@ -154,7 +155,7 @@ def _get_files(path: PathIsh) -> Iterable[Path]:
     """
     path = Path(path).expanduser()
     parts = path.parts[1:] if path.is_absolute() else path.parts
-    return Path(path.root).glob(str(Path("").joinpath(*parts)))
+    return Path(path.root).glob(str(Path("").joinpath(*parts)))  # noqa: PTH201
 
 
 def _harvest_db(db_path: PathIsh, msgs_query: str, locator_schema: str) -> Results:

promnesia/sources/website.py

@@ -2,12 +2,12 @@
 Clones a website with wget and indexes via sources.auto
 '''
 
-from pathlib import Path
 import re
+from collections.abc import Iterable
+from pathlib import Path
 from subprocess import run
-from typing import Iterable
 
-from ..common import Extraction, PathIsh, get_tmpdir, slugify, get_logger
+from promnesia.common import Extraction, PathIsh, get_logger, get_tmpdir, slugify
 
 
 def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
@@ -30,7 +30,7 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
     ]
     # TODO follow sitemap? e.g. gwern
     logger.info(' '.join(cmd))
-    res = run(cmd)
+    res = run(cmd, check=False)
 
     if res.returncode == 8:
         # man wget: 8 means server error (e.g. broken link)

promnesia/sources/zulip.py

@@ -2,12 +2,13 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Zulip data.
 '''
 
-from ..common import Results, Visit, Loc, iter_urls
+from promnesia.common import Loc, Results, Visit, iter_urls
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.zulip.organization as Z
+
     for m in Z.messages():
         if isinstance(m, Exception):
             yield m

promnesia/sqlite.py

@@ -1,6 +1,9 @@
-from contextlib import contextmanager
+from __future__ import annotations
+
 import sqlite3
-from typing import Callable, Optional, Any, Iterator, Union, Literal
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any, Callable, Literal, Union
 
 from .common import PathIsh
 
@@ -10,13 +13,13 @@ SqliteRowFactory = Callable[[sqlite3.Cursor, sqlite3.Row], Any]
 
 def dict_factory(cursor, row):
     fields = [column[0] for column in cursor.description]
-    return {key: value for key, value in zip(fields, row)}
+    return dict(zip(fields, row))
 
 
 Factory = Union[SqliteRowFactory, Literal['row', 'dict']]
 
 @contextmanager
-def sqlite_connection(db: PathIsh, *, immutable: bool=False, row_factory: Optional[Factory]=None) -> Iterator[sqlite3.Connection]:
+def sqlite_connection(db: PathIsh, *, immutable: bool=False, row_factory: Factory | None=None) -> Iterator[sqlite3.Connection]:
     dbp = f'file:{db}'
     # https://www.sqlite.org/draft/uri.html#uriimmutable
     if immutable:

promnesia/tests/common.py

@@ -1,16 +1,19 @@
-from contextlib import closing, contextmanager
+from __future__ import annotations
+
 import gc
 import inspect
 import os
-from pathlib import Path
 import socket
 import sys
+from collections.abc import Iterator
+from contextlib import closing, contextmanager
+from pathlib import Path
 from textwrap import dedent
-from typing import Iterator, NoReturn, TypeVar
+from typing import NoReturn, TypeVar
 
 import pytest
 
-from ..common import _is_windows, Res
+from ..common import Res, _is_windows
 
 
 def under_ci() -> bool:
@@ -25,7 +28,7 @@ def throw(x: Exception) -> NoReturn:
 
 
 @pytest.fixture
-def gc_control(gc_on: bool):
+def gc_control(*, gc_on: bool):
     if gc_on:
         # no need to do anything, should be on by default
         yield

promnesia/tests/server_helper.py

@@ -1,15 +1,18 @@
+from __future__ import annotations
+
+import sys
+import time
+from collections.abc import Iterator
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-import sys
-import time
-from typing import Any, Dict, Iterator, Optional
+from typing import Any
 
 import psutil
 import requests
 
 from ..common import PathIsh
-from .common import tmp_popen, promnesia_bin, free_port
+from .common import free_port, promnesia_bin, tmp_popen
 
 
 @dataclass
@@ -18,18 +21,18 @@ class Helper:
     port: str
     process: psutil.Popen
 
-    def get(self, path: str, *args):
+    def get(self, path: str):
         # check it's alive first so the error is cleaner
         assert self.process.poll() is None, self.process
         return requests.get(f'http://{self.host}:{self.port}' + path)
 
-    def post(self, path: str, *, json: Optional[Dict[str, Any]] = None):
+    def post(self, path: str, *, json: dict[str, Any] | None = None):
         assert self.process.poll() is None, self.process
         return requests.post(f'http://{self.host}:{self.port}' + path, json=json)
 
 
 @contextmanager
-def run_server(db: Optional[PathIsh] = None, *, timezone: Optional[str] = None) -> Iterator[Helper]:
+def run_server(db: PathIsh | None = None, *, timezone: str | None = None) -> Iterator[Helper]:
     # TODO not sure, perhaps best to use a thread or something?
     # but for some tests makes more sense to test in a separate process
     with free_port() as pp:
@@ -56,7 +59,7 @@ def run_server(db: Optional[PathIsh] = None, *, timezone: Optional[str] = None)
                     time.sleep(0.1)
             else:
                 raise RuntimeError("Cooldn't connect to '{st}' after 50 attempts")
-            print("Started server up, db: {db}".format(db=db), file=sys.stderr)
+            print(f"Started server up, db: {db}", file=sys.stderr)
 
             yield server
 

promnesia/tests/sources/test_auto.py

@@ -1,8 +1,7 @@
-from itertools import groupby
 import os
+from itertools import groupby
 
 from ...sources import auto
-
 from ..common import get_testdata, throw
 
 sa2464 = 'https://www.scottaaronson.com/blog/?p=2464'
@@ -19,7 +18,7 @@ def makemap(visits):
     def it():
         vit = (throw(v) if isinstance(v, Exception) else v for v in visits)
         for k, g in groupby(sorted(vit, key=key), key=key):
-            yield k, list(sorted(g))
+            yield k, sorted(g)
 
     return dict(it())
 

promnesia/tests/sources/test_filetypes.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
-from ...common import PathIsh, _is_windows as windows
+from ...common import PathIsh
+from ...common import _is_windows as windows
 from ...sources.auto import by_path
 
 

promnesia/tests/sources/test_hypothesis.py

@@ -1,10 +1,10 @@
 from pathlib import Path
 
-from ..common import write_config, get_testdata
+from my.core.cfg import tmp_config
+
 from ...__main__ import do_index
 from ...database.load import get_all_db_visits
-
-from my.core.cfg import tmp_config
+from ..common import get_testdata, write_config
 
 
 def index_hypothesis(tmp_path: Path) -> None:

promnesia/tests/sources/test_org.py

@@ -1,12 +1,11 @@
-from typing import Optional
+from __future__ import annotations
 
 from ...common import Visit
 from ...sources.org import extract_from_file
-
 from ..common import get_testdata, throw
 
 
-def delrf(s: Optional[str]) -> Optional[str]:
+def delrf(s: str | None) -> str | None:
     if s is None:
         return None
     # meh.. not sure how ot handle this properly, ideally should be via pytest?

promnesia/tests/sources/test_plaintext.py

@@ -1,7 +1,6 @@
 from ...common import Source
 from ...extract import extract_visits
 from ...sources import plaintext, shellcmd
-
 from ..common import get_testdata, unwrap
 
 

promnesia/tests/sources/test_shellcmd.py

@@ -1,9 +1,8 @@
-from ...common import _is_windows, Source
-from ...extract import extract_visits
-from ...sources import shellcmd
-
 import pytest
 
+from ...common import Source, _is_windows
+from ...extract import extract_visits
+from ...sources import shellcmd
 from ..common import get_testdata
 
 

promnesia/tests/sources/test_takeout.py

@@ -1,15 +1,13 @@
 from datetime import datetime, timezone
 
+import pytest
+from my.core.cfg import tmp_config
+
 from ...common import Source
 from ...extract import extract_visits
 from ...sources import takeout
-
-import pytest
-
 from ..common import get_testdata, unwrap
 
-from my.core.cfg import tmp_config
-
 
 # TODO apply in conftest so it's used in all tests?
 @pytest.fixture

promnesia/tests/test_cannon.py

@@ -2,7 +2,7 @@ from typing import cast
 
 import pytest
 
-from ..cannon import canonify, CanonifyException
+from ..cannon import CanonifyException, canonify
 
 # TODO should actually understand 'sequences'?
 # e.g.
@@ -134,7 +134,7 @@ def test_reddit(url, expected):
 def test_pocket(url, expected):
     assert canonify(url) == expected
 
-@pytest.mark.parametrize("url,expected", [
+@pytest.mark.parametrize(("url", "expected"), [
     # TODO ?? 'https://groups.google.com/a/list.hypothes.is/forum/#!topic/dev/kcmS7H8ssis',
     #
     # TODO FIXME fragment handling
@@ -295,7 +295,7 @@ def test(url, expected):
     },
 ])
 def test_same_norm(urls):
-    urls = list(sorted(urls))
+    urls = sorted(urls)
     u0 = urls[0]
     c0 = canonify(u0)
     for u in urls[1:]:
@@ -308,7 +308,7 @@ def test_error():
         # borrowed from https://bugs.mageia.org/show_bug.cgi?id=24640#c7
         canonify('https://example.com\uFF03@bing.com')
 
-@pytest.mark.parametrize("url,expected", [
+@pytest.mark.parametrize(("url", "expected"), [
     ('https://news.ycombinator.com/item?id=', 'news.ycombinator.com/item?id='),
     ('https://www.youtube.com/watch?v=hvoQiF0kBI8&list&index=2',
      'youtube.com/watch?v=hvoQiF0kBI8&list='),
@@ -316,7 +316,7 @@ def test_error():
 def test_empty_query_parameter(url, expected):
     assert canonify(url) == expected
 
-@pytest.mark.parametrize("url,expected", [
+@pytest.mark.parametrize(("url", "expected"), [
     ('http://www.isfdb.org/cgi-bin/title.cgi?2172', 'isfdb.org/cgi-bin/title.cgi?2172='),
     ('http://www.isfdb.org/cgi-bin/title.cgi?2172+1', 'isfdb.org/cgi-bin/title.cgi?2172%201='),
     ('http://www.isfdb.org/cgi-bin/title.cgi?2172&foo=bar&baz&quux', 'isfdb.org/cgi-bin/title.cgi?2172=&baz=&foo=bar&quux='),

promnesia/tests/test_cli.py

@@ -1,13 +1,11 @@
 import os
 import time
 
-from ..common import _is_windows
-
-from .common import get_testdata, promnesia_bin, tmp_popen
-
 import pytest
 import requests
 
+from ..common import _is_windows
+from .common import get_testdata, promnesia_bin, tmp_popen
 
 ox_hugo_data = get_testdata('ox-hugo/test/site')
 
@@ -22,12 +20,12 @@ def test_demo() -> None:
         # TODO why does it want post??
         time.sleep(2)  # meh.. need a generic helper to wait till ready...
         res = {}
-        for attempt in range(30):
+        for _attempt in range(30):
             time.sleep(1)
             try:
                 res = requests.post(
                     "http://localhost:16789/search",
-                    json=dict(url="https://github.com/kaushalmodi/ox-hugo/issues"),
+                    json={'url': "https://github.com/kaushalmodi/ox-hugo/issues"},
                 ).json()
                 break
             except:

promnesia/tests/test_compare.py

@@ -1,5 +1,5 @@
-from pathlib import Path
 import shutil
+from pathlib import Path
 
 from ..compare import compare_files
 from .utils import index_urls

promnesia/tests/test_config.py

@@ -1,15 +1,14 @@
+from __future__ import annotations
+
 from contextlib import contextmanager
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Union, List
-
-from ..common import Source
-from ..config import import_config, Config
-
 
-from more_itertools import ilen
 import pytest
+from more_itertools import ilen
 
+from ..common import Source
+from ..config import Config, import_config
 from .common import throw
 
 
@@ -22,7 +21,7 @@ def make(body: str) -> Config:
 
 
 @contextmanager
-def with_config(cfg: Union[str, Config]):
+def with_config(cfg: str | Config):
     from .. import config as C
 
     assert not C.has()
@@ -35,7 +34,7 @@ def with_config(cfg: Union[str, Config]):
         C.reset()
 
 
-def index(cfg: Union[str, Config], check=True) -> List[Exception]:
+def index(cfg: str | Config, *, check: bool = True) -> list[Exception]:
     from ..__main__ import _do_index
 
     with with_config(cfg):

promnesia/tests/test_db_dump.py

@@ -4,30 +4,29 @@ from concurrent.futures import ProcessPoolExecutor
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Any, Iterable
+from typing import Any
 
-
-from hypothesis import settings, given
-from hypothesis.strategies import from_type
 # NOTE: pytest ... -s --hypothesis-verbosity=debug is useful for seeing what hypothesis is doing
 import pytest
 import pytz
-
+from hypothesis import given, settings
+from hypothesis.strategies import from_type
 
 from ..common import Loc
 from ..database.common import DbVisit
 from ..database.dump import visits_to_sqlite
 from ..database.load import get_all_db_visits
 from ..sqlite import sqlite_connection
-
-from .common import gc_control, running_on_ci
-
-
-HSETTINGS: dict[str, Any] = dict(
-    derandomize=True,
-    deadline=timedelta(seconds=2),  # sometimes slow on ci
+from .common import (
+    gc_control,  # noqa: F401
+    running_on_ci,
 )
 
+HSETTINGS: dict[str, Any] = {
+    'derandomize': True,
+    'deadline': timedelta(seconds=2),  # sometimes slow on ci
+}
+
 
 def test_no_visits(tmp_path: Path) -> None:
     visits: list[DbVisit] = []

promnesia/tests/test_extract.py

@@ -1,12 +1,16 @@
 from datetime import datetime, timezone
 
-from ..common import Visit, DbVisit, Loc, Source
-from ..extract import extract_visits
-
-from .common import get_testdata, unwrap, running_on_ci, gc_control
-
-from more_itertools import ilen
 import pytest
+from more_itertools import ilen
+
+from ..common import DbVisit, Loc, Source, Visit
+from ..extract import extract_visits
+from .common import (
+    gc_control,  # noqa: F401
+    get_testdata,
+    running_on_ci,
+    unwrap,
+)
 
 
 def test_with_error() -> None: