promnesia 1.2.20240810__py3-none-any.whl → 1.3.20241021__py3-none-any.whl

promnesia/sources/org.py

@@ -1,16 +1,26 @@
-from datetime import datetime
+from __future__ import annotations
+
 import re
-from typing import Iterable, List, Set, Optional, Iterator, Tuple, NamedTuple, cast
+from collections.abc import Iterable, Iterator
+from datetime import datetime
 from pathlib import Path
-
-
-from ..common import Visit, get_logger, Results, Url, Loc, from_epoch, iter_urls, PathIsh, Res, file_mtime
-
+from typing import NamedTuple, Optional, cast
 
 import orgparse
-from orgparse.date import gene_timestamp_regex, OrgDate
+from orgparse.date import OrgDate, gene_timestamp_regex
 from orgparse.node import OrgNode
 
+from promnesia.common import (
+    Loc,
+    PathIsh,
+    Res,
+    Results,
+    Url,
+    Visit,
+    file_mtime,
+    get_logger,
+    iter_urls,
+)
 
 UPDATE_ORGPARSE_WARNING = 'WARNING: please update orgparse version to a more recent (pip3 install -U orgparse)'
 
@@ -36,7 +46,7 @@ CREATED_RGX = re.compile(gene_timestamp_regex(brtype='inactive'), re.VERBOSE)
 """
 
 class Parsed(NamedTuple):
-    dt: Optional[datetime]
+    dt: datetime | None
     heading: str
 
 
@@ -74,7 +84,7 @@ def _get_heading(n: OrgNode):
     return '' if n.is_root() else n.get_heading(format='raw')
 
 
-def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[Tuple[Parsed, OrgNode]]]:
+def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[tuple[Parsed, OrgNode]]]:
     try:
         parsed = _parse_node(node)
     except Exception as e:
@@ -98,7 +108,7 @@ def get_body_compat(node: OrgNode) -> str:
             # get_body was only added to root in 0.2.0
             for x in warn_old_orgparse_once():
                 # ugh. really crap, but it will at least only warn once... (becaue it caches)
-                raise x
+                raise x  # noqa: B904
             return UPDATE_ORGPARSE_WARNING
         else:
             raise e

promnesia/sources/plaintext.py

@@ -1,10 +1,9 @@
-from ..common import get_logger, get_tmpdir, PathIsh, _is_windows
-from ..compat import removeprefix
+from __future__ import annotations
 
 from functools import lru_cache
 from pathlib import Path
-import os
-from typing import List
+
+from promnesia.common import PathIsh, _is_windows, get_logger, get_tmpdir
 
 # https://linux-and-mac-hacks.blogspot.co.uk/2013/04/use-grep-and-regular-expressions-to.html
 _URL_REGEX = r'\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|]'
@@ -12,16 +11,16 @@ _URL_REGEX = r'\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@
 if _is_windows:
     # wtf? for some reason on windows (in cmd.exe specificaly) \b isn't working...
     # this will make the regex a bit less precise, but not end of the world
-    _URL_REGEX = removeprefix(_URL_REGEX, r'\b')
+    _URL_REGEX = _URL_REGEX.removeprefix(r'\b')
 
 
-@lru_cache()
+@lru_cache
 def _has_grep() -> bool:
     import shutil
     return shutil.which('grep') is not None
 
 
-Command = List[str]
+Command = list[str]
 
 
 _GREP_ARGS: Command = [
@@ -39,7 +38,7 @@ if not _is_windows:
 
 # NOTE: grep/findstr exit with code 1 on no matches...
 # we hack around it in shellcmd module (search 'grep')
-def _grep(*, paths: List[str], recursive: bool) -> Command:
+def _grep(*, paths: list[str], recursive: bool) -> Command:
     return [
         'grep',
         *(['-r'] if recursive else []),
@@ -91,26 +90,26 @@ def extract_from_path(path: PathIsh) -> Command:
     logger = get_logger()
     if pp.is_dir(): # TODO handle archives here???
         return _extract_from_dir(str(pp))
-    else:
-        if any(pp.suffix == ex for ex in (
-                '.xz',
-                '.bz2',
-                '.gz',
-                '.zip',
-        )):
-            # todo should be debug?
-            # or should delete it completely, feels like unpacking archives here is a bit too much
-            raise RuntimeError(f"Archives aren't supported yet: {path}")
-            logger.info(f"Extracting from compressed file {path}")
-            import lzma
-            from tempfile import NamedTemporaryFile
-            # TODO hopefully, no collisions
-            import os.path
-            fname = os.path.join(tdir.name, os.path.basename(path))
-            with open(fname, 'wb') as fo:
-                with lzma.open(path, 'r') as cf:
-                    fo.write(cf.read())
-                return _extract_from_file(fname)
-        else:
-            r = _extract_from_file(str(pp))
-            return r
+
+    if any(pp.suffix == ex for ex in (
+            '.xz',
+            '.bz2',
+            '.gz',
+            '.zip',
+    )):
+        # todo should be debug?
+        # or should delete it completely, feels like unpacking archives here is a bit too much
+        raise RuntimeError(f"Archives aren't supported yet: {path}")
+        # logger.info(f"Extracting from compressed file {path}")
+        # import lzma
+        # from tempfile import NamedTemporaryFile
+        # # TODO hopefully, no collisions
+        # import os.path
+        # fname = os.path.join(tdir.name, os.path.basename(path))
+        # with open(fname, 'wb') as fo:
+        #     with lzma.open(path, 'r') as cf:
+        #         fo.write(cf.read())
+        #     return _extract_from_file(fname)
+
+    r = _extract_from_file(str(pp))
+    return r

promnesia/sources/pocket.py

@@ -1,11 +1,12 @@
 '''
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Pocket highlights & bookmarks
 '''
-from ..common import Visit, Loc, Results
+
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.pocket import articles
 
     # TODO use docstring from my. module? E.g. describing which pocket format is expected

promnesia/sources/reddit.py

@@ -2,21 +2,27 @@
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myreddit][reddit]] module
 '''
 
+from __future__ import annotations
+
+import typing
 from itertools import chain
-from typing import Set, Optional, Type
 
-from ..common import Visit, Loc, extract_urls, Results, logger
+from promnesia.common import Loc, Results, Visit, extract_urls, logger
+
+if typing.TYPE_CHECKING:
+    from my.reddit.common import Comment, RedditBase, Save, Submission, Upvote
+
 
+def index(*, render_markdown: bool = False, renderer: type[RedditRenderer] | None = None) -> Results:
+    from . import hpi  # noqa: F401
 
-def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRenderer']] = None) -> Results:
-    from . import hpi
     try:
-        from my.reddit.all import submissions, comments, saved, upvoted
+        from my.reddit.all import comments, saved, submissions, upvoted
     except ModuleNotFoundError as e:
         if "No module named 'my.reddit.all'" in str(e):
             import warnings
             warnings.warn("DEPRECATED/reddit: Using an old version of HPI, please update")
-            from my.reddit import submissions, comments, saved, upvoted
+            from my.reddit import comments, saved, submissions, upvoted
         else:
             raise e
 
@@ -58,7 +64,7 @@ def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRende
 # mostly here so we can keep track of how the user
 # wants to render markdown
 class RedditRenderer:
-    def __init__(self, render_markdown: bool = False) -> None:
+    def __init__(self, *, render_markdown: bool = False) -> None:
         self._link_extractor = None
         self._parser_cls = None
         try:
@@ -77,7 +83,7 @@ class RedditRenderer:
         self.render_markdown = render_markdown
 
 
-    def _from_comment(self, i: 'Comment') -> Results:
+    def _from_comment(self, i: Comment) -> Results:
         locator = Loc.make(
             title='Reddit comment',
             href=i.url,
@@ -85,7 +91,7 @@ class RedditRenderer:
         yield from self._from_common(i, locator=locator)
 
 
-    def _from_submission(self, i: 'Submission') -> Results:
+    def _from_submission(self, i: Submission) -> Results:
         locator = Loc.make(
             title=f'Reddit submission: {i.title}',
             href=i.url,
@@ -93,7 +99,7 @@ class RedditRenderer:
         yield from self._from_common(i, locator=locator)
 
 
-    def _from_upvote(self, i: 'Upvote') -> Results:
+    def _from_upvote(self, i: Upvote) -> Results:
         locator = Loc.make(
             title='Reddit upvote',
             href=i.url,
@@ -101,7 +107,7 @@ class RedditRenderer:
         yield from self._from_common(i, locator=locator)
 
 
-    def _from_save(self, i: 'Save') -> Results:
+    def _from_save(self, i: Save) -> Results:
         locator = Loc.make(
             title='Reddit save',
             href=i.url,
@@ -117,7 +123,7 @@ class RedditRenderer:
             return text
 
 
-    def _from_common(self, i: 'RedditBase', locator: Loc) -> Results:
+    def _from_common(self, i: RedditBase, locator: Loc) -> Results:
         urls = [i.url]
         # TODO this should belong to HPI.. fix permalink handling I guess
         # ok, it's not present for all of them..
@@ -130,7 +136,7 @@ class RedditRenderer:
 
         context = self._render_body(i.text)
 
-        emitted: Set[str] = set()
+        emitted: set[str] = set()
 
         for url in chain(urls, extract_urls(i.text)):
             if url in emitted:
@@ -165,8 +171,3 @@ class RedditRenderer:
                 )
                 emitted.add(res.url)
 
-
-import typing
-if typing.TYPE_CHECKING:
-    from my.reddit.common import Submission, Comment, Save, Upvote, RedditBase
-

promnesia/sources/roamresearch.py

@@ -2,7 +2,7 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Roam Research data
 '''
 
-from ..common import Results, Visit, Loc, extract_urls
+from promnesia.common import Loc, Results, Visit, extract_urls
 
 
 def index() -> Results:
@@ -43,6 +43,7 @@ def _collect(node: 'RoamNode') -> Results:
 
 
 import typing
+
 if typing.TYPE_CHECKING:
     import my.roamresearch as RR
     RoamNode = RR.Node

promnesia/sources/rss.py

@@ -2,14 +2,12 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for RSS data.
 '''
 
-from itertools import chain
-
-from ..common import Visit, Loc, extract_urls, Results, get_logger
-
 from datetime import datetime
 
 import pytz
 
+from promnesia.common import Loc, Results, Visit
+
 # arbitrary,  2011-11-04 00:05:23.283+00:00
 default_datetime = datetime.fromtimestamp(1320365123, tz=pytz.utc)
 # TODO FIXME allow for visit not to have datetime?
@@ -17,6 +15,7 @@ default_datetime = datetime.fromtimestamp(1320365123, tz=pytz.utc)
 
 def index() -> Results:
     from my.rss.all import subscriptions
+
     for feed in subscriptions():
         # TODO locator should be optional too? although could use direct link in the rss reader interface
         locator = Loc.make(title='my.rss')

promnesia/sources/shellcmd.py

@@ -2,18 +2,31 @@
 Greps out URLs from an arbitrary shell command results.
 """
 
-from datetime import datetime
+from __future__ import annotations
+
 import os
 import re
-from subprocess import run, PIPE
-from typing import Union, Sequence
 import warnings
+from collections.abc import Sequence
+from datetime import datetime
+from subprocess import PIPE, run
+
+from promnesia.common import (
+    Loc,
+    PathIsh,
+    Results,
+    Visit,
+    _is_windows,
+    extract_urls,
+    file_mtime,
+    get_system_tz,
+    now_tz,
+)
 
-from ..common import Visit, Loc, Results, extract_urls, file_mtime, get_system_tz, now_tz, _is_windows, PathIsh
 from .plaintext import _has_grep
 
 
-def index(command: Union[str, Sequence[PathIsh]]) -> Results:
+def index(command: str | Sequence[PathIsh]) -> Results:
     cmd: Sequence[PathIsh]
     cmds: str
     if isinstance(command, str):
@@ -71,7 +84,7 @@ def index(command: Union[str, Sequence[PathIsh]]) -> Results:
                 context=context,
             )
 
-    r = run(cmd, stdout=PIPE)
+    r = run(cmd, stdout=PIPE, check=False)
     if r.returncode > 0:
         if not (cmd[0] in {'grep', 'findstr'} and r.returncode == 1): # ugh. grep returns 1 on no matches...
             r.check_returncode()

promnesia/sources/signal.py

@@ -1,23 +1,23 @@
 """
 Collects visits from Signal Desktop's encrypted SQLIite db(s).
 """
+from __future__ import annotations
 
 # Functions get their defaults from module-data.
 #
 # * Open-ciphered-db adapted from:
 #   https://github.com/carderne/signal-export/commit/2284c8f4
 # * Copyright (c) 2019 Chris Arderne, 2020 Kostis Anagnostopoulos
-
-
 import json
 import logging
 import platform
 import sqlite3
 import subprocess as sbp
+from collections.abc import Iterable, Iterator, Mapping
 from contextlib import contextmanager
 from pathlib import Path
 from textwrap import dedent, indent
-from typing import Any, Iterable, Iterator, Mapping, Union, Optional
+from typing import Any, Union
 
 from ..common import Loc, PathIsh, Results, Visit, extract_urls, from_epoch
 
@@ -29,7 +29,7 @@ def index(
     http_only: bool = False,
     locator_schema: str="editor",
     append_platform_path: bool = False,
-    override_key: Optional[str] = None,
+    override_key: str | None = None,
 ) -> Results:
     """
     :param db_paths:
@@ -109,10 +109,10 @@ messages_query = dedent(
             id,
             type,
             coalesce(
-                profileFullName, 
-                profileName, 
+                profileFullName,
+                profileName,
                 name,
-                profileFamilyName, 
+                profileFamilyName,
                 e164
             ) as aname,
             name,
@@ -237,11 +237,11 @@ def collect_db_paths(*db_paths: PathIsh, append: bool = False) -> Iterable[Path]
         platform_name = platform.system()
         try:
             plat_paths = platform_db_paths[platform_name]
-        except LookupError:
+        except LookupError as le:
             raise ValueError(
                 f"Unknown platform({platform_name}!"
                 f"\n  Expected one of {list(platform_db_paths.keys())}."
-            )
+            ) from le
 
         if db_paths and append:
             db_paths = [  # type: ignore[assignment]
@@ -261,7 +261,7 @@ def _config_for_dbfile(db_path: Path, default_key=None) -> Path:
 
 
 def _key_from_config(signal_desktop_config_path: PathIsh) -> str:
-    with open(signal_desktop_config_path, "r") as conf:
+    with Path(signal_desktop_config_path).open() as conf:
         return json.load(conf)["key"]
 
 
@@ -269,6 +269,7 @@ def _key_from_config(signal_desktop_config_path: PathIsh) -> str:
 def connect_db(
     db_path: Path,
     key,
+    *,
     decrypt_db: bool = False,
     sqlcipher_exe: PathIsh = "sqlcipher",
     **decryption_pragmas: Mapping[str, Any],
@@ -333,7 +334,7 @@ def connect_db(
                     check=True,
                     input=sql,
                     capture_output=True,
-                    universal_newlines=True,
+                    text=True,
                 )
             except sbp.CalledProcessError as ex:
                 prefix = " " * 4
@@ -380,7 +381,7 @@ def _handle_row(row: tuple, db_path: PathIsh, locator_schema: str) -> Results:
     if not urls:
         return
 
-    assert (
+    assert (  # noqa: PT018
         text and mid and sender and chatname
     ), f"should have eliminated messages without 'http' or missing ids: {row}"
 
@@ -400,7 +401,7 @@ def _harvest_db(
     db_path: Path,
     messages_query: str,
     *,
-    override_key: Optional[str] = None,
+    override_key: str | None = None,
     locator_schema: str = "editor",
     decrypt_db: bool = False,
     **decryption_pragmas,

promnesia/sources/smscalls.py

@@ -2,11 +2,11 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] smscalls module
 '''
 
-from promnesia.common import Visit, Loc, Results, extract_urls
+from promnesia.common import Loc, Results, Visit, extract_urls
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.smscalls import messages
 
     for m in messages():

promnesia/sources/stackexchange.py

@@ -2,12 +2,13 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Stackexchange data.
 '''
 
-from ..common import Results, Visit, Loc
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.stackexchange.gdpr as G
+
     for v in G.votes():
         if isinstance(v, Exception):
             yield v

promnesia/sources/takeout.py

@@ -1,11 +1,14 @@
 '''
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#mygoogletakeoutpaths][google.takeout]] module
 '''
-from typing import Iterable, Set, Any, NamedTuple
+
+from __future__ import annotations
+
 import warnings
+from collections.abc import Iterable
+from typing import Any, NamedTuple
 
-from ..common import Visit, Loc, Results, logger
-from ..compat import removeprefix
+from promnesia.common import Loc, Results, Visit, logger
 
 
 # incase user is using an old version of google_takeout_parser
@@ -14,13 +17,20 @@ class YoutubeCSVStub(NamedTuple):
 
 
 def index() -> Results:
-    from . import hpi
-    import json
+    from . import hpi  # noqa: F401
 
     try:
+        from google_takeout_parser.models import (
+            Activity,
+            ChromeHistory,
+            LikedYoutubeVideo,
+            YoutubeComment,
+        )
+        from google_takeout_parser.parse_csv import (
+            extract_comment_links,
+            reconstruct_comment_content,
+        )
         from my.google.takeout.parser import events
-        from google_takeout_parser.models import Activity, YoutubeComment, LikedYoutubeVideo, ChromeHistory
-        from google_takeout_parser.parse_csv import reconstruct_comment_content, extract_comment_links
     except ModuleNotFoundError as ex:
         logger.exception(ex)
         yield ex
@@ -32,7 +42,7 @@ def index() -> Results:
         return
 
 
-    _seen: Set[str] = {
+    _seen: set[str] = {
         # these are definitely not useful for promnesia
         'Location',
         'PlaceVisit',
@@ -54,7 +64,7 @@ def index() -> Results:
         if et_name in _seen:
             return
         _seen.add(et_name)
-        yield RuntimeError(f"Unhandled event {repr(type(e))}: {e}")
+        yield RuntimeError(f"Unhandled event {type(e)!r}: {e}")
 
     for e in events():
         if isinstance(e, Exception):
@@ -67,13 +77,13 @@ def index() -> Results:
                 # when you follow something from search the actual url goes after this
                 # e.g. https://www.google.com/url?q=https://en.wikipedia.org/wiki/Clapham
                 # note: also title usually starts with 'Visited ', in such case but perhaps fine to keep it
-                url = removeprefix(url, "https://www.google.com/url?q=")
+                url = url.removeprefix("https://www.google.com/url?q=")
                 title = e.title
 
                 if e.header == 'Chrome':
                     # title contains 'Visited <page title>' in this case
                     context = None
-                    title = removeprefix(title, 'Visited ')
+                    title = title.removeprefix('Visited ')
                 elif e.header in _CLEAR_CONTEXT_FOR_HEADERS:
                     # todo perhaps could add to some sort of metadata?
                     # only useful for debugging really
@@ -131,7 +141,7 @@ def index() -> Results:
                     url=url, dt=e.dt, context=e.content, locator=Loc(title=e.content, href=url)
                 )
         elif imported_yt_csv_models and isinstance(e, CSVYoutubeComment):
-            contentJSON = json.loads(e.contentJSON)
+            contentJSON = e.contentJSON
             content = reconstruct_comment_content(contentJSON, format='text')
             if isinstance(content, Exception):
                 yield content
@@ -149,7 +159,7 @@ def index() -> Results:
                 url=e.video_url, dt=e.dt, context=content, locator=Loc(title=context, href=e.video_url)
             )
         elif imported_yt_csv_models and isinstance(e, CSVYoutubeLiveChat):
-            contentJSON = json.loads(e.contentJSON)
+            contentJSON = e.contentJSON
             content = reconstruct_comment_content(contentJSON, format='text')
             if isinstance(content, Exception):
                 yield content

promnesia/sources/takeout_legacy.py

@@ -1,9 +1,13 @@
-from ..common import Visit, logger, PathIsh, Url, Loc, Results
+from __future__ import annotations
+
+from promnesia.common import Loc, Results, Visit, logger
+
 
 # TODO make an iterator, insert in db as we go? handle errors gracefully?
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.google.takeout.paths import get_takeouts
+
     takeouts = list(get_takeouts())
     # TODO if no takeouts, raise?
     # although could raise a warning on top level, when source emitted no takeouts
@@ -22,19 +26,17 @@ def index() -> Results:
 
 
 
-import pytz
-from itertools import chain
+import json
+from collections.abc import Iterable
 from datetime import datetime
-from typing import List, Optional, Iterable, TYPE_CHECKING
+from itertools import chain
 from pathlib import Path
-import json
-
-
-from .. import config
-
 
+import pytz
 from more_itertools import unique_everseen
 
+from promnesia import config
+
 try:
     from cachew import cachew
 except ModuleNotFoundError as me:
@@ -50,7 +52,9 @@ TakeoutPath = Path
 
 
 def _read_myactivity_html(takeout: TakeoutPath, kind: str) -> Iterable[Visit]:
+    # FIXME switch to actual kompress? and use CPath?
     from my.core.kompress import kexists
+
     # TODO glob
     # TODO not sure about windows path separators??
     spath = 'Takeout/My Activity/' + kind
@@ -61,7 +65,7 @@ def _read_myactivity_html(takeout: TakeoutPath, kind: str) -> Iterable[Visit]:
 
     locator = Loc.file(spath)
     from my.google.takeout.html import read_html
-    for dt, url, title in read_html(takeout, spath):
+    for dt, url, _title in read_html(takeout, spath):
         yield Visit(
             url=url,
             dt=dt,