promnesia 1.2.20240810__py3-none-any.whl → 1.4.20250909__py3-none-any.whl

promnesia/sources/markdown.py

@@ -1,14 +1,24 @@
-from pathlib import Path
-from typing import Iterator, NamedTuple, Optional
-
-from ..common import Extraction, Url, PathIsh, Res, Visit, Loc, file_mtime, logger
-
-
-import mistletoe  # type: ignore
-from mistletoe.span_token import AutoLink, Link  # type: ignore
-import mistletoe.block_token as BT  # type: ignore
-from mistletoe.html_renderer import HTMLRenderer  # type: ignore
+from __future__ import annotations
 
+from collections.abc import Iterator
+from pathlib import Path
+from typing import NamedTuple
+
+import mistletoe  # type: ignore[import-untyped]
+import mistletoe.block_token as BT  # type: ignore[import-untyped]
+from mistletoe.html_renderer import HTMLRenderer  # type: ignore[import-untyped]
+from mistletoe.span_token import AutoLink, Link  # type: ignore[import-untyped]
+
+from promnesia.common import (
+    Extraction,
+    Loc,
+    PathIsh,
+    Res,
+    Url,
+    Visit,
+    file_mtime,
+    logger,
+)
 
 renderer = HTMLRenderer()
 
@@ -18,7 +28,7 @@ block_tokens = tuple(getattr(BT, name) for name in BT.__all__)
 
 class Parsed(NamedTuple):
     url: Url
-    context: Optional[str]
+    context: str | None
 
 
 Result = Res[Parsed]
@@ -118,7 +128,7 @@ class TextParser(Parser):
             self._html = HTML_MARKER + _ashtml(self.doc)
         return self._html
 
-    def _extract(self, cur, last_block=None) -> Iterator[Parsed]:
+    def _extract(self, cur, last_block=None) -> Iterator[Parsed]:  # noqa: ARG002
         if not isinstance(cur, (AutoLink, Link)):
             return
 

promnesia/sources/org.py

@@ -1,20 +1,32 @@
-from datetime import datetime
+from __future__ import annotations
+
 import re
-from typing import Iterable, List, Set, Optional, Iterator, Tuple, NamedTuple, cast
+from collections.abc import Iterable, Iterator
+from datetime import datetime
 from pathlib import Path
-
-
-from ..common import Visit, get_logger, Results, Url, Loc, from_epoch, iter_urls, PathIsh, Res, file_mtime
-
+from typing import NamedTuple, cast
 
 import orgparse
-from orgparse.date import gene_timestamp_regex, OrgDate
+from orgparse.date import OrgDate, gene_timestamp_regex
 from orgparse.node import OrgNode
 
+from promnesia.common import (
+    Loc,
+    PathIsh,
+    Res,
+    Results,
+    Url,
+    Visit,
+    file_mtime,
+    get_logger,
+    iter_urls,
+)
 
 UPDATE_ORGPARSE_WARNING = 'WARNING: please update orgparse version to a more recent (pip3 install -U orgparse)'
 
 _warned = False
+
+
 def warn_old_orgparse_once() -> Iterable[Exception]:
     global _warned
     if _warned:
@@ -35,8 +47,9 @@ CREATED_RGX = re.compile(gene_timestamp_regex(brtype='inactive'), re.VERBOSE)
 ** subnote
 """
 
+
 class Parsed(NamedTuple):
-    dt: Optional[datetime]
+    dt: datetime | None
     heading: str
 
 
@@ -46,14 +59,14 @@ def _parse_node(n: OrgNode) -> Parsed:
 
     heading = n.get_heading('raw')
     pp = n.properties
-    createds = cast(Optional[str], pp.get('CREATED', None))
+    createds = cast(str | None, pp.get('CREATED', None))
     if createds is None:
         # TODO replace with 'match', but need to strip off priority etc first?
         # see _parse_heading in orgparse
         # todo maybe use n.get_timestamps(inactive=True, point=True)? only concern is that it's searching in the body as well?
         m = CREATED_RGX.search(heading)
         if m is not None:
-            createds = m.group(0) # could be None
+            createds = m.group(0)  # could be None
             # todo a bit hacky..
             heading = heading.replace(createds + ' ', '')
     if createds is not None:
@@ -62,7 +75,11 @@ def _parse_node(n: OrgNode) -> Parsed:
             dt = None
         else:
             [odt] = OrgDate.list_from_str(createds)
-            dt = odt.start
+            start = odt.start
+            if not isinstance(start, datetime):  # could be date
+                dt = datetime.combine(start, datetime.min.time())  # meh, but the best we can do?
+            else:
+                dt = start
     else:
         dt = None
     return Parsed(dt=dt, heading=heading)
@@ -74,7 +91,7 @@ def _get_heading(n: OrgNode):
     return '' if n.is_root() else n.get_heading(format='raw')
 
 
-def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[Tuple[Parsed, OrgNode]]]:
+def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[tuple[Parsed, OrgNode]]]:
     try:
         parsed = _parse_node(node)
     except Exception as e:
@@ -98,7 +115,7 @@ def get_body_compat(node: OrgNode) -> str:
             # get_body was only added to root in 0.2.0
             for x in warn_old_orgparse_once():
                 # ugh. really crap, but it will at least only warn once... (becaue it caches)
-                raise x
+                raise x  # noqa: B904
             return UPDATE_ORGPARSE_WARNING
         else:
             raise e
@@ -150,7 +167,7 @@ def extract_from_file(fname: PathIsh) -> Results:
 
         (parsed, node) = wr
         dt = parsed.dt
-        assert dt is not None # shouldn't be because of fallback
+        assert dt is not None  # shouldn't be because of fallback
         for r in iter_org_urls(node):
             # TODO get body recursively? not sure
             try:
@@ -160,7 +177,7 @@ def extract_from_file(fname: PathIsh) -> Results:
                 ctx = parsed.heading + tagss + '\n' + get_body_compat(node)
             except Exception as e:
                 yield e
-                ctx = 'ERROR' # TODO more context?
+                ctx = 'ERROR'  # TODO more context?
 
             if isinstance(r, Url):
                 yield Visit(
@@ -168,9 +185,11 @@ def extract_from_file(fname: PathIsh) -> Results:
                     dt=dt,
                     locator=Loc.file(
                         fname,
-                        line=getattr(node, 'linenumber', None), # make it defensive so it works against older orgparse (pre 0.2)
+                        line=getattr(
+                            node, 'linenumber', None
+                        ),  # make it defensive so it works against older orgparse (pre 0.2)
                     ),
                     context=ctx,
                 )
-            else: # error
+            else:  # error
                 yield r

promnesia/sources/plaintext.py

@@ -1,10 +1,9 @@
-from ..common import get_logger, get_tmpdir, PathIsh, _is_windows
-from ..compat import removeprefix
+from __future__ import annotations
 
 from functools import lru_cache
 from pathlib import Path
-import os
-from typing import List
+
+from promnesia.common import PathIsh, _is_windows
 
 # https://linux-and-mac-hacks.blogspot.co.uk/2013/04/use-grep-and-regular-expressions-to.html
 _URL_REGEX = r'\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|]'
@@ -12,23 +11,24 @@ _URL_REGEX = r'\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@
 if _is_windows:
     # wtf? for some reason on windows (in cmd.exe specificaly) \b isn't working...
     # this will make the regex a bit less precise, but not end of the world
-    _URL_REGEX = removeprefix(_URL_REGEX, r'\b')
+    _URL_REGEX = _URL_REGEX.removeprefix(r'\b')
 
 
-@lru_cache()
+@lru_cache
 def _has_grep() -> bool:
     import shutil
+
     return shutil.which('grep') is not None
 
 
-Command = List[str]
+Command = list[str]
 
 
 _GREP_ARGS: Command = [
     '--color=never',
-    '-H', # always show filename TODO not sure if works on osx
-    '-n', # print line numbers (to restore context)
-    '-I', # ignore binaries
+    '-H',  # always show filename TODO not sure if works on osx
+    '-n',  # print line numbers (to restore context)
+    '-I',  # ignore binaries
 ]
 
 if not _is_windows:
@@ -37,18 +37,20 @@ if not _is_windows:
         '--exclude-dir=".git"',
     ]
 
+
 # NOTE: grep/findstr exit with code 1 on no matches...
 # we hack around it in shellcmd module (search 'grep')
-def _grep(*, paths: List[str], recursive: bool) -> Command:
+def _grep(*, paths: list[str], recursive: bool) -> Command:
     return [
         'grep',
         *(['-r'] if recursive else []),
         *_GREP_ARGS,
-        '-E', # 'extended' syntax
+        '-E',  # 'extended' syntax
         _URL_REGEX,
         *paths,
     ]
 
+
 def _findstr(*, path: str, recursive: bool) -> Command:
     return [
         'findstr',
@@ -86,31 +88,31 @@ def _extract_from_file(path: str) -> Command:
 def extract_from_path(path: PathIsh) -> Command:
     pp = Path(path)
 
-    tdir = get_tmpdir()
-
-    logger = get_logger()
-    if pp.is_dir(): # TODO handle archives here???
+    if pp.is_dir():  # TODO handle archives here???
         return _extract_from_dir(str(pp))
-    else:
-        if any(pp.suffix == ex for ex in (
-                '.xz',
-                '.bz2',
-                '.gz',
-                '.zip',
-        )):
-            # todo should be debug?
-            # or should delete it completely, feels like unpacking archives here is a bit too much
-            raise RuntimeError(f"Archives aren't supported yet: {path}")
-            logger.info(f"Extracting from compressed file {path}")
-            import lzma
-            from tempfile import NamedTemporaryFile
-            # TODO hopefully, no collisions
-            import os.path
-            fname = os.path.join(tdir.name, os.path.basename(path))
-            with open(fname, 'wb') as fo:
-                with lzma.open(path, 'r') as cf:
-                    fo.write(cf.read())
-                return _extract_from_file(fname)
-        else:
-            r = _extract_from_file(str(pp))
-            return r
+
+    if any(
+        pp.suffix == ex
+        for ex in (
+            '.xz',
+            '.bz2',
+            '.gz',
+            '.zip',
+        )
+    ):
+        # todo should be debug?
+        # or should delete it completely, feels like unpacking archives here is a bit too much
+        raise RuntimeError(f"Archives aren't supported yet: {path}")
+        # logger.info(f"Extracting from compressed file {path}")
+        # import lzma
+        # from tempfile import NamedTemporaryFile
+        # # TODO hopefully, no collisions
+        # import os.path
+        # fname = os.path.join(tdir.name, os.path.basename(path))
+        # with open(fname, 'wb') as fo:
+        #     with lzma.open(path, 'r') as cf:
+        #         fo.write(cf.read())
+        #     return _extract_from_file(fname)
+
+    r = _extract_from_file(str(pp))
+    return r

promnesia/sources/pocket.py

@@ -1,11 +1,12 @@
 '''
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Pocket highlights & bookmarks
 '''
-from ..common import Visit, Loc, Results
+
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.pocket import articles
 
     # TODO use docstring from my. module? E.g. describing which pocket format is expected
@@ -15,7 +16,8 @@ def index() -> Results:
         loc = Loc.make(title=title, href=a.pocket_link)
         # Add a reverse locator so that the Promnesia browser extension shows a
         # link on the Pocket page back to the original URL.
-        loc_rev = Loc.make(title=title, href=a.url)
+        # FIXME need to actually use it
+        _loc_rev = Loc.make(title=title, href=a.url)
         hls = a.highlights
         excerpt = a.json.get('excerpt', None)
         if len(hls) == 0:

promnesia/sources/reddit.py

@@ -2,21 +2,28 @@
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myreddit][reddit]] module
 '''
 
+from __future__ import annotations
+
+import typing
 from itertools import chain
-from typing import Set, Optional, Type
 
-from ..common import Visit, Loc, extract_urls, Results, logger
+from promnesia.common import Loc, Results, Visit, extract_urls, logger
+
+if typing.TYPE_CHECKING:
+    from my.reddit.common import Comment, RedditBase, Save, Submission, Upvote
+
 
+def index(*, render_markdown: bool = False, renderer: type[RedditRenderer] | None = None) -> Results:
+    from . import hpi  # noqa: F401
 
-def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRenderer']] = None) -> Results:
-    from . import hpi
     try:
-        from my.reddit.all import submissions, comments, saved, upvoted
+        from my.reddit.all import comments, saved, submissions, upvoted
     except ModuleNotFoundError as e:
         if "No module named 'my.reddit.all'" in str(e):
             import warnings
+
             warnings.warn("DEPRECATED/reddit: Using an old version of HPI, please update")
-            from my.reddit import submissions, comments, saved, upvoted
+            from my.reddit import comments, saved, submissions, upvoted
         else:
             raise e
 
@@ -58,11 +65,12 @@ def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRende
 # mostly here so we can keep track of how the user
 # wants to render markdown
 class RedditRenderer:
-    def __init__(self, render_markdown: bool = False) -> None:
+    def __init__(self, *, render_markdown: bool = False) -> None:
         self._link_extractor = None
         self._parser_cls = None
         try:
             from .markdown import TextParser, extract_from_text
+
             self._link_extractor = extract_from_text
             self._parser_cls = TextParser
         except ImportError as import_err:
@@ -72,43 +80,40 @@ class RedditRenderer:
             # only send error if the user is trying to enable this feature
             if render_markdown:
                 logger.exception(import_err)
-                logger.critical("Could not import markdown module to render reddit markdown. Try 'python3 -m pip install mistletoe'")
+                logger.critical(
+                    "Could not import markdown module to render reddit markdown. Try 'python3 -m pip install mistletoe'"
+                )
             render_markdown = False  # force to be false, couldn't import
         self.render_markdown = render_markdown
 
-
-    def _from_comment(self, i: 'Comment') -> Results:
+    def _from_comment(self, i: Comment) -> Results:
         locator = Loc.make(
             title='Reddit comment',
             href=i.url,
         )
         yield from self._from_common(i, locator=locator)
 
-
-    def _from_submission(self, i: 'Submission') -> Results:
+    def _from_submission(self, i: Submission) -> Results:
         locator = Loc.make(
             title=f'Reddit submission: {i.title}',
             href=i.url,
         )
         yield from self._from_common(i, locator=locator)
 
-
-    def _from_upvote(self, i: 'Upvote') -> Results:
+    def _from_upvote(self, i: Upvote) -> Results:
         locator = Loc.make(
             title='Reddit upvote',
             href=i.url,
         )
         yield from self._from_common(i, locator=locator)
 
-
-    def _from_save(self, i: 'Save') -> Results:
+    def _from_save(self, i: Save) -> Results:
         locator = Loc.make(
             title='Reddit save',
             href=i.url,
         )
         yield from self._from_common(i, locator=locator)
 
-
     # to allow for possible subclassing by the user?
     def _render_body(self, text: str) -> str:
         if self.render_markdown and self._parser_cls is not None:
@@ -116,8 +121,7 @@ class RedditRenderer:
         else:
             return text
 
-
-    def _from_common(self, i: 'RedditBase', locator: Loc) -> Results:
+    def _from_common(self, i: RedditBase, locator: Loc) -> Results:
         urls = [i.url]
         # TODO this should belong to HPI.. fix permalink handling I guess
         # ok, it's not present for all of them..
@@ -130,7 +134,7 @@ class RedditRenderer:
 
         context = self._render_body(i.text)
 
-        emitted: Set[str] = set()
+        emitted: set[str] = set()
 
         for url in chain(urls, extract_urls(i.text)):
             if url in emitted:
@@ -164,9 +168,3 @@ class RedditRenderer:
                     locator=locator,
                 )
                 emitted.add(res.url)
-
-
-import typing
-if typing.TYPE_CHECKING:
-    from my.reddit.common import Submission, Comment, Save, Upvote, RedditBase
-

promnesia/sources/roamresearch.py

@@ -2,11 +2,12 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Roam Research data
 '''
 
-from ..common import Results, Visit, Loc, extract_urls
+from promnesia.common import Loc, Results, Visit, extract_urls
 
 
 def index() -> Results:
     import my.roamresearch as RR
+
     roam = RR.roam()
     for node in roam.traverse():
         yield from _collect(node)
@@ -14,7 +15,7 @@ def index() -> Results:
 
 def _collect(node: 'RoamNode') -> Results:
     title = node.title
-    body  = node.body or ''
+    body = node.body or ''
     if title is None:
         # most notes don't have title, so we just take the first line instead..
         lines = body.splitlines(keepends=True)
@@ -43,6 +44,8 @@ def _collect(node: 'RoamNode') -> Results:
 
 
 import typing
+
 if typing.TYPE_CHECKING:
     import my.roamresearch as RR
+
     RoamNode = RR.Node

promnesia/sources/rss.py

@@ -2,27 +2,25 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for RSS data.
 '''
 
-from itertools import chain
+from datetime import datetime, timezone
 
-from ..common import Visit, Loc, extract_urls, Results, get_logger
-
-from datetime import datetime
-
-import pytz
+from promnesia.common import Loc, Results, Visit
 
 # arbitrary,  2011-11-04 00:05:23.283+00:00
-default_datetime = datetime.fromtimestamp(1320365123, tz=pytz.utc)
+default_datetime = datetime.fromtimestamp(1320365123, tz=timezone.utc)
 # TODO FIXME allow for visit not to have datetime?
 # I.e. even having context is pretty good!
 
+
 def index() -> Results:
     from my.rss.all import subscriptions
+
     for feed in subscriptions():
         # TODO locator should be optional too? although could use direct link in the rss reader interface
         locator = Loc.make(title='my.rss')
         yield Visit(
             url=feed.url,
             dt=feed.created_at or default_datetime,
-            context='RSS subscription', # TODO use 'provider', etc?
+            context='RSS subscription',  # TODO use 'provider', etc?
             locator=locator,
         )

promnesia/sources/shellcmd.py

@@ -2,18 +2,30 @@
 Greps out URLs from an arbitrary shell command results.
 """
 
-from datetime import datetime
+from __future__ import annotations
+
 import os
 import re
-from subprocess import run, PIPE
-from typing import Union, Sequence
 import warnings
+from collections.abc import Sequence
+from datetime import datetime
+from subprocess import PIPE, run
+
+from promnesia.common import (
+    Loc,
+    PathIsh,
+    Results,
+    Visit,
+    _is_windows,
+    extract_urls,
+    file_mtime,
+    now_tz,
+)
 
-from ..common import Visit, Loc, Results, extract_urls, file_mtime, get_system_tz, now_tz, _is_windows, PathIsh
 from .plaintext import _has_grep
 
 
-def index(command: Union[str, Sequence[PathIsh]]) -> Results:
+def index(command: str | Sequence[PathIsh]) -> Results:
     cmd: Sequence[PathIsh]
     cmds: str
     if isinstance(command, str):
@@ -24,8 +36,6 @@ def index(command: Union[str, Sequence[PathIsh]]) -> Results:
         cmds = ' '.join(map(str, command))
         cmd = command
 
-    tz = get_system_tz()
-
     # ugh... on windows grep does something nasty? e.g:
     # grep --color=never -r -H -n -I -E http 'D:\\a\\promnesia\\promnesia\\tests\\testdata\\custom'
     # D:\a\promnesia\promnesia\tests\testdata\custom/file1.txt:1:Right, so this points at http://google.com
@@ -42,9 +52,9 @@ def index(command: Union[str, Sequence[PathIsh]]) -> Results:
             fname = None
             lineno = None
         else:
-            fname  = m.group(1)
+            fname = m.group(1)
             lineno = int(m.group(2))
-            line   = m.group(3)
+            line = m.group(3)
 
         if fname is not None and needs_windows_grep_patching:
             fname = fname.replace('/', os.sep)
@@ -71,9 +81,9 @@ def index(command: Union[str, Sequence[PathIsh]]) -> Results:
                 context=context,
             )
 
-    r = run(cmd, stdout=PIPE)
+    r = run(cmd, stdout=PIPE, check=False)
     if r.returncode > 0:
-        if not (cmd[0] in {'grep', 'findstr'} and r.returncode == 1): # ugh. grep returns 1 on no matches...
+        if not (cmd[0] in {'grep', 'findstr'} and r.returncode == 1):  # ugh. grep returns 1 on no matches...
             r.check_returncode()
     output = r.stdout
     assert output is not None