promnesia 1.2.20230515__py3-none-any.whl → 1.3.20241021__py3-none-any.whl

promnesia/sources/demo.py

@@ -3,18 +3,34 @@ A dummy source, used for testing
 Generates a sequence of fake evenly separated visits
 '''
 
+from __future__ import annotations
+
 from datetime import datetime, timedelta
 
-from ..common import Results, Visit, Loc
+from promnesia.common import Loc, Results, Visit
+
+IsoFormatDt = str
+Seconds = int
+
+
+# TODO allow passing isoformat string as base_dt?
+# and maybe something similar as delta? start with seconds maybe
+def index(
+    count: int = 100,
+    *,
+    base_dt: datetime | IsoFormatDt = datetime.min + timedelta(days=5000),
+    delta: timedelta | Seconds = timedelta(hours=1),
+) -> Results:
 
+    base_dt_ = base_dt if isinstance(base_dt, datetime) else datetime.fromisoformat(base_dt)
+    delta_ = delta if isinstance(delta, timedelta) else timedelta(seconds=delta)
 
-def index(count: int=100, *, base_dt: datetime=datetime.min + timedelta(days=5000), delta: timedelta=timedelta(hours=1)) -> Results:
     # todo with some errors too?
     # todo use data generation library suggested for HPI?
     for i in range(count):
         yield Visit(
             url=f'https://demo.com/page{i}.html',
-            dt=base_dt + delta * i,
+            dt=base_dt_ + delta_ * i,
             locator=Loc.make('demo'),
         )
         # todo add context?

promnesia/sources/fbmessenger.py

@@ -2,12 +2,13 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for the messages data.
 '''
 
-from ..common import Results, Visit, Loc, extract_urls
+from promnesia.common import Loc, Results, Visit, extract_urls
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.fbmessenger import messages
+
     for m in messages():
         if isinstance(m, Exception):
             yield m

promnesia/sources/filetypes.py

@@ -1,11 +1,12 @@
-#!/usr/bin/env python3
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, Callable, Optional, Sequence, NamedTuple, Union, Iterable
+from typing import Callable, NamedTuple, Union
 
 from ..common import Results, Url
 
-
 # TODO doesn't really belong here...
 Ctx = Sequence[str]
 
@@ -18,13 +19,13 @@ class EUrl(NamedTuple):
 # keys are mime types + extensions
 Ex = Callable[[Path], Union[Results, Iterable[EUrl]]]
 # None means unhandled
-TYPE2IDX: Dict[str, Optional[Ex]] = {}
+TYPE2IDX: dict[str, Ex | None] = {}
 # NOTE: there are some types in auto.py at the moment... it's a bit messy
 
 
 # TYPE2IDX only contains the 'prefixes', to speed up the lookup we are using cache..
 @lru_cache(None)
-def type2idx(t: str) -> Optional[Ex]:
+def type2idx(t: str) -> Ex | None:
     if len(t) == 0:
         return None # just in case?
     # first try exact match
@@ -67,6 +68,7 @@ CODE = {
     'text/vnd.graphviz',
     'text/x-diff',  # patch files
     'text/x-php',
+    'text/x-lilypond',
 
     # these didn't have a mime type, or were mistyped?
     'css',
@@ -96,9 +98,9 @@ audio/
 video/
 '''
 
-handle_later = lambda *args, **kwargs: ()
+handle_later = lambda *_args, **_kwargs: ()
 
-def ignore(*args, **kwargs):
+def ignore(*_args, **_kwargs):
     # TODO log (once?)
     yield from ()
 
@@ -115,6 +117,13 @@ TYPE2IDX.update({
     '.vcf' : ignore,
     'message/rfc822': ignore, # ??
 
+    # todo ignore all fonts?
+    'font/woff2': ignore,
+    'font/woff': ignore,
+    'text/x-Algol68': ignore,  # ugh some license file had this?? maybe always index text/ as text?
+    'text/x-bytecode.python': ignore,  # todo ignore all x-bytecode?
+    'text/calendar': ignore,
+
     # TODO not sure what to do about these..
     'application/octet-stream': handle_later,
     'application/zip'         : handle_later,

promnesia/sources/github.py

@@ -1,16 +1,14 @@
 '''
 Uses [[https://github.com/karlicoss/HPI][HPI]] github module
 '''
+from __future__ import annotations
 
 # Note: requires the 'mistletoe' module if you enable render_markdown
-
-from typing import Optional, Set
-
-from ..common import Results, Visit, Loc, iter_urls, logger
+from promnesia.common import Loc, Results, Visit, iter_urls, logger
 
 
 def index(*, render_markdown: bool = False) -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.github.all import events
 
     if render_markdown:
@@ -29,9 +27,9 @@ def index(*, render_markdown: bool = False) -> Results:
             continue
 
         # if enabled, convert the (markdown) body to HTML
-        context: Optional[str] = e.body
+        context: str | None = e.body
         if e.body is not None and render_markdown:
-            context = TextParser(e.body)._doc_ashtml()
+            context = TextParser(e.body)._doc_ashtml()  # type: ignore[possibly-undefined]
 
         # locator should link back to this event
         loc = Loc.make(title=e.summary, href=e.link)
@@ -59,7 +57,7 @@ def index(*, render_markdown: bool = False) -> Results:
         #
         # Note: this set gets reset every event, is here to
         # prevent duplicates between URLExtract and the markdown parser
-        emitted: Set[str] = set()
+        emitted: set[str] = set()
         for url in iter_urls(e.body):
             if url in emitted:
                 continue
@@ -74,7 +72,7 @@ def index(*, render_markdown: bool = False) -> Results:
         # extract from markdown links like [link text](https://...)
         # incase URLExtract missed any somehow
         if render_markdown:
-            for res in extract_from_text(e.body):
+            for res in extract_from_text(e.body):  # type: ignore[possibly-undefined]
                 if isinstance(res, Exception):
                     yield res
                     continue

promnesia/sources/guess.py

@@ -1,6 +1,7 @@
 # TODO eh. confusing how guess and auto are different...
 # maybe merge them later?
-from typing import Iterable, Any
+from collections.abc import Iterable
+from typing import Any
 
 from ..common import Extraction, PathIsh
 

promnesia/sources/hackernews.py

@@ -4,11 +4,11 @@ Uses [[https://github.com/karlicoss/HPI][HPI]] dogsheep module to import HackerN
 
 import textwrap
 
-from promnesia.common import Visit, Loc, Results
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.hackernews import dogsheep
 
     for item in dogsheep.items():

promnesia/sources/hpi.py

@@ -2,10 +2,10 @@
 Just a helper for a more humane error message when importing my.* dependencies
 '''
 
-from ..common import logger
+from promnesia.common import logger
 
 try:
-    import my
+    import my  # noqa: F401
 except ImportError as e:
     logger.exception(e)
     logger.critical("Failed during 'import my'. You probably need to install & configure HPI package first (see 'https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org')")

promnesia/sources/html.py

@@ -2,19 +2,21 @@
 Extracts links from HTML files
 '''
 
-from pathlib import Path
-from typing import Iterator, Tuple
+from __future__ import annotations
 
-from ..common import PathIsh, Visit, Loc, Results, file_mtime
+from collections.abc import Iterator
+from pathlib import Path
 
 from bs4 import BeautifulSoup
 
+from promnesia.common import Loc, PathIsh, Results, Visit, file_mtime
 
-# TODO present error summary in the very end; import errors -- makes sense to show 
+# TODO present error summary in the very end; import errors -- makes sense to show
 # TODO on some exceptions, request a fallback to text?
 
 
-Url = Tuple[str, str]
+Url = tuple[str, str]
+
 
 def extract_urls_from_html(s: str) -> Iterator[Url]:
     """

promnesia/sources/hypothesis.py

@@ -1,14 +1,15 @@
 """
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myhypothesis][hypothesis]] module
 """
-from ..common import Loc, Results, Visit, extract_urls, join_tags
+
+from promnesia.common import Loc, Results, Visit, extract_urls, join_tags
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.hypothesis as hyp
 
-    for h in hyp.get_highlights():
+    for h in hyp.highlights():
         if isinstance(h, Exception):
             yield h
             continue

promnesia/sources/instapaper.py

@@ -1,11 +1,11 @@
 '''
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myinstapaper][instapaper]] module
 '''
-from ..common import Results, logger, Visit, Loc
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.instapaper as ip
 
     for p in ip.pages():

promnesia/sources/markdown.py

@@ -1,14 +1,24 @@
-from pathlib import Path
-from typing import Iterator, NamedTuple, Optional
-
-from ..common import get_logger, Extraction, Url, PathIsh, Res, Visit, Loc, file_mtime, logger
-
-
-import mistletoe # type: ignore
-from mistletoe.span_token import AutoLink, Link # type: ignore
-import mistletoe.block_token as BT # type: ignore
-from mistletoe.html_renderer import HTMLRenderer # type: ignore
+from __future__ import annotations
 
+from collections.abc import Iterator
+from pathlib import Path
+from typing import NamedTuple
+
+import mistletoe  # type: ignore
+import mistletoe.block_token as BT  # type: ignore
+from mistletoe.html_renderer import HTMLRenderer  # type: ignore
+from mistletoe.span_token import AutoLink, Link  # type: ignore
+
+from promnesia.common import (
+    Extraction,
+    Loc,
+    PathIsh,
+    Res,
+    Url,
+    Visit,
+    file_mtime,
+    logger,
+)
 
 renderer = HTMLRenderer()
 
@@ -18,7 +28,7 @@ block_tokens = tuple(getattr(BT, name) for name in BT.__all__)
 
 class Parsed(NamedTuple):
     url: Url
-    context: Optional[str]
+    context: str | None
 
 
 Result = Res[Parsed]
@@ -42,7 +52,7 @@ HTML_MARKER = '!html '
 def _ashtml(block) -> str:
     res = renderer.render(block)
     if res.startswith('<p>') and res.endswith('</p>'):
-        res = res[3: -4] # meh, but for now fine
+        res = res[3:-4]  # meh, but for now fine
     return res
 
 
@@ -62,7 +72,6 @@ class Parser:
         context = None if last_block is None else HTML_MARKER + _ashtml(last_block)
         yield Parsed(url=url, context=context)
 
-
     def _walk(self, cur, last_block) -> Iterator[Result]:
         if isinstance(cur, block_tokens):
             last_block = cur
@@ -73,12 +82,14 @@ class Parser:
             logger.exception(e)
             yield e
 
-        children = getattr(cur, 'children', [])
+        # keeping getattr for compatibility in older versions of mistletoe, it was optional
+        children = getattr(cur, 'children', None)
+        if children is None:
+            return
         for c in children:
             yield from self._walk(c, last_block=last_block)
 
-
-    def walk(self):
+    def walk(self) -> Iterator[Result]:
         yield from self._walk(self.doc, last_block=None)
 
 
@@ -94,7 +105,7 @@ def extract_from_file(fname: PathIsh) -> Iterator[Extraction]:
             yield Visit(
                 url=r.url,
                 dt=fallback_dt,
-                locator=Loc.file(fname), # TODO line number
+                locator=Loc.file(fname),  # TODO line number
                 context=r.context,
             )
 
@@ -105,9 +116,9 @@ class TextParser(Parser):
     Instead of chunking blocks like for files, this returns the entire
     message rendered as the context
     '''
-    def __init__(self, text: str):
-        self.doc = mistletoe.Document(text)
 
+    def __init__(self, text: str) -> None:
+        self.doc = mistletoe.Document(text)
 
     def _doc_ashtml(self):
         '''
@@ -117,8 +128,7 @@ class TextParser(Parser):
             self._html = HTML_MARKER + _ashtml(self.doc)
         return self._html
 
-
-    def _extract(self, cur, last_block = None) -> Iterator[Parsed]:
+    def _extract(self, cur, last_block=None) -> Iterator[Parsed]:  # noqa: ARG002
         if not isinstance(cur, (AutoLink, Link)):
             return
 

promnesia/sources/org.py

@@ -1,16 +1,26 @@
-from datetime import datetime
+from __future__ import annotations
+
 import re
-from typing import Iterable, List, Set, Optional, Iterator, Tuple, NamedTuple, cast
+from collections.abc import Iterable, Iterator
+from datetime import datetime
 from pathlib import Path
-
-
-from ..common import Visit, get_logger, Results, Url, Loc, from_epoch, iter_urls, PathIsh, Res, file_mtime
-
+from typing import NamedTuple, Optional, cast
 
 import orgparse
-from orgparse.date import gene_timestamp_regex, OrgDate
+from orgparse.date import OrgDate, gene_timestamp_regex
 from orgparse.node import OrgNode
 
+from promnesia.common import (
+    Loc,
+    PathIsh,
+    Res,
+    Results,
+    Url,
+    Visit,
+    file_mtime,
+    get_logger,
+    iter_urls,
+)
 
 UPDATE_ORGPARSE_WARNING = 'WARNING: please update orgparse version to a more recent (pip3 install -U orgparse)'
 
@@ -36,7 +46,7 @@ CREATED_RGX = re.compile(gene_timestamp_regex(brtype='inactive'), re.VERBOSE)
 """
 
 class Parsed(NamedTuple):
-    dt: Optional[datetime]
+    dt: datetime | None
     heading: str
 
 
@@ -57,8 +67,12 @@ def _parse_node(n: OrgNode) -> Parsed:
             # todo a bit hacky..
             heading = heading.replace(createds + ' ', '')
     if createds is not None:
-        [odt] = OrgDate.list_from_str(createds)
-        dt = odt.start
+        if '<%%' in createds:
+            # sexp date, not supported
+            dt = None
+        else:
+            [odt] = OrgDate.list_from_str(createds)
+            dt = odt.start
     else:
         dt = None
     return Parsed(dt=dt, heading=heading)
@@ -70,7 +84,7 @@ def _get_heading(n: OrgNode):
     return '' if n.is_root() else n.get_heading(format='raw')
 
 
-def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[Tuple[Parsed, OrgNode]]]:
+def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[tuple[Parsed, OrgNode]]]:
     try:
         parsed = _parse_node(node)
     except Exception as e:
@@ -80,7 +94,7 @@ def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[Tuple[Parsed, OrgN
             parsed = parsed._replace(dt=dt)
         else:
             dt = parsed.dt
-    yield parsed, node
+        yield parsed, node
 
     for c in node.children:
         yield from walk_node(node=c, dt=dt)
@@ -94,7 +108,7 @@ def get_body_compat(node: OrgNode) -> str:
             # get_body was only added to root in 0.2.0
             for x in warn_old_orgparse_once():
                 # ugh. really crap, but it will at least only warn once... (becaue it caches)
-                raise x
+                raise x  # noqa: B904
             return UPDATE_ORGPARSE_WARNING
         else:
             raise e

promnesia/sources/plaintext.py

@@ -1,10 +1,9 @@
-from ..common import get_logger, get_tmpdir, PathIsh, _is_windows
-from ..compat import removeprefix
+from __future__ import annotations
 
 from functools import lru_cache
 from pathlib import Path
-import os
-from typing import List
+
+from promnesia.common import PathIsh, _is_windows, get_logger, get_tmpdir
 
 # https://linux-and-mac-hacks.blogspot.co.uk/2013/04/use-grep-and-regular-expressions-to.html
 _URL_REGEX = r'\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|]'
@@ -12,16 +11,16 @@ _URL_REGEX = r'\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@
 if _is_windows:
     # wtf? for some reason on windows (in cmd.exe specificaly) \b isn't working...
     # this will make the regex a bit less precise, but not end of the world
-    _URL_REGEX = removeprefix(_URL_REGEX, r'\b')
+    _URL_REGEX = _URL_REGEX.removeprefix(r'\b')
 
 
-@lru_cache()
+@lru_cache
 def _has_grep() -> bool:
     import shutil
     return shutil.which('grep') is not None
 
 
-Command = List[str]
+Command = list[str]
 
 
 _GREP_ARGS: Command = [
@@ -39,7 +38,7 @@ if not _is_windows:
 
 # NOTE: grep/findstr exit with code 1 on no matches...
 # we hack around it in shellcmd module (search 'grep')
-def _grep(*, paths: List[str], recursive: bool) -> Command:
+def _grep(*, paths: list[str], recursive: bool) -> Command:
     return [
         'grep',
         *(['-r'] if recursive else []),
@@ -91,24 +90,26 @@ def extract_from_path(path: PathIsh) -> Command:
     logger = get_logger()
     if pp.is_dir(): # TODO handle archives here???
         return _extract_from_dir(str(pp))
-    else:
-        if any(pp.suffix == ex for ex in (
-                '.xz',
-                '.bz2',
-                '.gz',
-                '.zip',
-        )):
-            logger.info(f"Extracting from compressed file {path}")
-            raise RuntimeError(f"Archives aren't supported yet: {path}")
-            import lzma
-            from tempfile import NamedTemporaryFile
-            # TODO hopefully, no collisions
-            import os.path
-            fname = os.path.join(tdir.name, os.path.basename(path))
-            with open(fname, 'wb') as fo:
-                with lzma.open(path, 'r') as cf:
-                    fo.write(cf.read())
-                return _extract_from_file(fname)
-        else:
-            r = _extract_from_file(str(pp))
-            return r
+
+    if any(pp.suffix == ex for ex in (
+            '.xz',
+            '.bz2',
+            '.gz',
+            '.zip',
+    )):
+        # todo should be debug?
+        # or should delete it completely, feels like unpacking archives here is a bit too much
+        raise RuntimeError(f"Archives aren't supported yet: {path}")
+        # logger.info(f"Extracting from compressed file {path}")
+        # import lzma
+        # from tempfile import NamedTemporaryFile
+        # # TODO hopefully, no collisions
+        # import os.path
+        # fname = os.path.join(tdir.name, os.path.basename(path))
+        # with open(fname, 'wb') as fo:
+        #     with lzma.open(path, 'r') as cf:
+        #         fo.write(cf.read())
+        #     return _extract_from_file(fname)
+
+    r = _extract_from_file(str(pp))
+    return r

promnesia/sources/pocket.py

@@ -1,11 +1,12 @@
 '''
 Uses [[https://github.com/karlicoss/HPI][HPI]] for Pocket highlights & bookmarks
 '''
-from ..common import Visit, Loc, Results
+
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.pocket import articles
 
     # TODO use docstring from my. module? E.g. describing which pocket format is expected

promnesia/sources/reddit.py

@@ -2,21 +2,27 @@
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myreddit][reddit]] module
 '''
 
+from __future__ import annotations
+
+import typing
 from itertools import chain
-from typing import Set, Optional, Type
 
-from ..common import Visit, Loc, extract_urls, Results, logger
+from promnesia.common import Loc, Results, Visit, extract_urls, logger
+
+if typing.TYPE_CHECKING:
+    from my.reddit.common import Comment, RedditBase, Save, Submission, Upvote
+
 
+def index(*, render_markdown: bool = False, renderer: type[RedditRenderer] | None = None) -> Results:
+    from . import hpi  # noqa: F401
 
-def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRenderer']] = None) -> Results:
-    from . import hpi
     try:
-        from my.reddit.all import submissions, comments, saved, upvoted
+        from my.reddit.all import comments, saved, submissions, upvoted
     except ModuleNotFoundError as e:
         if "No module named 'my.reddit.all'" in str(e):
             import warnings
             warnings.warn("DEPRECATED/reddit: Using an old version of HPI, please update")
-            from my.reddit import submissions, comments, saved, upvoted  # type: ignore[no-redef]
+            from my.reddit import comments, saved, submissions, upvoted
         else:
             raise e
 
@@ -58,7 +64,7 @@ def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRende
 # mostly here so we can keep track of how the user
 # wants to render markdown
 class RedditRenderer:
-    def __init__(self, render_markdown: bool = False) -> None:
+    def __init__(self, *, render_markdown: bool = False) -> None:
         self._link_extractor = None
         self._parser_cls = None
         try:
@@ -77,7 +83,7 @@ class RedditRenderer:
         self.render_markdown = render_markdown
 
 
-    def _from_comment(self, i: 'Comment') -> Results:
+    def _from_comment(self, i: Comment) -> Results:
         locator = Loc.make(
             title='Reddit comment',
             href=i.url,
@@ -85,7 +91,7 @@ class RedditRenderer:
         yield from self._from_common(i, locator=locator)
 
 
-    def _from_submission(self, i: 'Submission') -> Results:
+    def _from_submission(self, i: Submission) -> Results:
         locator = Loc.make(
             title=f'Reddit submission: {i.title}',
             href=i.url,
@@ -93,15 +99,15 @@ class RedditRenderer:
         yield from self._from_common(i, locator=locator)
 
 
-    def _from_upvote(self, i: 'Upvote') -> Results:
+    def _from_upvote(self, i: Upvote) -> Results:
         locator = Loc.make(
-            title=f'Reddit upvote',
+            title='Reddit upvote',
             href=i.url,
         )
         yield from self._from_common(i, locator=locator)
 
 
-    def _from_save(self, i: 'Save') -> Results:
+    def _from_save(self, i: Save) -> Results:
         locator = Loc.make(
             title='Reddit save',
             href=i.url,
@@ -117,7 +123,7 @@ class RedditRenderer:
             return text
 
 
-    def _from_common(self, i: 'RedditBase', locator: Loc) -> Results:
+    def _from_common(self, i: RedditBase, locator: Loc) -> Results:
         urls = [i.url]
         # TODO this should belong to HPI.. fix permalink handling I guess
         # ok, it's not present for all of them..
@@ -130,7 +136,7 @@ class RedditRenderer:
 
         context = self._render_body(i.text)
 
-        emitted: Set[str] = set()
+        emitted: set[str] = set()
 
         for url in chain(urls, extract_urls(i.text)):
             if url in emitted:
@@ -165,8 +171,3 @@ class RedditRenderer:
                 )
                 emitted.add(res.url)
 
-
-import typing
-if typing.TYPE_CHECKING:
-    from my.reddit.common import Submission, Comment, Save, Upvote, RedditBase
-