promnesia 1.2.20240810__py3-none-any.whl → 1.3.20241021__py3-none-any.whl

promnesia/sources/auto.py

@@ -5,33 +5,44 @@
 - autodetects Obsidian vault and adds `obsidian://` app protocol support [[file:../src/promnesia/sources/obsidian.py][promnesia.sources.obsidian]]
 - autodetects Logseq graph and adds `logseq://` app protocol support [[file:../src/promnesia/sources/logseq.py][promnesia.sources.logseq]]
 """
+from __future__ import annotations
 
 import csv
-from concurrent.futures import ProcessPoolExecutor as Pool
-from contextlib import nullcontext
-from datetime import datetime
 import itertools
 import json
 import os
-from typing import Optional, Iterable, Union, List, Tuple, NamedTuple, Sequence, Iterator, Iterable, Callable, Any, Dict, Set
+from collections.abc import Iterable, Iterator, Sequence
+from concurrent.futures import ProcessPoolExecutor as Pool
+from contextlib import nullcontext
 from fnmatch import fnmatch
+from functools import wraps
 from pathlib import Path
-from functools import lru_cache, wraps
-import warnings
-
-import pytz
-
-from ..common import Visit, Url, PathIsh, get_logger, Loc, get_tmpdir, extract_urls, Extraction, Result, Results, mime, traverse, file_mtime, echain, logger
-from ..common import warn_once
-from ..config import use_cores
+from typing import Any, Callable, NamedTuple, Optional
+
+from promnesia.common import (
+    Loc,
+    PathIsh,
+    Result,
+    Results,
+    Visit,
+    echain,
+    extract_urls,
+    file_mtime,
+    get_logger,
+    get_tmpdir,
+    logger,
+    mime,
+    traverse,
+    warn_once,
+)
+from promnesia.config import use_cores
 
-
-from .filetypes import EUrl, Ctx
-from .auto_obsidian import obsidian_replacer
 from .auto_logseq import logseq_replacer
+from .auto_obsidian import obsidian_replacer
+from .filetypes import Ctx, EUrl
 
 
-def _collect(thing, path: List[str], result: List[EUrl]) -> None:
+def _collect(thing, path: list[str], result: list[EUrl]) -> None:
     if isinstance(thing, str):
         ctx: Ctx = tuple(path)
         result.extend([EUrl(url=u, ctx=ctx) for u in extract_urls(thing)])
@@ -51,9 +62,9 @@ def _collect(thing, path: List[str], result: List[EUrl]) -> None:
 
 
 # TODO mm. okay, I suppose could use kython consuming thingy?..
-def collect_from(thing) -> List[EUrl]:
-    uuu: List[EUrl] = []
-    path: List[str] = []
+def collect_from(thing) -> list[EUrl]:
+    uuu: list[EUrl] = []
+    path: list[str] = []
     _collect(thing, path, uuu)
     return uuu
 
@@ -85,7 +96,7 @@ def _plaintext(path: Path) -> Results:
 def fallback(ex):
     """Falls back to plaintext in case of issues"""
 
-    fallback_active: Dict[Any, bool] = {}
+    fallback_active: dict[Any, bool] = {}
     @wraps(ex)
     def wrapped(path: Path):
         nonlocal fallback_active
@@ -99,7 +110,7 @@ def fallback(ex):
             except ModuleNotFoundError as me:
                 logger = get_logger()
                 logger.exception(me)
-                logger.warn('%s: %s not found, falling back to grep! "pip3 install --user %s" for better support!', path, me.name, me.name)
+                logger.warning('%s: %s not found, falling back to grep! "pip3 install --user %s" for better support!', path, me.name, me.name)
                 yield me
                 fallback_active[ex] = True
                 do_fallback = True
@@ -126,7 +137,7 @@ def _org(path: Path) -> Results:
     return org.extract_from_file(path)
 
 
-from .filetypes import TYPE2IDX, type2idx, IGNORE, CODE
+from .filetypes import CODE, IGNORE, TYPE2IDX, type2idx
 
 TYPE2IDX.update({
     'application/json': _json,
@@ -169,7 +180,7 @@ Replacer = Optional[Callable[[str, str], str]]
 
 def index(
         *paths: PathIsh,
-        ignored: Union[Sequence[str], str]=(),
+        ignored: Sequence[str] | str=(),
         follow: bool=True,
         replacer: Replacer=None,
 ) -> Results:
@@ -210,10 +221,10 @@ class Options(NamedTuple):
     # TODO option to add ignores? not sure..
     # TODO I don't like this replacer thing... think about removing it
     replacer: Replacer
-    root: Optional[Path]=None
+    root: Path | None=None
 
 
-def _index_file_aux(path: Path, opts: Options) -> Union[Exception, List[Result]]:
+def _index_file_aux(path: Path, opts: Options) -> Exception | list[Result]:
     # just a helper for the concurrent version (the generator isn't picklable)
     try:
         return list(_index_file(path, opts=opts))
@@ -248,7 +259,7 @@ def _index(path: Path, opts: Options) -> Results:
                 continue
 
             p = p.resolve()
-            if not os.path.exists(p):
+            if not os.path.exists(p):  # noqa: PTH110
                 logger.debug('ignoring %s: broken symlink?', p)
                 continue
 
@@ -266,8 +277,10 @@ def _index(path: Path, opts: Options) -> Results:
 
 
 Mime = str
-from .filetypes import Ex # meh
-def by_path(pp: Path) -> Tuple[Optional[Ex], Optional[Mime]]:
+from .filetypes import Ex  # meh
+
+
+def by_path(pp: Path) -> tuple[Ex | None, Mime | None]:
     suf = pp.suffix.lower()
     # firt check suffixes, it's faster
     s = type2idx(suf)
@@ -318,7 +331,7 @@ def _index_file(pp: Path, opts: Options) -> Results:
 
     logger.debug('indexing via %s: %s', ip.__name__, pp)
 
-    def indexer() -> Union[Urls, Results]:
+    def indexer() -> Urls | Results:
         # eh, annoying.. need to make more generic..
         idx = ip(pp)
         try:
@@ -353,7 +366,7 @@ def _index_file(pp: Path, opts: Options) -> Results:
             v = v._replace(locator=loc)
 
         if replacer is not None and root is not None:
-            upd: Dict[str, Any] = {}
+            upd: dict[str, Any] = {}
             href = v.locator.href
             if href is not None:
                 upd['locator'] = v.locator._replace(href=replacer(href, str(root)), title=replacer(v.locator.title, str(root)))

promnesia/sources/auto_logseq.py

@@ -1,14 +1,15 @@
 import os.path
 import urllib.parse
 
+
 def logseq_replacer(path: str, root: str) -> str:
-    if not path.startswith("editor://") or not (path.endswith('.md') or path.endswith('.org')):
+    if not path.startswith("editor://") or not (path.endswith((".md", ".org"))):
         return path
-        
-    graph = os.path.basename(root)
-    page_name = os.path.basename(path).rsplit('.', 1)[0]
+
+    graph = os.path.basename(root)  # noqa: PTH119
+    page_name = os.path.basename(path).rsplit('.', 1)[0]  # noqa: PTH119
     encoded_page_name = urllib.parse.quote(page_name)
-    
+
     uri = f"logseq://graph/{graph}?page={encoded_page_name}"
 
     return uri

promnesia/sources/auto_obsidian.py

@@ -1,8 +1,8 @@
 def obsidian_replacer(p: str, r: str) -> str:
     if not p.startswith("editor://") or not p.endswith('.md'):
         return p
-    
+
     path = p.split('/', 2)[-1]
-    
+
     uri = f"obsidian://{path}"
     return uri

promnesia/sources/browser.py

@@ -2,15 +2,18 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for visits from web browsers.
 '''
 
+from __future__ import annotations
+
 import re
-from typing import Optional, Iterator, Any, TYPE_CHECKING
 import warnings
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Any
 
-from promnesia.common import Results, Visit, Loc, Second, PathIsh, logger, is_sqlite_db
+from promnesia.common import Loc, PathIsh, Results, Second, Visit, is_sqlite_db, logger
 
 
-def index(p: Optional[PathIsh]=None) -> Results:
-    from . import hpi
+def index(p: PathIsh | None = None) -> Results:
+    from . import hpi  # noqa: F401,I001
 
     if p is None:
         from my.browser.all import history
@@ -24,10 +27,11 @@ def index(p: Optional[PathIsh]=None) -> Results:
     )
     try:
         yield from _index_new_with_adhoc_config(path=p)
-        return
     except Exception as e:
         logger.exception(e)
         warnings.warn("Hacking my.config.browser.export didn't work. You probably need to update HPI.")
+    else:
+        return
 
     logger.warning("Falling back onto legacy promnesia.sources.browser_legacy module")
     yield from _index_old(path=p)
@@ -35,11 +39,12 @@ def index(p: Optional[PathIsh]=None) -> Results:
 
 def _index_old(*, path: PathIsh) -> Results:
     from . import browser_legacy
+
     yield from browser_legacy.index(path)
 
 
 def _index_new_with_adhoc_config(*, path: PathIsh) -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
 
     ## previously, it was possible to index be called with multiple different db search paths
     ## this would result in each subsequent call to my.browser.export.history to invalidate cache every time
@@ -50,7 +55,7 @@ def _index_new_with_adhoc_config(*, path: PathIsh) -> Results:
     cache_override = None if hpi_cache_dir is None else hpi_cache_dir / sanitized_path
     ##
 
-    from my.core.common import classproperty, Paths, get_files
+    from my.core.common import Paths, classproperty, get_files
     class config:
         class core:
             cache_dir = cache_override
@@ -75,8 +80,8 @@ else:
 
 def _index_new(history: Iterator[BrowserMergeVisit]) -> Results:
     for v in history:
-        desc: Optional[str] = None
-        duration: Optional[Second] = None
+        desc: str | None = None
+        duration: Second | None = None
         metadata = v.metadata
         if metadata is not None:
             desc = metadata.title

promnesia/sources/browser_legacy.py

@@ -1,13 +1,14 @@
+from __future__ import annotations
+
+import sqlite3
 from datetime import datetime
 from pathlib import Path
 from urllib.parse import unquote
-import sqlite3
-from typing import List, Set, Optional
 
 import pytz
 
-from ..common import PathIsh, Results, Visit, Loc, logger, Second, is_sqlite_db
-from .. import config
+from promnesia import config
+from promnesia.common import Loc, PathIsh, Results, Second, Visit, is_sqlite_db, logger
 
 try:
     from cachew import cachew
@@ -35,21 +36,21 @@ def index(p: PathIsh) -> Results:
 
 
 
-def _index_dbs(dbs: List[Path], cachew_name: str):
+def _index_dbs(dbs: list[Path], cachew_name: str):
     # TODO right... not ideal, need to think how to handle it properly...
     import sys
     sys.setrecursionlimit(5000)
 
     cache_dir = config.get().cache_dir
     cpath = None if cache_dir is None else cache_dir / cachew_name
-    emitted: Set = set()
+    emitted: set = set()
     yield from _index_dbs_aux(cpath, dbs, emitted=emitted)
 
 
 # todo wow, stack traces are ridiculous here...
 # todo hmm, feels like it should be a class or something?
-@cachew(lambda cp, dbs, emitted: cp, depends_on=lambda cp, dbs, emitted: dbs) # , logger=logger)
-def _index_dbs_aux(cache_path: Optional[Path], dbs: List[Path], emitted: Set) -> Results:
+@cachew(lambda cp, dbs, emitted: cp, depends_on=lambda cp, dbs, emitted: dbs) # , logger=logger)  # noqa: ARG005
+def _index_dbs_aux(cache_path: Path | None, dbs: list[Path], emitted: set) -> Results:
     if len(dbs) == 0:
         return
 
@@ -75,7 +76,7 @@ def _index_dbs_aux(cache_path: Optional[Path], dbs: List[Path], emitted: Set) ->
         yield from _index_db(db, emitted=emitted)
 
 
-def _index_db(db: Path, emitted: Set):
+def _index_db(db: Path, emitted: set):
     logger.info('processing %s', db) # debug level?
 
     # todo schema check (not so critical for cachew though)
@@ -121,17 +122,20 @@ Col = str
 ColType = str
 
 
-from typing import Any, NamedTuple, Tuple, Union, Sequence, Optional
+from collections.abc import Sequence
+from typing import NamedTuple, Union
+
 
 class Schema(NamedTuple):
-    cols: Sequence[Tuple[Col, ColType]]
+    cols: Sequence[tuple[Col, ColType]]
     key: Sequence[str]
 
 
-SchemaCheck = Tuple[str, Union[str, Sequence[str]]] # todo Union: meh
+SchemaCheck = tuple[str, Union[str, Sequence[str]]] # todo Union: meh
 
 from dataclasses import dataclass
 
+
 # todo protocol?
 @dataclass
 class Extr:
@@ -179,7 +183,7 @@ class Chrome(Extr):
         dt = chrome_time_to_utc(int(ts))
         url = unquote(url) # chrome urls are all quoted
         dd = int(durs)
-        dur: Optional[Second] = None if dd == 0 else dd // 1_000_000
+        dur: Second | None = None if dd == 0 else dd // 1_000_000
         return Visit(
             url=url,
             dt=dt,

promnesia/sources/demo.py

@@ -3,11 +3,11 @@ A dummy source, used for testing
 Generates a sequence of fake evenly separated visits
 '''
 
-from datetime import datetime, timedelta
-from typing import Union
+from __future__ import annotations
 
-from ..common import Results, Visit, Loc
+from datetime import datetime, timedelta
 
+from promnesia.common import Loc, Results, Visit
 
 IsoFormatDt = str
 Seconds = int
@@ -16,10 +16,10 @@ Seconds = int
 # TODO allow passing isoformat string as base_dt?
 # and maybe something similar as delta? start with seconds maybe
 def index(
-        count: int=100,
-        *,
-        base_dt: Union[datetime, IsoFormatDt] = datetime.min + timedelta(days=5000),
-        delta: Union[timedelta, Seconds] = timedelta(hours=1),
+    count: int = 100,
+    *,
+    base_dt: datetime | IsoFormatDt = datetime.min + timedelta(days=5000),
+    delta: timedelta | Seconds = timedelta(hours=1),
 ) -> Results:
 
     base_dt_ = base_dt if isinstance(base_dt, datetime) else datetime.fromisoformat(base_dt)

promnesia/sources/fbmessenger.py

@@ -2,12 +2,13 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for the messages data.
 '''
 
-from ..common import Results, Visit, Loc, extract_urls
+from promnesia.common import Loc, Results, Visit, extract_urls
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.fbmessenger import messages
+
     for m in messages():
         if isinstance(m, Exception):
             yield m

promnesia/sources/filetypes.py

@@ -1,11 +1,12 @@
-#!/usr/bin/env python3
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, Callable, Optional, Sequence, NamedTuple, Union, Iterable
+from typing import Callable, NamedTuple, Union
 
 from ..common import Results, Url
 
-
 # TODO doesn't really belong here...
 Ctx = Sequence[str]
 
@@ -18,13 +19,13 @@ class EUrl(NamedTuple):
 # keys are mime types + extensions
 Ex = Callable[[Path], Union[Results, Iterable[EUrl]]]
 # None means unhandled
-TYPE2IDX: Dict[str, Optional[Ex]] = {}
+TYPE2IDX: dict[str, Ex | None] = {}
 # NOTE: there are some types in auto.py at the moment... it's a bit messy
 
 
 # TYPE2IDX only contains the 'prefixes', to speed up the lookup we are using cache..
 @lru_cache(None)
-def type2idx(t: str) -> Optional[Ex]:
+def type2idx(t: str) -> Ex | None:
     if len(t) == 0:
         return None # just in case?
     # first try exact match
@@ -97,9 +98,9 @@ audio/
 video/
 '''
 
-handle_later = lambda *args, **kwargs: ()
+handle_later = lambda *_args, **_kwargs: ()
 
-def ignore(*args, **kwargs):
+def ignore(*_args, **_kwargs):
     # TODO log (once?)
     yield from ()
 
@@ -121,6 +122,7 @@ TYPE2IDX.update({
     'font/woff': ignore,
     'text/x-Algol68': ignore,  # ugh some license file had this?? maybe always index text/ as text?
     'text/x-bytecode.python': ignore,  # todo ignore all x-bytecode?
+    'text/calendar': ignore,
 
     # TODO not sure what to do about these..
     'application/octet-stream': handle_later,

promnesia/sources/github.py

@@ -1,16 +1,14 @@
 '''
 Uses [[https://github.com/karlicoss/HPI][HPI]] github module
 '''
+from __future__ import annotations
 
 # Note: requires the 'mistletoe' module if you enable render_markdown
-
-from typing import Optional, Set
-
-from ..common import Results, Visit, Loc, iter_urls, logger
+from promnesia.common import Loc, Results, Visit, iter_urls, logger
 
 
 def index(*, render_markdown: bool = False) -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.github.all import events
 
     if render_markdown:
@@ -29,7 +27,7 @@ def index(*, render_markdown: bool = False) -> Results:
             continue
 
         # if enabled, convert the (markdown) body to HTML
-        context: Optional[str] = e.body
+        context: str | None = e.body
         if e.body is not None and render_markdown:
             context = TextParser(e.body)._doc_ashtml()  # type: ignore[possibly-undefined]
 
@@ -59,7 +57,7 @@ def index(*, render_markdown: bool = False) -> Results:
         #
         # Note: this set gets reset every event, is here to
         # prevent duplicates between URLExtract and the markdown parser
-        emitted: Set[str] = set()
+        emitted: set[str] = set()
         for url in iter_urls(e.body):
             if url in emitted:
                 continue

promnesia/sources/guess.py

@@ -1,6 +1,7 @@
 # TODO eh. confusing how guess and auto are different...
 # maybe merge them later?
-from typing import Iterable, Any
+from collections.abc import Iterable
+from typing import Any
 
 from ..common import Extraction, PathIsh
 

promnesia/sources/hackernews.py

@@ -4,11 +4,11 @@ Uses [[https://github.com/karlicoss/HPI][HPI]] dogsheep module to import HackerN
 
 import textwrap
 
-from promnesia.common import Visit, Loc, Results
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.hackernews import dogsheep
 
     for item in dogsheep.items():

promnesia/sources/hpi.py

@@ -2,10 +2,10 @@
 Just a helper for a more humane error message when importing my.* dependencies
 '''
 
-from ..common import logger
+from promnesia.common import logger
 
 try:
-    import my
+    import my  # noqa: F401
 except ImportError as e:
     logger.exception(e)
     logger.critical("Failed during 'import my'. You probably need to install & configure HPI package first (see 'https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org')")

promnesia/sources/html.py

@@ -2,19 +2,21 @@
 Extracts links from HTML files
 '''
 
-from pathlib import Path
-from typing import Iterator, Tuple
+from __future__ import annotations
 
-from ..common import PathIsh, Visit, Loc, Results, file_mtime
+from collections.abc import Iterator
+from pathlib import Path
 
 from bs4 import BeautifulSoup
 
+from promnesia.common import Loc, PathIsh, Results, Visit, file_mtime
 
-# TODO present error summary in the very end; import errors -- makes sense to show 
+# TODO present error summary in the very end; import errors -- makes sense to show
 # TODO on some exceptions, request a fallback to text?
 
 
-Url = Tuple[str, str]
+Url = tuple[str, str]
+
 
 def extract_urls_from_html(s: str) -> Iterator[Url]:
     """

promnesia/sources/hypothesis.py

@@ -1,11 +1,12 @@
 """
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myhypothesis][hypothesis]] module
 """
-from ..common import Loc, Results, Visit, extract_urls, join_tags
+
+from promnesia.common import Loc, Results, Visit, extract_urls, join_tags
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.hypothesis as hyp
 
     for h in hyp.highlights():

promnesia/sources/instapaper.py

@@ -1,11 +1,11 @@
 '''
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myinstapaper][instapaper]] module
 '''
-from ..common import Results, logger, Visit, Loc
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.instapaper as ip
 
     for p in ip.pages():

promnesia/sources/markdown.py

@@ -1,14 +1,24 @@
-from pathlib import Path
-from typing import Iterator, NamedTuple, Optional
-
-from ..common import Extraction, Url, PathIsh, Res, Visit, Loc, file_mtime, logger
+from __future__ import annotations
 
+from collections.abc import Iterator
+from pathlib import Path
+from typing import NamedTuple
 
 import mistletoe  # type: ignore
-from mistletoe.span_token import AutoLink, Link  # type: ignore
 import mistletoe.block_token as BT  # type: ignore
 from mistletoe.html_renderer import HTMLRenderer  # type: ignore
+from mistletoe.span_token import AutoLink, Link  # type: ignore
 
+from promnesia.common import (
+    Extraction,
+    Loc,
+    PathIsh,
+    Res,
+    Url,
+    Visit,
+    file_mtime,
+    logger,
+)
 
 renderer = HTMLRenderer()
 
@@ -18,7 +28,7 @@ block_tokens = tuple(getattr(BT, name) for name in BT.__all__)
 
 class Parsed(NamedTuple):
     url: Url
-    context: Optional[str]
+    context: str | None
 
 
 Result = Res[Parsed]
@@ -118,7 +128,7 @@ class TextParser(Parser):
             self._html = HTML_MARKER + _ashtml(self.doc)
         return self._html
 
-    def _extract(self, cur, last_block=None) -> Iterator[Parsed]:
+    def _extract(self, cur, last_block=None) -> Iterator[Parsed]:  # noqa: ARG002
         if not isinstance(cur, (AutoLink, Link)):
             return