promnesia 1.2.20240810__py3-none-any.whl → 1.4.20250909__py3-none-any.whl

promnesia/sources/browser_legacy.py

@@ -1,19 +1,19 @@
-from datetime import datetime
+from __future__ import annotations
+
+import sqlite3
+from datetime import datetime, timezone
 from pathlib import Path
 from urllib.parse import unquote
-import sqlite3
-from typing import List, Set, Optional
 
-import pytz
-
-from ..common import PathIsh, Results, Visit, Loc, logger, Second, is_sqlite_db
-from .. import config
+from promnesia import config
+from promnesia.common import Loc, PathIsh, Results, Second, Visit, is_sqlite_db, logger
 
 try:
     from cachew import cachew
 except ModuleNotFoundError as me:
     if me.name != 'cachew':
         raise me
+
     # this module is legacy anyway, so just make it defensive
     def cachew(*args, **kwargs):  # type: ignore[no-redef]
         return lambda f: f
@@ -21,7 +21,7 @@ except ModuleNotFoundError as me:
 
 def index(p: PathIsh) -> Results:
     pp = Path(p)
-    assert pp.exists(), pp # just in case of broken symlinks
+    assert pp.exists(), pp  # just in case of broken symlinks
 
     # todo warn if filtered out too many?
     # todo wonder how quickly mimes can be computed?
@@ -30,31 +30,31 @@ def index(p: PathIsh) -> Results:
 
     assert len(dbs) > 0, pp
     logger.info('processing %d databases', len(dbs))
-    cname = str('_'.join(pp.parts[1:])) # meh
+    cname = str('_'.join(pp.parts[1:]))  # meh
     yield from _index_dbs(dbs, cachew_name=cname)
 
 
-
-def _index_dbs(dbs: List[Path], cachew_name: str):
+def _index_dbs(dbs: list[Path], cachew_name: str):
     # TODO right... not ideal, need to think how to handle it properly...
     import sys
+
     sys.setrecursionlimit(5000)
 
     cache_dir = config.get().cache_dir
     cpath = None if cache_dir is None else cache_dir / cachew_name
-    emitted: Set = set()
+    emitted: set = set()
     yield from _index_dbs_aux(cpath, dbs, emitted=emitted)
 
 
 # todo wow, stack traces are ridiculous here...
 # todo hmm, feels like it should be a class or something?
-@cachew(lambda cp, dbs, emitted: cp, depends_on=lambda cp, dbs, emitted: dbs) # , logger=logger)
-def _index_dbs_aux(cache_path: Optional[Path], dbs: List[Path], emitted: Set) -> Results:
+@cachew(lambda cp, dbs, emitted: cp, depends_on=lambda cp, dbs, emitted: dbs)  # , logger=logger)  # noqa: ARG005
+def _index_dbs_aux(cache_path: Path | None, dbs: list[Path], emitted: set) -> Results:
     if len(dbs) == 0:
         return
 
     xs = dbs[:-1]
-    x  = dbs[-1:]
+    x = dbs[-1:]
 
     xs_res = _index_dbs_aux(cache_path, xs, emitted)
     xs_was_cached = False
@@ -65,36 +65,38 @@ def _index_dbs_aux(cache_path: Optional[Path], dbs: List[Path], emitted: Set) ->
             logger.debug('seems that %d first items were previously cached', len(xs))
         if xs_was_cached:
             key = str(r) if isinstance(r, Exception) else (r.url, r.dt)
-            assert key not in emitted, key # todo not sure if this assert is necessary?
+            assert key not in emitted, key  # todo not sure if this assert is necessary?
             # hmm ok it might happen if we messed up with indexing individual db?
             # alternatively, could abuse it to avoid messing with 'emitted' in _index_db?
             emitted.add(key)
-        yield r # todo not sure about exceptions?
+        yield r  # todo not sure about exceptions?
 
     for db in x:
         yield from _index_db(db, emitted=emitted)
 
 
-def _index_db(db: Path, emitted: Set):
-    logger.info('processing %s', db) # debug level?
+def _index_db(db: Path, emitted: set):
+    logger.info('processing %s', db)  # debug level?
 
     # todo schema check (not so critical for cachew though)
     total = 0
-    new   = 0
-    loc = Loc.file(db) # todo possibly needs to be optimized -- moving from within the loop considerably speeds everything up
+    new = 0
+    loc = Loc.file(
+        db
+    )  # todo possibly needs to be optimized -- moving from within the loop considerably speeds everything up
     with sqlite3.connect(f'file:{db}?immutable=1', uri=True) as c:
         browser = None
         for b in [Chrome, Firefox, FirefoxPhone, Safari]:
             try:
                 c.execute(f'SELECT * FROM {b.detector}')
-            except sqlite3.OperationalError: # not sure if the right kind?
+            except sqlite3.OperationalError:  # not sure if the right kind?
                 pass
             else:
                 browser = b
                 break
         assert browser is not None
 
-        proj  = ', '.join(c for c, _ in browser.schema.cols)
+        proj = ', '.join(c for c, _ in browser.schema.cols)
         query = browser.query.replace('chunk.', '')
 
         c.row_factory = sqlite3.Row
@@ -121,17 +123,20 @@ Col = str
 ColType = str
 
 
-from typing import Any, NamedTuple, Tuple, Union, Sequence, Optional
+from collections.abc import Sequence
+from typing import NamedTuple
+
 
 class Schema(NamedTuple):
-    cols: Sequence[Tuple[Col, ColType]]
+    cols: Sequence[tuple[Col, ColType]]
     key: Sequence[str]
 
 
-SchemaCheck = Tuple[str, Union[str, Sequence[str]]] # todo Union: meh
+SchemaCheck = tuple[str, str | Sequence[str]]  # todo Union: meh
 
 from dataclasses import dataclass
 
+
 # todo protocol?
 @dataclass
 class Extr:
@@ -147,14 +152,15 @@ class Extr:
 
 
 class Chrome(Extr):
-    detector='keyword_search_terms'
+    detector = 'keyword_search_terms'
+    # fmt: off
     schema_check=(
         'visits', [
             'visits', "id, url, visit_time, from_visit, transition, segment_id, visit_duration, incremented_omnibox_typed_score",
             'visits', "id, url, visit_time, from_visit, transition, segment_id, visit_duration"
         ]
     )
-    schema=Schema(cols=[
+    schema = Schema(cols=[
         ('U.url'                                  , 'TEXT'   ),
 
         # while these two are not very useful, might be good to have just in case for some debugging
@@ -168,18 +174,19 @@ class Chrome(Extr):
         ('V.visit_duration'                       , 'INTEGER NOT NULL'),
         # V.omnibox thing looks useless
     ], key=('url', 'visit_time', 'vid', 'urlid'))
-    query='FROM chunk.visits as V, chunk.urls as U WHERE V.url = U.id'
+    # fmt: on
+    query = 'FROM chunk.visits as V, chunk.urls as U WHERE V.url = U.id'
 
     @staticmethod
     def row2visit(row: sqlite3.Row, loc: Loc) -> Visit:
-        url  = row['url']
-        ts   = row['visit_time']
+        url = row['url']
+        ts = row['visit_time']
         durs = row['visit_duration']
 
         dt = chrome_time_to_utc(int(ts))
-        url = unquote(url) # chrome urls are all quoted
+        url = unquote(url)  # chrome urls are all quoted
         dd = int(durs)
-        dur: Optional[Second] = None if dd == 0 else dd // 1_000_000
+        dur: Second | None = None if dd == 0 else dd // 1_000_000
         return Visit(
             url=url,
             dt=dt,
@@ -192,12 +199,12 @@ class Chrome(Extr):
 # yep, tested it and looks like utc
 def chrome_time_to_utc(chrome_time: int) -> datetime:
     epoch = (chrome_time / 1_000_000) - 11644473600
-    return datetime.fromtimestamp(epoch, pytz.utc)
+    return datetime.fromtimestamp(epoch, timezone.utc)
 
 
 def _row2visit_firefox(row: sqlite3.Row, loc: Loc) -> Visit:
     url = row['url']
-    ts  = float(row['visit_date'])
+    ts = float(row['visit_date'])
     # ok, looks like it's unix epoch
     # https://stackoverflow.com/a/19430099/706389
 
@@ -210,17 +217,19 @@ def _row2visit_firefox(row: sqlite3.Row, loc: Loc) -> Visit:
     else:
         # milliseconds
         ts /= 1_000
-    dt = datetime.fromtimestamp(ts, pytz.utc)
-    url = unquote(url) # firefox urls are all quoted
+    dt = datetime.fromtimestamp(ts, timezone.utc)
+    url = unquote(url)  # firefox urls are all quoted
     return Visit(
         url=url,
         dt=dt,
         locator=loc,
     )
 
+
 # https://web.archive.org/web/20201026130310/http://fileformats.archiveteam.org/wiki/History.db
 class Safari(Extr):
-    detector='history_tombstones'
+    detector = 'history_tombstones'
+    # fmt: off
     schema_check=(
         'history_visits', [
             'history_visits', "id, history_item, visit_time",
@@ -241,13 +250,14 @@ class Safari(Extr):
         # ('V.visit_duration'                       , 'INTEGER NOT NULL'),
         # V.omnibox thing looks useless
     ], key=('url', 'visit_time', 'vid', 'urlid'))
-    query='FROM chunk.history_visits as V, chunk.history_items as U WHERE V.history_item = U.id'
+    # fmt: on
+    query = 'FROM chunk.history_visits as V, chunk.history_items as U WHERE V.history_item = U.id'
 
     @staticmethod
     def row2visit(row: sqlite3.Row, loc: Loc) -> Visit:
-        url  = row['url']
-        ts   = row['visit_time'] + 978307200 # https://stackoverflow.com/a/34546556/16645
-        dt = datetime.fromtimestamp(ts, pytz.utc)
+        url = row['url']
+        ts = row['visit_time'] + 978307200  # https://stackoverflow.com/a/34546556/16645
+        dt = datetime.fromtimestamp(ts, timezone.utc)
 
         return Visit(
             url=url,
@@ -255,10 +265,12 @@ class Safari(Extr):
             locator=loc,
         )
 
+
 # https://web.archive.org/web/20190730231715/https://www.forensicswiki.org/wiki/Mozilla_Firefox_3_History_File_Format#moz_historyvisits
 class Firefox(Extr):
-    detector='moz_meta'
-    schema_check=('moz_historyvisits', "id, from_visit, place_id, visit_date, visit_type")
+    detector = 'moz_meta'
+    schema_check = ('moz_historyvisits', "id, from_visit, place_id, visit_date, visit_type")
+    # fmt: off
     schema=Schema(cols=[
         ('P.url'       , 'TEXT'),
 
@@ -274,14 +286,16 @@ class Firefox(Extr):
         # needs to be defensive
         # ('V.session'   , 'INTEGER'),
     ], key=('url', 'visit_date', 'vid', 'pid'))
-    query='FROM chunk.moz_historyvisits as V, chunk.moz_places as P WHERE V.place_id = P.id'
+    # fmt: on
+    query = 'FROM chunk.moz_historyvisits as V, chunk.moz_places as P WHERE V.place_id = P.id'
 
-    row2visit = _row2visit_firefox
+    row2visit = _row2visit_firefox  # type: ignore[assignment]
 
 
 class FirefoxPhone(Extr):
-    detector='remote_devices'
-    schema_check=('visits', "_id, history_guid, visit_type, date, is_local")
+    detector = 'remote_devices'
+    schema_check = ('visits', "_id, history_guid, visit_type, date, is_local")
+    # fmt: off
     schema=Schema(cols=[
         ('H.url'               , 'TEXT NOT NULL'   ),
 
@@ -293,6 +307,7 @@ class FirefoxPhone(Extr):
         ('V.date as visit_date', 'INTEGER NOT NULL'),
         # ('is_local'    , 'INTEGER NOT NULL'),
     ], key=('url', 'date', 'vid', 'hid'))
-    query='FROM chunk.visits as V, chunk.history as H  WHERE V.history_guid = H.guid'
+    # fmt: on
+    query = 'FROM chunk.visits as V, chunk.history as H  WHERE V.history_guid = H.guid'
 
-    row2visit = _row2visit_firefox
+    row2visit = _row2visit_firefox  # type: ignore[assignment]

promnesia/sources/demo.py

@@ -3,11 +3,11 @@ A dummy source, used for testing
 Generates a sequence of fake evenly separated visits
 '''
 
-from datetime import datetime, timedelta
-from typing import Union
+from __future__ import annotations
 
-from ..common import Results, Visit, Loc
+from datetime import datetime, timedelta
 
+from promnesia.common import Loc, Results, Visit
 
 IsoFormatDt = str
 Seconds = int
@@ -16,12 +16,11 @@ Seconds = int
 # TODO allow passing isoformat string as base_dt?
 # and maybe something similar as delta? start with seconds maybe
 def index(
-        count: int=100,
-        *,
-        base_dt: Union[datetime, IsoFormatDt] = datetime.min + timedelta(days=5000),
-        delta: Union[timedelta, Seconds] = timedelta(hours=1),
+    count: int = 100,
+    *,
+    base_dt: datetime | IsoFormatDt = datetime.min + timedelta(days=5000),
+    delta: timedelta | Seconds = timedelta(hours=1),
 ) -> Results:
-
     base_dt_ = base_dt if isinstance(base_dt, datetime) else datetime.fromisoformat(base_dt)
     delta_ = delta if isinstance(delta, timedelta) else timedelta(seconds=delta)
 

promnesia/sources/fbmessenger.py

@@ -2,12 +2,13 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] for the messages data.
 '''
 
-from ..common import Results, Visit, Loc, extract_urls
+from promnesia.common import Loc, Results, Visit, extract_urls
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.fbmessenger import messages
+
     for m in messages():
         if isinstance(m, Exception):
             yield m
@@ -32,4 +33,3 @@ def index() -> Results:
                 context=m.text,
                 locator=loc,
             )
-

promnesia/sources/filetypes.py

@@ -1,38 +1,42 @@
-#!/usr/bin/env python3
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Sequence
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, Callable, Optional, Sequence, NamedTuple, Union, Iterable
+from typing import NamedTuple
 
 from ..common import Results, Url
 
-
 # TODO doesn't really belong here...
 Ctx = Sequence[str]
 
+
 class EUrl(NamedTuple):
     url: Url
-    ctx: Ctx # TODO ctx here is more like a Loc
+    ctx: Ctx  # TODO ctx here is more like a Loc
+
+
 ###
 
 
 # keys are mime types + extensions
-Ex = Callable[[Path], Union[Results, Iterable[EUrl]]]
+Ex = Callable[[Path], Results | Iterable[EUrl]]
 # None means unhandled
-TYPE2IDX: Dict[str, Optional[Ex]] = {}
+TYPE2IDX: dict[str, Ex | None] = {}
 # NOTE: there are some types in auto.py at the moment... it's a bit messy
 
 
 # TYPE2IDX only contains the 'prefixes', to speed up the lookup we are using cache..
 @lru_cache(None)
-def type2idx(t: str) -> Optional[Ex]:
+def type2idx(t: str) -> Ex | None:
     if len(t) == 0:
-        return None # just in case?
+        return None  # just in case?
     # first try exact match
-    e = TYPE2IDX.get(t, None)
+    e = TYPE2IDX.get(t)
     if e is not None:
         return e
     t = t.strip('.')
-    e = TYPE2IDX.get(t, None)
+    e = TYPE2IDX.get(t)
     if e is not None:
         return e
     # otherwise, try prefixes?
@@ -41,6 +45,7 @@ def type2idx(t: str) -> Optional[Ex]:
             return v
     return None
 
+
 # for now source code just indexed with grep, not sure if it's good enough?
 # if not, some fanceir library could be used...
 # e.g. https://github.com/karlicoss/promnesia/pull/152/commits/c2f00eb4ee4018b02c9bf3966a036db69a43373d
@@ -81,7 +86,7 @@ CODE = {
 
     '.ts', # most likely typescript.. otherwise determined as text/vnd.trolltech.linguist mime
     '.js',
-}
+}  # fmt: skip
 # TODO discover more extensions with mimetypes library?
 
 
@@ -97,9 +102,10 @@ audio/
 video/
 '''
 
-handle_later = lambda *args, **kwargs: ()
+handle_later = lambda *_args, **_kwargs: ()
 
-def ignore(*args, **kwargs):
+
+def ignore(*_args, **_kwargs):
     # TODO log (once?)
     yield from ()
 
@@ -121,13 +127,14 @@ TYPE2IDX.update({
     'font/woff': ignore,
     'text/x-Algol68': ignore,  # ugh some license file had this?? maybe always index text/ as text?
     'text/x-bytecode.python': ignore,  # todo ignore all x-bytecode?
+    'text/calendar': ignore,
 
     # TODO not sure what to do about these..
     'application/octet-stream': handle_later,
     'application/zip'         : handle_later,
     'application/x-tar'       : handle_later,
     'application/gzip'        : handle_later,
-})
+})  # fmt: skip
 
 
 # TODO use some existing file for initial gitignore..
@@ -146,5 +153,4 @@ IGNORE = [
     # TODO not sure about these:
     '.gitignore',
     '.babelrc',
-]
-
+]  # fmt: skip

promnesia/sources/github.py

@@ -2,15 +2,14 @@
 Uses [[https://github.com/karlicoss/HPI][HPI]] github module
 '''
 
-# Note: requires the 'mistletoe' module if you enable render_markdown
-
-from typing import Optional, Set
+from __future__ import annotations
 
-from ..common import Results, Visit, Loc, iter_urls, logger
+# Note: requires the 'mistletoe' module if you enable render_markdown
+from promnesia.common import Loc, Results, Visit, iter_urls, logger
 
 
 def index(*, render_markdown: bool = False) -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.github.all import events
 
     if render_markdown:
@@ -18,7 +17,9 @@ def index(*, render_markdown: bool = False) -> Results:
             from .markdown import TextParser, extract_from_text
         except ImportError as import_err:
             logger.exception(import_err)
-            logger.critical("Could not import markdown module to render github body markdown. Try 'python3 -m pip install mistletoe'")
+            logger.critical(
+                "Could not import markdown module to render github body markdown. Try 'python3 -m pip install mistletoe'"
+            )
             render_markdown = False
 
     for e in events():
@@ -29,7 +30,7 @@ def index(*, render_markdown: bool = False) -> Results:
             continue
 
         # if enabled, convert the (markdown) body to HTML
-        context: Optional[str] = e.body
+        context: str | None = e.body
         if e.body is not None and render_markdown:
             context = TextParser(e.body)._doc_ashtml()  # type: ignore[possibly-undefined]
 
@@ -59,7 +60,7 @@ def index(*, render_markdown: bool = False) -> Results:
         #
         # Note: this set gets reset every event, is here to
         # prevent duplicates between URLExtract and the markdown parser
-        emitted: Set[str] = set()
+        emitted: set[str] = set()
         for url in iter_urls(e.body):
             if url in emitted:
                 continue

promnesia/sources/guess.py

@@ -1,6 +1,7 @@
 # TODO eh. confusing how guess and auto are different...
 # maybe merge them later?
-from typing import Iterable, Any
+from collections.abc import Iterable
+from typing import Any
 
 from ..common import Extraction, PathIsh
 
@@ -21,14 +22,17 @@ def index(path: PathIsh, *args, **kwargs) -> Iterable[Extraction]:
     ps = str(path)
     # TODO better url detection
 
-    index_: Any # meh
+    index_: Any  # meh
     if is_git_repo(ps):
         from . import vcs
+
         index_ = vcs.index
     elif is_website(ps):
         from . import website
+
         index_ = website.index
     else:
         from . import auto
+
         index_ = auto.index
     yield from index_(path, *args, **kwargs)

promnesia/sources/hackernews.py

@@ -4,11 +4,11 @@ Uses [[https://github.com/karlicoss/HPI][HPI]] dogsheep module to import HackerN
 
 import textwrap
 
-from promnesia.common import Visit, Loc, Results
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     from my.hackernews import dogsheep
 
     for item in dogsheep.items():
@@ -21,9 +21,7 @@ def index() -> Results:
             title = item.title
         elif item.text_html:
             title = item.text_html
-            title = textwrap.shorten(
-                    title, width=79, placeholder="…",
-                    break_long_words=True)
+            title = textwrap.shorten(title, width=79, placeholder="…", break_long_words=True)
         # The locator is always the HN story. If the story is a link (as
         # opposed to a text post), we insert a visit such that the link
         # will point back to the corresponding HN story.
@@ -33,8 +31,8 @@ def index() -> Results:
             urls.append(item.url)
         for url in urls:
             yield Visit(
-                    url=url,
-                    dt=item.created,
-                    locator=loc,
-                    context=title,
+                url=url,
+                dt=item.created,
+                locator=loc,
+                context=title,
             )

promnesia/sources/hpi.py

@@ -2,10 +2,12 @@
 Just a helper for a more humane error message when importing my.* dependencies
 '''
 
-from ..common import logger
+from promnesia.common import logger
 
 try:
-    import my
+    import my  # noqa: F401
 except ImportError as e:
     logger.exception(e)
-    logger.critical("Failed during 'import my'. You probably need to install & configure HPI package first (see 'https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org')")
+    logger.critical(
+        "Failed during 'import my'. You probably need to install & configure HPI package first (see 'https://github.com/karlicoss/HPI/blob/master/doc/SETUP.org')"
+    )

promnesia/sources/html.py

@@ -2,19 +2,21 @@
 Extracts links from HTML files
 '''
 
-from pathlib import Path
-from typing import Iterator, Tuple
+from __future__ import annotations
 
-from ..common import PathIsh, Visit, Loc, Results, file_mtime
+from collections.abc import Iterator
+from pathlib import Path
 
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, Tag
 
+from promnesia.common import Loc, PathIsh, Results, Visit, file_mtime
 
-# TODO present error summary in the very end; import errors -- makes sense to show 
+# TODO present error summary in the very end; import errors -- makes sense to show
 # TODO on some exceptions, request a fallback to text?
 
 
-Url = Tuple[str, str]
+Url = tuple[str, str]
+
 
 def extract_urls_from_html(s: str) -> Iterator[Url]:
     """
@@ -23,11 +25,13 @@ def extract_urls_from_html(s: str) -> Iterator[Url]:
     """
     soup = BeautifulSoup(s, 'lxml')
     for a in soup.find_all('a'):
+        assert isinstance(a, Tag), a  # make mypy happy
         href = a.attrs.get('href')
         if href is None or ('://' not in href):
             # second condition means relative link
             continue
-        text = a.text
+        assert isinstance(href, str), href  # make mypy happy
+        text: str = a.text
         yield (href, text)
 
 

promnesia/sources/hypothesis.py

@@ -1,11 +1,12 @@
 """
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myhypothesis][hypothesis]] module
 """
-from ..common import Loc, Results, Visit, extract_urls, join_tags
+
+from promnesia.common import Loc, Results, Visit, extract_urls, join_tags
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.hypothesis as hyp
 
     for h in hyp.highlights():

promnesia/sources/instapaper.py

@@ -1,11 +1,12 @@
 '''
 Uses HPI [[https://github.com/karlicoss/HPI/blob/master/doc/MODULES.org#myinstapaper][instapaper]] module
 '''
-from ..common import Results, logger, Visit, Loc
+
+from promnesia.common import Loc, Results, Visit
 
 
 def index() -> Results:
-    from . import hpi
+    from . import hpi  # noqa: F401,I001
     import my.instapaper as ip
 
     for p in ip.pages():