promnesia 1.1.20230129__py3-none-any.whl → 1.2.20240810__py3-none-any.whl

promnesia/sources/browser_legacy.py

@@ -0,0 +1,298 @@
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import unquote
+import sqlite3
+from typing import List, Set, Optional
+
+import pytz
+
+from ..common import PathIsh, Results, Visit, Loc, logger, Second, is_sqlite_db
+from .. import config
+
+try:
+    from cachew import cachew
+except ModuleNotFoundError as me:
+    if me.name != 'cachew':
+        raise me
+    # this module is legacy anyway, so just make it defensive
+    def cachew(*args, **kwargs):  # type: ignore[no-redef]
+        return lambda f: f
+
+
+def index(p: PathIsh) -> Results:
+    pp = Path(p)
+    assert pp.exists(), pp # just in case of broken symlinks
+
+    # todo warn if filtered out too many?
+    # todo wonder how quickly mimes can be computed?
+    # todo ugh, dunno, maybe this really belongs to hpi?? need get_files etc...
+    dbs = [p for p in sorted(pp.rglob('*')) if is_sqlite_db(p)]
+
+    assert len(dbs) > 0, pp
+    logger.info('processing %d databases', len(dbs))
+    cname = str('_'.join(pp.parts[1:])) # meh
+    yield from _index_dbs(dbs, cachew_name=cname)
+
+
+
+def _index_dbs(dbs: List[Path], cachew_name: str):
+    # TODO right... not ideal, need to think how to handle it properly...
+    import sys
+    sys.setrecursionlimit(5000)
+
+    cache_dir = config.get().cache_dir
+    cpath = None if cache_dir is None else cache_dir / cachew_name
+    emitted: Set = set()
+    yield from _index_dbs_aux(cpath, dbs, emitted=emitted)
+
+
+# todo wow, stack traces are ridiculous here...
+# todo hmm, feels like it should be a class or something?
+@cachew(lambda cp, dbs, emitted: cp, depends_on=lambda cp, dbs, emitted: dbs) # , logger=logger)
+def _index_dbs_aux(cache_path: Optional[Path], dbs: List[Path], emitted: Set) -> Results:
+    if len(dbs) == 0:
+        return
+
+    xs = dbs[:-1]
+    x  = dbs[-1:]
+
+    xs_res = _index_dbs_aux(cache_path, xs, emitted)
+    xs_was_cached = False
+    for r in xs_res:
+        # if it was cached, emitted would be empty
+        if len(emitted) == 0:
+            xs_was_cached = True
+            logger.debug('seems that %d first items were previously cached', len(xs))
+        if xs_was_cached:
+            key = str(r) if isinstance(r, Exception) else (r.url, r.dt)
+            assert key not in emitted, key # todo not sure if this assert is necessary?
+            # hmm ok it might happen if we messed up with indexing individual db?
+            # alternatively, could abuse it to avoid messing with 'emitted' in _index_db?
+            emitted.add(key)
+        yield r # todo not sure about exceptions?
+
+    for db in x:
+        yield from _index_db(db, emitted=emitted)
+
+
+def _index_db(db: Path, emitted: Set):
+    logger.info('processing %s', db) # debug level?
+
+    # todo schema check (not so critical for cachew though)
+    total = 0
+    new   = 0
+    loc = Loc.file(db) # todo possibly needs to be optimized -- moving from within the loop considerably speeds everything up
+    with sqlite3.connect(f'file:{db}?immutable=1', uri=True) as c:
+        browser = None
+        for b in [Chrome, Firefox, FirefoxPhone, Safari]:
+            try:
+                c.execute(f'SELECT * FROM {b.detector}')
+            except sqlite3.OperationalError: # not sure if the right kind?
+                pass
+            else:
+                browser = b
+                break
+        assert browser is not None
+
+        proj  = ', '.join(c for c, _ in browser.schema.cols)
+        query = browser.query.replace('chunk.', '')
+
+        c.row_factory = sqlite3.Row
+        for r in c.execute(f'select {proj} {query}'):
+            v = browser.row2visit(r, loc)
+            total += 1
+
+            key = (v.url, v.dt)
+            # todo how to keep keys compatible?
+            if key in emitted:
+                continue
+            yield v
+            emitted.add(key)
+            new += 1
+
+            # eh, ok, almost 2x faster if I don't construct Visit first
+            # maybe it's Loc.file that's too slow?
+            # yeah, seems like it, 4.1 s after computing it only once
+
+    logger.info('%s: %d/%d new visits', db, new, total)
+
+
+Col = str
+ColType = str
+
+
+from typing import Any, NamedTuple, Tuple, Union, Sequence, Optional
+
+class Schema(NamedTuple):
+    cols: Sequence[Tuple[Col, ColType]]
+    key: Sequence[str]
+
+
+SchemaCheck = Tuple[str, Union[str, Sequence[str]]] # todo Union: meh
+
+from dataclasses import dataclass
+
+# todo protocol?
+@dataclass
+class Extr:
+    detector: str
+    schema_check: SchemaCheck
+    schema: Schema
+    query: str
+
+    # todo calllable?
+    @staticmethod
+    def row2visit(row: sqlite3.Row, loc: Loc) -> Visit:
+        raise NotImplementedError
+
+
+class Chrome(Extr):
+    detector='keyword_search_terms'
+    schema_check=(
+        'visits', [
+            'visits', "id, url, visit_time, from_visit, transition, segment_id, visit_duration, incremented_omnibox_typed_score",
+            'visits', "id, url, visit_time, from_visit, transition, segment_id, visit_duration"
+        ]
+    )
+    schema=Schema(cols=[
+        ('U.url'                                  , 'TEXT'   ),
+
+        # while these two are not very useful, might be good to have just in case for some debugging
+        ('U.id AS urlid'                          , 'INTEGER'),
+        ('V.id AS vid'                            , 'INTEGER'),
+
+        ('V.visit_time'                           , 'INTEGER NOT NULL'),
+        ('V.from_visit'                           , 'INTEGER'         ),
+        ('V.transition'                           , 'INTEGER NOT NULL'),
+        # V.segment_id looks useless
+        ('V.visit_duration'                       , 'INTEGER NOT NULL'),
+        # V.omnibox thing looks useless
+    ], key=('url', 'visit_time', 'vid', 'urlid'))
+    query='FROM chunk.visits as V, chunk.urls as U WHERE V.url = U.id'
+
+    @staticmethod
+    def row2visit(row: sqlite3.Row, loc: Loc) -> Visit:
+        url  = row['url']
+        ts   = row['visit_time']
+        durs = row['visit_duration']
+
+        dt = chrome_time_to_utc(int(ts))
+        url = unquote(url) # chrome urls are all quoted
+        dd = int(durs)
+        dur: Optional[Second] = None if dd == 0 else dd // 1_000_000
+        return Visit(
+            url=url,
+            dt=dt,
+            locator=loc,
+            duration=dur,
+        )
+
+
+# should be utc? https://stackoverflow.com/a/26226771/706389
+# yep, tested it and looks like utc
+def chrome_time_to_utc(chrome_time: int) -> datetime:
+    epoch = (chrome_time / 1_000_000) - 11644473600
+    return datetime.fromtimestamp(epoch, pytz.utc)
+
+
+def _row2visit_firefox(row: sqlite3.Row, loc: Loc) -> Visit:
+    url = row['url']
+    ts  = float(row['visit_date'])
+    # ok, looks like it's unix epoch
+    # https://stackoverflow.com/a/19430099/706389
+
+    # NOTE: ugh. on Fenix (experimental Android version) it uses milliseconds, not nanos...
+    # about year 2001... if someone has browser history exports before that -- please let me know, I'm impressed
+    threshold = 1000000000
+    if ts > threshold * 1_000_000:
+        # presumably it's in microseconds
+        ts /= 1_000_000
+    else:
+        # milliseconds
+        ts /= 1_000
+    dt = datetime.fromtimestamp(ts, pytz.utc)
+    url = unquote(url) # firefox urls are all quoted
+    return Visit(
+        url=url,
+        dt=dt,
+        locator=loc,
+    )
+
+# https://web.archive.org/web/20201026130310/http://fileformats.archiveteam.org/wiki/History.db
+class Safari(Extr):
+    detector='history_tombstones'
+    schema_check=(
+        'history_visits', [
+            'history_visits', "id, history_item, visit_time",
+            'history_items', "id, url"
+        ]
+    )
+    schema=Schema(cols=[
+        ('U.url'                                  , 'TEXT'   ),
+
+        # while these two are not very useful, might be good to have just in case for some debugging
+        ('U.id AS urlid'                          , 'INTEGER'),
+        ('V.id AS vid'                            , 'INTEGER'),
+
+        ('V.visit_time'                           , 'INTEGER NOT NULL'),
+        # ('V.from_visit'                           , 'INTEGER'         ),
+        # ('V.transition'                           , 'INTEGER NOT NULL'),
+        # V.segment_id looks useless
+        # ('V.visit_duration'                       , 'INTEGER NOT NULL'),
+        # V.omnibox thing looks useless
+    ], key=('url', 'visit_time', 'vid', 'urlid'))
+    query='FROM chunk.history_visits as V, chunk.history_items as U WHERE V.history_item = U.id'
+
+    @staticmethod
+    def row2visit(row: sqlite3.Row, loc: Loc) -> Visit:
+        url  = row['url']
+        ts   = row['visit_time'] + 978307200 # https://stackoverflow.com/a/34546556/16645
+        dt = datetime.fromtimestamp(ts, pytz.utc)
+
+        return Visit(
+            url=url,
+            dt=dt,
+            locator=loc,
+        )
+
+# https://web.archive.org/web/20190730231715/https://www.forensicswiki.org/wiki/Mozilla_Firefox_3_History_File_Format#moz_historyvisits
+class Firefox(Extr):
+    detector='moz_meta'
+    schema_check=('moz_historyvisits', "id, from_visit, place_id, visit_date, visit_type")
+    schema=Schema(cols=[
+        ('P.url'       , 'TEXT'),
+
+        ('P.id AS pid' , 'INTEGER'),
+        ('V.id AS vid' , 'INTEGER'),
+
+        ('V.from_visit', 'INTEGER'),
+        ('V.visit_date', 'INTEGER'),
+        ('V.visit_type', 'INTEGER'),
+
+        # not sure what session is form but could be useful?..
+        # NOTE(20210410): for now, commented it out since some older databases from phone have this column commented?
+        # needs to be defensive
+        # ('V.session'   , 'INTEGER'),
+    ], key=('url', 'visit_date', 'vid', 'pid'))
+    query='FROM chunk.moz_historyvisits as V, chunk.moz_places as P WHERE V.place_id = P.id'
+
+    row2visit = _row2visit_firefox
+
+
+class FirefoxPhone(Extr):
+    detector='remote_devices'
+    schema_check=('visits', "_id, history_guid, visit_type, date, is_local")
+    schema=Schema(cols=[
+        ('H.url'               , 'TEXT NOT NULL'   ),
+
+        ('H.guid AS guid'      , 'TEXT'            ),
+        ('H._id  AS hid'       , 'INTEGER'         ),
+        ('V._id  AS vid'       , 'INTEGER'         ),
+
+        ('V.visit_type'        , 'INTEGER NOT NULL'),
+        ('V.date as visit_date', 'INTEGER NOT NULL'),
+        # ('is_local'    , 'INTEGER NOT NULL'),
+    ], key=('url', 'date', 'vid', 'hid'))
+    query='FROM chunk.visits as V, chunk.history as H  WHERE V.history_guid = H.guid'
+
+    row2visit = _row2visit_firefox

promnesia/sources/demo.py

@@ -4,17 +4,33 @@ Generates a sequence of fake evenly separated visits
 '''
 
 from datetime import datetime, timedelta
+from typing import Union
 
 from ..common import Results, Visit, Loc
 
 
-def index(count: int=100, *, base_dt: datetime=datetime.min + timedelta(days=5000), delta: timedelta=timedelta(hours=1)) -> Results:
+IsoFormatDt = str
+Seconds = int
+
+
+# TODO allow passing isoformat string as base_dt?
+# and maybe something similar as delta? start with seconds maybe
+def index(
+        count: int=100,
+        *,
+        base_dt: Union[datetime, IsoFormatDt] = datetime.min + timedelta(days=5000),
+        delta: Union[timedelta, Seconds] = timedelta(hours=1),
+) -> Results:
+
+    base_dt_ = base_dt if isinstance(base_dt, datetime) else datetime.fromisoformat(base_dt)
+    delta_ = delta if isinstance(delta, timedelta) else timedelta(seconds=delta)
+
     # todo with some errors too?
     # todo use data generation library suggested for HPI?
     for i in range(count):
         yield Visit(
             url=f'https://demo.com/page{i}.html',
-            dt=base_dt + delta * i,
+            dt=base_dt_ + delta_ * i,
             locator=Loc.make('demo'),
         )
         # todo add context?

promnesia/sources/filetypes.py

@@ -66,6 +66,8 @@ CODE = {
     'text/x-lisp',
     'text/vnd.graphviz',
     'text/x-diff',  # patch files
+    'text/x-php',
+    'text/x-lilypond',
 
     # these didn't have a mime type, or were mistyped?
     'css',
@@ -114,6 +116,12 @@ TYPE2IDX.update({
     '.vcf' : ignore,
     'message/rfc822': ignore, # ??
 
+    # todo ignore all fonts?
+    'font/woff2': ignore,
+    'font/woff': ignore,
+    'text/x-Algol68': ignore,  # ugh some license file had this?? maybe always index text/ as text?
+    'text/x-bytecode.python': ignore,  # todo ignore all x-bytecode?
+
     # TODO not sure what to do about these..
     'application/octet-stream': handle_later,
     'application/zip'         : handle_later,

promnesia/sources/github.py

@@ -31,7 +31,7 @@ def index(*, render_markdown: bool = False) -> Results:
         # if enabled, convert the (markdown) body to HTML
         context: Optional[str] = e.body
         if e.body is not None and render_markdown:
-            context = TextParser(e.body)._doc_ashtml()
+            context = TextParser(e.body)._doc_ashtml()  # type: ignore[possibly-undefined]
 
         # locator should link back to this event
         loc = Loc.make(title=e.summary, href=e.link)
@@ -74,7 +74,7 @@ def index(*, render_markdown: bool = False) -> Results:
         # extract from markdown links like [link text](https://...)
         # incase URLExtract missed any somehow
         if render_markdown:
-            for res in extract_from_text(e.body):
+            for res in extract_from_text(e.body):  # type: ignore[possibly-undefined]
                 if isinstance(res, Exception):
                     yield res
                     continue

promnesia/sources/hackernews.py

@@ -1,6 +1,5 @@
 '''
-Uses [[https://github.com/karlicoss/HPI][HPI]] dogsheep module to import
-Hacker News items.
+Uses [[https://github.com/karlicoss/HPI][HPI]] dogsheep module to import HackerNews items.
 '''
 
 import textwrap

promnesia/sources/hypothesis.py

@@ -8,7 +8,7 @@ def index() -> Results:
     from . import hpi
     import my.hypothesis as hyp
 
-    for h in hyp.get_highlights():
+    for h in hyp.highlights():
         if isinstance(h, Exception):
             yield h
             continue

promnesia/sources/markdown.py

@@ -1,13 +1,13 @@
 from pathlib import Path
 from typing import Iterator, NamedTuple, Optional
 
-from ..common import get_logger, Extraction, Url, PathIsh, Res, Visit, Loc, file_mtime, logger
+from ..common import Extraction, Url, PathIsh, Res, Visit, Loc, file_mtime, logger
 
 
-import mistletoe # type: ignore
-from mistletoe.span_token import AutoLink, Link # type: ignore
-import mistletoe.block_token as BT # type: ignore
-from mistletoe.html_renderer import HTMLRenderer # type: ignore
+import mistletoe  # type: ignore
+from mistletoe.span_token import AutoLink, Link  # type: ignore
+import mistletoe.block_token as BT  # type: ignore
+from mistletoe.html_renderer import HTMLRenderer  # type: ignore
 
 
 renderer = HTMLRenderer()
@@ -42,7 +42,7 @@ HTML_MARKER = '!html '
 def _ashtml(block) -> str:
     res = renderer.render(block)
     if res.startswith('<p>') and res.endswith('</p>'):
-        res = res[3: -4] # meh, but for now fine
+        res = res[3:-4]  # meh, but for now fine
     return res
 
 
@@ -62,7 +62,6 @@ class Parser:
         context = None if last_block is None else HTML_MARKER + _ashtml(last_block)
         yield Parsed(url=url, context=context)
 
-
     def _walk(self, cur, last_block) -> Iterator[Result]:
         if isinstance(cur, block_tokens):
             last_block = cur
@@ -73,12 +72,14 @@ class Parser:
             logger.exception(e)
             yield e
 
-        children = getattr(cur, 'children', [])
+        # keeping getattr for compatibility in older versions of mistletoe, it was optional
+        children = getattr(cur, 'children', None)
+        if children is None:
+            return
         for c in children:
             yield from self._walk(c, last_block=last_block)
 
-
-    def walk(self):
+    def walk(self) -> Iterator[Result]:
         yield from self._walk(self.doc, last_block=None)
 
 
@@ -94,7 +95,7 @@ def extract_from_file(fname: PathIsh) -> Iterator[Extraction]:
             yield Visit(
                 url=r.url,
                 dt=fallback_dt,
-                locator=Loc.file(fname), # TODO line number
+                locator=Loc.file(fname),  # TODO line number
                 context=r.context,
             )
 
@@ -105,9 +106,9 @@ class TextParser(Parser):
     Instead of chunking blocks like for files, this returns the entire
     message rendered as the context
     '''
-    def __init__(self, text: str):
-        self.doc = mistletoe.Document(text)
 
+    def __init__(self, text: str) -> None:
+        self.doc = mistletoe.Document(text)
 
     def _doc_ashtml(self):
         '''
@@ -117,8 +118,7 @@ class TextParser(Parser):
             self._html = HTML_MARKER + _ashtml(self.doc)
         return self._html
 
-
-    def _extract(self, cur, last_block = None) -> Iterator[Parsed]:
+    def _extract(self, cur, last_block=None) -> Iterator[Parsed]:
         if not isinstance(cur, (AutoLink, Link)):
             return
 

promnesia/sources/org.py

@@ -57,8 +57,12 @@ def _parse_node(n: OrgNode) -> Parsed:
             # todo a bit hacky..
             heading = heading.replace(createds + ' ', '')
     if createds is not None:
-        [odt] = OrgDate.list_from_str(createds)
-        dt = odt.start
+        if '<%%' in createds:
+            # sexp date, not supported
+            dt = None
+        else:
+            [odt] = OrgDate.list_from_str(createds)
+            dt = odt.start
     else:
         dt = None
     return Parsed(dt=dt, heading=heading)
@@ -80,7 +84,7 @@ def walk_node(*, node: OrgNode, dt: datetime) -> Iterator[Res[Tuple[Parsed, OrgN
             parsed = parsed._replace(dt=dt)
         else:
             dt = parsed.dt
-    yield parsed, node
+        yield parsed, node
 
     for c in node.children:
         yield from walk_node(node=c, dt=dt)

promnesia/sources/plaintext.py

@@ -98,8 +98,10 @@ def extract_from_path(path: PathIsh) -> Command:
                 '.gz',
                 '.zip',
         )):
-            logger.info(f"Extracting from compressed file {path}")
+            # todo should be debug?
+            # or should delete it completely, feels like unpacking archives here is a bit too much
             raise RuntimeError(f"Archives aren't supported yet: {path}")
+            logger.info(f"Extracting from compressed file {path}")
             import lzma
             from tempfile import NamedTemporaryFile
             # TODO hopefully, no collisions

promnesia/sources/reddit.py

@@ -16,7 +16,7 @@ def index(*, render_markdown: bool = False, renderer: Optional[Type['RedditRende
         if "No module named 'my.reddit.all'" in str(e):
             import warnings
             warnings.warn("DEPRECATED/reddit: Using an old version of HPI, please update")
-            from my.reddit import submissions, comments, saved, upvoted  # type: ignore[no-redef]
+            from my.reddit import submissions, comments, saved, upvoted
         else:
             raise e
 
@@ -95,7 +95,7 @@ class RedditRenderer:
 
     def _from_upvote(self, i: 'Upvote') -> Results:
         locator = Loc.make(
-            title=f'Reddit upvote',
+            title='Reddit upvote',
             href=i.url,
         )
         yield from self._from_common(i, locator=locator)

promnesia/sources/rss.py

@@ -1,3 +1,7 @@
+'''
+Uses [[https://github.com/karlicoss/HPI][HPI]] for RSS data.
+'''
+
 from itertools import chain
 
 from ..common import Visit, Loc, extract_urls, Results, get_logger
@@ -19,6 +23,6 @@ def index() -> Results:
         yield Visit(
             url=feed.url,
             dt=feed.created_at or default_datetime,
-            context=f'RSS subscription', # TODO use 'provider', etc?
+            context='RSS subscription', # TODO use 'provider', etc?
             locator=locator,
         )

promnesia/sources/shellcmd.py

@@ -1,10 +1,14 @@
+"""
+Greps out URLs from an arbitrary shell command results.
+"""
+
 from datetime import datetime
 import os
 import re
-from typing import Optional, Union, Sequence
+from subprocess import run, PIPE
+from typing import Union, Sequence
 import warnings
 
-from ..compat import run, PIPE
 from ..common import Visit, Loc, Results, extract_urls, file_mtime, get_system_tz, now_tz, _is_windows, PathIsh
 from .plaintext import _has_grep