promnesia 1.1.20230129__py3-none-any.whl → 1.2.20240810__py3-none-any.whl

promnesia/__main__.py

@@ -1,21 +1,25 @@
+from __future__ import annotations
+
 import argparse
-import logging
+import ast
+import importlib
 import inspect
-import sys
-from typing import List, Tuple, Optional, Dict, Sequence, Iterable, Iterator, Union
+import os
 from pathlib import Path
-from datetime import datetime
-from .compat import check_call, register_argparse_extend_action_in_pre_py38
-from tempfile import TemporaryDirectory
+import shutil
+from subprocess import run, check_call, Popen
+import sys
+from tempfile import TemporaryDirectory, gettempdir
+from typing import Callable, Sequence, Iterable, Iterator, Union
 
 
 from . import config
 from . import server
 from .misc import install_server
-from .common import PathIsh, logger, get_tmpdir, DbVisit, Res
+from .common import Extractor, PathIsh, logger, get_tmpdir, DbVisit, Res
 from .common import Source, get_system_tz, user_config_file, default_config_path
-from .dump import visits_to_sqlite
-from .extract import extract_visits, make_filter
+from .database.dump import visits_to_sqlite
+from .extract import extract_visits
 
 
 def iter_all_visits(sources_subset: Iterable[Union[str, int]]=()) -> Iterator[Res[DbVisit]]:
@@ -36,7 +40,7 @@ def iter_all_visits(sources_subset: Iterable[Union[str, int]]=()) -> Iterator[Re
 
     for i, source in enumerate(sources):
         # TODO why would it not be present??
-        name = getattr(source, "name", None)
+        name: str | None = getattr(source, "name", None)
         if name and is_subset_sources:
             matched = name in sources_subset or i in sources_subset
             if matched:
@@ -72,7 +76,7 @@ def iter_all_visits(sources_subset: Iterable[Union[str, int]]=()) -> Iterator[Re
 
 def _do_index(dry: bool=False, sources_subset: Iterable[Union[str, int]]=(), overwrite_db: bool=False) -> Iterable[Exception]:
     # also keep & return errors for further display
-    errors: List[Exception] = []
+    errors: list[Exception] = []
     def it() -> Iterable[Res[DbVisit]]:
         for v in iter_all_visits(sources_subset):
             if isinstance(v, Exception):
@@ -93,38 +97,38 @@ def _do_index(dry: bool=False, sources_subset: Iterable[Union[str, int]]=(), ove
 
 
 def do_index(
-        config_file: Path,
-        dry: bool=False,
-        sources_subset: Iterable[Union[str, int]]=(),
-        overwrite_db: bool=False,
-    ) -> None:
+    config_file: Path,
+    dry: bool=False,
+    sources_subset: Iterable[Union[str, int]]=(),
+    overwrite_db: bool=False,
+) -> Sequence[Exception]:
     config.load_from(config_file) # meh.. should be cleaner
     try:
         errors = list(_do_index(dry=dry, sources_subset=sources_subset, overwrite_db=overwrite_db))
     finally:
+        # this reset is mainly for tests, so we don't end up reusing the same config by accident
         config.reset()
     if len(errors) > 0:
         logger.error('%d errors, printing them out:', len(errors))
         for e in errors:
             logger.exception(e)
         logger.error('%d errors, exit code 1', len(errors))
-        sys.exit(1)
+    return errors
 
 
-def demo_sources():
-    def lazy(name: str):
+def demo_sources() -> dict[str, Callable[[], Extractor]]:
+    def lazy(name: str) -> Callable[[], Extractor]:
         # helper to avoid failed imports etc, since people might be lacking necessary dependencies
-        def inner(*args, **kwargs):
+        def inner() -> Extractor:
             from . import sources
-            import importlib
-            module = importlib.import_module('promnesia.sources' + '.' + name)
+            module = importlib.import_module(f'promnesia.sources.{name}')
             return getattr(module, 'index')
         return inner
 
     res = {}
-    import ast
     import promnesia.sources
-    for p in promnesia.sources.__path__: # type: ignore[attr-defined] # should be present
+    path: list[str] = getattr(promnesia.sources, '__path__')  # should be present
+    for p in path:
         for x in sorted(Path(p).glob('*.py')):
             a = ast.parse(x.read_text())
             candidates = [c for c in a.body if getattr(c, 'name', None) == 'index']
@@ -137,14 +141,13 @@ def do_demo(
         *,
         index_as: str,
         params: Sequence[str],
-        port: Optional[str],
-        config_file: Optional[Path],
+        port: str | None,
+        config_file: Path | None,
         dry: bool=False,
         name: str='demo',
         sources_subset: Iterable[Union[str, int]]=(),
         overwrite_db: bool=False,
     ) -> None:
-    from pprint import pprint
     with TemporaryDirectory() as tdir:
         outdir = Path(tdir)
 
@@ -185,7 +188,6 @@ def do_demo(
 
 
 def read_example_config() -> str:
-    import inspect
     from .misc import config_example
     return inspect.getsource(config_example)
 
@@ -214,19 +216,25 @@ def config_check(args: argparse.Namespace) -> None:
 
 
 def _config_check(cfg: Path) -> Iterable[Exception]:
-    from .compat import run
-
     logger.info('config: %s', cfg)
 
-    def check(cmd) -> Iterable[Exception]:
+    def check(cmd: list[str | Path], **kwargs) -> Iterable[Exception]:
         logger.debug(' '.join(map(str, cmd)))
-        res = run(cmd)
+        res = run(cmd, **kwargs)
         if res.returncode > 0:
             yield Exception()
 
     logger.info('Checking syntax...')
-    cmd = [sys.executable, '-m', 'compileall', cfg]
-    yield from check(cmd)
+    cmd: list[str | Path] = [sys.executable, '-m', 'compileall', cfg]
+    yield from check(
+        cmd,
+        env={
+            **os.environ,
+            # if config is on read only partition, the command would fail due to generated bytecode
+            # so put it in the temporary directory instead
+            'PYTHONPYCACHEPREFIX': gettempdir()
+        },
+    )
 
     # todo not sure if should be more defensive than check_call here
     logger.info('Checking type safety...')
@@ -265,18 +273,16 @@ def cli_doctor_db(args: argparse.Namespace) -> None:
     check_call(cmd)
 
     bro = 'sqlitebrowser'
-    import shutil
     if not shutil.which(bro):
         logger.error(f'Install {bro} to inspect the database!')
         sys.exit(1)
     cmd = [bro, str(db)]
     logger.debug(f'Running {cmd}')
-    from .compat import Popen
     Popen(cmd)
 
 
 def cli_doctor_server(args: argparse.Namespace) -> None:
-    port = args.port
+    port: str = args.port
     endpoint = f'http://localhost:{port}/status'
     cmd = ['curl', endpoint]
     logger.info(f'Running {cmd}')
@@ -296,12 +302,11 @@ def _ordinal_or_name(s: str) -> Union[str, int]:
 def main() -> None:
     # TODO longer, literate description?
 
-    def add_index_args(parser: argparse.ArgumentParser, default_config_path: Optional[PathIsh]=None) -> None:
+    def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh | None = None) -> None:
         """
         :param default_config_path:
             if not given, all :func:`demo_sources()` are run
         """
-        register_argparse_extend_action_in_pre_py38(parser)
         parser.add_argument('--config', type=Path, default=default_config_path, help='Config path')
         parser.add_argument('--dry', action='store_true', help="Dry run, won't touch the database, only print the results out")
         parser.add_argument(
@@ -322,14 +327,14 @@ def main() -> None:
         )
 
     F = lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=120)
-    p = argparse.ArgumentParser(formatter_class=F) # type: ignore
+    p = argparse.ArgumentParser(formatter_class=F)
     subp = p.add_subparsers(dest='mode', )
     ep = subp.add_parser('index', help='Create/update the link database', formatter_class=F)
     add_index_args(ep, default_config_path())
     # TODO use some way to override or provide config only via cmdline?
     ep.add_argument('--intermediate', required=False, help="Used for development, you don't need it")
 
-    sp = subp.add_parser('serve', help='Serve a link database', formatter_class=F) # type: ignore
+    sp = subp.add_parser('serve', help='Serve a link database', formatter_class=F)
     server.setup_parser(sp)
 
     ap = subp.add_parser('demo', help='Demo mode: index and serve a directory in single command', formatter_class=F)
@@ -378,8 +383,9 @@ def main() -> None:
 
     args = p.parse_args()
 
+    mode: str | None = args.mode
     # TODO is there a way to print full help? i.e. for all subparsers
-    if args.mode is None:
+    if mode is None:
         print('ERROR: Please specify a mode', file=sys.stderr)
         p.print_help(sys.stderr)
         sys.exit(1)
@@ -391,16 +397,18 @@ def main() -> None:
     # worst case -- could use database?
 
     with get_tmpdir() as tdir: # TODO??
-        if args.mode == 'index':
-            do_index(
+        if mode == 'index':
+            errors = do_index(
                 config_file=args.config,
                 dry=args.dry,
                 sources_subset=args.sources,
                 overwrite_db=args.overwrite,
             )
-        elif args.mode == 'serve':
+            if len(errors) > 0:
+                sys.exit(1)
+        elif mode == 'serve':
             server.run(args)
-        elif args.mode == 'demo':
+        elif mode == 'demo':
             # TODO not sure if 'as' is that useful
             # something like Telegram/Takeout is too hard to setup to justify adhoc mode like this?
             do_demo(
@@ -413,14 +421,14 @@ def main() -> None:
                 sources_subset=args.sources,
                 overwrite_db=args.overwrite,
                 )
-        elif args.mode == 'install-server': # todo rename to 'autostart' or something?
+        elif mode == 'install-server': # todo rename to 'autostart' or something?
             install_server.install(args)
-        elif args.mode == 'config':
+        elif mode == 'config':
             args.func(args)
-        elif args.mode == 'doctor':
+        elif mode == 'doctor':
             args.func(args)
         else:
-            raise AssertionError(f'unexpected mode {args.mode}')
+            raise AssertionError(f'unexpected mode {mode}')
 
 if __name__ == '__main__':
     main()

promnesia/cannon.py

@@ -422,7 +422,7 @@ def canonify(url: str) -> str:
     qq = [(k, v) for i, k, v in sorted(iqq)]
     # TODO still not sure what we should do..
     # quote_plus replaces %20 with +, not sure if we want it...
-    query = urlencode(qq, quote_via=quote_via) # type: ignore[type-var]
+    query = urlencode(qq, quote_via=quote_via)
 
     path = _quote_path(path)
 
@@ -683,7 +683,7 @@ def domains(it): # pragma: no cover
         try:
             nurl = canonify(url)
         except CanonifyException as e:
-            print(f"ERROR while normalising! {nurl} {e}")
+            print(f"ERROR while normalising! {url} {e}")
             c['ERROR'] += 1
             continue
         else:
@@ -718,7 +718,7 @@ def groups(it, args): # pragma: no cover
         try:
             nurl = canonify(url)
         except CanonifyException as e:
-            print(f"ERROR while normalising! {nurl} {e}")
+            print(f"ERROR while normalising! {url} {e}")
             continue
         udom = nurl[:nurl.find('/')]
         usplit = udom.split('.')
@@ -818,7 +818,7 @@ def main() -> None: # pragma: no cover
 
 - running comparison
   sqlite3 promnesia.sqlite 'select distinct orig_url from visits where norm_url like "%twitter%" order by orig_url' | src/promnesia/cannon.py
-''', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=100) # type: ignore
+''', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=100)
     )
     p.add_argument('input', nargs='?')
     p.add_argument('--human', action='store_true')

promnesia/common.py

@@ -1,22 +1,25 @@
-from collections.abc import Sized
+from __future__ import annotations
+
 from contextlib import contextmanager
 from datetime import datetime, date
-import os
-from typing import NamedTuple, Set, Iterable, Dict, TypeVar, Callable, List, Optional, Union, Any, Collection, Sequence, Tuple, TypeVar, TYPE_CHECKING
-from pathlib import Path
+from functools import lru_cache
 from glob import glob
 import itertools
-from more_itertools import intersperse
 import logging
-from functools import lru_cache
+import os
+from pathlib import Path
 import shutil
+from subprocess import run, PIPE, Popen
 from timeit import default_timer as timer
 from types import ModuleType
+from typing import NamedTuple, Iterable, TypeVar, Callable, List, Optional, Union, TypeVar
 import warnings
 
+from more_itertools import intersperse
 import pytz
 
 from .cannon import canonify
+from .compat import removeprefix
 
 
 _is_windows = os.name == 'nt'
@@ -74,14 +77,26 @@ class Loc(NamedTuple):
     # but generally, it will be
     # (url|file)(linenumber|json_path|anchor)
 
+
+@lru_cache(None)
+def warn_once(message: str) -> None:
+    # you'd think that warnings module already logs warnings only once per line..
+    # but sadly it's not the case
+    # see https://github.com/karlicoss/python_duplicate_warnings_investigation/blob/master/test.py
+    warnings.warn(message, stacklevel=2)
+
+
+def _warn_no_xdg_mime() -> None:
+    warn_once("No xdg-mime on your OS! If you're on OSX, perhaps you can help me! https://github.com/karlicoss/open-in-editor/issues/1")
+
+
 @lru_cache(1)
 def _detect_mime_handler() -> str:
     def exists(what: str) -> bool:
-        from .compat import run, PIPE
         try:
             r = run(f'xdg-mime query default x-scheme-handler/{what}'.split(), stdout=PIPE)
-        except FileNotFoundError:
-            warnings.warn("No xdg-mime on your OS! If you're on OSX, perhaps you can help me! https://github.com/karlicoss/open-in-editor/issues/1")
+        except (FileNotFoundError, NotADirectoryError):  # ugh seems that osx might throw NotADirectory for some reason
+            _warn_no_xdg_mime()
             return False
         if r.returncode > 0:
             warnings.warn('xdg-mime failed') # hopefully rest is in stderr
@@ -101,6 +116,7 @@ def _detect_mime_handler() -> str:
         result = 'emacs:'
 
     # 2. now try to use newer editor:// thing
+    # TODO flip order here? should rely on editor:// first?
 
     # TODO would be nice to collect warnings and display at the end
     if not exists('editor'):
@@ -191,7 +207,7 @@ def get_logger() -> logging.Logger:
 import tempfile
 # kinda singleton
 @lru_cache(1)
-def get_tmpdir() -> tempfile.TemporaryDirectory:
+def get_tmpdir() -> tempfile.TemporaryDirectory[str]:
     # todo use appdirs?
     tdir = tempfile.TemporaryDirectory(suffix="promnesia")
     return tdir
@@ -284,9 +300,10 @@ def _guess_name(thing: PreSource) -> str:
         guess = thing.__module__
 
     dflt = 'promnesia.sources.'
-    if guess.startswith(dflt):
-        # meh
-        guess = guess[len(dflt):]
+    guess = removeprefix(guess, prefix=dflt)
+    if guess == 'config':
+        # this happens when we define a lambda in config or something without properly wrapping in Source
+        logger.warning(f'Inferred source name "config" for {thing}. This might be misleading TODO')
     return guess
 
 
@@ -296,7 +313,7 @@ def _get_index_function(sourceish: PreSource) -> PreExtractor:
     if hasattr(sourceish, 'index'):  # must be a module
         res = getattr(sourceish, 'index')
     else:
-        res = sourceish  # type: ignore[assignment]
+        res = sourceish
     return res
 
 
@@ -316,12 +333,17 @@ class Source:
         self.extractor: Extractor = lambda: self.ff(*self.args, **self.kwargs)
         if src is not None:
             warnings.warn("'src' argument is deprecated, please use 'name' instead", DeprecationWarning)
-        try:
-            name_guess = _guess_name(ff)
-        except:
-            # todo warn?
-            name_guess = ''
-        self.name = name or src or name_guess
+        if name != '':
+            self.name = name
+        elif src != '':
+            self.name = src
+        else:
+            try:
+                name_guess = _guess_name(ff)
+            except:
+                # todo warn?
+                name_guess = ''
+            self.name = name_guess
 
     @property
     def description(self) -> str:
@@ -370,7 +392,7 @@ def appdirs():
     under_test = os.environ.get('PYTEST_CURRENT_TEST') is not None
     # todo actually use test name?
     name = 'promnesia-test' if under_test else 'promnesia'
-    import appdirs as ad # type: ignore[import]
+    import appdirs as ad # type: ignore[import-untyped]
     return ad.AppDirs(appname=name)
 
 
@@ -460,13 +482,13 @@ def fdfind_args(root: Path, follow: bool, ignore: List[str]=[]) -> List[str]:
     ignore_args = []
     if ignore:
         # Add a statement that excludes the folder
-        ignore_args = [['--exclude', f'{n}'] for n in ignore]
+        _ignore_args = [['--exclude', f'{n}'] for n in ignore]
         # Flatten the list of lists
-        ignore_args_l = list(itertools.chain(*ignore_args))
+        ignore_args = list(itertools.chain(*_ignore_args))
 
     return [
         *extra_fd_args(),
-        *ignore_args_l,
+        *ignore_args,
         *(['--follow'] if follow else []),
         '--type', 'f',
         '.',
@@ -491,7 +513,6 @@ def traverse(root: Path, *, follow: bool=True, ignore: List[str]=[]) -> Iterable
             yield from (Path(r) / f for f in files if f not in ignore)
         return
 
-    from .compat import Popen, PIPE
     cmd = ['find', *find_args(root, follow=follow, ignore=ignore)]
     # try to use fd.. it cooperates well with gitignore etc, also faster than find
     for x in ('fd', 'fd-find', 'fdfind'): # has different names on different dists..
@@ -516,17 +537,7 @@ def traverse(root: Path, *, follow: bool=True, ignore: List[str]=[]) -> Iterable
 def get_system_zone() -> str:
     try:
         import tzlocal
-        # note: tzlocal mypy stubs aren't aware of api change yet (see https://github.com/python/typeshed/issues/6038)
-        try:
-            # 4.0 way
-            return tzlocal.get_localzone_name() # type: ignore[attr-defined]
-        except AttributeError as e:
-            # 2.0 way
-            zone = tzlocal.get_localzone().zone  # type: ignore[attr-defined]
-            # see https://github.com/python/typeshed/blame/968fd6d01d23470e0c8368e7ee7c43f54aaedc0e/stubs/pytz/pytz/tzinfo.pyi#L6
-            # it says all concrete instances should not be None
-            assert zone is not None
-            return zone
+        return tzlocal.get_localzone_name()
     except Exception as e:
         logger.exception(e)
         logger.error("Couldn't determine system timezone. Falling back to UTC. Please report this as a bug!")
@@ -540,7 +551,7 @@ def get_system_tz() -> pytz.BaseTzInfo:
         return pytz.timezone(zone)
     except Exception as e:
         logger.exception(e)
-        logger.error(f"Unknown time zone %s. Falling back to UTC. Please report this as a bug!", zone)
+        logger.error("Unknown time zone %s. Falling back to UTC. Please report this as a bug!", zone)
         return pytz.utc
 
 # used in misc/install_server.py
@@ -578,7 +589,7 @@ def default_config_path() -> Path:
 
 
 @contextmanager
-def measure(tag: str='', *, logger, unit: str='ms'):
+def measure(tag: str='', *, logger: logging.Logger, unit: str='ms'):
     before = timer()
     yield lambda: timer() - before
     after = timer()
@@ -586,3 +597,11 @@ def measure(tag: str='', *, logger, unit: str='ms'):
     mult = {'s': 1, 'ms': 10**3, 'us': 10**6}[unit]
     xx = secs * mult
     logger.debug(f'[{tag}]: {xx:.1f}{unit} elapsed')
+
+
+def is_sqlite_db(x: Path) -> bool:
+    return x.is_file() and mime(x) in {
+        'application/x-sqlite3',
+        'application/vnd.sqlite3',
+        # TODO this mime can also match wal files/journals, not sure
+    }

promnesia/compare.py

@@ -8,6 +8,7 @@ from typing import Dict, List, Any, NamedTuple, Optional, Iterator, Set, Tuple
 
 
 from .common import DbVisit, Url, PathWithMtime # TODO ugh. figure out pythonpath
+from .database.load import row_to_db_visit
 
 # TODO include latest too?
 # from cconfig import ignore, filtered
@@ -139,10 +140,10 @@ def compare_files(*files: Path, log=True) -> Iterator[Tuple[str, DbVisit]]:
         this_dts = name[0: name.index('.')] # can't use stem due to multiple extensions..
 
         from promnesia.server import _get_stuff # TODO ugh
-        engine, binder, table = _get_stuff(PathWithMtime.make(f))
+        engine, table = _get_stuff(PathWithMtime.make(f))
 
         with engine.connect() as conn:
-            vis = [binder.from_row(row) for row in conn.execute(table.select())]  # type: ignore[var-annotated]
+            vis = [row_to_db_visit(row) for row in conn.execute(table.select())]
 
         if last is not None:
             between = f'{last_dts}:{this_dts}'

promnesia/compat.py

@@ -1,71 +1,12 @@
-from pathlib import Path
-import sys
-from typing import Union, Sequence, List, TYPE_CHECKING
-
-
-PathIsh = Union[Path, str]
-Paths = Sequence[PathIsh]
-
-# TLDR: py37 on windows has an annoying bug.. https://github.com/karlicoss/promnesia/issues/91#issuecomment-701051074
-def _fix(args: Paths) -> List[str]:
-    assert not isinstance(args, str), args # just to prevent shell=True calls...
-    return list(map(str, args))
-
-
-import argparse
-
-def register_argparse_extend_action_in_pre_py38(parser: argparse.ArgumentParser):
-    import sys
-
-    if sys.version_info < (3, 8):
-
-        class ExtendAction(argparse.Action):
-
-            def __call__(self, parser, namespace, values, option_string=None):
-                items = getattr(namespace, self.dest) or []
-                items.extend(values)
-                setattr(namespace, self.dest, items)
-
-
-        parser.register('action', 'extend', ExtendAction)
-
-
-import subprocess
-from subprocess import PIPE  # for convenience?
-
-
-if TYPE_CHECKING:
-    from subprocess import run, check_call, check_output, Popen
-else:
-    def run(args: Paths, **kwargs) -> subprocess.CompletedProcess:
-        return subprocess.run(_fix(args), **kwargs)
-
-    def check_call(args: Paths, **kwargs) -> None:
-        subprocess.check_call(_fix(args), **kwargs)
-
-    def check_output(args: Paths, **kwargs) -> bytes:
-        return subprocess.check_output(_fix(args), **kwargs)
-
-    def Popen(args: Paths, **kwargs) -> subprocess.Popen:
-        return subprocess.Popen(_fix(args), **kwargs)
+## we used to have compat fixes here for these for python3.7
+## keeping in case any sources depended on compat functions
+from subprocess import PIPE, run, check_call, check_output, Popen
+from typing import Protocol, Literal
+## 
 
 
 # can remove after python3.9
 def removeprefix(text: str, prefix: str) -> str:
     if text.startswith(prefix):
         return text[len(prefix):]
-    return text
-
-
-# TODO Deprecate instead, they shouldn't be exported form this module
-# del PathIsh
-# del Paths
-
-if sys.version_info[:2] >= (3, 8):
-    from typing import Protocol
-else:
-    if TYPE_CHECKING:
-        from typing_extensions import Protocol  # type: ignore[misc]
-    else:
-        # todo could also use NamedTuple?
-        Protocol = object
+    return text

promnesia/config.py

@@ -6,7 +6,7 @@ import importlib
 import importlib.util
 import warnings
 
-from .common import PathIsh, get_tmpdir, appdirs, default_output_dir, default_cache_dir, user_config_file
+from .common import PathIsh, default_output_dir, default_cache_dir
 from .common import Res, Source, DbVisit
 
 
@@ -69,6 +69,8 @@ class Config(NamedTuple):
 
     @property
     def cache_dir(self) -> Optional[Path]:
+        # TODO we used to use this for cachew, but it's best to rely on HPI modules etc to cofigure this
+        # keeping just in case for now
         cd = self.CACHE_DIR
         cpath: Optional[Path]
         if cd is None:
@@ -127,7 +129,7 @@ def import_config(config_file: PathIsh) -> Config:
     spec = importlib.util.spec_from_file_location(name, p); assert spec is not None
     mod = importlib.util.module_from_spec(spec); assert mod is not None
     loader = spec.loader; assert loader is not None
-    loader.exec_module(mod) # type: ignore[attr-defined]
+    loader.exec_module(mod)
 
     d = {}
     for f in Config._fields: