sqlite-utils 3.36__tar.gz → 3.38__tar.gz

{sqlite-utils-3.36/sqlite_utils.egg-info → sqlite_utils-3.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlite-utils
-Version: 3.36
+Version: 3.38
 Summary: CLI tool and Python library for manipulating SQLite databases
 Home-page: https://github.com/simonw/sqlite-utils
 Author: Simon Willison
@@ -16,13 +16,13 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: End Users/Desktop
 Classifier: Topic :: Database
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.7
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: sqlite-fts4
@@ -33,7 +33,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: pluggy
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
-Requires-Dist: black; extra == "test"
+Requires-Dist: black>=24.1.1; extra == "test"
 Requires-Dist: hypothesis; extra == "test"
 Requires-Dist: cogapp; extra == "test"
 Provides-Extra: docs

{sqlite-utils-3.36 → sqlite_utils-3.38}/docs/changelog.rst

@@ -4,6 +4,25 @@
  Changelog
 ===========
 
+.. _v3_38:
+
+3.38 (2024-11-23)
+-----------------
+
+- Plugins can now reuse the implementation of the ``sqlite-utils memory`` CLI command with the new ``return_db=True`` parameter. (:issue:`643`)
+- ``table.transform()`` now recreates indexes after transforming a table. A new ``sqlite_utils.db.TransformError`` exception is raised if these indexes cannot be recreated due to conflicting changes to the table such as a column rename. Thanks, `Mat Miller <https://github.com/matdmiller>`__. (:issue:`633`)
+- ``table.search()`` now accepts a ``include_rank=True`` parameter, causing the resulting rows to have a ``rank`` column showing the calculated relevance score. Thanks, `liunux4odoo <https://github.com/liunux4odoo>`__. (`#628 <https://github.com/simonw/sqlite-utils/pull/628>`__)
+- Fixed an error that occurred when creating a strict table with at least one floating point column. These ``FLOAT`` columns are now correctly created as ``REAL`` as well, but only for strict tables. (:issue:`644`)
+
+.. _v3_37:
+
+3.37 (2024-07-18)
+-----------------
+
+- The ``create-table`` and ``insert-files`` commands all now accept multiple ``--pk`` options for compound primary keys. (:issue:`620`)
+- Now tested against Python 3.13 pre-release. (`#619 <https://github.com/simonw/sqlite-utils/pull/619>`__)
+- Fixed a crash that can occur in environments with a broken ``numpy`` installation, producing a ``module 'numpy' has no attribute 'int8'``. (:issue:`632`)
+
 .. _v3_36:
 
 3.36 (2023-12-07)

{sqlite-utils-3.36 → sqlite_utils-3.38}/docs/cli-reference.rst

@@ -85,6 +85,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command.
         cog.out("::\n\n")
         result = CliRunner().invoke(cli.cli, [command, "--help"])
         output = result.output.replace("Usage: cli ", "Usage: sqlite-utils ")
+        output = output.replace('\b', '')
         cog.out(textwrap.indent(output, '    '))
         cog.out("\n\n")
 .. ]]]
@@ -603,9 +604,9 @@ See :ref:`cli_convert`.
 
       Convert columns using Python code you supply. For example:
 
-          sqlite-utils convert my.db mytable mycolumn \
-              '"\n".join(textwrap.wrap(value, 10))' \
-              --import=textwrap
+      sqlite-utils convert my.db mytable mycolumn \
+          '"\n".join(textwrap.wrap(value, 10))' \
+          --import=textwrap
 
       "value" is a variable with the column value to be converted.
 
@@ -615,30 +616,30 @@ See :ref:`cli_convert`.
 
       r.jsonsplit(value, delimiter=',', type=<class 'str'>)
 
-          Convert a string like a,b,c into a JSON array ["a", "b", "c"]
+      Convert a string like a,b,c into a JSON array ["a", "b", "c"]
 
       r.parsedate(value, dayfirst=False, yearfirst=False, errors=None)
 
-          Parse a date and convert it to ISO date format: yyyy-mm-dd
-          
-          - dayfirst=True: treat xx as the day in xx/yy/zz
-          - yearfirst=True: treat xx as the year in xx/yy/zz
-          - errors=r.IGNORE to ignore values that cannot be parsed
-          - errors=r.SET_NULL to set values that cannot be parsed to null
+      Parse a date and convert it to ISO date format: yyyy-mm-dd
+  
+      - dayfirst=True: treat xx as the day in xx/yy/zz
+      - yearfirst=True: treat xx as the year in xx/yy/zz
+      - errors=r.IGNORE to ignore values that cannot be parsed
+      - errors=r.SET_NULL to set values that cannot be parsed to null
 
       r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None)
 
-          Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS
-          
-          - dayfirst=True: treat xx as the day in xx/yy/zz
-          - yearfirst=True: treat xx as the year in xx/yy/zz
-          - errors=r.IGNORE to ignore values that cannot be parsed
-          - errors=r.SET_NULL to set values that cannot be parsed to null
+      Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS
+  
+      - dayfirst=True: treat xx as the day in xx/yy/zz
+      - yearfirst=True: treat xx as the year in xx/yy/zz
+      - errors=r.IGNORE to ignore values that cannot be parsed
+      - errors=r.SET_NULL to set values that cannot be parsed to null
 
       You can use these recipes like so:
 
-          sqlite-utils convert my.db mytable mycolumn \
-              'r.jsonsplit(value, delimiter=":")'
+      sqlite-utils convert my.db mytable mycolumn \
+          'r.jsonsplit(value, delimiter=":")'
 
     Options:
       --import TEXT                   Python modules to import

{sqlite-utils-3.36 → sqlite_utils-3.38}/docs/cli.rst

@@ -1088,11 +1088,13 @@ You can import all three records into an automatically created ``dogs`` table an
 
     sqlite-utils insert dogs.db dogs dogs.json --pk=id
 
+Pass ``--pk`` multiple times to define a compound primary key.
+
 You can skip inserting any records that have a primary key that already exists using ``--ignore``:
 
 .. code-block:: bash
 
-    sqlite-utils insert dogs.db dogs dogs.json --ignore
+    sqlite-utils insert dogs.db dogs dogs.json --pk=id --ignore
 
 You can delete all the existing rows in the table before inserting the new records using ``--truncate``:
 
@@ -1909,6 +1911,8 @@ This will create a table called ``mytable`` with two columns - an integer ``id``
 
 You can pass as many column-name column-type pairs as you like. Valid types are ``integer``, ``text``, ``float`` and ``blob``.
 
+Pass ``--pk`` more than once for a compound primary key that covers multiple columns.
+
 You can specify columns that should be NOT NULL using ``--not-null colname``. You can specify default values for columns using ``--default colname defaultvalue``.
 
 .. code-block:: bash
@@ -2080,7 +2084,7 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi
 ``--drop-foreign-key column``
     Drop the specified foreign key.
 
-``--add-foregn-key column other_table other_column``
+``--add-foreign-key column other_table other_column``
     Add a foreign key constraint to ``column`` pointing to ``other_table.other_column``.
 
 If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example:
@@ -2792,7 +2796,7 @@ To enable this feature you will need to install the ``trogon`` dependency. You c
 
 .. code-block:: bash
 
-    sqite-utils install trogon
+    sqlite-utils install trogon
 
 Once installed, running the ``sqlite-utils tui`` command will launch the TUI interface:
 

{sqlite-utils-3.36 → sqlite_utils-3.38}/docs/plugins.rst

@@ -115,6 +115,26 @@ Example implementation:
             "Say hello world"
             click.echo("Hello world!")
 
+New commands implemented by plugins can invoke existing commands using the `context.invoke <https://click.palletsprojects.com/en/stable/api/#click.Context.invoke>`__ mechanism.
+
+As a special niche feature, if your plugin needs to import some files and then act against an in-memory database containing those files you can forward to the :ref:`sqlite-utils memory command <cli_memory>` and pass it ``return_db=True``:
+
+.. code-block:: python
+
+    @cli.command()
+    @click.pass_context
+    @click.argument(
+        "paths",
+        type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+        required=False,
+        nargs=-1,
+    )
+    def show_schema_for_files(ctx, paths):
+        from sqlite_utils.cli import memory
+        db = ctx.invoke(memory, paths=paths, return_db=True)
+        # Now do something with that database
+        click.echo(db.schema)
+
 .. _plugins_hooks_prepare_connection:
 
 prepare_connection(conn)

{sqlite-utils-3.36 → sqlite_utils-3.38}/docs/python-api.rst

@@ -890,7 +890,8 @@ You can delete all records in a table that match a specific WHERE statement usin
 
     >>> db = sqlite_utils.Database("dogs.db")
     >>> # Delete every dog with age less than 3
-    >>> db["dogs"].delete_where("age < ?", [3])
+    >>> with db.conn:
+    >>>     db["dogs"].delete_where("age < ?", [3])
 
 Calling ``table.delete_where()`` with no other arguments will delete every row in the table.
 
@@ -1248,6 +1249,9 @@ If you pass a Python type, it will be mapped to SQLite types as shown here::
     np.float32: "FLOAT"
     np.float64: "FLOAT"
 
+.. note::
+    In sqlite-utils 3.x ``FLOAT`` is used for floating point columns when the correct column type is actually ``REAL``. If you specify ``strict=True`` tables created in strict mode will use the correct column type of ``REAL`` instead. We plan to change this behavior in ``sqlite-utils`` 4.x to always use ``REAL``, but this will represent a minor breaking change and so is being held for the next major release, see issue :issue:`645`.
+
 You can also add a column that is a foreign key reference to another table using the ``fk`` parameter:
 
 .. code-block:: python
@@ -1401,6 +1405,8 @@ To keep the original table around instead of dropping it, pass the ``keep_table=
 
     table.transform(types={"age": int}, keep_table="original_table")
 
+This method raises a ``sqlite_utils.db.TransformError`` exception if the table cannot be transformed, usually because there are existing constraints or indexes that are incompatible with modifications to the columns.
+
 .. _python_api_transform_alter_column_types:
 
 Altering column types
@@ -2177,7 +2183,7 @@ The ``.has_counts_triggers`` property shows if a table has been configured with
     >>> db["authors"].has_counts_triggers
     True
 
-.. _python_api_introspection_supports_strict
+.. _python_api_introspection_supports_strict:
 
 db.supports_strict
 ------------------
@@ -2308,6 +2314,9 @@ The ``.search()`` method also accepts the following optional parameters:
 ``where_args`` dictionary
     Arguments to use for ``:param`` placeholders in the extra WHERE clause
 
+``include_rank`` bool
+    If set a ``rank`` column will be included with the BM25 ranking score - for FTS5 tables only.
+
 ``quote`` bool
     Apply :ref:`FTS quoting rules <python_api_quote_fts>` to the search query, disabling advanced query syntax in a way that avoids surprising errors.
 

{sqlite-utils-3.36 → sqlite_utils-3.38}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import io
 import os
 
-VERSION = "3.36"
+VERSION = "3.38"
 
 
 def get_long_description():
@@ -32,7 +32,7 @@ setup(
         "pluggy",
     ],
     extras_require={
-        "test": ["pytest", "black", "hypothesis", "cogapp"],
+        "test": ["pytest", "black>=24.1.1", "hypothesis", "cogapp"],
         "docs": [
             "furo",
             "sphinx-autobuild",
@@ -64,7 +64,7 @@ setup(
         "Issues": "https://github.com/simonw/sqlite-utils/issues",
         "CI": "https://github.com/simonw/sqlite-utils/actions",
     },
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
@@ -72,12 +72,12 @@ setup(
         "Intended Audience :: End Users/Desktop",
         "Topic :: Database",
         "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
     ],
     # Needed to bundle py.typed so mypy can see it:
     zip_safe=False,

{sqlite-utils-3.36 → sqlite_utils-3.38}/sqlite_utils/cli.py

@@ -1489,7 +1489,7 @@ def create_database(path, enable_wal, init_spatialite, load_extension):
 )
 @click.argument("table")
 @click.argument("columns", nargs=-1, required=True)
-@click.option("--pk", help="Column to use as primary key")
+@click.option("pks", "--pk", help="Column to use as primary key", multiple=True)
 @click.option(
     "--not-null",
     multiple=True,
@@ -1532,7 +1532,7 @@ def create_table(
     path,
     table,
     columns,
-    pk,
+    pks,
     not_null,
     default,
     fk,
@@ -1581,7 +1581,7 @@ def create_table(
             )
     db[table].create(
         coltypes,
-        pk=pk,
+        pk=pks[0] if len(pks) == 1 else pks,
         not_null=not_null,
         defaults=dict(default),
         foreign_keys=fk,
@@ -1894,6 +1894,7 @@ def memory(
     save,
     analyze,
     load_extension,
+    return_db=False,
 ):
     """Execute SQL query against an in-memory database, optionally populated by imported data
 
@@ -1922,6 +1923,7 @@ def memory(
         sqlite-utils memory animals.csv --schema
     """
     db = sqlite_utils.Database(memory=True)
+
     # If --dump or --save or --analyze used but no paths detected, assume SQL query is a path:
     if (dump or save or schema or analyze) and not paths:
         paths = [sql]
@@ -1954,6 +1956,7 @@ def memory(
             rows = tracker.wrap(rows)
         if flatten:
             rows = (_flatten(row) for row in rows)
+
         db[file_table].insert_all(rows, alter=True)
         if tracker is not None:
             db[file_table].transform(types=tracker.types)
@@ -1964,6 +1967,7 @@ def memory(
         for view_name in view_names:
             if not db[view_name].exists():
                 db.create_view(view_name, "select * from [{}]".format(file_table))
+
         if fp:
             fp.close()
 
@@ -1994,6 +1998,9 @@ def memory(
     if functions:
         _register_functions(db, functions)
 
+    if return_db:
+        return db
+
     _execute_query(
         db,
         sql,
@@ -2594,7 +2601,7 @@ def extract(
     multiple=True,
     help="Column definitions for the table",
 )
-@click.option("--pk", type=str, help="Column to use as primary key")
+@click.option("pks", "--pk", help="Column to use as primary key", multiple=True)
 @click.option("--alter", is_flag=True, help="Alter table to add missing columns")
 @click.option("--replace", is_flag=True, help="Replace files with matching primary key")
 @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key")
@@ -2611,7 +2618,7 @@ def insert_files(
     table,
     file_or_dir,
     column,
-    pk,
+    pks,
     alter,
     replace,
     upsert,
@@ -2641,8 +2648,8 @@ def insert_files(
             column = ["path:path", "content_text:content_text", "size:size"]
         else:
             column = ["path:path", "content:content", "size:size"]
-        if not pk:
-            pk = "path"
+        if not pks:
+            pks = ["path"]
 
     def yield_paths_and_relative_paths():
         for f_or_d in file_or_dir:
@@ -2712,7 +2719,11 @@ def insert_files(
         try:
             with db.conn:
                 db[table].insert_all(
-                    to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert
+                    to_insert(),
+                    pk=pks[0] if len(pks) == 1 else pks,
+                    alter=alter,
+                    replace=replace,
+                    upsert=upsert,
                 )
         except UnicodeDecodeErrorForPath as e:
             raise click.ClickException(
@@ -2871,9 +2882,9 @@ def _generate_convert_help():
     Convert columns using Python code you supply. For example:
 
     \b
-        sqlite-utils convert my.db mytable mycolumn \\
-            '"\\n".join(textwrap.wrap(value, 10))' \\
-            --import=textwrap
+    sqlite-utils convert my.db mytable mycolumn \\
+        '"\\n".join(textwrap.wrap(value, 10))' \\
+        --import=textwrap
 
     "value" is a variable with the column value to be converted.
 
@@ -2892,7 +2903,7 @@ def _generate_convert_help():
     for name in recipe_names:
         fn = getattr(recipes, name)
         help += "\n\nr.{}{}\n\n\b{}".format(
-            name, str(inspect.signature(fn)), fn.__doc__.rstrip()
+            name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip())
         )
     help += "\n\n"
     help += textwrap.dedent(
@@ -2900,8 +2911,8 @@ def _generate_convert_help():
     You can use these recipes like so:
 
     \b
-        sqlite-utils convert my.db mytable mycolumn \\
-            'r.jsonsplit(value, delimiter=":")'
+    sqlite-utils convert my.db mytable mycolumn \\
+        'r.jsonsplit(value, delimiter=":")'
     """
     ).strip()
     return help

{sqlite-utils-3.36 → sqlite_utils-3.38}/sqlite_utils/db.py

@@ -156,6 +156,10 @@ XIndexColumn = namedtuple(
 Trigger = namedtuple("Trigger", ("name", "table", "sql"))
 
 
+class TransformError(Exception):
+    pass
+
+
 ForeignKeyIndicator = Union[
     str,
     ForeignKey,
@@ -206,21 +210,25 @@ COLUMN_TYPE_MAPPING = {
 }
 # If numpy is available, add more types
 if np:
-    COLUMN_TYPE_MAPPING.update(
-        {
-            np.int8: "INTEGER",
-            np.int16: "INTEGER",
-            np.int32: "INTEGER",
-            np.int64: "INTEGER",
-            np.uint8: "INTEGER",
-            np.uint16: "INTEGER",
-            np.uint32: "INTEGER",
-            np.uint64: "INTEGER",
-            np.float16: "FLOAT",
-            np.float32: "FLOAT",
-            np.float64: "FLOAT",
-        }
-    )
+    try:
+        COLUMN_TYPE_MAPPING.update(
+            {
+                np.int8: "INTEGER",
+                np.int16: "INTEGER",
+                np.int32: "INTEGER",
+                np.int64: "INTEGER",
+                np.uint8: "INTEGER",
+                np.uint16: "INTEGER",
+                np.uint32: "INTEGER",
+                np.uint64: "INTEGER",
+                np.float16: "FLOAT",
+                np.float32: "FLOAT",
+                np.float64: "FLOAT",
+            }
+        )
+    except AttributeError:
+        # https://github.com/simonw/sqlite-utils/issues/632
+        pass
 
 # If pandas is available, add more types
 if pd:
@@ -321,6 +329,8 @@ class Database:
         execute_plugins: bool = True,
         strict: bool = False,
     ):
+        self.memory_name = None
+        self.memory = False
         assert (filename_or_conn is not None and (not memory and not memory_name)) or (
             filename_or_conn is None and (memory or memory_name)
         ), "Either specify a filename_or_conn or pass memory=True"
@@ -331,8 +341,11 @@ class Database:
                 uri=True,
                 check_same_thread=False,
             )
+            self.memory = True
+            self.memory_name = memory_name
         elif memory or filename_or_conn == ":memory:":
             self.conn = sqlite3.connect(":memory:")
+            self.memory = True
         elif isinstance(filename_or_conn, (str, pathlib.Path)):
             if recreate and os.path.exists(filename_or_conn):
                 try:
@@ -457,8 +470,7 @@ class Database:
                         fn_name, arity, fn, **dict(kwargs, deterministic=True)
                     )
                     registered = True
-                except (sqlite3.NotSupportedError, TypeError):
-                    # TypeError is Python 3.7 "function takes at most 3 arguments"
+                except sqlite3.NotSupportedError:
                     pass
             if not registered:
                 self.conn.create_function(fn_name, arity, fn, **kwargs)
@@ -926,13 +938,18 @@ class Database:
                         other_column=foreign_keys_by_column[column_name].other_column,
                     )
                 )
+            column_type_str = COLUMN_TYPE_MAPPING[column_type]
+            # Special case for strict tables to map FLOAT to REAL
+            # Refs https://github.com/simonw/sqlite-utils/issues/644
+            if strict and column_type_str == "FLOAT":
+                column_type_str = "REAL"
             column_defs.append(
                 "   [{column_name}] {column_type}{column_extras}".format(
                     column_name=column_name,
-                    column_type=COLUMN_TYPE_MAPPING[column_type],
-                    column_extras=(" " + " ".join(column_extras))
-                    if column_extras
-                    else "",
+                    column_type=column_type_str,
+                    column_extras=(
+                        (" " + " ".join(column_extras)) if column_extras else ""
+                    ),
                 )
             )
         extra_pk = ""
@@ -1481,9 +1498,11 @@ class Table(Queryable):
     def __repr__(self) -> str:
         return "<Table {}{}>".format(
             self.name,
-            " (does not exist yet)"
-            if not self.exists()
-            else " ({})".format(", ".join(c.name for c in self.columns)),
+            (
+                " (does not exist yet)"
+                if not self.exists()
+                else " ({})".format(", ".join(c.name for c in self.columns))
+            ),
         )
 
     @property
@@ -1962,6 +1981,30 @@ class Table(Queryable):
         sqls.append(
             "ALTER TABLE [{}] RENAME TO [{}];".format(new_table_name, self.name)
         )
+        # Re-add existing indexes
+        for index in self.indexes:
+            if index.origin != "pk":
+                index_sql = self.db.execute(
+                    """SELECT sql FROM sqlite_master WHERE type = 'index' AND name = :index_name;""",
+                    {"index_name": index.name},
+                ).fetchall()[0][0]
+                if index_sql is None:
+                    raise TransformError(
+                        f"Index '{index.name}' on table '{self.name}' does not have a "
+                        "CREATE INDEX statement. You must manually drop this index prior to running this "
+                        "transformation and manually recreate the new index after running this transformation."
+                    )
+                if keep_table:
+                    sqls.append(f"DROP INDEX IF EXISTS [{index.name}];")
+                for col in index.columns:
+                    if col in rename.keys() or col in drop:
+                        raise TransformError(
+                            f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. "
+                            f"You must manually drop this index prior to running this transformation "
+                            f"and manually recreate the new index after running this transformation. "
+                            f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table."
+                        )
+                sqls.append(index_sql)
         return sqls
 
     def extract(
@@ -2640,6 +2683,7 @@ class Table(Queryable):
         offset: Optional[int] = None,
         where: Optional[str] = None,
         where_args: Optional[Union[Iterable, dict]] = None,
+        include_rank: bool = False,
         quote: bool = False,
     ) -> Generator[dict, None, None]:
         """
@@ -2653,6 +2697,7 @@ class Table(Queryable):
         :param offset: Optional integer SQL offset.
         :param where: Extra SQL fragment for the WHERE clause
         :param where_args: Arguments to use for :param placeholders in the extra WHERE clause
+        :param include_rank: Select the search rank column in the final query
         :param quote: Apply quoting to disable any special characters in the search query
 
         See :ref:`python_api_fts_search`.
@@ -2672,6 +2717,7 @@ class Table(Queryable):
                 limit=limit,
                 offset=offset,
                 where=where,
+                include_rank=include_rank,
             ),
             args,
         )
@@ -2940,9 +2986,11 @@ class Table(Queryable):
                 value = jsonify_if_needed(
                     record.get(
                         key,
-                        None
-                        if key != hash_id
-                        else hash_record(record, hash_id_columns),
+                        (
+                            None
+                            if key != hash_id
+                            else hash_record(record, hash_id_columns)
+                        ),
                     )
                 )
                 if key in extracts:

{sqlite-utils-3.36 → sqlite_utils-3.38}/sqlite_utils/utils.py

@@ -304,10 +304,7 @@ def rows_from_file(
         rows = rows_from_file(
             fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
         )[0]
-        return (
-            _extra_key_strategy(rows, ignore_extras, extras_key),
-            Format.TSV,
-        )
+        return _extra_key_strategy(rows, ignore_extras, extras_key), Format.TSV
     elif format is None:
         # Detect the format, then call this recursively
         buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096)

{sqlite-utils-3.36 → sqlite_utils-3.38/sqlite_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlite-utils
-Version: 3.36
+Version: 3.38
 Summary: CLI tool and Python library for manipulating SQLite databases
 Home-page: https://github.com/simonw/sqlite-utils
 Author: Simon Willison
@@ -16,13 +16,13 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: End Users/Desktop
 Classifier: Topic :: Database
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.7
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: sqlite-fts4
@@ -33,7 +33,7 @@ Requires-Dist: python-dateutil
 Requires-Dist: pluggy
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
-Requires-Dist: black; extra == "test"
+Requires-Dist: black>=24.1.1; extra == "test"
 Requires-Dist: hypothesis; extra == "test"
 Requires-Dist: cogapp; extra == "test"
 Provides-Extra: docs

{sqlite-utils-3.36 → sqlite_utils-3.38}/sqlite_utils.egg-info/requires.txt

@@ -26,7 +26,7 @@ data-science-types
 
 [test]
 pytest
-black
+black>=24.1.1
 hypothesis
 cogapp
 

{sqlite-utils-3.36 → sqlite_utils-3.38}/tests/test_cli.py

@@ -1180,6 +1180,14 @@ def test_upsert_alter(db_path, tmpdir):
             ["age", "integer", "--default", "age", "3"],
             ("CREATE TABLE [t] (\n" "   [age] INTEGER DEFAULT '3'\n" ")"),
         ),
+        # Compound primary key
+        (
+            ["category", "text", "name", "text", "--pk", "category", "--pk", "name"],
+            (
+                "CREATE TABLE [t] (\n   [category] TEXT,\n   [name] TEXT,\n"
+                "   PRIMARY KEY ([category], [name])\n)"
+            ),
+        ),
     ],
 )
 def test_create_table(args, schema):
@@ -2082,11 +2090,10 @@ def test_schema(tmpdir, options, expected):
 def test_long_csv_column_value(tmpdir):
     db_path = str(tmpdir / "test.db")
     csv_path = str(tmpdir / "test.csv")
-    csv_file = open(csv_path, "w")
-    long_string = "a" * 131073
-    csv_file.write("id,text\n")
-    csv_file.write("1,{}\n".format(long_string))
-    csv_file.close()
+    with open(csv_path, "w") as csv_file:
+        long_string = "a" * 131073
+        csv_file.write("id,text\n")
+        csv_file.write("1,{}\n".format(long_string))
     result = CliRunner().invoke(
         cli.cli,
         ["insert", db_path, "bigtable", csv_path, "--csv"],
@@ -2110,11 +2117,10 @@ def test_long_csv_column_value(tmpdir):
 def test_import_no_headers(tmpdir, args, tsv):
     db_path = str(tmpdir / "test.db")
     csv_path = str(tmpdir / "test.csv")
-    csv_file = open(csv_path, "w")
-    sep = "\t" if tsv else ","
-    csv_file.write("Cleo{sep}Dog{sep}5\n".format(sep=sep))
-    csv_file.write("Tracy{sep}Spider{sep}7\n".format(sep=sep))
-    csv_file.close()
+    with open(csv_path, "w") as csv_file:
+        sep = "\t" if tsv else ","
+        csv_file.write("Cleo{sep}Dog{sep}5\n".format(sep=sep))
+        csv_file.write("Tracy{sep}Spider{sep}7\n".format(sep=sep))
     result = CliRunner().invoke(
         cli.cli,
         ["insert", db_path, "creatures", csv_path] + args,
@@ -2410,11 +2416,13 @@ def test_create_table_strict(strict):
         db = Database("test.db")
         result = runner.invoke(
             cli.cli,
-            ["create-table", "test.db", "items", "id", "integer"]
+            ["create-table", "test.db", "items", "id", "integer", "w", "float"]
             + (["--strict"] if strict else []),
         )
         assert result.exit_code == 0
         assert db["items"].strict == strict or not db.supports_strict
+        # Should have a floating point column
+        assert db["items"].columns_dict == {"id": int, "w": float}
 
 
 @pytest.mark.parametrize("method", ("insert", "upsert"))

{sqlite-utils-3.36 → sqlite_utils-3.38}/tests/test_cli_insert.py

@@ -77,19 +77,24 @@ def test_insert_json_flatten_nl(tmpdir):
     ]
 
 
-def test_insert_with_primary_key(db_path, tmpdir):
+@pytest.mark.parametrize(
+    "args,expected_pks",
+    (
+        (["--pk", "id"], ["id"]),
+        (["--pk", "id", "--pk", "name"], ["id", "name"]),
+    ),
+)
+def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks):
     json_path = str(tmpdir / "dog.json")
     with open(json_path, "w") as fp:
         fp.write(json.dumps({"id": 1, "name": "Cleo", "age": 4}))
-    result = CliRunner().invoke(
-        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
-    )
+    result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path] + args)
     assert result.exit_code == 0
     assert [{"id": 1, "age": 4, "name": "Cleo"}] == list(
         Database(db_path).query("select * from dogs")
     )
     db = Database(db_path)
-    assert ["id"] == db["dogs"].pks
+    assert db["dogs"].pks == expected_pks
 
 
 def test_insert_multiple_with_primary_key(db_path, tmpdir):

{sqlite-utils-3.36 → sqlite_utils-3.38}/tests/test_cli_memory.py

@@ -1,5 +1,5 @@
+import click
 import json
-
 import pytest
 from click.testing import CliRunner
 
@@ -305,3 +305,16 @@ def test_memory_functions():
     )
     assert result.exit_code == 0
     assert result.output.strip() == '[{"hello()": "Hello"}]'
+
+
+def test_memory_return_db(tmpdir):
+    # https://github.com/simonw/sqlite-utils/issues/643
+    from sqlite_utils.cli import cli
+
+    path = str(tmpdir / "dogs.csv")
+    open(path, "w").write("id,name\n1,Cleo")
+
+    with click.Context(cli) as ctx:
+        db = ctx.invoke(cli.commands["memory"], paths=(path,), return_db=True)
+
+    assert db.table_names() == ["dogs"]

{sqlite-utils-3.36 → sqlite_utils-3.38}/tests/test_create.py

@@ -1350,8 +1350,14 @@ def test_insert_upsert_strict(fresh_db, method_name, strict):
 
 @pytest.mark.parametrize("strict", (False, True))
 def test_create_table_strict(fresh_db, strict):
-    table = fresh_db.create_table("t", {"id": int}, strict=strict)
+    table = fresh_db.create_table("t", {"id": int, "f": float}, strict=strict)
     assert table.strict == strict or not fresh_db.supports_strict
+    expected_schema = "CREATE TABLE [t] (\n" "   [id] INTEGER,\n" "   [f] FLOAT\n" ")"
+    if strict and not fresh_db.supports_strict:
+        return
+    if strict:
+        expected_schema = "CREATE TABLE [t] (\n   [id] INTEGER,\n   [f] REAL\n) STRICT"
+    assert table.schema == expected_schema
 
 
 @pytest.mark.parametrize("strict", (False, True))

{sqlite-utils-3.36 → sqlite_utils-3.38}/tests/test_fts.py

@@ -1,6 +1,7 @@
 import pytest
 from sqlite_utils import Database
 from sqlite_utils.utils import sqlite3
+from unittest.mock import ANY
 
 search_records = [
     {
@@ -126,6 +127,32 @@ def test_search_where_args_disallows_query(fresh_db):
     )
 
 
+def test_search_include_rank(fresh_db):
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version="FTS5")
+    results = list(table.search("are", include_rank=True))
+    assert results == [
+        {
+            "rowid": 1,
+            "text": "tanuki are running tricksters",
+            "country": "Japan",
+            "not_searchable": "foo",
+            "rank": ANY,
+        },
+        {
+            "rowid": 2,
+            "text": "racoons are biting trash pandas",
+            "country": "USA",
+            "not_searchable": "bar",
+            "rank": ANY,
+        },
+    ]
+    assert isinstance(results[0]["rank"], float)
+    assert isinstance(results[1]["rank"], float)
+    assert results[0]["rank"] < results[1]["rank"]
+
+
 def test_enable_fts_table_names_containing_spaces(fresh_db):
     table = fresh_db["test"]
     table.insert({"column with spaces": "in its name"})

{sqlite-utils-3.36 → sqlite_utils-3.38}/tests/test_insert_files.py

@@ -7,7 +7,14 @@ import sys
 
 
 @pytest.mark.parametrize("silent", (False, True))
-def test_insert_files(silent):
+@pytest.mark.parametrize(
+    "pk_args,expected_pks",
+    (
+        (["--pk", "path"], ["path"]),
+        (["--pk", "path", "--pk", "name"], ["path", "name"]),
+    ),
+)
+def test_insert_files(silent, pk_args, expected_pks):
     runner = CliRunner()
     with runner.isolated_filesystem():
         tmpdir = pathlib.Path(".")
@@ -42,7 +49,7 @@ def test_insert_files(silent):
             cli.cli,
             ["insert-files", db_path, "files", str(tmpdir)]
             + cols
-            + ["--pk", "path"]
+            + pk_args
             + (["--silent"] if silent else []),
             catch_exceptions=False,
         )
@@ -105,6 +112,7 @@ def test_insert_files(silent):
         for colname, expected_type in expected_types.items():
             for row in (one, two, three):
                 assert isinstance(row[colname], expected_type)
+        assert set(db["files"].pks) == set(expected_pks)
 
 
 @pytest.mark.parametrize(

{sqlite-utils-3.36 → sqlite_utils-3.38}/tests/test_transform.py

@@ -1,4 +1,4 @@
-from sqlite_utils.db import ForeignKey
+from sqlite_utils.db import ForeignKey, TransformError
 from sqlite_utils.utils import OperationalError
 import pytest
 
@@ -539,3 +539,125 @@ def test_transform_strict(fresh_db, strict):
     assert dogs.strict == strict or not fresh_db.supports_strict
     dogs.transform(not_null={"name"})
     assert dogs.strict == strict or not fresh_db.supports_strict
+
+
+@pytest.mark.parametrize(
+    "indexes, transform_params",
+    [
+        ([["name"]], {"types": {"age": str}}),
+        ([["name"], ["age", "breed"]], {"types": {"age": str}}),
+        ([], {"types": {"age": str}}),
+        ([["name"]], {"types": {"age": str}, "keep_table": "old_dogs"}),
+    ],
+)
+def test_transform_indexes(fresh_db, indexes, transform_params):
+    # https://github.com/simonw/sqlite-utils/issues/633
+    # New table should have same indexes as old table after transformation
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5, "breed": "Labrador"}, pk="id")
+
+    for index in indexes:
+        dogs.create_index(index)
+
+    indexes_before_transform = dogs.indexes
+
+    dogs.transform(**transform_params)
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+    if "keep_table" in transform_params:
+        assert all(
+            index.origin == "pk"
+            for index in fresh_db[transform_params["keep_table"]].indexes
+        )
+
+
+def test_transform_retains_indexes_with_foreign_keys(fresh_db):
+    dogs = fresh_db["dogs"]
+    owners = fresh_db["owners"]
+
+    dogs.insert({"id": 1, "name": "Cleo", "owner_id": 1}, pk="id")
+    owners.insert({"id": 1, "name": "Alice"}, pk="id")
+
+    dogs.create_index(["name"])
+
+    indexes_before_transform = dogs.indexes
+
+    fresh_db.add_foreign_keys([("dogs", "owner_id", "owners", "id")])  # calls transform
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+
+
+@pytest.mark.parametrize(
+    "transform_params",
+    [
+        {"rename": {"age": "dog_age"}},
+        {"drop": ["age"]},
+    ],
+)
+def test_transform_with_indexes_errors(fresh_db, transform_params):
+    # Should error with a compound (name, age) index if age is renamed or dropped
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id")
+
+    dogs.create_index(["name", "age"])
+
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(**transform_params)
+
+    assert (
+        "Index 'idx_dogs_name_age' column 'age' is not in updated table 'dogs'. "
+        "You must manually drop this index prior to running this transformation"
+        in str(excinfo.value)
+    )
+
+
+def test_transform_with_unique_constraint_implicit_index(fresh_db):
+    dogs = fresh_db["dogs"]
+    # Create a table with a UNIQUE constraint on 'name', which creates an implicit index
+    fresh_db.execute(
+        """
+        CREATE TABLE dogs (
+            id INTEGER PRIMARY KEY,
+            name TEXT UNIQUE,
+            age INTEGER
+        );
+    """
+    )
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5})
+
+    # Attempt to transform the table without modifying 'name'
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(types={"age": str})
+
+    assert (
+        "Index 'sqlite_autoindex_dogs_1' on table 'dogs' does not have a CREATE INDEX statement."
+        in str(excinfo.value)
+    )
+    assert (
+        "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation."
+        in str(excinfo.value)
+    )