sqlite-utils 3.37__tar.gz → 3.38__tar.gz

{sqlite_utils-3.37/sqlite_utils.egg-info → sqlite_utils-3.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlite-utils
-Version: 3.37
+Version: 3.38
 Summary: CLI tool and Python library for manipulating SQLite databases
 Home-page: https://github.com/simonw/sqlite-utils
 Author: Simon Willison

{sqlite_utils-3.37 → sqlite_utils-3.38}/docs/changelog.rst

@@ -4,6 +4,16 @@
  Changelog
 ===========
 
+.. _v3_38:
+
+3.38 (2024-11-23)
+-----------------
+
+- Plugins can now reuse the implementation of the ``sqlite-utils memory`` CLI command with the new ``return_db=True`` parameter. (:issue:`643`)
+- ``table.transform()`` now recreates indexes after transforming a table. A new ``sqlite_utils.db.TransformError`` exception is raised if these indexes cannot be recreated due to conflicting changes to the table such as a column rename. Thanks, `Mat Miller <https://github.com/matdmiller>`__. (:issue:`633`)
+- ``table.search()`` now accepts a ``include_rank=True`` parameter, causing the resulting rows to have a ``rank`` column showing the calculated relevance score. Thanks, `liunux4odoo <https://github.com/liunux4odoo>`__. (`#628 <https://github.com/simonw/sqlite-utils/pull/628>`__)
+- Fixed an error that occurred when creating a strict table with at least one floating point column. These ``FLOAT`` columns are now correctly created as ``REAL`` as well, but only for strict tables. (:issue:`644`)
+
 .. _v3_37:
 
 3.37 (2024-07-18)

{sqlite_utils-3.37 → sqlite_utils-3.38}/docs/cli.rst

@@ -2084,7 +2084,7 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi
 ``--drop-foreign-key column``
     Drop the specified foreign key.
 
-``--add-foregn-key column other_table other_column``
+``--add-foreign-key column other_table other_column``
     Add a foreign key constraint to ``column`` pointing to ``other_table.other_column``.
 
 If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example:
@@ -2796,7 +2796,7 @@ To enable this feature you will need to install the ``trogon`` dependency. You c
 
 .. code-block:: bash
 
-    sqite-utils install trogon
+    sqlite-utils install trogon
 
 Once installed, running the ``sqlite-utils tui`` command will launch the TUI interface:
 

{sqlite_utils-3.37 → sqlite_utils-3.38}/docs/plugins.rst

@@ -115,6 +115,26 @@ Example implementation:
             "Say hello world"
             click.echo("Hello world!")
 
+New commands implemented by plugins can invoke existing commands using the `context.invoke <https://click.palletsprojects.com/en/stable/api/#click.Context.invoke>`__ mechanism.
+
+As a special niche feature, if your plugin needs to import some files and then act against an in-memory database containing those files you can forward to the :ref:`sqlite-utils memory command <cli_memory>` and pass it ``return_db=True``:
+
+.. code-block:: python
+
+    @cli.command()
+    @click.pass_context
+    @click.argument(
+        "paths",
+        type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+        required=False,
+        nargs=-1,
+    )
+    def show_schema_for_files(ctx, paths):
+        from sqlite_utils.cli import memory
+        db = ctx.invoke(memory, paths=paths, return_db=True)
+        # Now do something with that database
+        click.echo(db.schema)
+
 .. _plugins_hooks_prepare_connection:
 
 prepare_connection(conn)

{sqlite_utils-3.37 → sqlite_utils-3.38}/docs/python-api.rst

@@ -1249,6 +1249,9 @@ If you pass a Python type, it will be mapped to SQLite types as shown here::
     np.float32: "FLOAT"
     np.float64: "FLOAT"
 
+.. note::
+    In sqlite-utils 3.x ``FLOAT`` is used for floating point columns when the correct column type is actually ``REAL``. If you specify ``strict=True`` tables created in strict mode will use the correct column type of ``REAL`` instead. We plan to change this behavior in ``sqlite-utils`` 4.x to always use ``REAL``, but this will represent a minor breaking change and so is being held for the next major release, see issue :issue:`645`.
+
 You can also add a column that is a foreign key reference to another table using the ``fk`` parameter:
 
 .. code-block:: python
@@ -1402,6 +1405,8 @@ To keep the original table around instead of dropping it, pass the ``keep_table=
 
     table.transform(types={"age": int}, keep_table="original_table")
 
+This method raises a ``sqlite_utils.db.TransformError`` exception if the table cannot be transformed, usually because there are existing constraints or indexes that are incompatible with modifications to the columns.
+
 .. _python_api_transform_alter_column_types:
 
 Altering column types
@@ -2309,6 +2314,9 @@ The ``.search()`` method also accepts the following optional parameters:
 ``where_args`` dictionary
     Arguments to use for ``:param`` placeholders in the extra WHERE clause
 
+``include_rank`` bool
+    If set a ``rank`` column will be included with the BM25 ranking score - for FTS5 tables only.
+
 ``quote`` bool
     Apply :ref:`FTS quoting rules <python_api_quote_fts>` to the search query, disabling advanced query syntax in a way that avoids surprising errors.
 

{sqlite_utils-3.37 → sqlite_utils-3.38}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import io
 import os
 
-VERSION = "3.37"
+VERSION = "3.38"
 
 
 def get_long_description():

{sqlite_utils-3.37 → sqlite_utils-3.38}/sqlite_utils/cli.py

@@ -1894,6 +1894,7 @@ def memory(
     save,
     analyze,
     load_extension,
+    return_db=False,
 ):
     """Execute SQL query against an in-memory database, optionally populated by imported data
 
@@ -1922,6 +1923,7 @@ def memory(
         sqlite-utils memory animals.csv --schema
     """
     db = sqlite_utils.Database(memory=True)
+
     # If --dump or --save or --analyze used but no paths detected, assume SQL query is a path:
     if (dump or save or schema or analyze) and not paths:
         paths = [sql]
@@ -1954,6 +1956,7 @@ def memory(
             rows = tracker.wrap(rows)
         if flatten:
             rows = (_flatten(row) for row in rows)
+
         db[file_table].insert_all(rows, alter=True)
         if tracker is not None:
             db[file_table].transform(types=tracker.types)
@@ -1964,6 +1967,7 @@ def memory(
         for view_name in view_names:
             if not db[view_name].exists():
                 db.create_view(view_name, "select * from [{}]".format(file_table))
+
         if fp:
             fp.close()
 
@@ -1994,6 +1998,9 @@ def memory(
     if functions:
         _register_functions(db, functions)
 
+    if return_db:
+        return db
+
     _execute_query(
         db,
         sql,

{sqlite_utils-3.37 → sqlite_utils-3.38}/sqlite_utils/db.py

@@ -156,6 +156,10 @@ XIndexColumn = namedtuple(
 Trigger = namedtuple("Trigger", ("name", "table", "sql"))
 
 
+class TransformError(Exception):
+    pass
+
+
 ForeignKeyIndicator = Union[
     str,
     ForeignKey,
@@ -325,6 +329,8 @@ class Database:
         execute_plugins: bool = True,
         strict: bool = False,
     ):
+        self.memory_name = None
+        self.memory = False
         assert (filename_or_conn is not None and (not memory and not memory_name)) or (
             filename_or_conn is None and (memory or memory_name)
         ), "Either specify a filename_or_conn or pass memory=True"
@@ -335,8 +341,11 @@ class Database:
                 uri=True,
                 check_same_thread=False,
             )
+            self.memory = True
+            self.memory_name = memory_name
         elif memory or filename_or_conn == ":memory:":
             self.conn = sqlite3.connect(":memory:")
+            self.memory = True
         elif isinstance(filename_or_conn, (str, pathlib.Path)):
             if recreate and os.path.exists(filename_or_conn):
                 try:
@@ -929,10 +938,15 @@ class Database:
                         other_column=foreign_keys_by_column[column_name].other_column,
                     )
                 )
+            column_type_str = COLUMN_TYPE_MAPPING[column_type]
+            # Special case for strict tables to map FLOAT to REAL
+            # Refs https://github.com/simonw/sqlite-utils/issues/644
+            if strict and column_type_str == "FLOAT":
+                column_type_str = "REAL"
             column_defs.append(
                 "   [{column_name}] {column_type}{column_extras}".format(
                     column_name=column_name,
-                    column_type=COLUMN_TYPE_MAPPING[column_type],
+                    column_type=column_type_str,
                     column_extras=(
                         (" " + " ".join(column_extras)) if column_extras else ""
                     ),
@@ -1967,6 +1981,30 @@ class Table(Queryable):
         sqls.append(
             "ALTER TABLE [{}] RENAME TO [{}];".format(new_table_name, self.name)
         )
+        # Re-add existing indexes
+        for index in self.indexes:
+            if index.origin != "pk":
+                index_sql = self.db.execute(
+                    """SELECT sql FROM sqlite_master WHERE type = 'index' AND name = :index_name;""",
+                    {"index_name": index.name},
+                ).fetchall()[0][0]
+                if index_sql is None:
+                    raise TransformError(
+                        f"Index '{index.name}' on table '{self.name}' does not have a "
+                        "CREATE INDEX statement. You must manually drop this index prior to running this "
+                        "transformation and manually recreate the new index after running this transformation."
+                    )
+                if keep_table:
+                    sqls.append(f"DROP INDEX IF EXISTS [{index.name}];")
+                for col in index.columns:
+                    if col in rename.keys() or col in drop:
+                        raise TransformError(
+                            f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. "
+                            f"You must manually drop this index prior to running this transformation "
+                            f"and manually recreate the new index after running this transformation. "
+                            f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table."
+                        )
+                sqls.append(index_sql)
         return sqls
 
     def extract(
@@ -2645,6 +2683,7 @@ class Table(Queryable):
         offset: Optional[int] = None,
         where: Optional[str] = None,
         where_args: Optional[Union[Iterable, dict]] = None,
+        include_rank: bool = False,
         quote: bool = False,
     ) -> Generator[dict, None, None]:
         """
@@ -2658,6 +2697,7 @@ class Table(Queryable):
         :param offset: Optional integer SQL offset.
         :param where: Extra SQL fragment for the WHERE clause
         :param where_args: Arguments to use for :param placeholders in the extra WHERE clause
+        :param include_rank: Select the search rank column in the final query
         :param quote: Apply quoting to disable any special characters in the search query
 
         See :ref:`python_api_fts_search`.
@@ -2677,6 +2717,7 @@ class Table(Queryable):
                 limit=limit,
                 offset=offset,
                 where=where,
+                include_rank=include_rank,
             ),
             args,
         )

{sqlite_utils-3.37 → sqlite_utils-3.38/sqlite_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlite-utils
-Version: 3.37
+Version: 3.38
 Summary: CLI tool and Python library for manipulating SQLite databases
 Home-page: https://github.com/simonw/sqlite-utils
 Author: Simon Willison

{sqlite_utils-3.37 → sqlite_utils-3.38}/tests/test_cli.py

@@ -2416,11 +2416,13 @@ def test_create_table_strict(strict):
         db = Database("test.db")
         result = runner.invoke(
             cli.cli,
-            ["create-table", "test.db", "items", "id", "integer"]
+            ["create-table", "test.db", "items", "id", "integer", "w", "float"]
             + (["--strict"] if strict else []),
         )
         assert result.exit_code == 0
         assert db["items"].strict == strict or not db.supports_strict
+        # Should have a floating point column
+        assert db["items"].columns_dict == {"id": int, "w": float}
 
 
 @pytest.mark.parametrize("method", ("insert", "upsert"))

{sqlite_utils-3.37 → sqlite_utils-3.38}/tests/test_cli_memory.py

@@ -1,5 +1,5 @@
+import click
 import json
-
 import pytest
 from click.testing import CliRunner
 
@@ -305,3 +305,16 @@ def test_memory_functions():
     )
     assert result.exit_code == 0
     assert result.output.strip() == '[{"hello()": "Hello"}]'
+
+
+def test_memory_return_db(tmpdir):
+    # https://github.com/simonw/sqlite-utils/issues/643
+    from sqlite_utils.cli import cli
+
+    path = str(tmpdir / "dogs.csv")
+    open(path, "w").write("id,name\n1,Cleo")
+
+    with click.Context(cli) as ctx:
+        db = ctx.invoke(cli.commands["memory"], paths=(path,), return_db=True)
+
+    assert db.table_names() == ["dogs"]

{sqlite_utils-3.37 → sqlite_utils-3.38}/tests/test_create.py

@@ -1350,8 +1350,14 @@ def test_insert_upsert_strict(fresh_db, method_name, strict):
 
 @pytest.mark.parametrize("strict", (False, True))
 def test_create_table_strict(fresh_db, strict):
-    table = fresh_db.create_table("t", {"id": int}, strict=strict)
+    table = fresh_db.create_table("t", {"id": int, "f": float}, strict=strict)
     assert table.strict == strict or not fresh_db.supports_strict
+    expected_schema = "CREATE TABLE [t] (\n" "   [id] INTEGER,\n" "   [f] FLOAT\n" ")"
+    if strict and not fresh_db.supports_strict:
+        return
+    if strict:
+        expected_schema = "CREATE TABLE [t] (\n   [id] INTEGER,\n   [f] REAL\n) STRICT"
+    assert table.schema == expected_schema
 
 
 @pytest.mark.parametrize("strict", (False, True))

{sqlite_utils-3.37 → sqlite_utils-3.38}/tests/test_fts.py

@@ -1,6 +1,7 @@
 import pytest
 from sqlite_utils import Database
 from sqlite_utils.utils import sqlite3
+from unittest.mock import ANY
 
 search_records = [
     {
@@ -126,6 +127,32 @@ def test_search_where_args_disallows_query(fresh_db):
     )
 
 
+def test_search_include_rank(fresh_db):
+    table = fresh_db["t"]
+    table.insert_all(search_records)
+    table.enable_fts(["text", "country"], fts_version="FTS5")
+    results = list(table.search("are", include_rank=True))
+    assert results == [
+        {
+            "rowid": 1,
+            "text": "tanuki are running tricksters",
+            "country": "Japan",
+            "not_searchable": "foo",
+            "rank": ANY,
+        },
+        {
+            "rowid": 2,
+            "text": "racoons are biting trash pandas",
+            "country": "USA",
+            "not_searchable": "bar",
+            "rank": ANY,
+        },
+    ]
+    assert isinstance(results[0]["rank"], float)
+    assert isinstance(results[1]["rank"], float)
+    assert results[0]["rank"] < results[1]["rank"]
+
+
 def test_enable_fts_table_names_containing_spaces(fresh_db):
     table = fresh_db["test"]
     table.insert({"column with spaces": "in its name"})

{sqlite_utils-3.37 → sqlite_utils-3.38}/tests/test_transform.py

@@ -1,4 +1,4 @@
-from sqlite_utils.db import ForeignKey
+from sqlite_utils.db import ForeignKey, TransformError
 from sqlite_utils.utils import OperationalError
 import pytest
 
@@ -539,3 +539,125 @@ def test_transform_strict(fresh_db, strict):
     assert dogs.strict == strict or not fresh_db.supports_strict
     dogs.transform(not_null={"name"})
     assert dogs.strict == strict or not fresh_db.supports_strict
+
+
+@pytest.mark.parametrize(
+    "indexes, transform_params",
+    [
+        ([["name"]], {"types": {"age": str}}),
+        ([["name"], ["age", "breed"]], {"types": {"age": str}}),
+        ([], {"types": {"age": str}}),
+        ([["name"]], {"types": {"age": str}, "keep_table": "old_dogs"}),
+    ],
+)
+def test_transform_indexes(fresh_db, indexes, transform_params):
+    # https://github.com/simonw/sqlite-utils/issues/633
+    # New table should have same indexes as old table after transformation
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5, "breed": "Labrador"}, pk="id")
+
+    for index in indexes:
+        dogs.create_index(index)
+
+    indexes_before_transform = dogs.indexes
+
+    dogs.transform(**transform_params)
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+    if "keep_table" in transform_params:
+        assert all(
+            index.origin == "pk"
+            for index in fresh_db[transform_params["keep_table"]].indexes
+        )
+
+
+def test_transform_retains_indexes_with_foreign_keys(fresh_db):
+    dogs = fresh_db["dogs"]
+    owners = fresh_db["owners"]
+
+    dogs.insert({"id": 1, "name": "Cleo", "owner_id": 1}, pk="id")
+    owners.insert({"id": 1, "name": "Alice"}, pk="id")
+
+    dogs.create_index(["name"])
+
+    indexes_before_transform = dogs.indexes
+
+    fresh_db.add_foreign_keys([("dogs", "owner_id", "owners", "id")])  # calls transform
+
+    assert sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in dogs.indexes
+        ],
+        key=lambda x: x["name"],
+    ) == sorted(
+        [
+            {k: v for k, v in idx._asdict().items() if k != "seq"}
+            for idx in indexes_before_transform
+        ],
+        key=lambda x: x["name"],
+    ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}"
+
+
+@pytest.mark.parametrize(
+    "transform_params",
+    [
+        {"rename": {"age": "dog_age"}},
+        {"drop": ["age"]},
+    ],
+)
+def test_transform_with_indexes_errors(fresh_db, transform_params):
+    # Should error with a compound (name, age) index if age is renamed or dropped
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id")
+
+    dogs.create_index(["name", "age"])
+
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(**transform_params)
+
+    assert (
+        "Index 'idx_dogs_name_age' column 'age' is not in updated table 'dogs'. "
+        "You must manually drop this index prior to running this transformation"
+        in str(excinfo.value)
+    )
+
+
+def test_transform_with_unique_constraint_implicit_index(fresh_db):
+    dogs = fresh_db["dogs"]
+    # Create a table with a UNIQUE constraint on 'name', which creates an implicit index
+    fresh_db.execute(
+        """
+        CREATE TABLE dogs (
+            id INTEGER PRIMARY KEY,
+            name TEXT UNIQUE,
+            age INTEGER
+        );
+    """
+    )
+    dogs.insert({"id": 1, "name": "Cleo", "age": 5})
+
+    # Attempt to transform the table without modifying 'name'
+    with pytest.raises(TransformError) as excinfo:
+        dogs.transform(types={"age": str})
+
+    assert (
+        "Index 'sqlite_autoindex_dogs_1' on table 'dogs' does not have a CREATE INDEX statement."
+        in str(excinfo.value)
+    )
+    assert (
+        "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation."
+        in str(excinfo.value)
+    )