clear-skies 2.0.4__py3-none-any.whl → 2.0.5__py3-none-any.whl

clearskies/backends/memory_backend.py

@@ -1,797 +0,0 @@
-import inspect
-from functools import cmp_to_key
-from typing import Any, Callable
-
-import clearskies.model
-import clearskies.query
-from clearskies import functional
-from clearskies.autodoc.schema import Integer as AutoDocInteger
-from clearskies.autodoc.schema import Schema as AutoDocSchema
-from clearskies.backends.backend import Backend
-from clearskies.di import InjectableProperties, inject
-
-
-class Null:
-    def __lt__(self, other):
-        return True
-
-    def __gt__(self, other):
-        return False
-
-    def __eq__(self, other):
-        return isinstance(other, Null) or other is None
-
-
-# for some comparisons we prefer comparing floats, but we need to be able to
-# fall back on string comparison
-def gentle_float_conversion(value):
-    try:
-        return float(value)
-    except:
-        return value
-
-
-def _sort(row_a: Any, row_b: Any, sorts: list[clearskies.query.Sort], default_table_name: str) -> int:
-    for sort in sorts:
-        # so, if we've done a join then the rows will have data from all joined tables via a dict of dicts.
-        # if there wasn't a join then we'll just have the data
-        if sort.table_name in row_a and isinstance(row_a[sort.table_name], dict):
-            sort_data_a = row_a[sort.table_name]
-        elif not sort.table_name and default_table_name in row_a and isinstance(row_a[default_table_name], dict):
-            sort_data_a = row_a[default_table_name]
-        else:
-            sort_data_a = row_a
-
-        if sort.table_name in row_b and isinstance(row_b[sort.table_name], dict):
-            sort_data_b = row_b[sort.table_name]
-        elif not sort.table_name and default_table_name in row_b and isinstance(row_b[default_table_name], dict):
-            sort_data_b = row_b[default_table_name]
-        else:
-            sort_data_b = row_b
-
-        reverse = 1 if sort.direction.lower() == "asc" else -1
-        value_a = sort_data_a[sort.column_name] if sort.column_name in sort_data_a else None
-        value_b = sort_data_b[sort.column_name] if sort.column_name in sort_data_b else None
-        if value_a == value_b:
-            continue
-        if value_a is None:
-            return -1 * reverse
-        if value_b is None:
-            return 1 * reverse
-        return reverse * (1 if value_a > value_b else -1)
-    return 0
-
-
-def cheating_equals(column, values, null):
-    """
-    Cheating because this solves a very specific problem that likely is a generic issue.
-
-    The memory backend has some matching failures because boolean columns stay boolean in the
-    memory store, but the incoming search values are not converted to boolean and tend to be
-    str(1) or str(0).  The issue is that save data goes through the `to_backend` flow, but search
-    data doesn't.  This doesn't matter most of the time because, in practice, the backend itself
-    often does its own type conversion, but it causes problems for the memory backend.  I can't
-    decide if fixing this will cause more problems than it solves, so for now I'm just cheating
-    and putting in a hack for this specific use case :shame:.
-    """
-
-    def inner(row):
-        backend_value = row[column] if column in row else null
-        if isinstance(backend_value, bool):
-            return backend_value == bool(values[0])
-        return str(backend_value) == str(values[0])
-
-    return inner
-
-
-class MemoryTable:
-    _table_name: str = ""
-    _column_names: list[str] = []
-    _rows: list[dict[str, Any] | None] = []
-    null: Null = Null()
-    _id_index: dict[int | str, int] = {}
-    id_column_name: str = ""
-    _next_id: int = 1
-    _model_class: type[clearskies.model.Model] = None  # type:  ignore
-
-    # here be dragons.  This is not a 100% drop-in replacement for the equivalent SQL operators
-    # https://codereview.stackexchange.com/questions/259198/in-memory-table-filtering-in-python
-    _operator_lambda_builders = {
-        "<=>": lambda column, values, null: lambda row: row.get(column, null) == values[0],
-        "!=": lambda column, values, null: lambda row: row.get(column, null) != values[0],
-        "<=": (
-            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-            <= gentle_float_conversion(values[0])
-        ),
-        ">=": (
-            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-            >= gentle_float_conversion(values[0])
-        ),
-        ">": (
-            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-            > gentle_float_conversion(values[0])
-        ),
-        "<": (
-            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-            < gentle_float_conversion(values[0])
-        ),
-        "=": cheating_equals,
-        "is not null": lambda column, values, null: lambda row: (column in row and row[column] is not None),
-        "is null": lambda column, values, null: lambda row: (column not in row or row[column] is None),
-        "is not": lambda column, values, null: lambda row: row.get(column, null) != values[0],
-        "is": lambda column, values, null: lambda row: row.get(column, null) == str(values[0]),
-        "like": lambda column, values, null: lambda row: row.get(column, null) == str(values[0]),
-        "in": lambda column, values, null: lambda row: row.get(column, null) in values,
-    }
-
-    def __init__(self, model_class: type[clearskies.model.Model]) -> None:
-        self._rows = []
-        self._id_index = {}
-        self.id_column_name = model_class.id_column_name
-        self._next_id = 1
-        self._model_class = model_class
-
-        self._table_name = model_class.destination_name()
-        self._column_names = list(model_class.get_columns().keys())
-        if self.id_column_name not in self._column_names:
-            self._column_names.append(self.id_column_name)
-
-    def update(self, id: int | str, data: dict[str, Any]) -> dict[str, Any]:
-        if id not in self._id_index:
-            raise ValueError(f"Attempt to update non-existent record with '{self.id_column_name}' of '{id}'")
-        index = self._id_index[id]
-        row = self._rows[index]
-        if row is None:
-            raise ValueError(
-                f"Cannot update record with '{self.id_column_name}' of '{id}' because it was already deleted"
-            )
-        for column_name in data.keys():
-            if column_name not in self._column_names:
-                raise ValueError(
-                    f"Cannot update record: column '{column_name}' does not exist in table '{self._table_name}'"
-                )
-        self._rows[index] = {
-            **self._rows[index],  # type: ignore
-            **data,
-        }
-        return self._rows[index]  # type: ignore
-
-    def create(self, data: dict[str, Any]) -> dict[str, Any]:
-        for column_name in data.keys():
-            if column_name not in self._column_names:
-                raise ValueError(
-                    f"Cannot create record: column '{column_name}' does not exist for model '{self._model_class.__name__}'"
-                )
-        incoming_id = data.get(self.id_column_name)
-        if not incoming_id:
-            incoming_id = self._next_id
-            data[self.id_column_name] = incoming_id
-            self._next_id += 1
-        try:
-            incoming_as_int = int(incoming_id)
-            if incoming_as_int >= self._next_id:
-                self._next_id = incoming_as_int + 1
-        except:
-            pass
-        if incoming_id in self._id_index and self._rows[self._id_index[data[self.id_column_name]]] is not None:
-            return self.update(data[self.id_column_name], data)
-        for column_name in self._column_names:
-            if column_name not in data:
-                data[column_name] = None
-        self._rows.append({**data})
-        self._id_index[data[self.id_column_name]] = len(self._rows) - 1
-        return data
-
-    def delete(self, id):
-        if id not in self._id_index:
-            return True
-        index = self._id_index[id]
-        if self._rows[index] is None:
-            return True
-        # we set the row to None because if we remove it we'll change the indexes of the rest
-        # of the rows, and I like being able to calculate the index from the id
-        self._rows[index] = None
-        return True
-
-    def count(self, query: clearskies.query.Query):
-        return len(self.rows(query, query.conditions, filter_only=True))
-
-    def rows(
-        self,
-        query: clearskies.query.Query,
-        conditions: list[clearskies.query.Condition],
-        filter_only: bool = False,
-        next_page_data: dict[str, Any] | None = None,
-    ):
-        rows = [row for row in self._rows if row is not None]
-        for condition in conditions:
-            rows = list(filter(self._condition_as_filter(condition), rows))
-        rows = [*rows]
-        if filter_only:
-            return rows
-        if query.sorts:
-            rows = sorted(
-                rows,
-                key=cmp_to_key(
-                    lambda row_a, row_b: _sort(row_a, row_b, query.sorts, query.model_class.destination_name())
-                ),
-            )
-        if query.limit or query.pagination.get("start"):
-            number_rows = len(rows)
-            start = int(query.pagination.get("start", 0))
-            if not start:
-                start = 0
-            if int(start) >= number_rows:
-                start = number_rows - 1
-            end = len(rows)
-            if query.limit and start + int(query.limit) <= number_rows:
-                end = start + int(query.limit)
-            if end < number_rows and type(next_page_data) == dict:
-                next_page_data["start"] = start + int(query.limit)
-            rows = rows[start:end]
-        return rows
-
-    @classmethod
-    def _condition_as_filter(cls, condition: clearskies.query.Condition) -> Callable:
-        column = condition.column_name
-        values = condition.values
-        return cls._operator_lambda_builders[condition.operator.lower()](column, values, cls.null)
-
-
-class MemoryBackend(Backend, InjectableProperties):
-    """
-    Store data in an in-memory store built in to clearskies.
-
-    ## Usage
-
-    Since the memory backend is built into clearskies, there's no configuration necessary to make it work:
-    simply attach it to any of your models and it will manage data for you.  If you want though, you can declare
-    a binding named `memory_backend_default_data` which you fill with records for your models to pre-populate
-    the memory backend.  This can be helpful for tests as well as tables with fixed values.
-
-    ### Testing
-
-    A primary use case of the memory backend is for building unit tests of your code.  You can use the dependency
-    injection system to override other backends with the memory backend.  You can still operate with model classes
-    in the exact same way, so this can be an easy way to mock out databases/api endpoints/etc...  Of course,
-    there can be behavioral differences between the memory backend and other backends, so this isn't always perfect.
-    Hence why this works well for unit tests, but can't replace all testing, especially integration tests or
-    end-to-end tests.  Here's an example of that.  Note that the UserPreference model uses the cursor backend,
-    but when you run this code it will actually end up with the memory backend, so the code will run even without
-    attempting to connect to a database.
-
-    ```python
-    import clearskies
-
-
-    class UserPreference(clearskies.Model):
-        id_column_name = "id"
-        backend = clearskies.backends.CursorBackend()
-        id = clearskies.columns.Uuid()
-
-
-    cli = clearskies.contexts.Cli(
-        clearskies.endpoints.Callable(
-            lambda user_preferences: user_preferences.create(no_data=True).id,
-        ),
-        classes=[UserPreference],
-        class_overrides={
-            clearskies.backends.CursorBackend: clearskies.backends.MemoryBackend(),
-        },
-    )
-
-    if __name__ == "__main__":
-        cli()
-    ```
-
-    Note that the model requests the CursorBackend, but then we switch that for the memory backend via the
-    `class_overrides` kwarg to `clearskies.contexts.Cli`.  Therefore, the above code works regardless of
-    whether or not a database is running.  Since the models are used to interact with the backend
-    (rather than using the cursor directly), the above code works the same despite the change in backend.
-
-    Again, this is helpful as a quick way to manage tests - swap out a database backend (or any other backend)
-    for the memory backend, and then pre-populate records to test your application logic.  Obviously, this
-    won't cach backend-specific issues (e.g. forgetting to add a column to your database, mismatches between
-    column schema and backend schema, missing indexes, etc...), which is why this helps for unit tests
-    but not for integration tests.
-
-    ### Production Usage
-
-    You can use the memory backend in production if you want, although there are some important caveats to keep
-    in mind:
-
-     1. There is limited attempts at performance optimization, so you should test it before putting it under
-        substantial loads.
-     2. There's no concept of replication.  If you have multiple workers, then write operations won't be
-        persisted between them.
-
-    So, while there may be some cases where this is useful in production, it's by no means a replacement for
-    more typical in-memory data stores.
-
-    ### Predefined Records
-
-    You can declare a binding named `memory_backend_default_data` to seed the memory backend with records.  This
-    can be helpful in testing to setup your tests, and is occassionally helpful for keeping track of data in
-    fixed, read-only tables.  Here's an example:
-
-    ```python
-    import clearskies
-
-
-    class Owner(clearskies.Model):
-        id_column_name = "id"
-        backend = clearskies.backends.MemoryBackend()
-
-        id = clearskies.columns.Uuid()
-        name = clearskies.columns.String()
-        phone = clearskies.columns.Phone()
-
-
-    class Pet(clearskies.Model):
-        id_column_name = "id"
-        backend = clearskies.backends.MemoryBackend()
-
-        id = clearskies.columns.Uuid()
-        name = clearskies.columns.String()
-        species = clearskies.columns.String()
-        owner_id = clearskies.columns.BelongsToId(Owner, readable_parent_columns=["id", "name"])
-        owner = clearskies.columns.BelongsToModel("owner_id")
-
-
-    wsgi = clearskies.contexts.WsgiRef(
-        clearskies.endpoints.List(
-            model_class=Pet,
-            readable_column_names=["id", "name", "species", "owner"],
-            sortable_column_names=["id", "name", "species"],
-            default_sort_column_name="name",
-        ),
-        classes=[Owner, Pet],
-        bindings={
-            "memory_backend_default_data": [
-                {
-                    "model_class": Owner,
-                    "records": [
-                        {"id": "1-2-3-4", "name": "John Doe", "phone": "555-123-4567"},
-                        {"id": "5-6-7-8", "name": "Jane Doe", "phone": "555-321-0987"},
-                    ],
-                },
-                {
-                    "model_class": Pet,
-                    "records": [
-                        {"id": "a-b-c-d", "name": "Fido", "species": "Dog", "owner_id": "1-2-3-4"},
-                        {"id": "e-f-g-h", "name": "Spot", "species": "Dog", "owner_id": "1-2-3-4"},
-                        {
-                            "id": "i-j-k-l",
-                            "name": "Puss in Boots",
-                            "species": "Cat",
-                            "owner_id": "5-6-7-8",
-                        },
-                    ],
-                },
-            ],
-        },
-    )
-
-    if __name__ == "__main__":
-        wsgi()
-    ```
-
-    And if you invoke it:
-
-    ```bash
-    $ curl 'http://localhost:8080' | jq
-    {
-        "status": "success",
-        "error": "",
-        "data": [
-            {
-                "id": "a-b-c-d",
-                "name": "Fido",
-                "species": "Dog",
-                "owner": {
-                    "id": "1-2-3-4",
-                    "name": "John Doe"
-                }
-            },
-            {
-                "id": "i-j-k-l",
-                "name": "Puss in Boots",
-                "species": "Cat",
-                "owner": {
-                    "id": "5-6-7-8",
-                    "name": "Jane Doe"
-                }
-            },
-            {
-                "id": "e-f-g-h",
-                "name": "Spot",
-                "species": "Dog",
-                "owner": {
-                    "id": "1-2-3-4",
-                    "name": "John Doe"
-                }
-            }
-        ],
-        "pagination": {
-            "number_results": 3,
-            "limit": 50,
-            "next_page": {}
-        },
-        "input_errors": {}
-    }
-    ```
-    """
-
-    default_data = inject.ByName("memory_backend_default_data")
-    default_data_loaded = False
-    _tables: dict[str, MemoryTable] = {}
-    _silent_on_missing_tables: bool = True
-
-    def __init__(self, silent_on_missing_tables=True):
-        self.__class__._tables = {}
-        self._silent_on_missing_tables = silent_on_missing_tables
-
-    @classmethod
-    def clear_table_cache(cls):
-        cls._tables = {}
-
-    def load_default_data(self):
-        if self.default_data_loaded:
-            return
-        self.default_data_loaded = True
-        if not isinstance(self.default_data, list):
-            raise TypeError(
-                f"'memory_backend_default_data' should be populated with a list, but I received a value of type '{self.default_data.__class__.__name__}'"
-            )
-        for index, table_data in enumerate(self.default_data):
-            if "model_class" not in table_data:
-                raise TypeError(
-                    f"Each entry in the 'memory_backend_default_data' list should have a key named 'model_class', but entry #{index + 1} is missing this key."
-                )
-            model_class = table_data["model_class"]
-            if not functional.validations.is_model_class(table_data["model_class"]):
-                raise TypeError(
-                    f"The 'model_class' key in 'memory_backend_default_data' for entry #{index + 1} is not a model class."
-                )
-            if "records" not in table_data:
-                raise TypeError(
-                    f"Each entry in the 'memory_backend_default_data' list should have a key named 'records', but entry #{index + 1} is missing this key."
-                )
-            records = table_data["records"]
-            if not isinstance(records, list):
-                raise TypeError(
-                    f"The 'records' key in 'memory_backend_default_data' for entry #{index + 1} was not a list."
-                )
-            self.create_table(model_class)
-            for record in records:
-                self.create_with_model_class(record, model_class)
-
-    def silent_on_missing_tables(self, silent=True):
-        self._silent_on_missing_tables = silent
-
-    def create_table(self, model_class: type[clearskies.model.Model]):
-        self.load_default_data()
-        table_name = model_class.destination_name()
-        if table_name in self.__class__._tables:
-            return
-        self.__class__._tables[table_name] = MemoryTable(model_class)
-
-    def has_table(self, model_class: type[clearskies.model.Model]) -> bool:
-        self.load_default_data()
-        table_name = model_class.destination_name()
-        return table_name in self.__class__._tables
-
-    def get_table(self, model_class: type[clearskies.model.Model], create_if_missing=False) -> MemoryTable:
-        table_name = model_class.destination_name()
-        if table_name not in self.__class__._tables:
-            if create_if_missing:
-                self.create_table(model_class)
-            else:
-                raise ValueError(
-                    f"The memory backend was asked to work with the model '{model_class.__name__}' but this model hasn't been explicitly added to the memory backend.  This typically means that you are querying for records in a model but haven't created any yet."
-                )
-        return self.__class__._tables[table_name]
-
-    def create_with_model_class(
-        self, data: dict[str, Any], model_class: type[clearskies.model.Model]
-    ) -> dict[str, Any]:
-        self.create_table(model_class)
-        return self.get_table(model_class).create(data)
-
-    def update(self, id: int | str, data: dict[str, Any], model: clearskies.model.Model) -> dict[str, Any]:
-        self.create_table(model.__class__)
-        return self.get_table(model.__class__).update(id, data)
-
-    def create(self, data: dict[str, Any], model: clearskies.model.Model) -> dict[str, Any]:
-        self.create_table(model.__class__)
-        return self.get_table(model.__class__).create(data)
-
-    def delete(self, id: int | str, model: clearskies.model.Model) -> bool:
-        self.create_table(model.__class__)
-        return self.get_table(model.__class__).delete(id)
-
-    def count(self, query: clearskies.query.Query) -> int:
-        self.check_query(query)
-        if not self.has_table(query.model_class):
-            if self._silent_on_missing_tables:
-                return 0
-
-            raise ValueError(
-                f"Attempt to count records for model '{query.model_class.__name__}' that hasn't yet been loaded into the MemoryBackend"
-            )
-
-        # this is easy if we have no joins, so just return early so I don't have to think about it
-        if not query.joins:
-            return self.get_table(query.model_class).count(query)
-
-        # we can ignore left joins when counting
-        query.joins = [join for join in query.joins if join.join_type != "LEFT"]
-        return len(self.rows_with_joins(query))
-
-    def records(
-        self, query: clearskies.query.Query, next_page_data: dict[str, str | int] | None = None
-    ) -> list[dict[str, Any]]:
-        self.check_query(query)
-        if not self.has_table(query.model_class):
-            if self._silent_on_missing_tables:
-                return []
-
-            raise ValueError(
-                f"Attempt to fetch records for model '{query.model_class.__name__} that hasn't yet been loaded into the MemoryBackend"
-            )
-
-        # this is easy if we have no joins, so just return early so I don't have to think about it
-        if not query.joins:
-            return self.get_table(query.model_class).rows(query, query.conditions, next_page_data=next_page_data)
-        rows = self.rows_with_joins(query)
-
-        if query.sorts:
-            default_table_name = query.model_class.destination_name()
-            rows = sorted(
-                rows, key=cmp_to_key(lambda row_a, row_b: _sort(row_a, row_b, query.sorts, default_table_name))
-            )
-
-        # currently we don't do much with selects, so just limit results down to the data from the original
-        # table.
-        rows = [row[query.model_class.destination_name()] for row in rows]
-
-        if "start" in query.pagination or query.limit:
-            number_rows = len(rows)
-            start = query.pagination.get("start", 0)
-            if start >= number_rows:
-                start = number_rows - 1
-            end = len(rows)
-            if query.limit and start + query.limit <= number_rows:
-                end = start + query.limit
-            rows = rows[start:end]
-            if end < number_rows and type(next_page_data) == dict:
-                next_page_data["start"] = start + query.limit
-        return rows
-
-    def rows_with_joins(self, query: clearskies.query.Query) -> list[dict[str, Any]]:
-        joins = [*query.joins]
-        conditions = [*query.conditions]
-        # quick sanity check
-        for join in query.joins:
-            if join.unaliased_table_name not in self.__class__._tables:
-                if not self._silent_on_missing_tables:
-                    raise ValueError(
-                        f"Join '{join._raw_join}' refrences table '{join.unaliased_table_name}' which does not exist in MemoryBackend"
-                    )
-                return []
-
-        # start with the matches in the main table
-        left_table_name = query.model_class.destination_name()
-        left_conditions = self.conditions_for_table(left_table_name, conditions, is_left=True)
-        main_rows = self.get_table(query.model_class).rows(query, left_conditions, filter_only=True)
-        # and now adjust the way data is stored in our rows list to support the joining process.
-        # we're going to go from something like: `[{row_1}, {row_2}]` to something like:
-        # [{table_1: table_1_row_1, table_2: table_2_row_1}, {table_1: table_1_row_2, table_2: table_2_row_2}]
-        # etc...
-        rows = [{left_table_name: row} for row in main_rows]
-        joined_tables = [left_table_name]
-
-        # and now work through our joins.  The tricky part is order - we need to manage the joins in the
-        # proper order, but they may not be in the correcet order in our join list.  I still don't feel like building
-        # a full graph, so cheat and be dumb: loop through them all and join in the ones we can, skipping the ones
-        # we can't.  If we get to the end and there are still joins left in the queue, then repeat, and eventually
-        # complain (since the joins may not be a valid object graph)
-        for i in range(10):
-            for index, join in enumerate(joins):
-                left_table_name = join.left_table_name
-                alias = join.alias
-                right_table_name = join.right_table_name
-                table_name_for_join = alias if alias else right_table_name
-                if left_table_name not in joined_tables:
-                    continue
-
-                join_rows = self.__class__._tables[join.unaliased_table_name].rows(query, [], filter_only=True)
-
-                rows = self.join_rows(rows, join_rows, join, joined_tables)
-
-                # done with this one!
-                del joins[index]
-                joined_tables.append(table_name_for_join)
-
-            # are we done yet?
-            if not joins:
-                break
-
-        if joins:
-            raise ValueError(
-                "Unable to fulfill joins for query - perhaps a necessary join is missing? "
-                + "One way to get this error is if you tried to join on another table which hasn't been "
-                + "joined itself.  e.g.: SELECT * FROM users JOIN type ON type.id=categories.type_id"
-            )
-
-        # now apply any remaining conditions.
-        left_condition_ids = [id(condition) for condition in left_conditions]
-        for condition in [condition for condition in conditions if id(condition) not in left_condition_ids]:
-            condition_filter = MemoryTable._condition_as_filter(condition)
-            rows = list(
-                filter(lambda row: condition.table_name in row and condition_filter(row[condition.table_name]), rows)
-            )
-
-        return rows
-
-    def all_rows(self, table_name: str) -> list[dict[str, Any]]:
-        if table_name not in self.__class__._tables:
-            if self._silent_on_missing_tables:
-                return []
-
-            raise ValueError(f"Cannot return rows for unknown table '{table_name}'")
-        return list(filter(None, self.__class__._tables[table_name]._rows))
-
-    def check_query(self, query: clearskies.query.Query) -> None:
-        if query.group_by:
-            raise KeyError(
-                f"MemoryBackend does not support group_by clauses in queries. You may be using the wrong backend."
-            )
-
-    def conditions_for_table(
-        self, table_name: str, conditions: list[clearskies.query.Condition], is_left=False
-    ) -> list[clearskies.query.Condition]:
-        """
-        Return only the conditions for the given table.
-
-        If you set is_left=True then it assumes this is the "default" table and so will also return conditions
-        without a table name.
-        """
-        return [
-            condition
-            for condition in conditions
-            if condition.table_name == table_name or (is_left and not condition.table_name)
-        ]
-
-    def join_rows(
-        self,
-        rows: list[dict[str, Any]],
-        join_rows: list[dict[str, Any]],
-        join: clearskies.query.Join,
-        joined_tables: list[str],
-    ) -> list[dict[str, Any]]:
-        """
-        Add the rows in `join_rows` in to the `rows` holder.
-
-        `rows` should be something like:
-
-        ```python
-        [
-            {
-                "table_1": {"table_1_row_1"},
-                "table_2": {"table_2_row_1"},
-            },
-            {
-                "table_1": {"table_1_row_2"},
-                "table_2": {"table_2_row_2"},
-            },
-        ]
-        ```
-
-        and join_rows should be the rows for the new table, something like:
-
-        `[{table_3_row_1}, {table_3_row_2}]`
-
-        which will then get merged into the rows variable properly (which it will return as a new list)
-        """
-        join_table_name = join.alias if join.alias else join.right_table_name
-        join_type = join.join_type
-
-        #######
-        ########
-        ## our problem is here.  When we join rows we can end up with multiple copies of the records from the left table because
-        # there can be more than one matching record in the right table.  This isn't happening, and so we're not getting the
-        # proper results because the one record that is chosen to match with the left table doesn't meet the where condition
-        # that is applied at the very end.  If we have multiple records that match, they all need to get retunred in the
-        # final list of rows here, so we can properly search everything.
-
-        # loop through each entry in rows, find a matching table in join_rows, and take action depending on join type
-        rows = [*rows]
-        matched_right_row_indexes = set()
-        left_table_name = join.left_table_name
-        left_column_name = join.left_column_name
-        # we're
-        for row_index in range(len(rows)):
-            row = rows[row_index]
-            matching_row = None
-            if left_table_name not in row:
-                raise ValueError("Attempted to check join data from unjoined table, which should not happen...")
-            left_value = (
-                row[left_table_name][left_column_name]
-                if (row[left_table_name] is not None and left_column_name in row[left_table_name])
-                else None
-            )
-            matching_rows = []
-            for join_index, join_row in enumerate(join_rows):
-                right_value = join_row[join.right_column_name] if join.right_column_name in join_row else None
-                # for now we are assuming the operator for the matching is `=`.  This is mainly because
-                # our join parsing doesn't bother checking for the matching operator, because it is `=` in
-                # 99% of cases.  We can always adjust down the line.
-                if (right_value is None and left_value is None) or (right_value == left_value):
-                    matching_rows.append(join_row)
-                    matched_right_row_indexes.add(right_value)
-
-            # if we have matching rows then join them in.
-            for index, matching_row in enumerate(matching_rows):
-                if not index:
-                    rows[row_index][join_table_name] = matching_row
-                else:
-                    rows.append({**row, **{join_table_name: matching_row}})
-
-            # if we don't have matching rows then remove them for an inner or right join
-            if not matching_rows:
-                if join_type == "LEFT" or join_type == "OUTER":
-                    rows[row_index][join_table_name] = matching_row = None
-                else:
-                    # we can't immediately delete the row because we're looping over the array it is in,
-                    # so just mark it as None and remove it later
-                    rows[row_index] = None  # type: ignore
-
-        rows = [row for row in rows if row is not None]
-
-        # now for outer/right rows we add on any unmatched rows
-        if (join_type == "OUTER" or join_type == "RIGHT") and len(matched_right_row_indexes) < len(join_rows):
-            for join_index in set(range(len(join_rows))) - matched_right_row_indexes:
-                rows.append(
-                    {
-                        join_table_name: join_rows[join_index],
-                        **{table_name: None for table_name in joined_tables},
-                    }
-                )
-
-        return rows
-
-    def validate_pagination_data(self, data: dict[str, Any], case_mapping: Callable) -> str:
-        extra_keys = set(data.keys()) - set(self.allowed_pagination_keys())
-        if len(extra_keys):
-            key_name = case_mapping("start")
-            return "Invalid pagination key(s): '" + "','".join(extra_keys) + f"'.  Only '{key_name}' is allowed"
-        if "start" not in data:
-            key_name = case_mapping("start")
-            return f"You must specify '{key_name}' when setting pagination"
-        start = data["start"]
-        try:
-            start = int(start)
-        except:
-            key_name = case_mapping("start")
-            return f"Invalid pagination data: '{key_name}' must be a number"
-        return ""
-
-    def allowed_pagination_keys(self) -> list[str]:
-        return ["start"]
-
-    def documentation_pagination_next_page_response(self, case_mapping: Callable[[str], str]) -> list[Any]:
-        return [AutoDocInteger(case_mapping("start"), example=0)]
-
-    def documentation_pagination_next_page_example(self, case_mapping: Callable[[str], str]) -> dict[str, Any]:
-        return {case_mapping("start"): 0}
-
-    def documentation_pagination_parameters(
-        self, case_mapping: Callable[[str], str]
-    ) -> list[tuple[AutoDocSchema, str]]:
-        return [
-            (
-                AutoDocInteger(case_mapping("start"), example=0),
-                "The zero-indexed record number to start listing results from",
-            )
-        ]