clear-skies 1.22.31__py3-none-any.whl → 2.0.0__py3-none-any.whl

clearskies/backends/memory_backend.py

@@ -1,10 +1,14 @@
-from .backend import Backend
-from collections import OrderedDict
-from functools import cmp_to_key
 import inspect
-from typing import Any, Callable, Dict, List, Tuple
-from ..autodoc.schema import Integer as AutoDocInteger
-from .. import model
+from functools import cmp_to_key
+from typing import Any, Callable
+
+import clearskies.model
+import clearskies.query
+from clearskies import functional
+from clearskies.autodoc.schema import Integer as AutoDocInteger
+from clearskies.autodoc.schema import Schema as AutoDocSchema
+from clearskies.backends.backend import Backend
+from clearskies.di import InjectableProperties, inject
 
 
 class Null:
@@ -27,11 +31,27 @@ def gentle_float_conversion(value):
         return value
 
 
-def _sort(row_a, row_b, sorts):
+def _sort(row_a: Any, row_b: Any, sorts: list[clearskies.query.Sort], default_table_name: str) -> int:
     for sort in sorts:
-        reverse = 1 if sort["direction"].lower() == "asc" else -1
-        value_a = row_a[sort["column"]] if sort["column"] in row_a else None
-        value_b = row_b[sort["column"]] if sort["column"] in row_b else None
+        # so, if we've done a join then the rows will have data from all joined tables via a dict of dicts.
+        # if there wasn't a join then we'll just have the data
+        if sort.table_name in row_a and isinstance(row_a[sort.table_name], dict):
+            sort_data_a = row_a[sort.table_name]
+        elif not sort.table_name and default_table_name in row_a and isinstance(row_a[default_table_name], dict):
+            sort_data_a = row_a[default_table_name]
+        else:
+            sort_data_a = row_a
+
+        if sort.table_name in row_b and isinstance(row_b[sort.table_name], dict):
+            sort_data_b = row_b[sort.table_name]
+        elif not sort.table_name and default_table_name in row_b and isinstance(row_b[default_table_name], dict):
+            sort_data_b = row_b[default_table_name]
+        else:
+            sort_data_b = row_b
+
+        reverse = 1 if sort.direction.lower() == "asc" else -1
+        value_a = sort_data_a[sort.column_name] if sort.column_name in sort_data_a else None
+        value_b = sort_data_b[sort.column_name] if sort.column_name in sort_data_b else None
         if value_a == value_b:
             continue
         if value_a is None:
@@ -41,68 +61,82 @@ def _sort(row_a, row_b, sorts):
         return reverse * (1 if value_a > value_b else -1)
     return 0
 
-def like_check(column, values, null):
-    def matches(row):
-        value = row.get(column)
-        search = values[0].lower()
-        if not value or isinstance(value, Null):
-            return bool(search)
-
-        value = value.lower()
-        if search[0] == "%" and search[-1] == "%":
-            return search.strip('%') in value
-        if search[0] == "%":
-            return value[0:len(search)] == search.lstrip('%')
-        if search[-1] == "%":
-            return value[-1*len(search):] == search.rstrip('%')
-        return value == search
-    return matches
+
+def cheating_equals(column, values, null):
+    """
+    Cheating because this solves a very specific problem that likely is a generic issue.
+
+    The memory backend has some matching failures because boolean columns stay boolean in the
+    memory store, but the incoming search values are not converted to boolean and tend to be
+    str(1) or str(0).  The issue is that save data goes through the `to_backend` flow, but search
+    data doesn't.  This doesn't matter most of the time because, in practice, the backend itself
+    often does its own type conversion, but it causes problems for the memory backend.  I can't
+    decide if fixing this will cause more problems than it solves, so for now I'm just cheating
+    and putting in a hack for this specific use case :shame:.
+    """
+
+    def inner(row):
+        backend_value = row[column] if column in row else null
+        if isinstance(backend_value, bool):
+            return backend_value == bool(values[0])
+        return str(backend_value) == str(values[0])
+
+    return inner
+
 
 class MemoryTable:
-    _table_name = None
-    _column_names = None
-    _rows = None
-    null = None
-    _id_index = None
-    id_column_name = None
-    _next_id = None
+    _table_name: str = ""
+    _column_names: list[str] = []
+    _rows: list[dict[str, Any] | None] = []
+    null: Null = Null()
+    _id_index: dict[int | str, int] = {}
+    id_column_name: str = ""
+    _next_id: int = 1
+    _model_class: type[clearskies.model.Model] = None  # type:  ignore
 
     # here be dragons.  This is not a 100% drop-in replacement for the equivalent SQL operators
     # https://codereview.stackexchange.com/questions/259198/in-memory-table-filtering-in-python
     _operator_lambda_builders = {
         "<=>": lambda column, values, null: lambda row: row.get(column, null) == values[0],
         "!=": lambda column, values, null: lambda row: row.get(column, null) != values[0],
-        "<=": lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-        <= gentle_float_conversion(values[0]),
-        ">=": lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-        >= gentle_float_conversion(values[0]),
-        ">": lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-        > gentle_float_conversion(values[0]),
-        "<": lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
-        < gentle_float_conversion(values[0]),
-        "=": lambda column, values, null: lambda row: (str(row[column]) if column in row else null) == str(values[0]),
+        "<=": (
+            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
+            <= gentle_float_conversion(values[0])
+        ),
+        ">=": (
+            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
+            >= gentle_float_conversion(values[0])
+        ),
+        ">": (
+            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
+            > gentle_float_conversion(values[0])
+        ),
+        "<": (
+            lambda column, values, null: lambda row: gentle_float_conversion(row.get(column, null))
+            < gentle_float_conversion(values[0])
+        ),
+        "=": cheating_equals,
         "is not null": lambda column, values, null: lambda row: (column in row and row[column] is not None),
         "is null": lambda column, values, null: lambda row: (column not in row or row[column] is None),
         "is not": lambda column, values, null: lambda row: row.get(column, null) != values[0],
         "is": lambda column, values, null: lambda row: row.get(column, null) == str(values[0]),
-        "like": like_check,
+        "like": lambda column, values, null: lambda row: row.get(column, null) == str(values[0]),
         "in": lambda column, values, null: lambda row: row.get(column, null) in values,
     }
 
-    def __init__(self, model):
-        self.null = Null()
-        self._column_names = []
+    def __init__(self, model_class: type[clearskies.model.Model]) -> None:
         self._rows = []
         self._id_index = {}
-        self.id_column_name = model.id_column_name
+        self.id_column_name = model_class.id_column_name
         self._next_id = 1
+        self._model_class = model_class
 
-        self._table_name = model.table_name()
-        self._column_names.extend(model.columns_configuration().keys())
+        self._table_name = model_class.destination_name()
+        self._column_names = list(model_class.get_columns().keys())
         if self.id_column_name not in self._column_names:
             self._column_names.append(self.id_column_name)
 
-    def update(self, id, data):
+    def update(self, id: int | str, data: dict[str, Any]) -> dict[str, Any]:
         if id not in self._id_index:
             raise ValueError(f"Attempt to update non-existent record with '{self.id_column_name}' of '{id}'")
         index = self._id_index[id]
@@ -117,16 +151,16 @@ class MemoryTable:
                     f"Cannot update record: column '{column_name}' does not exist in table '{self._table_name}'"
                 )
         self._rows[index] = {
-            **self._rows[index],
+            **self._rows[index],  # type: ignore
             **data,
         }
-        return self._rows[index]
+        return self._rows[index]  # type: ignore
 
-    def create(self, data):
+    def create(self, data: dict[str, Any]) -> dict[str, Any]:
         for column_name in data.keys():
             if column_name not in self._column_names:
                 raise ValueError(
-                    f"Cannot create record: column '{column_name}' does not exist in table '{self._table_name}'"
+                    f"Cannot create record: column '{column_name}' does not exist for model '{self._model_class.__name__}'"
                 )
         incoming_id = data.get(self.id_column_name)
         if not incoming_id:
@@ -134,7 +168,7 @@ class MemoryTable:
             data[self.id_column_name] = incoming_id
             self._next_id += 1
         try:
-            incoming_as_int = int(incoming_as_int)
+            incoming_as_int = int(incoming_id)
             if incoming_as_int >= self._next_id:
                 self._next_id = incoming_as_int + 1
         except:
@@ -159,166 +193,403 @@ class MemoryTable:
         self._rows[index] = None
         return True
 
-    def count(self, configuration, wheres):
-        return len(self.rows(configuration, wheres, filter_only=True))
-
-    def rows(self, configuration, wheres, filter_only=False, next_page_data=None):
-        rows = list(filter(None, self._rows))
-        for where in wheres:
-            rows = filter(self._where_as_filter(where), rows)
-        rows = list(rows)
+    def count(self, query: clearskies.query.Query):
+        return len(self.rows(query, query.conditions, filter_only=True))
+
+    def rows(
+        self,
+        query: clearskies.query.Query,
+        conditions: list[clearskies.query.Condition],
+        filter_only: bool = False,
+        next_page_data: dict[str, Any] | None = None,
+    ):
+        rows = [row for row in self._rows if row is not None]
+        for condition in conditions:
+            rows = list(filter(self._condition_as_filter(condition), rows))
+        rows = [*rows]
         if filter_only:
             return rows
-        if "sorts" in configuration and configuration["sorts"]:
-            rows = sorted(rows, key=cmp_to_key(lambda row_a, row_b: _sort(row_a, row_b, configuration["sorts"])))
-        if "limit" in configuration or ("pagination" in configuration and configuration["pagination"].get("start")):
+        if query.sorts:
+            rows = sorted(
+                rows,
+                key=cmp_to_key(
+                    lambda row_a, row_b: _sort(row_a, row_b, query.sorts, query.model_class.destination_name())
+                ),
+            )
+        if query.limit or query.pagination.get("start"):
             number_rows = len(rows)
-            start = int(configuration.get("pagination", {}).get("start", 0))
+            start = int(query.pagination.get("start", 0))
             if not start:
                 start = 0
             if int(start) >= number_rows:
                 start = number_rows - 1
             end = len(rows)
-            if configuration.get("limit") and configuration.get("limit") > 0 and start + int(configuration["limit"]) <= number_rows:
-                end = start + int(configuration["limit"])
+            if query.limit and start + int(query.limit) <= number_rows:
+                end = start + int(query.limit)
             if end < number_rows and type(next_page_data) == dict:
-                next_page_data["start"] = start + configuration["limit"]
+                next_page_data["start"] = start + int(query.limit)
             rows = rows[start:end]
         return rows
 
-    def _where_as_filter(self, where):
-        column = where["column"]
-        values = where["values"]
-        return self._operator_lambda_builders[where["operator"].lower()](column, values, self.null)
+    @classmethod
+    def _condition_as_filter(cls, condition: clearskies.query.Condition) -> Callable:
+        column = condition.column_name
+        values = condition.values
+        return cls._operator_lambda_builders[condition.operator.lower()](column, values, cls.null)
+
+
+class MemoryBackend(Backend, InjectableProperties):
+    """
+    Store data in an in-memory store built in to clearskies.
+
+    ## Usage
 
+    Since the memory backend is built into clearskies, there's no configuration necessary to make it work:
+    simply attach it to any of your models and it will manage data for you.  If you want though, you can declare
+    a binding named `memory_backend_default_data` which you fill with records for your models to pre-populate
+    the memory backend.  This can be helpful for tests as well as tables with fixed values.
 
-class MemoryBackend(Backend):
-    _tables = None
-    _silent_on_missing_tables = False
+    ### Testing
 
-    _allowed_configs = [
-        "table_name",
-        "wheres",
-        "joins",
-        "sorts",
-        "pagination",
-        "length",
-        "selects",
-        "select_all",
-        "model_columns",
-    ]
+    A primary use case of the memory backend is for building unit tests of your code.  You can use the dependency
+    injection system to override other backends with the memory backend.  You can still operate with model classes
+    in the exact same way, so this can be an easy way to mock out databases/api endpoints/etc...  Of course,
+    there can be behavioral differences between the memory backend and other backends, so this isn't always perfect.
+    Hence why this works well for unit tests, but can't replace all testing, especially integration tests or
+    end-to-end tests.  Here's an example of that.  Note that the UserPreference model uses the cursor backend,
+    but when you run this code it will actually end up with the memory backend, so the code will run even without
+    attempting to connect to a database.
+
+    ```python
+    import clearskies
+
+
+    class UserPreference(clearskies.Model):
+        id_column_name = "id"
+        backend = clearskies.backends.CursorBackend()
+        id = clearskies.columns.Uuid()
+
+
+    cli = clearskies.contexts.Cli(
+        clearskies.endpoints.Callable(
+            lambda user_preferences: user_preferences.create(no_data=True).id,
+        ),
+        classes=[UserPreference],
+        class_overrides={
+            clearskies.backends.CursorBackend: clearskies.backends.MemoryBackend(),
+        },
+    )
+
+    if __name__ == "__main__":
+        cli()
+    ```
+
+    Note that the model requests the CursorBackend, but then we switch that for the memory backend via the
+    `class_overrides` kwarg to `clearskies.contexts.Cli`.  Therefore, the above code works regardless of
+    whether or not a database is running.  Since the models are used to interact with the backend
+    (rather than using the cursor directly), the above code works the same despite the change in backend.
+
+    Again, this is helpful as a quick way to manage tests - swap out a database backend (or any other backend)
+    for the memory backend, and then pre-populate records to test your application logic.  Obviously, this
+    won't cach backend-specific issues (e.g. forgetting to add a column to your database, mismatches between
+    column schema and backend schema, missing indexes, etc...), which is why this helps for unit tests
+    but not for integration tests.
+
+    ### Production Usage
+
+    You can use the memory backend in production if you want, although there are some important caveats to keep
+    in mind:
+
+     1. There is limited attempts at performance optimization, so you should test it before putting it under
+        substantial loads.
+     2. There's no concept of replication.  If you have multiple workers, then write operations won't be
+        persisted between them.
+
+    So, while there may be some cases where this is useful in production, it's by no means a replacement for
+    more typical in-memory data stores.
+
+    ### Predefined Records
+
+    You can declare a binding named `memory_backend_default_data` to seed the memory backend with records.  This
+    can be helpful in testing to setup your tests, and is occassionally helpful for keeping track of data in
+    fixed, read-only tables.  Here's an example:
+
+    ```python
+    import clearskies
+
+
+    class Owner(clearskies.Model):
+        id_column_name = "id"
+        backend = clearskies.backends.MemoryBackend()
+
+        id = clearskies.columns.Uuid()
+        name = clearskies.columns.String()
+        phone = clearskies.columns.Phone()
+
+
+    class Pet(clearskies.Model):
+        id_column_name = "id"
+        backend = clearskies.backends.MemoryBackend()
+
+        id = clearskies.columns.Uuid()
+        name = clearskies.columns.String()
+        species = clearskies.columns.String()
+        owner_id = clearskies.columns.BelongsToId(Owner, readable_parent_columns=["id", "name"])
+        owner = clearskies.columns.BelongsToModel("owner_id")
+
+
+    wsgi = clearskies.contexts.WsgiRef(
+        clearskies.endpoints.List(
+            model_class=Pet,
+            readable_column_names=["id", "name", "species", "owner"],
+            sortable_column_names=["id", "name", "species"],
+            default_sort_column_name="name",
+        ),
+        classes=[Owner, Pet],
+        bindings={
+            "memory_backend_default_data": [
+                {
+                    "model_class": Owner,
+                    "records": [
+                        {"id": "1-2-3-4", "name": "John Doe", "phone": "555-123-4567"},
+                        {"id": "5-6-7-8", "name": "Jane Doe", "phone": "555-321-0987"},
+                    ],
+                },
+                {
+                    "model_class": Pet,
+                    "records": [
+                        {"id": "a-b-c-d", "name": "Fido", "species": "Dog", "owner_id": "1-2-3-4"},
+                        {"id": "e-f-g-h", "name": "Spot", "species": "Dog", "owner_id": "1-2-3-4"},
+                        {
+                            "id": "i-j-k-l",
+                            "name": "Puss in Boots",
+                            "species": "Cat",
+                            "owner_id": "5-6-7-8",
+                        },
+                    ],
+                },
+            ],
+        },
+    )
+
+    if __name__ == "__main__":
+        wsgi()
+    ```
+
+    And if you invoke it:
+
+    ```bash
+    $ curl 'http://localhost:8080' | jq
+    {
+        "status": "success",
+        "error": "",
+        "data": [
+            {
+                "id": "a-b-c-d",
+                "name": "Fido",
+                "species": "Dog",
+                "owner": {
+                    "id": "1-2-3-4",
+                    "name": "John Doe"
+                }
+            },
+            {
+                "id": "i-j-k-l",
+                "name": "Puss in Boots",
+                "species": "Cat",
+                "owner": {
+                    "id": "5-6-7-8",
+                    "name": "Jane Doe"
+                }
+            },
+            {
+                "id": "e-f-g-h",
+                "name": "Spot",
+                "species": "Dog",
+                "owner": {
+                    "id": "1-2-3-4",
+                    "name": "John Doe"
+                }
+            }
+        ],
+        "pagination": {
+            "number_results": 3,
+            "limit": 50,
+            "next_page": {}
+        },
+        "input_errors": {}
+    }
+    ```
+    """
 
-    _required_configs = [
-        "table_name",
-    ]
+    default_data = inject.ByName("memory_backend_default_data")
+    default_data_loaded = False
+    _tables: dict[str, MemoryTable] = {}
+    _silent_on_missing_tables: bool = True
 
-    def __init__(self):
-        self._tables = {}
-        self._silent_on_missing_tables = True
+    def __init__(self, silent_on_missing_tables=True):
+        self.__class__._tables = {}
+        self._silent_on_missing_tables = silent_on_missing_tables
+
+    @classmethod
+    def clear_table_cache(cls):
+        cls._tables = {}
+
+    def load_default_data(self):
+        if self.default_data_loaded:
+            return
+        self.default_data_loaded = True
+        if not isinstance(self.default_data, list):
+            raise TypeError(
+                f"'memory_backend_default_data' should be populated with a list, but I received a value of type '{self.default_data.__class__.__name__}'"
+            )
+        for index, table_data in enumerate(self.default_data):
+            if "model_class" not in table_data:
+                raise TypeError(
+                    f"Each entry in the 'memory_backend_default_data' list should have a key named 'model_class', but entry #{index + 1} is missing this key."
+                )
+            model_class = table_data["model_class"]
+            if not functional.validations.is_model_class(table_data["model_class"]):
+                raise TypeError(
+                    f"The 'model_class' key in 'memory_backend_default_data' for entry #{index + 1} is not a model class."
+                )
+            if "records" not in table_data:
+                raise TypeError(
+                    f"Each entry in the 'memory_backend_default_data' list should have a key named 'records', but entry #{index + 1} is missing this key."
+                )
+            records = table_data["records"]
+            if not isinstance(records, list):
+                raise TypeError(
+                    f"The 'records' key in 'memory_backend_default_data' for entry #{index + 1} was not a list."
+                )
+            self.create_table(model_class)
+            for record in records:
+                self.create_with_model_class(record, model_class)
 
     def silent_on_missing_tables(self, silent=True):
         self._silent_on_missing_tables = silent
 
-    def configure(self):
-        pass
-
-    def create_table(self, model):
-        """
-        Accepts either a model or a model class and creates a "table" for it
-        """
-        model = self.cheez_model(model)
-        if model.table_name() in self._tables:
+    def create_table(self, model_class: type[clearskies.model.Model]):
+        self.load_default_data()
+        table_name = model_class.destination_name()
+        if table_name in self.__class__._tables:
             return
-        self._tables[model.table_name()] = MemoryTable(model)
+        self.__class__._tables[table_name] = MemoryTable(model_class)
+
+    def has_table(self, model_class: type[clearskies.model.Model]) -> bool:
+        self.load_default_data()
+        table_name = model_class.destination_name()
+        return table_name in self.__class__._tables
+
+    def get_table(self, model_class: type[clearskies.model.Model], create_if_missing=False) -> MemoryTable:
+        table_name = model_class.destination_name()
+        if table_name not in self.__class__._tables:
+            if create_if_missing:
+                self.create_table(model_class)
+            else:
+                raise ValueError(
+                    f"The memory backend was asked to work with the model '{model_class.__name__}' but this model hasn't been explicitly added to the memory backend.  This typically means that you are querying for records in a model but haven't created any yet."
+                )
+        return self.__class__._tables[table_name]
+
+    def create_with_model_class(
+        self, data: dict[str, Any], model_class: type[clearskies.model.Model]
+    ) -> dict[str, Any]:
+        self.create_table(model_class)
+        return self.get_table(model_class).create(data)
 
-    def update(self, id, data, model):
-        self.create_table(model)
-        return self._tables[model.table_name()].update(id, data)
+    def update(self, id: int | str, data: dict[str, Any], model: clearskies.model.Model) -> dict[str, Any]:
+        self.create_table(model.__class__)
+        return self.get_table(model.__class__).update(id, data)
 
-    def create(self, data, model):
-        self.create_table(model)
-        return self._tables[model.table_name()].create(data)
+    def create(self, data: dict[str, Any], model: clearskies.model.Model) -> dict[str, Any]:
+        self.create_table(model.__class__)
+        return self.get_table(model.__class__).create(data)
 
-    def delete(self, id, model):
-        self.create_table(model)
-        return self._tables[model.table_name()].delete(id)
+    def delete(self, id: int | str, model: clearskies.model.Model) -> bool:
+        self.create_table(model.__class__)
+        return self.get_table(model.__class__).delete(id)
 
-    def count(self, configuration, model):
-        if configuration["table_name"] not in self._tables:
+    def count(self, query: clearskies.query.Query) -> int:
+        self.check_query(query)
+        if not self.has_table(query.model_class):
             if self._silent_on_missing_tables:
                 return 0
 
             raise ValueError(
-                f"Attempt to count records in non-existent table '{configuration['table_name']} via MemoryBackend"
+                f"Attempt to count records for model '{query.model_class.__name__}' that hasn't yet been loaded into the MemoryBackend"
             )
 
         # this is easy if we have no joins, so just return early so I don't have to think about it
-        if "joins" not in configuration or not configuration["joins"]:
-            wheres = configuration["wheres"] if "wheres" in configuration else []
-            return self._tables[configuration["table_name"]].count(configuration, wheres)
+        if not query.joins:
+            return self.get_table(query.model_class).count(query)
 
         # we can ignore left joins when counting
-        configuration = {**configuration}
-        configuration["joins"] = [join for join in configuration["joins"] if join["type"] != "LEFT"]
-        return len(self.rows_with_joins(configuration))
-
-    def records(self, configuration, model, next_page_data=None):
-        table_name = configuration["table_name"]
-        if table_name not in self._tables:
+        query.joins = [join for join in query.joins if join.join_type != "LEFT"]
+        return len(self.rows_with_joins(query))
+
+    def records(
+        self, query: clearskies.query.Query, next_page_data: dict[str, str | int] | None = None
+    ) -> list[dict[str, Any]]:
+        self.check_query(query)
+        if not self.has_table(query.model_class):
             if self._silent_on_missing_tables:
                 return []
 
             raise ValueError(
-                f"Attempt to fetch records from non-existent table '{configuration['table_name']} via MemoryBackend"
+                f"Attempt to fetch records for model '{query.model_class.__name__} that hasn't yet been loaded into the MemoryBackend"
             )
 
         # this is easy if we have no joins, so just return early so I don't have to think about it
-        if "joins" not in configuration or not configuration["joins"]:
-            wheres = configuration["wheres"] if "wheres" in configuration else []
-            return self._tables[table_name].rows(configuration, wheres, next_page_data=next_page_data)
-
-        rows = self.rows_with_joins(configuration)
+        if not query.joins:
+            return self.get_table(query.model_class).rows(query, query.conditions, next_page_data=next_page_data)
+        rows = self.rows_with_joins(query)
+
+        if query.sorts:
+            default_table_name = query.model_class.destination_name()
+            rows = sorted(
+                rows, key=cmp_to_key(lambda row_a, row_b: _sort(row_a, row_b, query.sorts, default_table_name))
+            )
 
         # currently we don't do much with selects, so just limit results down to the data from the original
         # table.
-        rows = [row[table_name] for row in rows]
+        rows = [row[query.model_class.destination_name()] for row in rows]
 
-        if "sorts" in configuration and configuration["sorts"]:
-            rows = sorted(rows, key=cmp_to_key(lambda row_a, row_b: _sort(row_a, row_b, configuration["sorts"])))
-        if "start" in configuration.get("pagination", {}) or "limit" in configuration:
+        if "start" in query.pagination or query.limit:
             number_rows = len(rows)
-            start = configuration.get("pagination", {}).get("start", 0)
+            start = query.pagination.get("start", 0)
             if start >= number_rows:
                 start = number_rows - 1
             end = len(rows)
-            if configuration.get("limit") and start + configuration.get("limit") <= number_rows:
-                end = start + configuration.get("limit")
+            if query.limit and start + query.limit <= number_rows:
+                end = start + query.limit
             rows = rows[start:end]
             if end < number_rows and type(next_page_data) == dict:
-                next_page_data["start"] = start + configuration["limit"]
+                next_page_data["start"] = start + query.limit
         return rows
 
-    def rows_with_joins(self, configuration):
-        joins = configuration["joins"]
-        wheres = configuration["wheres"] if "wheres" in configuration else []
+    def rows_with_joins(self, query: clearskies.query.Query) -> list[dict[str, Any]]:
+        joins = [*query.joins]
+        conditions = [*query.conditions]
         # quick sanity check
-        for join in configuration["joins"]:
-            if join["table"] not in self._tables:
-                raise ValueError(
-                    f"Join '{join['raw']}' refrences table '{join['table']}' which does not exist in MemoryBackend"
-                )
+        for join in query.joins:
+            if join.unaliased_table_name not in self.__class__._tables:
+                if not self._silent_on_missing_tables:
+                    raise ValueError(
+                        f"Join '{join._raw_join}' refrences table '{join.unaliased_table_name}' which does not exist in MemoryBackend"
+                    )
+                return []
 
         # start with the matches in the main table
-        left_table = configuration["table_name"]
-        main_rows = self._tables[left_table].rows(
-            configuration, self._wheres_for_table(left_table, wheres, is_left=True), filter_only=True
-        )
+        left_table_name = query.model_class.destination_name()
+        left_conditions = self.conditions_for_table(left_table_name, conditions, is_left=True)
+        main_rows = self.get_table(query.model_class).rows(query, left_conditions, filter_only=True)
         # and now adjust the way data is stored in our rows list to support the joining process.
         # we're going to go from something like: `[{row_1}, {row_2}]` to something like:
         # [{table_1: table_1_row_1, table_2: table_2_row_1}, {table_1: table_1_row_2, table_2: table_2_row_2}]
         # etc...
-        rows = [{left_table: row} for row in main_rows]
-        joined_tables = [left_table]
+        rows = [{left_table_name: row} for row in main_rows]
+        joined_tables = [left_table_name]
 
         # and now work through our joins.  The tricky part is order - we need to manage the joins in the
         # proper order, but they may not be in the correcet order in our join list.  I still don't feel like building
@@ -327,16 +598,14 @@ class MemoryBackend(Backend):
         # complain (since the joins may not be a valid object graph)
         for i in range(10):
             for index, join in enumerate(joins):
-                left_table = join["left_table"]
-                alias = join["alias"]
-                right_table = join["right_table"]
-                table_name_for_join = alias if alias else right_table
-                if left_table not in joined_tables:
+                left_table_name = join.left_table_name
+                alias = join.alias
+                right_table_name = join.right_table_name
+                table_name_for_join = alias if alias else right_table_name
+                if left_table_name not in joined_tables:
                     continue
 
-                join_rows = self._tables[right_table].rows(
-                    configuration, self._wheres_for_table(table_name_for_join, wheres, joined_tables), filter_only=True
-                )
+                join_rows = self.__class__._tables[join.unaliased_table_name].rows(query, [], filter_only=True)
 
                 rows = self.join_rows(rows, join_rows, join, joined_tables)
 
@@ -355,56 +624,67 @@ class MemoryBackend(Backend):
                 + "joined itself.  e.g.: SELECT * FROM users JOIN type ON type.id=categories.type_id"
             )
 
+        # now apply any remaining conditions.
+        left_condition_ids = [id(condition) for condition in left_conditions]
+        for condition in [condition for condition in conditions if id(condition) not in left_condition_ids]:
+            condition_filter = MemoryTable._condition_as_filter(condition)
+            rows = list(
+                filter(lambda row: condition.table_name in row and condition_filter(row[condition.table_name]), rows)
+            )
+
         return rows
 
-    def all_rows(self, table_name):
-        if table_name not in self._tables:
+    def all_rows(self, table_name: str) -> list[dict[str, Any]]:
+        if table_name not in self.__class__._tables:
             if self._silent_on_missing_tables:
                 return []
 
             raise ValueError(f"Cannot return rows for unknown table '{table_name}'")
-        return self._tables[table_name]._rows
-
-    def _check_query_configuration(self, configuration):
-        for key in configuration.keys():
-            if key not in self._allowed_configs:
-                raise KeyError(f"MemoryBackend does not support config '{key}'. You may be using the wrong backend")
-        for key in self._required_configs:
-            if key not in configuration:
-                raise KeyError(f"Missing required configuration key {key}")
-
-        for key in self._allowed_configs:
-            if not key in configuration:
-                configuration[key] = [] if key[-1] == "s" else ""
-        if "pagination" not in configuration or "start" not in configuration["pagination"]:
-            configuration["pagination"] = {"start": 0}
-        return configuration
-
-    def _wheres_for_table(self, table_name, wheres, is_left=False):
+        return list(filter(None, self.__class__._tables[table_name]._rows))
+
+    def check_query(self, query: clearskies.query.Query) -> None:
+        if query.group_by:
+            raise KeyError(
+                f"MemoryBackend does not support group_by clauses in queries. You may be using the wrong backend."
+            )
+
+    def conditions_for_table(
+        self, table_name: str, conditions: list[clearskies.query.Condition], is_left=False
+    ) -> list[clearskies.query.Condition]:
         """
-        Returns only the where conditions for the current table
+        Return only the conditions for the given table.
 
         If you set is_left=True then it assumes this is the "default" table and so will also return conditions
         without a table name.
         """
-        return [where for where in wheres if where["table"] == table_name or (is_left and not where["table"])]
+        return [
+            condition
+            for condition in conditions
+            if condition.table_name == table_name or (is_left and not condition.table_name)
+        ]
 
-    def join_rows(self, rows, join_rows, join_config, joined_tables):
+    def join_rows(
+        self,
+        rows: list[dict[str, Any]],
+        join_rows: list[dict[str, Any]],
+        join: clearskies.query.Join,
+        joined_tables: list[str],
+    ) -> list[dict[str, Any]]:
         """
-        Adds the rows in `join_rows` in to the `rows` holder.
+        Add the rows in `join_rows` in to the `rows` holder.
 
         `rows` should be something like:
 
-        ```
+        ```python
         [
             {
-                'table_1': {'table_1_row_1'},
-                'table_2': {'table_2_row_1'},
+                "table_1": {"table_1_row_1"},
+                "table_2": {"table_2_row_1"},
             },
             {
-                'table_1': {'table_1_row_2'},
-                'table_2': {'table_2_row_2'},
-            }
+                "table_1": {"table_1_row_2"},
+                "table_2": {"table_2_row_2"},
+            },
         ]
         ```
 
@@ -414,53 +694,64 @@ class MemoryBackend(Backend):
 
         which will then get merged into the rows variable properly (which it will return as a new list)
         """
-        join_table_name = join_config["alias"] if join_config["alias"] else join_config["right_table"]
-        join_type = join_config["type"]
+        join_table_name = join.alias if join.alias else join.right_table_name
+        join_type = join.join_type
+
+        #######
+        ########
+        ## our problem is here.  When we join rows we can end up with multiple copies of the records from the left table because
+        # there can be more than one matching record in the right table.  This isn't happening, and so we're not getting the
+        # proper results because the one record that is chosen to match with the left table doesn't meet the where condition
+        # that is applied at the very end.  If we have multiple records that match, they all need to get retunred in the
+        # final list of rows here, so we can properly search everything.
 
         # loop through each entry in rows, find a matching table in join_rows, and take action depending on join type
         rows = [*rows]
-        matched_right_row_indexes = []
-        left_table = join_config["left_table"]
-        left_column = join_config["left_column"]
-        for row_index, row in enumerate(rows):
+        matched_right_row_indexes = set()
+        left_table_name = join.left_table_name
+        left_column_name = join.left_column_name
+        # we're
+        for row_index in range(len(rows)):
+            row = rows[row_index]
             matching_row = None
-            if left_table not in row:
+            if left_table_name not in row:
                 raise ValueError("Attempted to check join data from unjoined table, which should not happen...")
             left_value = (
-                row[left_table][left_column]
-                if (row[left_table] is not None and left_column in row[left_table])
+                row[left_table_name][left_column_name]
+                if (row[left_table_name] is not None and left_column_name in row[left_table_name])
                 else None
             )
-            for join_index, join in enumerate(join_rows):
-                right_value = join[join_config["right_column"]] if join_config["right_column"] in join else None
+            matching_rows = []
+            for join_index, join_row in enumerate(join_rows):
+                right_value = join_row[join.right_column_name] if join.right_column_name in join_row else None
                 # for now we are assuming the operator for the matching is `=`.  This is mainly because
                 # our join parsing doesn't bother checking for the matching operator, because it is `=` in
                 # 99% of cases.  We can always adjust down the line.
                 if (right_value is None and left_value is None) or (right_value == left_value):
-                    matching_row = join
-                    matched_right_row_indexes.append(right_value)
-                    break
-
-            # next action depends on the join type and match success
-            # for left and outer joins we always preserve records in the main table, so just plop in our match
-            # (even if it is None)
-            if join_type == "LEFT" or join_type == "OUTER":
-                rows[row_index][join_table_name] = matching_row
-
-            # for inner and right joins we delete the row if we don't have a match
-            elif join_type == "INNER" or join_type == "RIGHT":
-                if matching_row is not None:
+                    matching_rows.append(join_row)
+                    matched_right_row_indexes.add(right_value)
+
+            # if we have matching rows then join them in.
+            for index, matching_row in enumerate(matching_rows):
+                if not index:
                     rows[row_index][join_table_name] = matching_row
+                else:
+                    rows.append({**row, **{join_table_name: matching_row}})
+
+            # if we don't have matching rows then remove them for an inner or right join
+            if not matching_rows:
+                if join_type == "LEFT" or join_type == "OUTER":
+                    rows[row_index][join_table_name] = matching_row = None
                 else:
                     # we can't immediately delete the row because we're looping over the array it is in,
                     # so just mark it as None and remove it later
-                    rows[row_index] = None
+                    rows[row_index] = None  # type: ignore
 
         rows = [row for row in rows if row is not None]
 
         # now for outer/right rows we add on any unmatched rows
         if (join_type == "OUTER" or join_type == "RIGHT") and len(matched_right_row_indexes) < len(join_rows):
-            for join_index in set(enumerate(join_rows)) - set(matched_right_row_indexes):
+            for join_index in set(range(len(join_rows))) - matched_right_row_indexes:
                 rows.append(
                     {
                         join_table_name: join_rows[join_index],
@@ -470,15 +761,15 @@ class MemoryBackend(Backend):
 
         return rows
 
-    def validate_pagination_kwargs(self, kwargs: Dict[str, Any], case_mapping: Callable) -> str:
-        extra_keys = set(kwargs.keys()) - set(self.allowed_pagination_keys())
+    def validate_pagination_data(self, data: dict[str, Any], case_mapping: Callable) -> str:
+        extra_keys = set(data.keys()) - set(self.allowed_pagination_keys())
         if len(extra_keys):
             key_name = case_mapping("start")
             return "Invalid pagination key(s): '" + "','".join(extra_keys) + f"'.  Only '{key_name}' is allowed"
-        if "start" not in kwargs:
+        if "start" not in data:
             key_name = case_mapping("start")
             return f"You must specify '{key_name}' when setting pagination"
-        start = kwargs["start"]
+        start = data["start"]
         try:
             start = int(start)
         except:
@@ -486,16 +777,18 @@ class MemoryBackend(Backend):
             return f"Invalid pagination data: '{key_name}' must be a number"
         return ""
 
-    def allowed_pagination_keys(self) -> List[str]:
+    def allowed_pagination_keys(self) -> list[str]:
         return ["start"]
 
-    def documentation_pagination_next_page_response(self, case_mapping: Callable) -> List[Any]:
+    def documentation_pagination_next_page_response(self, case_mapping: Callable[[str], str]) -> list[Any]:
         return [AutoDocInteger(case_mapping("start"), example=0)]
 
-    def documentation_pagination_next_page_example(self, case_mapping: Callable) -> Dict[str, Any]:
+    def documentation_pagination_next_page_example(self, case_mapping: Callable[[str], str]) -> dict[str, Any]:
         return {case_mapping("start"): 0}
 
-    def documentation_pagination_parameters(self, case_mapping: Callable) -> List[Tuple[Any]]:
+    def documentation_pagination_parameters(
+        self, case_mapping: Callable[[str], str]
+    ) -> list[tuple[AutoDocSchema, str]]:
         return [
             (
                 AutoDocInteger(case_mapping("start"), example=0),