sera-2 1.24.1__py3-none-any.whl → 1.25.0__py3-none-any.whl

sera/libs/base_service.py

@@ -10,8 +10,8 @@ from sqlalchemy.orm import contains_eager, load_only
 
 from sera.libs.base_orm import BaseORM
 from sera.libs.search_helper import Query, QueryOp
-from sera.misc import assert_not_null, to_snake_case
-from sera.models import Cardinality, Class, DataProperty, ObjectProperty
+from sera.misc import assert_isinstance, assert_not_null, to_snake_case
+from sera.models import Cardinality, Class, DataProperty, IndexType, ObjectProperty
 
 R = TypeVar("R", bound=BaseORM)
 ID = TypeVar("ID")  # ID of a class
@@ -20,6 +20,7 @@ SqlResult = TypeVar("SqlResult", bound=Result)
 
 class QueryResult(NamedTuple, Generic[R]):
     records: Sequence[R]
+    extra_columns: Mapping[str, Sequence]
     total: Optional[int]
 
 
@@ -36,6 +37,7 @@ class BaseAsyncService(Generic[ID, R]):
         self._cls_id_prop = getattr(self.orm_cls, self.id_prop.name)
         self.is_id_auto_increment = assert_not_null(self.id_prop.db).is_auto_increment
 
+        # mapping from property name to ORM class for object properties
         self.prop2orm: dict[str, type] = {
             prop.name: orm_classes[prop.target.name]
             for prop in cls.properties.values()
@@ -103,15 +105,15 @@ class BaseAsyncService(Generic[ID, R]):
                 # A -> B is either 1:1 or N:1, we will store the foreign key is in A
                 # .join(B, A.<foreign_key> == B.id)
                 self.join_clauses[prop.name] = [
-                    {
-                        "class": target_tbl,
-                        "condition": getattr(
-                            target_tbl,
-                            assert_not_null(target_cls.get_id_property()).name,
-                        )
-                        == getattr(self.orm_cls, source_fk),
-                        "contains_eager": getattr(self.orm_cls, source_relprop),
-                    },
+                    #     {
+                    #         "class": target_tbl,
+                    #         "condition": getattr(
+                    #             target_tbl,
+                    #             assert_not_null(target_cls.get_id_property()).name,
+                    #         )
+                    #         == getattr(self.orm_cls, source_fk),
+                    #         "contains_eager": getattr(self.orm_cls, source_relprop),
+                    #     },
                 ]
 
     @classmethod
@@ -135,6 +137,7 @@ class BaseAsyncService(Generic[ID, R]):
             session: The database session
         """
         q = self._select()
+        extra_cols = []
 
         if len(query.fields) > 0:
             q = q.options(
@@ -195,12 +198,60 @@ class BaseAsyncService(Generic[ID, R]):
                 q = q.where(~getattr(self.orm_cls, clause.field).in_(clause.value))
             else:
                 assert clause.op == QueryOp.fuzzy
-                # Assuming fuzzy search is implemented as a full-text search
-                q = q.where(
-                    func.to_tsvector(getattr(self.orm_cls, clause.field)).match(
-                        clause.value
-                    )
+                clause_prop = self.cls.properties[clause.field]
+                assert (
+                    isinstance(clause_prop, DataProperty) and clause_prop.db is not None
                 )
+                clause_orm_field = getattr(self.orm_cls, clause.field)
+                extra_cols.append(f"{clause.field}_score")
+
+                if clause_prop.db.index_type == IndexType.POSTGRES_FTS_SEVI:
+                    # fuzzy search is implemented using Postgres Full-Text Search
+                    # sevi is a custom text search configuration that we defined in `configs/postgres-fts.sql`
+                    q = q.where(
+                        func.to_tsvector("sevi", clause_orm_field).bool_op("@@")(
+                            func.plainto_tsquery("sevi", clause.value)
+                        )
+                    )
+                    # TODO: figure out which rank function is better
+                    # https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-RANKING
+                    q = q.order_by(
+                        func.ts_rank_cd(
+                            func.to_tsvector("sevi", clause_orm_field),
+                            func.plainto_tsquery("sevi", clause.value),
+                        ).desc()
+                    )
+                    q = q.add_columns(
+                        func.ts_rank_cd(
+                            func.to_tsvector("sevi", clause_orm_field),
+                            func.plainto_tsquery("sevi", clause.value),
+                        ).label(f"{clause.field}_score")
+                    )
+                elif clause_prop.db.index_type == IndexType.POSTGRES_TRIGRAM:
+                    # fuzzy search is implemented using Postgres trigram index
+                    # using a custom function f_unaccent to ignore accents -- see `configs/postgres-fts.sql`
+                    q = q.where(
+                        func.f_unaccent(clause_orm_field).bool_op("%>")(
+                            func.f_unaccent(clause.value)
+                        )
+                    )
+                    q = q.order_by(
+                        func.f_unaccent(clause_orm_field).op("<->>")(
+                            func.f_unaccent(clause.value)
+                        )
+                    )
+                    q = q.add_columns(
+                        (
+                            1
+                            - func.f_unaccent(clause_orm_field).op("<->>")(
+                                func.f_unaccent(clause.value)
+                            )
+                        ).label(f"{clause.field}_score")
+                    )
+                else:
+                    raise NotImplementedError(
+                        f"Fuzzy search is not implemented for index type {clause_prop.db.index_type}"
+                    )
 
         for join_condition in query.join_conditions:
             for join_clause in self.join_clauses[join_condition.prop]:
@@ -211,16 +262,29 @@ class BaseAsyncService(Generic[ID, R]):
                     full=join_condition.join_type == "full",
                 ).options(contains_eager(join_clause["contains_eager"]))
 
-                print(">>>", join_clause)
-
-        cq = select(func.count()).select_from(q.subquery())
+        # Create count query without order_by clause to improve performance
+        cq = select(func.count()).select_from(q.order_by(None).subquery())
         rq = q.limit(query.limit).offset(query.offset)
-        records = self._process_result(await session.execute(rq)).scalars().all()
+
+        if len(extra_cols) == 0:
+            records = self._process_result(await session.execute(rq)).scalars().all()
+            extra_columns = {}
+        else:
+            records = []
+            raw_extra_columns = [[] for col in extra_cols]
+            for row in self._process_result(await session.execute(rq)):
+                records.append(row[0])
+                for i in range(len(extra_cols)):
+                    raw_extra_columns[i].append(row[i + 1])
+            extra_columns = {
+                col: raw_extra_columns[i] for i, col in enumerate(extra_cols)
+            }
+
         if query.return_total:
             total = (await session.execute(cq)).scalar_one()
         else:
             total = None
-        return QueryResult(records, total)
+        return QueryResult(records, extra_columns, total)
 
     async def get_by_id(self, id: ID, session: AsyncSession) -> Optional[R]:
         """Retrieving a record by ID."""

sera/libs/search_helper.py

@@ -253,9 +253,9 @@ class Query(msgspec.Struct):
             else:
                 if join_clause.join_type != "inner":
                     output[target_name].extend(
-                        deser_func((val := getattr(record, source_relprop)))
+                        deser_func(val)
                         for record in result.records
-                        if val is not None
+                        if (val := getattr(record, source_relprop)) is not None
                     )
                 else:
                     output[target_name].extend(
@@ -265,6 +265,8 @@ class Query(msgspec.Struct):
 
         deser_func = dataschema[cls.name].from_db
         output[cls.name] = [deser_func(record) for record in result.records]
+        # include extra columns such as fuzzy search scores
+        output.update(result.extra_columns)
 
         return output
 

sera/make/make_python_model.py

@@ -22,6 +22,7 @@ from sera.models import (
     Cardinality,
     Class,
     DataProperty,
+    IndexType,
     ObjectProperty,
     Package,
     PyTypeWithDep,
@@ -1207,14 +1208,79 @@ def make_python_relational_model(
         )
 
         index_stmts = []
-        if len(cls.db.indices) > 0:
+
+        if len(cls.db.indices) > 0 or any(
+            isinstance(prop, DataProperty)
+            and prop.db is not None
+            and prop.db.is_indexed
+            and (
+                prop.db.index_type == IndexType.POSTGRES_FTS_SEVI
+                or prop.db.index_type == IndexType.POSTGRES_TRIGRAM
+            )
+            for prop in cls.properties.values()
+        ):
             program.import_("sqlalchemy.Index", True)
+
+            fts_index = []
+            for prop in cls.properties.values():
+                if (
+                    not isinstance(prop, DataProperty)
+                    or prop.db is None
+                    or not prop.db.is_indexed
+                ):
+                    continue
+                if prop.db.index_type == IndexType.POSTGRES_FTS_SEVI:
+                    fts_index.append(
+                        expr.ExprFuncCall(
+                            expr.ExprIdent("Index"),
+                            [
+                                expr.ExprConstant(
+                                    f"ix_{cls.db.table_name}_{get_property_name(prop)}_gin"
+                                ),
+                                expr.ExprFuncCall(
+                                    ident_manager.use("text"),
+                                    [
+                                        expr.ExprConstant(
+                                            f"to_tsvector('sevi', {get_property_name(prop)})"
+                                        )
+                                    ],
+                                ),
+                                PredefinedFn.keyword_assignment(
+                                    "postgresql_using", expr.ExprConstant("gin")
+                                ),
+                            ],
+                        )
+                    )
+                if prop.db.index_type == IndexType.POSTGRES_TRIGRAM:
+                    fts_index.append(
+                        expr.ExprFuncCall(
+                            expr.ExprIdent("Index"),
+                            [
+                                expr.ExprConstant(
+                                    f"ix_{cls.db.table_name}_{get_property_name(prop)}_gist"
+                                ),
+                                expr.ExprFuncCall(
+                                    expr.ExprIdent("text"),
+                                    [
+                                        expr.ExprConstant(
+                                            f"f_unaccent({get_property_name(prop)}) gist_trgm_ops(siglen=256)"
+                                        )
+                                    ],
+                                ),
+                                PredefinedFn.keyword_assignment(
+                                    "postgresql_using", expr.ExprConstant("gist")
+                                ),
+                            ],
+                        )
+                    )
+
             index_stmts.append(
                 stmt.DefClassVarStatement(
                     "_table_args__",
                     None,
                     PredefinedFn.tuple(
-                        [
+                        fts_index
+                        + [
                             expr.ExprFuncCall(
                                 expr.ExprIdent("Index"),
                                 [expr.ExprConstant(index.name)]
@@ -1317,7 +1383,8 @@ def make_python_relational_model(
                                 "unique", expr.ExprConstant(True)
                             )
                         )
-                    elif prop.db.is_indexed:
+                    elif prop.db.is_indexed and prop.db.index_type == IndexType.DEFAULT:
+                        # only add index=True for default index type
                         propvalargs.append(
                             PredefinedFn.keyword_assignment(
                                 "index", expr.ExprConstant(True)

sera/models/__init__.py

@@ -5,7 +5,13 @@ from sera.models._enum import Enum
 from sera.models._module import App, Module, Package
 from sera.models._multi_lingual_string import MultiLingualString
 from sera.models._parse import parse_schema
-from sera.models._property import Cardinality, DataProperty, ObjectProperty, Property
+from sera.models._property import (
+    Cardinality,
+    DataProperty,
+    IndexType,
+    ObjectProperty,
+    Property,
+)
 from sera.models._schema import Schema
 
 __all__ = [
@@ -14,6 +20,7 @@ __all__ = [
     "Property",
     "DataProperty",
     "ObjectProperty",
+    "IndexType",
     "Class",
     "Cardinality",
     "DataType",

sera/models/_class.py

@@ -5,7 +5,7 @@ from typing import Optional
 
 from sera.misc import to_kebab_case, to_snake_case
 from sera.models._multi_lingual_string import MultiLingualString
-from sera.models._property import DataProperty, ObjectProperty
+from sera.models._property import DataProperty, IndexType, ObjectProperty
 
 
 @dataclass(kw_only=True)
@@ -13,6 +13,7 @@ class Index:
     name: str
     columns: list[str]
     unique: bool = False
+    index_type: IndexType = IndexType.DEFAULT
 
 
 @dataclass(kw_only=True)

sera/models/_parse.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import re
 from copy import deepcopy
+from operator import index
 from pathlib import Path
 from typing import Sequence
 
@@ -30,6 +31,7 @@ from sera.models._property import (
     ForeignKeyOnDelete,
     ForeignKeyOnUpdate,
     GetSCPropValueFunc,
+    IndexType,
     ObjectPropDBInfo,
     ObjectProperty,
     PropDataAttrs,
@@ -150,7 +152,7 @@ def _parse_property(
 
     assert isinstance(prop, dict), prop
     if "datatype" in prop:
-        return DataProperty(
+        return_prop = DataProperty(
             name=prop_name,
             label=_parse_multi_lingual_string(prop.get("label", prop_name)),
             description=_parse_multi_lingual_string(prop.get("desc", "")),
@@ -164,6 +166,9 @@ def _parse_property(
                     is_indexed=db.get("is_indexed", False)
                     or db.get("is_unique", False)
                     or db.get("is_primary_key", False),
+                    index_type=(
+                        IndexType(db["index_type"]) if "index_type" in db else None
+                    ),
                     foreign_key=schema.classes.get(db.get("foreign_key")),
                 )
                 if "db" in prop
@@ -173,6 +178,10 @@ def _parse_property(
             default_value=_parse_default_value(prop.get("default_value", None)),
             default_factory=_parse_default_factory(prop.get("default_factory", None)),
         )
+        if return_prop.db is not None and return_prop.db.is_indexed:
+            if return_prop.db.index_type is None:
+                return_prop.db.index_type = IndexType.DEFAULT
+        return return_prop
 
     assert "target" in prop, prop
     return ObjectProperty(

sera/models/_property.py

@@ -16,6 +16,12 @@ if TYPE_CHECKING:
     from sera.models._class import Class
 
 
+class IndexType(str, Enum):
+    DEFAULT = "default"
+    POSTGRES_FTS_SEVI = "postgres_fts_sevi"
+    POSTGRES_TRIGRAM = "postgres_trigram"
+
+
 class ForeignKeyOnDelete(str, Enum):
     CASCADE = "cascade"
     SET_NULL = "set null"
@@ -150,6 +156,8 @@ class DataPropDBInfo:
     is_unique: bool = False
     # whether this property is indexed or not
     is_indexed: bool = False
+    # type of the index if it is indexed --- if is_indexed is True, this must be not None
+    index_type: Optional[IndexType] = None
     # this is used in conjunction with is_primary_key = True for the case of
     # extending a table with frequently updated properties. The value for the `foreign_key`
     # will be a target class. The cardinality is one-to-one, on target class deletion,

sera/typing.py

@@ -39,4 +39,5 @@ GLOBAL_IDENTS = {
     "UNSET": "sera.typing.UNSET",
     "ForeignKey": "sqlalchemy.ForeignKey",
     "Optional": "typing.Optional",
+    "text": "sqlalchemy.text",
 }

{sera_2-1.24.1.dist-info → sera_2-1.25.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: sera-2
-Version: 1.24.1
+Version: 1.25.0
 Summary: 
 Author: Binh Vu
 Author-email: bvu687@gmail.com
@@ -8,8 +8,9 @@ Requires-Python: >=3.12,<4.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: black (==25.1.0)
-Requires-Dist: codegen-2 (>=2.14.0,<3.0.0)
+Requires-Dist: codegen-2 (>=2.15.1,<3.0.0)
 Requires-Dist: graph-wrapper (>=1.7.3,<2.0.0)
 Requires-Dist: isort (==6.0.1)
 Requires-Dist: litestar (>=2.15.1,<3.0.0)

{sera_2-1.24.1.dist-info → sera_2-1.25.0.dist-info}/RECORD

@@ -7,7 +7,7 @@ sera/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sera/libs/api_helper.py,sha256=SkdgZkoYlX6HCyD_D6EiBvE0dWH0QW40xh4opIy737M,3237
 sera/libs/api_test_helper.py,sha256=SVGhl9UXPyCOw3IRK2znV6_c5tl1lTYYsb9qXJnJHoE,1344
 sera/libs/base_orm.py,sha256=5hOH_diUeaABm3cpE2-9u50VRqG1QW2osPQnvVHIhIA,3365
-sera/libs/base_service.py,sha256=wJ1x0xrxjczRU6Y43EjQb7RlkZ0tAY58tFayDXWBzLs,10773
+sera/libs/base_service.py,sha256=I3mPufl0A0HKKkmYbs_9X0651bkedks4dRQgNnSa7ns,14114
 sera/libs/directed_computing_graph/__init__.py,sha256=xiF5_I1y9HtQ-cyq02iwkRYgEZvxBB8YIvysCHCLBco,1290
 sera/libs/directed_computing_graph/_dcg.py,sha256=nQf9MhnTkFU2-dxhv_PFThz9L61Nn8cvYIOoMq0OVb8,15352
 sera/libs/directed_computing_graph/_edge.py,sha256=iBq6cpLWWyuD99QWTHVEh8naWUJrR4WJJuq5iuCrwHo,1026
@@ -19,12 +19,12 @@ sera/libs/directed_computing_graph/_type_conversion.py,sha256=_XGvDidOJVmHS4gqdP
 sera/libs/middlewares/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sera/libs/middlewares/auth.py,sha256=r6aix1ZBwxMd1Jv5hMCTB8a_gFOJQ6egvxIrf3DWEOs,2323
 sera/libs/middlewares/uscp.py,sha256=DRy99nmS3qS5HLjRMIGP0oNUtQIci_a4hL5xQh-lXNY,2322
-sera/libs/search_helper.py,sha256=G7wig3K-mT0xWtn1lkG_YffZq8SzlBkot3H42HUz1sw,13675
+sera/libs/search_helper.py,sha256=3ADhO6s6iIL3TWuH2zicAm_NmJfIm7wPY0cRQjESx2Y,13779
 sera/make/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sera/make/__main__.py,sha256=HRfOR53p351h6KblVvYm3DLhDIfEtk6R0kjl78_S_S8,1453
 sera/make/make_app.py,sha256=dTzpJRPGoCojCdJr1TAzwbaquctwwitrGPxkRm9skpo,6033
 sera/make/make_python_api.py,sha256=sfaYnX6Vwj0xWPpvpWbBhq8GyNHgNkvwlyZr7NDCO0k,24770
-sera/make/make_python_model.py,sha256=YHtZtHWN9X6s8IFiG-5w8sZSbsVWhU_sHtqVrkZkfgA,69686
+sera/make/make_python_model.py,sha256=FYqtzfVMfJxRPuEVhFuYCG5-DzVOS2aW_5RUtgiQY7Y,72685
 sera/make/make_python_services.py,sha256=SvdU--QF23FN_d-Ao07CeXIK9d5eL_-mdXYwXx69dRE,2102
 sera/make/make_typescript_model.py,sha256=Ax6xVrZ-x3xaKCsv6X4NKLdaMsAOau_GDrZK1TJ8NL8,57543
 sera/make/ts_frontend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -34,8 +34,8 @@ sera/make/ts_frontend/misc.py,sha256=R96Tm4Sz05YPVk-gJ8vGW_GbMoUM7sLStar1LVnEn-I
 sera/misc/__init__.py,sha256=ee7OODdWPKP8pCDxsfg1EOdxxKFMJCoNMljgV0NCxBw,767
 sera/misc/_formatter.py,sha256=aCGYL08l8f3aLODHxSocxBBwkRYEo3K1QzCDEn3suj0,1685
 sera/misc/_utils.py,sha256=_zWGjihxTEo3N1gzdxxB5CPCBBHxsviKWhQSCEonzVQ,13420
-sera/models/__init__.py,sha256=vJC5Kzo_N7wd16ocNPy1VvAZDGNiWeiAhWJ4ihATKvA,780
-sera/models/_class.py,sha256=_jLbewobv0Q6KLNd_jH-Z8vnDfwkRBbhdJbRDtKyE6g,2865
+sera/models/__init__.py,sha256=ZCnwzgUXhmHEcx5a7nYLopsBXJmSnfSkjt4kUH_vPnM,833
+sera/models/_class.py,sha256=495nP6j4pZIX1j0ZElUHNxe9doUX7vzSIwVrp_lAOHE,2922
 sera/models/_collection.py,sha256=72ZBdVa3UE8UyghJ8V-BPimO0f68zFRJvXdzfNaKDMo,2874
 sera/models/_constraints.py,sha256=SwUkvV8sESfuO3En6keA_r8GxKiarXYxMb5biml63lU,2021
 sera/models/_datatype.py,sha256=OlCnQlEpB4w9doz0hPhaZk5d4sF664CdGznSBvx0X9g,10293
@@ -43,10 +43,10 @@ sera/models/_default.py,sha256=ABggW6qdPR4ZDqIPJdJ0GCGQ-7kfsfZmQ_DchgZEa-I,137
 sera/models/_enum.py,sha256=FRtEfto2httDB308W8OAuHh2LSazV3v16DHixlz1IZA,2088
 sera/models/_module.py,sha256=r9_3nEbLaZKO-jvbZuFYy9DkM6E9jY8hBsBhQBULpVU,5428
 sera/models/_multi_lingual_string.py,sha256=JETN6k00VH4wrA4w5vAHMEJV8fp3SY9bJebskFTjQLA,1186
-sera/models/_parse.py,sha256=GL1a5HAlH4S389AyioLjz_9QFpH2rDTMOfXYXFC3fB4,12806
-sera/models/_property.py,sha256=9yMDxrmbyuF6-29lQjiq163Xzwbk75TlmGBpu0NLpkI,7485
+sera/models/_parse.py,sha256=nihBEDni-jKwecSBagd6bupDOuCzGviuoD9u6-rT64Q,13203
+sera/models/_property.py,sha256=Qo23KZl5OQNbuceygicgC3_Yv5ve1KxiFiMoWq6Ljj0,7758
 sera/models/_schema.py,sha256=VxJEiqgVvbXgcSUK4UW6JnRcggk4nsooVSE6MyXmfNY,1636
-sera/typing.py,sha256=m4rir-fB6Cgcm7_ZSXXcNdla2LJgq96WXxtTTrDaJno,1058
-sera_2-1.24.1.dist-info/METADATA,sha256=1DHbHN_a6VlNhVu8xrWJNig7IZCE2WzGvsRLGPnstGM,936
-sera_2-1.24.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-sera_2-1.24.1.dist-info/RECORD,,
+sera/typing.py,sha256=7S6Ah-Yfcl8a0xAJ4lxEcguq3rYVM7SBeOiuxyJhC04,1089
+sera_2-1.25.0.dist-info/METADATA,sha256=7NpIOo2Jk7y1K6hK81XLFOJ0oJiP765h1M5Jch841V0,987
+sera_2-1.25.0.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
+sera_2-1.25.0.dist-info/RECORD,,

{sera_2-1.24.1.dist-info → sera_2-1.25.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.0
 Root-Is-Purelib: true
 Tag: py3-none-any