sqlengine-lite 2.2.0__py3-none-any.whl → 2.2.1__py3-none-any.whl

sqlengine/__init__.py

@@ -1,7 +1,11 @@
-from .core import sqlgen, types
-from .core.types import Schema, Primary
+from ._internal import sqlgen
+from ._internal import ConnectionManager
+from ._internal.types import Schema, Primary, register_type
 from .sqltable import SqlTableMixin
 
 __author__  = "suffermuffin"
 
-__all__ = ["types", "sqlgen", "Schema", "SqlTableMixin", "Primary"]
+__all__ = [
+    "sqlgen", "Schema", "SqlTableMixin",
+    "Primary", "ConnectionManager", "register_type"
+]

sqlengine/_internal/__init__.py

@@ -0,0 +1,4 @@
+from .statements import Select, Delete, Update
+from .connection_manager import ConnectionManager
+
+__all__ = ["Select", "Delete", "Update", "ConnectionManager"]

sqlengine/{core/connection.py → _internal/connection_manager.py}

@@ -6,6 +6,7 @@ from typing import overload, Literal, Sequence
 from contextlib import contextmanager
 
 from .types import SqlValue, SqlRow
+from ..exceptions import TransactionError, NestedTransactionError, OutsideTransactionError
 
 
 logger = logging.getLogger("sqlengine")
@@ -14,6 +15,16 @@ logger.setLevel(os.getenv("SQL_ENGINE_LOG_LEVEL", "WARNING").upper())
 
 class ConnectionManager:
 
+    """ 
+    Connection manager for sqlite3 
+    
+    Args:
+
+        database (str): database filename to connect to. If `":memory:"` is passed, then database will be set in memory and you will have to
+            create table manually with `create_table()` method inside `transaction()` block.
+        **connection_params: Params to create connection with. Reference: https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
+    """
+
     _trans : sqlite3.Connection
     _trans_cursor : sqlite3.Cursor
 
@@ -172,7 +183,7 @@ class ConnectionManager:
     def open(self) -> None:
         """ Opens unmanaged transaction """
         if self.in_transaction():
-            raise RuntimeError("Can't re-open existing connection")
+            raise NestedTransactionError("Can't re-open existing connection")
         
         self._trans = self.connect()
         self._trans_cursor = self._trans.cursor()
@@ -184,7 +195,7 @@ class ConnectionManager:
             return
         
         if self._is_managed_transaction:
-            raise RuntimeError("Can't manually close managed transaction")
+            raise TransactionError("Can't manually close managed transaction")
         
         self._trans_cursor.close()
         self._trans.close()
@@ -194,14 +205,14 @@ class ConnectionManager:
 
     def commit(self) -> None:
         if not self.in_transaction():
-            raise RuntimeError("Can't commit outside transaction mode")
+            raise OutsideTransactionError("Can't commit outside transaction mode")
         
         self._trans.commit()
 
 
     def rollback(self) -> None:
         if not self.in_transaction():
-            raise RuntimeError("Can't rollback outside transaction mode")
+            raise OutsideTransactionError("Can't rollback outside transaction mode")
         
         self._trans.rollback()
     
@@ -242,7 +253,7 @@ class ConnectionManager:
     def tx_conn(self) -> sqlite3.Connection:
         """ Gives access to connection while in transaction """
         if not self.in_transaction():
-            raise RuntimeError("`tx_conn` is not available outside the transaction mode")
+            raise OutsideTransactionError("`tx_conn` is not available outside the transaction mode")
         return self._trans
     
     
@@ -250,6 +261,6 @@ class ConnectionManager:
     def tx_cursor(self) -> sqlite3.Cursor:
         """ Gives access to connection cursor while in transaction """
         if not self.in_transaction():
-            raise RuntimeError("`tx_cursor` is not available outside the transaction mode")
+            raise OutsideTransactionError("`tx_cursor` is not available outside the transaction mode")
         return self._trans_cursor
 

sqlengine/{core → _internal}/statements.py

@@ -2,10 +2,11 @@ from typing import Sequence, Literal, Generator, Self
 from abc    import ABC, abstractmethod
 
 from . import sqlgen as sql
-from .connection import ConnectionManager
+from .connection_manager import ConnectionManager
 
 from .types import SqlValue, SqlRow, Schema
 from .repr  import to_html
+from ..exceptions import SqlEngineError, OutsideTransactionError
 
 
 class Where[T : "Statement"]:
@@ -146,7 +147,7 @@ class Statement(ABC):
 
         self._tableschema = tableschema
         self._connection  = connection
-        self._where: Where[Self] = Where(self)
+        self._where = Where(self)
 
         self._custom_query : str | None = None
         self._custom_args  : tuple[SqlValue, ...] = ()
@@ -206,6 +207,7 @@ class Statement(ABC):
 class MutationalStatement(Statement, ABC):
     
     def execute(self) -> None:
+        """ Execute built statement """
         query, args = self.build()
         self._connection.execute(query, *args)
 
@@ -222,46 +224,52 @@ class Select(Statement):
 
 
     def __call__(self, *columns : str) -> Self:
+        """ Shortcut to columns selector """
         return self.columns(*columns)
     
 
     def columns(self, *columns : str) -> Self:
-        """ Column selector """
+        """ Columns selector """
         self._columns.extend(columns)
         return self
     
     
     def aggregate(self, by : Literal['COUNT', 'SUM', 'AVG', 'MIN', 'MAX']) -> Self:
-        
+        """ Aggregate by provided method """
         if self._aggregate:
-            raise ValueError("Can't aggregate columns multiple times")
+            raise SqlEngineError("Can't aggregate columns multiple times")
         
         self._aggregate = by
         return self
 
     
     def order_by(self, column : str, ascending : bool = True) -> Self:
+        """ Orders returned rows by provided column """
         order = "ASC" if ascending else "DESC"
         self._order_by.append(f"{column} {order}")
         return self
     
 
     def limit(self, n : int) -> Self:
+        """ Limit number of returned rows """
         self._limit = n
         return self
     
 
     def fetchone(self) -> SqlRow:
+        """ Fetch first row """
         query, args = self.build()
         return self._connection.fetchone(query, *args)
         
 
     def fetchmany(self, size : int = 1) -> list[SqlRow]:
+        """ Fetch first `size` rows """
         query, args = self.build()
         return self._connection.fetchmany(query, *args, size=size)
 
     
     def fetchall(self) -> list[SqlRow]:
+        """ Fetch all rows """
         query, args = self.build()
         return self._connection.fetchall(query, *args)
     
@@ -275,13 +283,15 @@ class Select(Statement):
 
         Examples:
 
-            >>> with table.transaction():
-            >>>     for batch in table.select.where.gt("Age", 30).then.fetchmany_iterator(1000):
-            >>>         process_batch(batch)
+        ```python
+        with table.transaction():
+            for batch in table.select.where.gt("Age", 30).then.fetchmany_iterator(1000):
+                process_batch(batch)
+        ```
         """
         if not self._connection.in_transaction():
-            raise RuntimeError("To use the `fetchall_iterator()` method you have \
-                    to keep open the transaction of the table with `transaction()` manager")
+            raise OutsideTransactionError("To use the `fetchall_iterator()` method you have \
+                    to keep open the transaction of the table")
         
         query, exec_args = self.build()
 
@@ -293,11 +303,22 @@ class Select(Statement):
 
     
     def __iter__(self) -> Generator[SqlRow, None, None]:
-        """ Select statement rows iterator """
+        """ 
+        Select statement rows iterator
+
+        Examples:
+        
+        ```python
+        with table.transaction():
+            # here `then` is used to link back to the `select` instance from `where` object
+            for row in table.select.where.gt("Age", 30).then: 
+                process_row(row)
+        ```
+        """
         
         if not self._connection.in_transaction():
-            raise RuntimeError("To use the __iter__ method you have \
-                to keep open the transaction of the table with `transaction()` manager")
+            raise OutsideTransactionError("To use the __iter__ method you have \
+                to keep open the transaction of the table")
         
         query, exec_args = self.build()
         
@@ -360,7 +381,7 @@ class Delete(MutationalStatement):
     def _build(self, where_clause : str, *args : SqlValue) -> tuple[str, tuple[SqlValue, ...]]:
         
         if not where_clause:
-            raise ValueError("Delete statement must have a where clause")
+            raise SqlEngineError("Delete statement must have a where clause")
         
         query = sql.delete_rows(self._tableschema["tablename"], where_clause)
         return query, args
@@ -379,6 +400,7 @@ class Update(MutationalStatement):
 
     
     def __call__(self, column : str, value : SqlValue) -> Self:
+        """ Shortcut to set value to a column """
         return self.set(column, value)
 
 

sqlengine/{core → _internal}/types.py

@@ -12,15 +12,16 @@ class CustomType(Protocol):
         ...
 
 
-type SqlValue = str | int | float | bytes | None | CustomType
-type SqlRow   = tuple[SqlValue, ...]
-type SqlType  = type[str | int | float | bytes | CustomType]
+type SqlValue   = str | int | float | bytes | None | CustomType
+type SqlRow     = tuple[SqlValue, ...]
+type SqlType    = type[str | int | float | bytes | CustomType]
+type ColumnType = SqlType | str | UnionType
 
 
 class Schema(TypedDict):
     tablename : str
     columns   : list[str]
-    types     : list[SqlType | str]
+    types     : list[ColumnType]
     primary   : list[str]
 
 
@@ -41,7 +42,7 @@ _TYPES_MAP : dict[type | UnionType, str] = {
 }
 
 
-def is_custom_type(type_: SqlType) -> TypeGuard[type[CustomType]]:
+def is_custom_type(type_: SqlType | UnionType) -> TypeGuard[type[CustomType]]:
     return (
         type_ not in (str, int, float, bytes)
         and isinstance(type_, type)
@@ -52,7 +53,7 @@ def is_custom_type(type_: SqlType) -> TypeGuard[type[CustomType]]:
 
 def register_type(cls : type[CustomType], type_name : str | None = None) -> None:
     """ 
-    Register custom type to be able to store it in tables 
+    Register custom type into sqlite3 to be able to store it in tables
     
     Args:
         cls (CustomType): Class that implements `from_sql(cls, sql : bytes) -> Self` and `
@@ -65,7 +66,7 @@ def register_type(cls : type[CustomType], type_name : str | None = None) -> None
     sqlite3.register_converter(type_name, cls.from_sql)
 
 
-def pytype_to_sqltype(type_ : type) -> str:
+def pytype_to_sqltype(type_ : type | UnionType) -> str:
     """ Converts python type to sql type """
     if type_ not in _TYPES_MAP:
         raise TypeError(f"{type_} is not natively supported by sqlite3")
@@ -73,7 +74,7 @@ def pytype_to_sqltype(type_ : type) -> str:
     return _TYPES_MAP[type_]
 
 
-def register_resolve_types(types : list[SqlType | str], **connection_params) -> tuple[list[str], dict[str, Any]]:
+def register_resolve_types(types : list[ColumnType], **connection_params) -> tuple[list[str], dict[str, Any]]:
     """ Converts py types to sql types, registers custom types, resolves type names, updates connection params """
 
     resolved : list[str] = []

sqlengine/exceptions.py

@@ -0,0 +1,20 @@
+
+class SqlEngineError(Exception):
+    """ Errors linked to sqlengine module """
+    pass
+
+class TransactionError(SqlEngineError):
+    """ Errors within transactions """
+    pass
+
+class OutsideTransactionError(TransactionError):
+    """ Errors of prohibited outside tranasctions operations """
+    pass
+
+class NestedTransactionError(TransactionError):
+    """ Errors of nested transaction operations """
+    pass
+
+class TableDeclarationError(SqlEngineError):
+    """ Errors of table declaration """
+    pass

sqlengine/schema.py

@@ -3,7 +3,7 @@ import sqlite3
 from typing import overload
 
 from .sqltable import SqlTableMixin
-from .core.types import Schema
+from ._internal.types import Schema
 
 
 def get_database_tablenames(database : str, cursor : sqlite3.Cursor | None = None) -> list[str]:

sqlengine/sqltable.py

@@ -5,13 +5,15 @@ import sqlite3
 from typing import Sequence, Literal, Any
 from typing import get_origin, get_args, overload, get_type_hints
 
-from .core import sqlgen as sql
-from .core.repr import to_html
+from .exceptions import TableDeclarationError
 
-from .core import ConnectionManager, Select, Update, Delete
-from .core.types import SqlRow, SqlValue, SqlType
-from .core.types import Schema, Primary
-from .core.types import register_resolve_types
+from ._internal import sqlgen as sql
+from ._internal.repr import to_html
+
+from ._internal import ConnectionManager, Select, Update, Delete
+from ._internal.types import SqlRow, SqlValue, ColumnType
+from ._internal.types import Schema, Primary
+from ._internal.types import register_resolve_types
 
 logger = logging.getLogger("sqlengine")
 logger.setLevel(os.getenv("SQL_ENGINE_LOG_LEVEL", "WARNING").upper())
@@ -23,50 +25,44 @@ class SqlTableMixin:
     
     Args:
         database (str): database filename to connect to. If it not exists - will create new one first.
-            If `":memory:"` is passed, then database will be created in memory and you will have to
+            If `":memory:"` is passed, then database will be set in memory and you will have to
             create table manually with `create_table()` method inside `transaction()` block.
         force_drop (bool): If `True` - will drop existing table.
-        **connection_params (dict): Params to create connection with. 
-            Reference: https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
+        **connection_params: Params to create connection with. Reference: https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
 
     Attributes:
         __tablename__ (Optional[str]): Name of the table that will be used in queries. 
             If omitted in inherited class declaration, then it will take the class name.
-        __columns__ (list[str]): Colum names of the table
-        __types__ (list[SqlType | str]): Colum types of the table
+        __columns__ (list[str]): Column names of the table
+        __types__ (list[ColumnType]): Column types of the table
         __primary__ (list[str]): List of primary keys
 
     Examples:
-        >>> from sqlengine import SqlTableMixin, Primary
-        >>>
-        >>> class Employees(SqlTableMixin):
-        >>>     __columns__   = ["ID", "name", "surname", "salary", "position"]
-        >>>     __types__     = [int, str, str, float, "TEXT NOT NULL"]
-        >>>     __primary__   = ["ID", "name"]
-        >>> 
-        >>> table = Employees(":memory:")
-        >>>
-        >>>
-        >>> class Employees(SqlTableMixin):
-        >>>     ID       : Primary[int]
-        >>>     name     : Primary[str]
-        >>>     surname  : str | None
-        >>>     salary   : float | None
-        >>>     position : str
-        >>> 
-        >>> table = Employees("mydb.sqlite3")
+    ```python
+    from sqlengine import SqlTableMixin, Primary
+    
+    class Employees(SqlTableMixin):
+        ID       : Primary[int]
+        name     : Primary[str]
+        surname  : str   | None
+        salary   : float | None
+        position : str
+        
+    table = Employees("mydb.sqlite3")
+    ```
     """
 
     __tablename__ : str
     __columns__   : list[str]
-    __types__     : list[SqlType | str]
+    __types__     : list[ColumnType]
     __primary__   : list[str]
 
     def __init__(self, database: str | Literal[":memory:"], force_drop : bool = False, **connection_params) -> None:
         
+        self.database = database
+
         resolved_types, connection_params = register_resolve_types(self.__types__, **connection_params)
 
-        self.database            = database
         self._connection_manager = ConnectionManager(database, **connection_params)
         self.__types_sql__       = resolved_types
 
@@ -90,7 +86,7 @@ class SqlTableMixin:
                 continue
             
             if name in columns:
-                raise AttributeError(f"Annotated column `{name}` is already in __columns__")
+                raise TableDeclarationError(f"Annotated column `{name}` is already in __columns__")
             
             columns.append(name)
             
@@ -99,12 +95,13 @@ class SqlTableMixin:
                 continue
 
             if name in primary:
-                raise AttributeError(f"Annotated primary column `{name}` is already in __primary__")
+                raise TableDeclarationError(f"Annotated primary column `{name}` is already in __primary__")
             
             primary_type = get_args(type_)[0]
 
             if not primary_type:
-                raise AttributeError("Primary type was declared without the type. Usage: `my_column : Primary[T]`, where T is desired type")
+                raise TableDeclarationError(("Primary type was declared without the type. "
+                "Usage: `my_column : Primary[T]`, where T is desired type"))
             
             types.append(primary_type)
             primary.append(name)
@@ -128,15 +125,15 @@ class SqlTableMixin:
         ]
 
         if missing_attrs:
-            raise AttributeError(f'{cls.__name__} is missing attributes: {missing_attrs}')
+            raise TableDeclarationError(f'{cls.__name__} is missing attributes: {missing_attrs}')
         
         n_types, n_cols = len(cls.__types__), len(cls.__columns__)
 
         if not n_types == n_cols:
-            raise AttributeError(f'`__types__` and `__columns__`: length mismatch: types = {n_types}, columns = {n_cols}')
+            raise TableDeclarationError((f'`__types__` and `__columns__`: length mismatch: types = {n_types}, columns = {n_cols}'))
         
         if len(cls.__primary__) < 1:
-            raise AttributeError(f'`__primary__`: Number of primary keys must be at least 1')
+            raise TableDeclarationError(f'`__primary__`: Number of primary keys must be at least 1')
         
         wrong_primaries = [
             prim for prim in cls.__primary__ if
@@ -144,7 +141,7 @@ class SqlTableMixin:
         ]
 
         if wrong_primaries:
-            raise AttributeError(f'`__primary__`: Keys {wrong_primaries} can\'t be primaries as they are not declared in __columns__')
+            raise TableDeclarationError(f'`__primary__`: Keys {wrong_primaries} can\'t be primaries as they are not declared in __columns__')
         
 
     def _write_db(self, force_drop : bool) -> None:
@@ -193,10 +190,12 @@ class SqlTableMixin:
             autocommit (bool): If `True`, will commit changes at the end of transaction
         
         Examples:
-
-            >>> with table.transaction():
-            >>>     for idx, age in table.select("ID", "Age"):
-            >>>         table.update("Age", age + 1).where.eq("ID", idx).then.execute()
+            
+        ```python
+        with table.transaction():
+            for idx, age in table.select("ID", "Age"):
+                table.update("Age", age + 1).where.eq("ID", idx).then.execute()
+        ```
         """
         
         return self._connection_manager.transaction(autocommit)
@@ -207,17 +206,22 @@ class SqlTableMixin:
         Insert single row
 
         Args:
-            *args (Any): Arguments in order of declared __columns__
-            **kwargs (Any): Unused
+            *args (SqlValue): Arguments in order of declared __columns__
+            **kwargs (SqlValue): Column to value mapping
 
-        Example:
-            >>> table = MyTable("mydb.db")
-            >>> table.columns 
-            >>> # ["ID", "Name", "Age"]
-            >>> table.insert(0, "Daniel", 27)
+        Examples:
+
+        ```python
+        table = MyTable("mydb.db")
+        table.columns # -> ["ID", "Name", "Age"]
+        table.insert(0, "Daniel", 27)
+        table.insert(ID=1, name="Boris", age=26)
+        ```
         """
-        query = sql.insert_row(self.tablename, self.columns)
-        self._connection_manager.execute(query, *args)
+        columns = self.columns[:len(args)]
+        columns.extend(kwargs.keys())
+        query = sql.insert_row(self.tablename, columns)
+        self._connection_manager.execute(query, *args, *kwargs.values())
 
     
     def upsert(self, *args, **kwargs) -> None:
@@ -226,18 +230,22 @@ class SqlTableMixin:
         via the declared `primary` key
         
         Args:
-            *args (Any): Arguments in order of declared __columns__
-            **kwargs (Any): Unused
+            **args (SqlValue): Arguments in order of declared __columns__
+            **kwargs (SqlValue): Column to value mapping
 
         Example:
-            >>> table = MyTable("mydb.db")
-            >>> table.columns 
-            >>> # ["ID", "Name", "Age"]
-            >>> table.upsert(0, "Daniel", 27)
-            >>> table.upsert(0, "Daniel", 21)
+
+        ```python
+        table = MyTable("mydb.db")
+        table.columns # -> ["ID", "Name", "Age"]
+        table.upsert(0, "Daniel", 27)
+        table.upsert(ID=0, age=21)
+        ```
         """
-        query = sql.upsert(self.tablename, self.columns, self.primary)
-        self._connection_manager.execute(query, *args)
+        columns = self.columns[:len(args)]
+        columns.extend(kwargs.keys())
+        query = sql.upsert(self.tablename, columns, self.primary)
+        self._connection_manager.execute(query, *args, *kwargs.values())
 
     
     def insert_many(self, rows: Sequence[SqlRow]) -> None:
@@ -386,19 +394,43 @@ class SqlTableMixin:
 
     @property
     def update(self) -> Update:
-        """ UPDATE statement builder and executor """
+        """ 
+        UPDATE statement builder and executor 
+        
+        Examples:
+        
+        ```python
+        table.update.set("City", "Karaganda").where.eq("Country", "Czech Republic").then.execute()
+        ```
+        """
         return Update(self.conn, self.schema)
 
     
     @property
     def delete(self) -> Delete:
-        """ DELETE statement builder and executor """
+        """ 
+        DELETE statement builder and executor 
+        
+        Examples:
+        
+        ```python
+        table.delete.where.eq("ID", 0).then.execute()
+        ```
+        """
         return Delete(self.conn, self.schema)
     
 
     @property
     def select(self) -> Select:
-        """ SELECT statement builder and fetcher """
+        """ 
+        SELECT statement builder and fetcher
+
+        Examples:
+        
+        ```python
+        table.select("Email").where.eq("SupportRepId", 3).then.aggregate("COUNT").fetchone()
+        ```
+        """
         return Select(self.conn, self.schema)
 
 
@@ -409,7 +441,7 @@ class SqlTableMixin:
 
     
     @property
-    def types(self) -> list[SqlType | str]:
+    def types(self) -> list[ColumnType]:
         """ List of table column dtypes as declared"""
         return self.__types__
     
@@ -452,7 +484,7 @@ class SqlTableMixin:
 
     @property
     def conn(self) -> ConnectionManager:
-        """  Access connection manager instance """
+        """ Access connection manager instance """
         return self._connection_manager
     
 
@@ -476,4 +508,4 @@ class SqlTableMixin:
     
     @connection_params.setter
     def connection_params(self, value : dict[str, Any]) -> None:
-        self._connection_manager.connection_params = value
+        self._connection_manager.connection_params = value

sqlengine/utils/__init__.py

@@ -1,4 +1,4 @@
 from .connection import shared_connection
-from .convert    import to_csv
+from .convert    import to_csv, to_dicts, to_dicts_stream
 
-__all__ = ["shared_connection", "to_csv"]
+__all__ = ["shared_connection", "to_csv", "to_dicts", "to_dicts_stream"]

sqlengine/utils/connection.py

@@ -4,8 +4,9 @@ import sqlite3
 
 from contextlib import contextmanager
 
-from ..sqltable import SqlTableMixin
-from ..core.connection import ConnectionManager
+from ..sqltable   import SqlTableMixin
+from .._internal  import ConnectionManager
+from ..exceptions import NestedTransactionError
 
 
 logger = logging.getLogger("sqlengine")
@@ -26,11 +27,13 @@ def shared_connection(*args : SqlTableMixin, autocommit : bool = True, **connect
     
     Examples:
 
-        >>> from sqlengine.utils import shared_connection
-        >>> with shared_connection(table1, table2, **table1.connection_params):
-        >>>     for (id1,), (id2, temp) in zip(table1.select("ID").limit(20), table2.select("ID", "Temperature").limit(20)):
-        >>>         if id1 == id2:
-        >>>             table.update.where.eq("ID", id2).then.set("Salary", temp).execute()
+    ```python
+    from sqlengine.utils import shared_connection
+    with shared_connection(table1, table2, **table1.connection_params):
+        for (id1,), (id2, temp) in zip(table1.select("ID").limit(20), table2.select("ID", "Temperature").limit(20)):
+            if id1 == id2:
+                table.update.where.eq("ID", id2).then.set("Salary", temp).execute()
+    ```
     """
 
     tables_in_trans = [
@@ -38,7 +41,7 @@ def shared_connection(*args : SqlTableMixin, autocommit : bool = True, **connect
     ]
     
     if tables_in_trans:
-        raise RuntimeError(f"Tables {tables_in_trans} are already in transaction")
+        raise NestedTransactionError(f"Tables {tables_in_trans} are already in transaction")
     
     unique_databases = set(table.database for table in args)
     database_map : dict[str, list[ConnectionManager]] = {}

sqlengine/utils/convert.py

@@ -1,24 +1,129 @@
 import csv
+from typing import Generator
 
 from ..sqltable import SqlTableMixin
-from ..core.statements import Where, Select
+from .._internal.statements import Where, Select
+from .._internal.types import SqlValue
 
 
-def to_csv(builder : Select | Where[Select] | SqlTableMixin, path : str) -> None:
+def _get_select(builder : Select | Where[Select] | SqlTableMixin) -> Select:
     
     match builder:
         case Where():
-            builder = builder.then
+            return builder.then
         case SqlTableMixin():
-            builder = builder.select
+            return builder.select
+        case Select():
+            return builder
+        case _:
+            raise TypeError(f"Unexpected builder type {type(builder)}")
+
+
+def to_csv(
+        builder : Select | Where[Select] | SqlTableMixin, 
+        filename : str, 
+        stream_batch_size : int | None = None
+    ) -> None:
+    """ 
+    Writes query or whole table to a csv file
+    
+    Args:
+        builder (Select | Where[Select] | SqlTableMixin): Object to convert to csv
+        filename (str): Path to write to
+        stream_bach_size (int | None): If not None or 0, will stream all rows to csv in batches of provided size
+    """
+    
+    builder = _get_select(builder)
 
     if builder._aggregate:
         raise AssertionError("Aggregated queries are not supported")
     
-    columns   = builder._resolve_columns()
-    repr_rows = builder.fetchall()
-
-    with open(path, 'w', newline='') as file:
+    if stream_batch_size and not builder._connection.in_transaction():
+        raise RuntimeError("To stream to csv you have to keep open the `transaction`")
+    
+    columns = builder._resolve_columns()
+    
+    with open(filename, 'w', newline='') as file:
         writer = csv.writer(file)
         writer.writerow(columns)
-        writer.writerows(repr_rows)
+        
+        if not stream_batch_size:
+            repr_rows = builder.fetchall()
+            writer.writerows(repr_rows)
+            return
+        
+        for rows in builder.fetchmany_iterator(stream_batch_size):
+            writer.writerows(rows)
+
+
+def to_dicts(builder : Select | Where[Select] | SqlTableMixin) -> list[dict[str, SqlValue]]:
+    """ 
+    Converts query or whole table to pandas friendly format
+
+    Args:
+        builder (Select | Where[Select] | SqlTableMixin): Object to convert to list of dicts
+
+    Returns:
+        out (list[dict[str, SqlValue]]): list of rows mappings
+    
+    Examples:
+
+    ```python
+    import pandas as pd
+    
+    df = pd.DataFrame(to_dict(table))
+    ```
+    """
+    
+    builder = _get_select(builder)
+
+    if builder._aggregate:
+        raise AssertionError("Aggregated queries are not supported")
+    
+    columns = builder._resolve_columns()
+    rows = builder.fetchall()
+
+    return [dict(zip(columns, row)) for row in rows]
+
+
+def to_dicts_stream(
+        builder : Select | Where[Select] | SqlTableMixin, 
+        batch_size : int
+    ) -> Generator[list[dict[str, SqlValue]], None, None]:
+    """
+    Converts query or whole table to pandas friendly format and yields it in batches
+
+    Args:
+        builder (Select | Where[Select] | SqlTableMixin): Object to convert to list of dicts
+        batch_size (int): Size of each yiedled batch
+
+    Yields:
+        batch (list[dict[str, SqlValue]]): list of rows mappings
+    
+    Examples:
+
+    ```python
+    import pandas as pd
+    
+    df = pd.DataFrame(columns=table.columns)
+    
+    with table.transaction():
+        for batch in to_dict_stream(table, 100):
+            df = pd.concat([df, pd.DataFrame(batch)], axis=0)
+    
+    df.set_index("ID", inplace=True)
+    ```
+    """
+    
+    builder = _get_select(builder)
+
+    if builder._aggregate:
+        raise AssertionError("Aggregated queries are not supported")
+    
+    if not builder._connection.in_transaction():
+        raise RuntimeError("To stream to csv you have to keep open the `transaction`")
+    
+    columns = builder._resolve_columns()
+
+    for rows in builder.fetchmany_iterator(batch_size):
+        yield [dict(zip(columns, row)) for row in rows]

{sqlengine_lite-2.2.0.dist-info → sqlengine_lite-2.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlengine-lite
-Version: 2.2.0
+Version: 2.2.1
 Summary: Cute sqlite3 wrapper for sql tables
 Project-URL: Homepage, https://github.com/suffermuffin/SQL-Engine
 Project-URL: Repository, https://github.com/suffermuffin/SQL-Engine.git
@@ -27,6 +27,7 @@ Dynamic: license-file
   - [Get Item](#get-item)
   - [Custom Types](#custom-types)
   - [Csv Converter](#csv-converter)
+  - [Pandas-like Converter](#pandas-like-converter)
 - [Full Documentation](#full-documentation)
 
 
@@ -127,25 +128,19 @@ More details at [Declaration](https://github.com/suffermuffin/SQL-Engine/blob/ma
 from sqlengine import SqlTableMixin, Primary
 
 # Helper constants for column names
-ID = "ID"
-Name = "Name"
+ID         = "ID"
+Name       = "Name"
 Occupation = "Occupation"
-Salary = "Salary"
+Salary     = "Salary"
 
 
 class Employees(SqlTableMixin):
 
     ID         : Primary[int]
-    Name       : str
+    Name       : str | None
     Occupation : str
     Salary     : float
 
-    # You may overwrite your insert methods for type consistency
-    def insert(self, id : int, name : str, occupation : str, salary : float) -> None:
-        return super().insert(id, name, occupation, salary)
-    
-    def upsert(self, id : int, name : str, occupation : str, salary : float) -> None:
-        return super().upsert(id, name, occupation, salary)
 ```
 
 ## Instantiation
@@ -168,11 +163,16 @@ table = Employees("temp/data.db", force_drop=True)
 table.insert(1, "John Doe", "Software Engineer", 75000.0)
 ```
 
+```py
+# Use kwargs mapping to insert/upsert one row
+
+table.insert(2, salary=80000.0, name="Jane Smith", occupation="Data Scientist")
+```
+
 ```py
 # Bulk insert multiple rows
 
 employees_data = [
-    (2, "Jane Smith", "Data Scientist", 80000.0),
     (3, "Alice Johnson", "Product Manager", 90000.0),
     (4, "Bob Brown", "Project Manager", 78000.0),
     (5, "Charlie Davis", "UI/UX Designer", 65000.0),
@@ -322,10 +322,40 @@ to_csv(table, "temp/table.csv")
 
 ```py
 # Save query result to csv
-
 to_csv(table.select.where.gt(Salary, 70_000), "temp/query.csv")
 ```
 
+```py
+# Stream to csv
+with table.transaction():
+    to_csv(table, "temp/query.csv", stream_batch_size=1000)
+```
+
+## Pandas-like Converter
+
+```py
+# via one shot
+import pandas as pd
+from sqlengine.utils import to_dicts
+
+df = pd.DataFrame(to_dicts(table))
+df.set_index("ID", inplace=True)
+```
+
+```py
+# via generator
+import pandas as pd
+from sqlengine.utils import to_dicts_stream
+
+df = pd.DataFrame(columns=table.columns)
+
+with table.transaction():
+    for batch in to_dicts_stream(table, batch_size=1000):
+        df = pd.concat([df, pd.DataFrame(batch)], axis=0)
+
+df.set_index("ID", inplace=True)
+```
+
 # Full Documentation
 
 For detailed usage, API reference, and advanced examples, see the [full documentation](https://github.com/suffermuffin/SQL-Engine/blob/main/docs/index.md).

sqlengine_lite-2.2.1.dist-info/RECORD

@@ -0,0 +1,18 @@
+sqlengine/__init__.py,sha256=_t3mutixVp60g1MwD6BZ6WcvXZPFz7JkV29nXXl1bOw,305
+sqlengine/exceptions.py,sha256=Sd12r8dHgtKr6RKUaiHNhaEpdJr3Jr-JXePwrBQojC4,506
+sqlengine/schema.py,sha256=HUk11gqOSzQTdiKBYU65EEQ52iCbUj3fxv3jigi1tG8,3692
+sqlengine/sqltable.py,sha256=FOwXyTghcIxF8vhxMgzFeF6wAZNRjVt6r7Ebhqs-iJ0,16198
+sqlengine/_internal/__init__.py,sha256=9cQIPDgLssRJNeROpuDYA0rJzY9B_N9VsY4rJbgzBnk,159
+sqlengine/_internal/connection_manager.py,sha256=2ooDsAUvSfXTBpUJtNDS2GIO96tJo1GnE_R7dRBVflY,8466
+sqlengine/_internal/repr.py,sha256=PEwC3MdOoTE8ORvcVXTCmY502boJfr5V2LU5jJ6qWG0,1478
+sqlengine/_internal/sqlgen.py,sha256=b8usEqBAX5JDEdMR6Xeaona0ZuOi7vtptl858qXhD78,3473
+sqlengine/_internal/statements.py,sha256=ntJrOeomPW15KowXUbnma9_ah-H5_ZmWKjemCIsf3Rw,12395
+sqlengine/_internal/types.py,sha256=bA-tsS8r13pvRB3M13wFNsd3Fc9KAaqGW47aXhkKkDQ,3265
+sqlengine/utils/__init__.py,sha256=EGg576qs61VsXlXd1bxF0oDytlmpcgqouVLdPxwqTrI,173
+sqlengine/utils/connection.py,sha256=H86Y2PUJcTIc9iD398MJoRR-IyMYSVjOSfu_bk9UuoM,3119
+sqlengine/utils/convert.py,sha256=Il5KAbc0Mbe2SIxBWyvj9rJ68XTJmpDFboJc1vifaaI,3694
+sqlengine_lite-2.2.1.dist-info/licenses/LICENSE,sha256=aepvve4t1ho5uHMQG88dErp5L5CgoNAXctIjMdoceqY,1069
+sqlengine_lite-2.2.1.dist-info/METADATA,sha256=XAAXT6IKmRIUkYtVkqij11U1W5UMZNwjfGg9sgbT81w,20593
+sqlengine_lite-2.2.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+sqlengine_lite-2.2.1.dist-info/top_level.txt,sha256=KG_FG0LCB_mIEu2qJ5LGLQMpan4QmF5d8xLEcqTghCE,10
+sqlengine_lite-2.2.1.dist-info/RECORD,,

sqlengine/core/__init__.py

@@ -1,4 +0,0 @@
-from .statements import Select, Delete, Update
-from .connection import ConnectionManager
-
-__all__ = ["Select", "Delete", "Update", "ConnectionManager"]

sqlengine_lite-2.2.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-sqlengine/__init__.py,sha256=CxA4Z-rx9RfDNBntCbY7wQ7IQGxKc9bcBjHY9g7T8YQ,207
-sqlengine/schema.py,sha256=2PW1RsBgZejozS3xIz63Mu80tSKpJ7pcKyWt8mgWgKA,3687
-sqlengine/sqltable.py,sha256=Oa0CPqBbYZT5TsMK53gRrxIo4vFfhsbLnOYEDgsTUXQ,15652
-sqlengine/core/__init__.py,sha256=1HPCmL4ca9WLFVsgZVtEEirYUF7PwoEGm4G9yrBIiX4,151
-sqlengine/core/connection.py,sha256=Fz9_P8-xKggAEOSk-7Dlqm2so-qXXHyi8yIiCjGBNI8,7876
-sqlengine/core/repr.py,sha256=PEwC3MdOoTE8ORvcVXTCmY502boJfr5V2LU5jJ6qWG0,1478
-sqlengine/core/sqlgen.py,sha256=b8usEqBAX5JDEdMR6Xeaona0ZuOi7vtptl858qXhD78,3473
-sqlengine/core/statements.py,sha256=mAvaM3poFDEWl6cu8FemeCLpRDFGwqM-QKGQym446xM,11683
-sqlengine/core/types.py,sha256=o-P9o4KRjWeubrwDdkEKsSaU7KVEBxazHY6BGaNG2Xw,3185
-sqlengine/utils/__init__.py,sha256=2D55vx-Gzlz0Yjl09zVn3S0tPh3NjbTHbrqw3BZbGL4,115
-sqlengine/utils/connection.py,sha256=bNGMN360FUoxscWv_-YuJA2UEQ0ZQLwjrSwBINXhFmw,3082
-sqlengine/utils/convert.py,sha256=aVpGy0ZC25ZTdeu36gbHxwzdP6je-N8bPVcFrq6HZms,654
-sqlengine_lite-2.2.0.dist-info/licenses/LICENSE,sha256=aepvve4t1ho5uHMQG88dErp5L5CgoNAXctIjMdoceqY,1069
-sqlengine_lite-2.2.0.dist-info/METADATA,sha256=i77esz2My03c_7zqcNYnqvR7CEa9zhQJOrzQjCZUfcs,20190
-sqlengine_lite-2.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-sqlengine_lite-2.2.0.dist-info/top_level.txt,sha256=KG_FG0LCB_mIEu2qJ5LGLQMpan4QmF5d8xLEcqTghCE,10
-sqlengine_lite-2.2.0.dist-info/RECORD,,