myrtille 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

myrtille/lib/cfg.py

@@ -0,0 +1,12 @@
+import datetime
+import pydantic
+
+
+class Database(pydantic.BaseModel):
+    user: str
+    password: str
+    host: str
+    port: int
+    echo: bool | None = None
+    pool_size: int | None = None
+    timeout: datetime.timedelta | None = None

myrtille/lib/db.py

@@ -0,0 +1,120 @@
+import contextlib
+import time
+import typing
+
+import asyncmy
+import pydantic
+
+from myrtille.lib import cfg
+
+ParamsType: typing.TypeAlias = typing.Collection[typing.Any]
+Querriable: typing.TypeAlias = 'Database | Connection'
+
+
+def _format_request(stmt: str, *, params: ParamsType | None = None):
+    if params is not None:
+        try:
+            stmt = stmt % tuple(params)
+        except Exception:
+            if len(params) == 0:
+                param_part = 'no params'
+            else:
+                param_part = f'params {", ".join(map(repr, params))}'
+            stmt = f"Invalid stmt '{stmt}' with {param_part}"
+    return f"Request '{' '.join(stmt.split())}'"
+
+
+class Database(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
+
+    config: cfg.Database
+    pool: asyncmy.pool.Pool = pydantic.Field(exclude=True)
+
+    @contextlib.asynccontextmanager
+    async def acquire(self):
+        async with self.pool.acquire() as cnx:
+            yield Connection(database=self, cnx=cnx)
+            await cnx.rollback()
+
+    async def execute(self, stmt: str, *, params: ParamsType | None = None):
+        async with self.acquire() as cnx:
+            await cnx.execute(stmt, params=params)
+            await cnx.commit()
+
+    async def fetch_all(self, stmt: str, *, params: typing.Sequence[typing.Any] | None = None):
+        async with self.acquire() as cnx:
+            return await cnx.fetch_all(stmt, params=params)
+
+    async def fetch_optional(self, stmt: str, *, params: typing.Sequence[typing.Any] | None = None):
+        async with self.acquire() as cnx:
+            return await cnx.fetch_optional(stmt, params=params)
+
+
+class Connection(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
+
+    database: Database
+    cnx: asyncmy.connection.Connection
+
+    async def execute(self, stmt: str, *, params: ParamsType | None = None):
+        t0 = time.perf_counter()
+        try:
+            async with self.cnx.cursor() as cursor:
+                await cursor.execute(stmt, params)
+                if self.database.config.echo:
+                    print(
+                        f'Log: {_format_request(stmt, params=params)}: {time.perf_counter() - t0:2.2f}s'
+                    )
+        except Exception as e:
+            e.add_note(f'In {_format_request(stmt, params=params)}')
+            raise
+
+    async def fetch_all(self, stmt: str, params: typing.Sequence[typing.Any] | None = None):
+        t0 = time.perf_counter()
+        try:
+            async with self.cnx.cursor() as cursor:
+                await cursor.execute(stmt, params)
+                rows = await cursor.fetchall()
+                if self.database.config.echo:
+                    print(
+                        f'Log: {_format_request(stmt, params=params)}: {time.perf_counter() - t0:2.2f}s'
+                    )
+                return rows
+        except Exception as e:
+            e.add_note(f'In {_format_request(stmt, params=params)}')
+            raise
+
+    async def fetch_optional(self, stmt: str, params: typing.Sequence[typing.Any] | None = None):
+        rows = await self.fetch_all(stmt, params)
+        if len(rows) > 1:
+            raise Exception(
+                f'{_format_request(stmt, params=params)} returned {len(rows)} (!= 1) rows'
+            )
+        elif len(rows) == 0:
+            return None
+        return rows[0]
+
+    async def commit(self):
+        await self.cnx.commit()
+
+
+@contextlib.asynccontextmanager
+async def make_database(db_config: cfg.Database):
+    async with asyncmy.pool.create_pool(
+        host=db_config.host,
+        port=db_config.port,
+        user=db_config.user,
+        password=db_config.password,
+        autocommit=False,
+        echo=db_config.echo or False,
+        connect_timeout=db_config.timeout.total_seconds()
+        if db_config.timeout is not None
+        else 31536000,
+        minsize=db_config.pool_size or 1,
+        maxsize=db_config.pool_size or 1,
+    ) as pool:
+        try:
+            yield Database(config=db_config, pool=pool)
+        finally:
+            pool.close()
+            await pool.wait_closed()

myrtille/lib/util.py

@@ -0,0 +1,9 @@
+import typing
+
+
+def as_any(v: typing.Any, /):
+    return v
+
+
+def snake_to_pascal(s: str) -> str:
+    return ''.join(word.capitalize() for word in s.split('_'))

myrtille/mysql/export.py

@@ -0,0 +1,59 @@
+import asyncio
+import re
+import typing
+
+import pydantic
+
+from myrtille.lib import cfg, db
+
+
+def _correct_ddl(ddl: str):
+    # Removes display width on interer types
+    # Removes floating point precision on time functions
+    for s in [
+        'tinyint',
+        'smallint',
+        'int',
+        'bigint',
+        'DEFAULT CURRENT_TIMESTAMP',
+        'ON UPDATE CURRENT_TIMESTAMP',
+    ]:
+        ddl = re.sub(rf' {s}\([0-9]*\)', f' {s}', ddl, flags=re.IGNORECASE)
+
+    # Removes non standart float precision
+    for s in ['float', 'double']:
+        ddl = re.sub(rf' {s}\([0-9]*,[0-9]*\)', f' {s}', ddl, flags=re.IGNORECASE)
+
+    return ddl
+
+
+class _Table(pydantic.BaseModel):
+    schema_name: str
+    table_name: str
+
+    async def get_ddl(self, database: db.Database) -> str:
+        show_create_response = await database.fetch_optional(
+            f'SHOW CREATE TABLE `{self.schema_name}`.`{self.table_name}`'
+        )
+        assert show_create_response is not None
+        (_, ddl) = show_create_response
+        assert isinstance(ddl, str)
+        return _correct_ddl(ddl)
+
+
+async def export(schema_name: str, config: cfg.Database) -> typing.Sequence[str]:
+    async with db.make_database(config) as database:
+        return await asyncio.gather(
+            *(
+                _Table(schema_name=schema_name, table_name=table_name).get_ddl(database)
+                for (table_name,) in await database.fetch_all(
+                    """
+                SELECT TABLE_NAME
+                    FROM INFORMATION_SCHEMA.TABLES
+                    WHERE TABLE_TYPE = "BASE TABLE"
+                    AND TABLE_SCHEMA = %s
+                """,
+                    params=[schema_name],
+                )
+            )
+        )

myrtille/mysql/generator.py

@@ -0,0 +1,353 @@
+from myrtille.mysql import types
+
+
+def generate_ternary_option(option: bool):
+    return '1' if option else '0'
+
+
+def generate_ref_action(action: types.RefAction):
+    match action:
+        case types.RefAction.RESTRICT:
+            return 'RESTRICT'
+        case types.RefAction.CASCADE:
+            return 'CASCADE'
+        case types.RefAction.SET_NULL:
+            return 'SET NULL'
+
+
+def generate_key_part(key: types.KeyPart):
+    res = f'`{key.identifier}`'
+    if key.length is not None:
+        res += f'({key.length})'
+    if key.direction is not None:
+        res += f' {key.direction.name}'
+    return res
+
+
+def generate_data_type(data_type: types.DataType):
+    arguments: list[str] = []
+
+    if (
+        isinstance(data_type, types.Datetime | types.Timestamp | types.Time)
+        and data_type.precision is not None
+    ):
+        arguments = [f'{data_type.precision}']
+
+    if isinstance(
+        data_type, types.Bit | types.Char | types.Varchar | types.Binary | types.Varbinary
+    ):
+        arguments = [f'{data_type.length}']
+
+    if isinstance(data_type, types.Set | types.Enum):
+        arguments = [f"'{v}'" for v in data_type.values]
+
+    match data_type:
+        case types.Tinyint():
+            data_types_name = 'tinyint'
+        case types.Smallint():
+            data_types_name = 'smallint'
+        case types.Mediumint():
+            data_types_name = 'mediumint'
+        case types.Int():
+            data_types_name = 'int'
+        case types.Bigint():
+            data_types_name = 'bigint'
+        case types.Decimal():
+            data_types_name = 'decimal'
+            if data_type.precision is not None:
+                arguments = [f'{data_type.precision}', f'{data_type.scale or 0}']
+        case types.Float():
+            data_types_name = 'float'
+        case types.Double():
+            data_types_name = 'double'
+        case types.Bit():
+            data_types_name = 'bit'
+        case types.Datetime():
+            data_types_name = 'datetime'
+        case types.Timestamp():
+            data_types_name = 'timestamp'
+        case types.Time():
+            data_types_name = 'time'
+        case types.Date():
+            data_types_name = 'date'
+        case types.Year():
+            data_types_name = 'year'
+        case types.Char():
+            data_types_name = 'char'
+        case types.Varchar():
+            data_types_name = 'varchar'
+        case types.TinyText():
+            data_types_name = 'tinytext'
+        case types.Text():
+            data_types_name = 'text'
+        case types.MediumText():
+            data_types_name = 'mediumtext'
+        case types.LongText():
+            data_types_name = 'longtext'
+        case types.Enum():
+            data_types_name = 'enum'
+        case types.Set():
+            data_types_name = 'set'
+        case types.Binary():
+            data_types_name = 'binary'
+        case types.Varbinary():
+            data_types_name = 'varbinary'
+        case types.TinyBlob():
+            data_types_name = 'tinyblob'
+        case types.Blob():
+            data_types_name = 'blob'
+        case types.MediumBlob():
+            data_types_name = 'mediumblob'
+        case types.LongBlob():
+            data_types_name = 'longblob'
+        case types.Json():
+            data_types_name = 'json'
+        case types.Geometry():
+            data_types_name = 'geometry'
+        case types.Point():
+            data_types_name = 'point'
+        case types.Linestring():
+            data_types_name = 'linestring'
+        case types.Polygon():
+            data_types_name = 'polygon'
+        case types.Geometrycollection():
+            data_types_name = 'geomcollection'
+        case types.Multipoint():
+            data_types_name = 'multipoint'
+        case types.Multilinestring():
+            data_types_name = 'multilinestring'
+        case types.Multipolygon():
+            data_types_name = 'multipolygon'
+    return data_types_name + (f'({",".join(arguments)})' if len(arguments) > 0 else '')
+
+
+def generate_literal(literal: types.Literal):
+    match literal:
+        case types.TextLiteral():
+            text = f"'{literal.text}'"
+            return text
+        case types.NullLiteral():
+            return 'NULL'
+
+
+def generate_default(default_value: types.DefaultValue):
+    match default_value:
+        case types.ExprDefaultAttribute():
+            return f'({default_value.expr})'
+        case types.LiteralDefaultAttribute():
+            return generate_literal(default_value.value)
+
+
+def generate_column(column: types.Column):
+    attributes: list[str] = []
+
+    if isinstance(column.data_type, types.IntegerDataType) and column.data_type.unsigned:
+        attributes.append('unsigned')
+
+    if isinstance(column.data_type, types.TextDataType):
+        if column.data_type.charset is not None:
+            attributes.append(f'CHARACTER SET {column.data_type.charset}')
+        if column.data_type.collate is not None:
+            attributes.append(f'COLLATE {column.data_type.collate}')
+
+    if column.format != types.ColumnFormat.DEFAULT:
+        attributes.append(f'/*!50606 COLUMN_FORMAT {column.format.name} */')
+
+    if column.storage_media != types.StorageMedia.DEFAULT:
+        attributes.append(f'/*!50606 STORAGE {column.storage_media.name} */')
+
+    if column.generated is not None:
+        attributes.append(
+            f'GENERATED ALWAYS AS ({column.generated.expr}) {column.generated.type.name}'
+        )
+
+    if column.non_nullable is not None:
+        attributes.append(('NOT ' if column.non_nullable else '') + 'NULL')
+
+    if isinstance(column.data_type, types.SpatialDataType) and column.data_type.srid is not None:
+        attributes.append(f'/*!80003 SRID {column.data_type.srid} */')
+
+    if isinstance(column.data_type, types.IntegerDataType) and column.data_type.auto_increment:
+        attributes.append('AUTO_INCREMENT')
+
+    if column.default_value is not None:
+        attributes.append(f'DEFAULT {generate_default(column.default_value)}')
+
+    if isinstance(column.data_type, types.Datetime | types.Timestamp):
+        if column.data_type.default_now:
+            attributes.append('DEFAULT CURRENT_TIMESTAMP')
+        if column.data_type.on_update_now:
+            attributes.append('ON UPDATE CURRENT_TIMESTAMP')
+
+    if column.comment is not None:
+        attributes.append(f"COMMENT '{column.comment}'")
+    if column.invisible:
+        attributes.append('/*!80023 INVISIBLE */')
+
+    return f'`{column.name}` {generate_data_type(column.data_type)}' + ''.join(
+        f' {a}' for a in attributes
+    )
+
+
+def generate_constraint(constraint: types.Constraint):
+    match constraint:
+        case types.ForeignConstraint():
+            col_names = f'({", ".join(f"`{c}`" for c in constraint.columns)})'
+            ref_col_names = f'({", ".join(f"`{c}`" for c in constraint.references.ref_columns)})'
+            parts = [
+                f'CONSTRAINT `{constraint.name}` FOREIGN KEY {col_names}',
+                f'REFERENCES `{constraint.references.ref_table}` {ref_col_names}',
+            ]
+            if constraint.references.on_delete is not None:
+                parts.append(f'ON DELETE {generate_ref_action(constraint.references.on_delete)}')
+            if constraint.references.on_update is not None:
+                parts.append(f'ON UPDATE {generate_ref_action(constraint.references.on_update)}')
+            return ' '.join(parts)
+        case types.CheckConstraint():
+            return f'CONSTRAINT `{constraint.name}` CHECK ({constraint.expr})'
+        case _:
+            key_list = f'({",".join(generate_key_part(key) for key in constraint.key_list)})'
+            match constraint:
+                case types.PrimaryConstraint():
+                    return f'PRIMARY KEY {key_list}'
+                case types.UniqueConstraint():
+                    return f'UNIQUE KEY `{constraint.name}` {key_list}'
+                case types.IndexConstraint():
+                    return f'KEY `{constraint.name}` {key_list}'
+                case types.FulltextConstraint():
+                    return f'FULLTEXT KEY `{constraint.name}` {key_list}'
+                case types.SpatialConstraint():
+                    return f'SPATIAL KEY `{constraint.name}` {key_list}'
+
+
+def generate_create_options(create_options: types.CreateOptions):
+    attributes: list[str] = []
+    if create_options.tablespace is not None:
+        attributes.append(f'/*!50100 TABLESPACE `{create_options.tablespace}` */')
+    if create_options.engine is not None:
+        attributes.append(f'ENGINE={create_options.engine}')
+    if create_options.auto_increment is not None:
+        attributes.append(f'AUTO_INCREMENT={create_options.auto_increment}')
+    if create_options.charset is not None:
+        attributes.append(f'DEFAULT CHARSET={create_options.charset}')
+    if create_options.collate is not None:
+        attributes.append(f'COLLATE={create_options.collate}')
+    if create_options.avg_row_length is not None:
+        attributes.append(f'AVG_ROW_LENGTH={create_options.avg_row_length}')
+    if create_options.stats_persistent is not None:
+        attributes.append(
+            f'STATS_PERSISTENT={generate_ternary_option(create_options.stats_persistent)}'
+        )
+    if create_options.row_format is not None:
+        attributes.append(f'ROW_FORMAT={create_options.row_format.name}')
+    if create_options.comment is not None:
+        attributes.append(f"COMMENT='{create_options.comment}'")
+
+    return ' '.join(attributes)
+
+
+def generate_key_hash_partitioning_method(
+    partitioning: types.KeyPartitionType | types.HashPartitionType,
+):
+    match partitioning:
+        case types.KeyPartitionType():
+            s = 'KEY'
+        case types.HashPartitionType():
+            s = f'HASH ({partitioning.expr})'
+    return ('LINEAR ' if partitioning.linear else '') + s
+
+
+def generate_partitioning_method(partitioning: types.Partitioning):
+    match partitioning:
+        case types.ListPartitioning() | types.RangePartitioning():
+            match partitioning.expr_or_columns:
+                case str():
+                    args = f'({partitioning.expr_or_columns})'
+                case list():
+                    args = f'COLUMNS ({", ".join(partitioning.expr_or_columns)})'
+            match partitioning:
+                case types.RangePartitioning():
+                    return f'RANGE {args}'
+                case types.ListPartitioning():
+                    return f'LIST {args}'
+        case types.KeyHashPartitioning():
+            return generate_key_hash_partitioning_method(partitioning.type)
+
+
+def generate_partitioning(partitioning: types.Partitioning):
+    return f'PARTITION BY {generate_partitioning_method(partitioning)}'
+
+
+def generate_subpartitioning(subpartitioning: types.Subpartitioning):
+    return f'SUBPARTITION BY {generate_key_hash_partitioning_method(subpartitioning.type)}'
+
+
+def generate_partition_options(options: types.PartitionOptions):
+    parts: dict[str, str] = {}
+    if options.engine is not None:
+        parts['ENGINE'] = options.engine
+    return [f'{k} = {v}' for k, v in parts.items()]
+
+
+def generate_key_hash_partition_def(partition: types.KeyHashPartition):
+    return ' '.join((partition.name, *generate_partition_options(partition.options)))
+
+
+def generate_partition_lt_values(values: types.ValuesLessThan):
+    if values == [None]:
+        args = 'MAXVALUE'
+    else:
+        args = f'({", ".join((v if v is not None else "MAXVALUE") for v in values)})'
+
+    return f'VALUES LESS THAN {args}'
+
+
+def generate_partition_in_values(values: types.ValuesIn):
+    elements = [(f'({", ".join(v)})' if isinstance(v, list) else v) for v in values]
+    return f'VALUES IN ({", ".join(elements)})'
+
+
+def generate_subpartition(subparition: types.KeyHashPartition):
+    return f'SUBPARTITION {generate_key_hash_partition_def(subparition)}'
+
+
+def generate_partition(
+    partition: types.RangePartition | types.ListPartition | types.KeyHashPartition,
+):
+    match partition:
+        case types.RangePartition() | types.ListPartition():
+            match partition:
+                case types.RangePartition():
+                    values = generate_partition_lt_values(partition.values)
+                case types.ListPartition():
+                    values = generate_partition_in_values(partition.values)
+            parts = [partition.name, values, *generate_partition_options(partition.options)]
+            definition = ' '.join(parts)
+
+            subparitions = [generate_subpartition(sub) for sub in partition.subpartitions]
+            if len(subparitions) > 0:
+                definition += f'\n ({",\n  ".join(subparitions)})'
+        case types.KeyHashPartition():
+            definition = generate_key_hash_partition_def(partition)
+    return f'PARTITION {definition}'
+
+
+def generate_partitioning_clause(partitioning: types.Partitioning):
+    parts = [generate_partitioning(partitioning)]
+    if partitioning.sub is not None:
+        parts.append(generate_subpartitioning(partitioning.sub))
+    parts.append(f'({",\n ".join(generate_partition(p) for p in partitioning.partitions)})')
+    return f'/*!50100 {"\n".join(parts)} */'
+
+
+def generate(table: types.Table):
+    elements = [generate_column(c) for c in table.columns] + [
+        generate_constraint(c) for c in table.constraints
+    ]
+    options = generate_create_options(table.options)
+    statement = (
+        f'CREATE TABLE `{table.name}` (\n{",\n".join(f"  {e}" for e in elements)}\n) {options}'
+    )
+    if table.partitioning is not None:
+        statement += '\n' + generate_partitioning_clause(table.partitioning)
+    return statement