databao 0.3.3.dev2__tar.gz → 0.3.3.dev3__tar.gz

{databao-0.3.3.dev2 → databao-0.3.3.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databao
-Version: 0.3.3.dev2
+Version: 0.3.3.dev3
 License-File: LICENSE
 Requires-Python: >=3.11
 Requires-Dist: adbc-driver-manager>=1.10.0

databao-0.3.3.dev3/e2e-tests/src/databases/duckdb_utils.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import duckdb
+
+
+@dataclass(frozen=True)
+class DuckdbDB:
+    datasource_name: str | None = "test duckdb conn"
+    datasource_type: str = "duckdb"
+    database_path: Path | None = None
+    check_connection: bool = False
+    check_connection_succeed: bool = True
+
+    @classmethod
+    def get_database(cls, db_file: Path):
+        return cls(database_path=db_file, check_connection=True)
+
+    def add_table(self, create_table_sql: str, fill_table_sql: str):
+        conn = duckdb.connect(database=str(self.database_path))
+        try:
+            with conn:
+                conn.execute(create_table_sql)
+                conn.execute(fill_table_sql)
+        finally:
+            conn.close()
+
+
+CREATE_DUCKDB_TABLE_SQL = """
+                           CREATE TABLE all_types_demo
+                           (
+                               -- Integer types
+                               id                INTEGER PRIMARY KEY,
+                               tinyint_col       TINYINT,
+                               smallint_col      SMALLINT,
+                               int_col           INTEGER,
+                               bigint_col        BIGINT,
+                               hugeint_col       HUGEINT,
+
+                               -- Unsigned integers
+                               utinyint_col      UTINYINT,
+                               usmallint_col     USMALLINT,
+                               uinteger_col      UINTEGER,
+                               ubigint_col       UBIGINT,
+
+                               -- Floating point
+                               real_col          REAL,
+                               double_col DOUBLE,
+
+                               -- Decimal
+                               decimal_col       DECIMAL(10, 2),
+
+                               -- Boolean
+                               boolean_col       BOOLEAN,
+
+                               -- Character / string
+                               char_col          CHAR(5),
+                               varchar_col       VARCHAR,
+                               text_col          TEXT,
+
+                               -- Binary
+                               blob_col          BLOB,
+
+                               -- Date & time
+                               date_col          DATE,
+                               time_col          TIME,
+                               timestamp_col     TIMESTAMP,
+                               timestamptz_col   TIMESTAMPTZ,
+                               interval_col INTERVAL,
+
+                               -- UUID
+                               uuid_col          UUID,
+
+                               -- JSON
+                               json_col          JSON,
+
+                               -- Arrays (LIST)
+                               int_array_col     INTEGER[],
+                               varchar_array_col VARCHAR[],
+
+                               -- Struct
+                               struct_col        STRUCT(name VARCHAR, age INTEGER),
+
+                               -- Map
+                               map_col           MAP(VARCHAR, INTEGER)
+                           );"""
+
+FILL_DUCKDB_TABLE_SQL = """
+                         INSERT INTO all_types_demo
+                         VALUES (1,
+                                 10,
+                                 100,
+                                 1000,
+                                 10000000000,
+                                 123456789012345678901234567890::HUGEINT,
+                                 255,
+                                 65535,
+                                 4000000000,
+                                 10000000000000000000,
+                                 3.14,
+                                 3.1415926535,
+                                 12345.67,
+                                 TRUE,
+                                 'ABCDE',
+                                 'Hello DuckDB',
+                                 'This is a text column',
+                                 'DEADBEEF'::BLOB,
+                                 DATE '2025-01-01',
+                                 TIME '12:30:00',
+                                 TIMESTAMP '2025-01-01 12:30:00',
+                                 TIMESTAMPTZ '2025-01-01 12:30:00+02',
+                                 INTERVAL '2 days',
+                                 '550e8400-e29b-41d4-a716-446655440000'::UUID,
+                                 '{"key": "value"}'::JSON,
+                                    [1, 2, 3],
+                                    ['a', 'b', 'c'],
+                                    {'name' : 'Alice', 'age': 30},
+                                 MAP(['a', 'b'], [1, 2]));"""

databao-0.3.3.dev3/e2e-tests/src/databases/mysql_utils.py

@@ -0,0 +1,168 @@
+from dataclasses import dataclass
+
+import pymysql
+from testcontainers.mysql import MySqlContainer
+
+
+@dataclass(frozen=True)
+class MysqlDB:
+    datasource_name: str | None = "test_mysql_conn"
+    datasource_type: str = "mysql"
+    host: str | None = None
+    port: int | None = None
+    database: str | None = None
+    user: str | None = None
+    password: str | None = None
+    check_connection: bool = False
+    check_connection_succeed: bool = True
+
+    @classmethod
+    def get_database(cls, container: MySqlContainer):
+        return cls(
+            host=container.get_container_host_ip(),
+            port=int(container.get_exposed_port(container.port)),
+            database=container.dbname,
+            user=container.username,
+            password=container.password,
+            check_connection=True,
+        )
+
+    def add_table(self, create_table_sql: str, fill_table_sql: str):
+        conn = pymysql.connect(
+            host=self.host,
+            port=self.port,
+            user=self.user,
+            password=self.password,
+            database=self.database,
+            cursorclass=pymysql.cursors.DictCursor,
+            autocommit=True,
+        )
+        try:
+            with conn.cursor() as cur:
+                cur.execute(create_table_sql)
+                cur.execute(fill_table_sql)
+        finally:
+            conn.close()
+
+
+CREATE_MYSQL_TABLE_SQL = """
+                          CREATE TABLE all_types_demo
+                          (
+                              -- Numeric types
+                              id             INT AUTO_INCREMENT PRIMARY KEY,
+                              tinyint_col    TINYINT,
+                              smallint_col   SMALLINT,
+                              mediumint_col  MEDIUMINT,
+                              int_col        INT,
+                              bigint_col     BIGINT,
+                              decimal_col    DECIMAL(10, 2),
+                              float_col      FLOAT,
+                              double_col DOUBLE,
+                              bit_col        BIT(4),
+                              boolean_col    BOOLEAN,
+
+                              -- Date and time types
+                              date_col       DATE,
+                              time_col       TIME,
+                              datetime_col   DATETIME,
+                              timestamp_col  TIMESTAMP NULL,
+                              year_col YEAR,
+
+                              -- String types
+                              char_col       CHAR(5),
+                              varchar_col    VARCHAR(50),
+                              binary_col     BINARY(4),
+                              varbinary_col  VARBINARY(10),
+
+                              -- Text types
+                              tinytext_col   TINYTEXT,
+                              text_col       TEXT,
+                              mediumtext_col MEDIUMTEXT,
+                              longtext_col   LONGTEXT,
+
+                              -- Blob types
+                              tinyblob_col   TINYBLOB,
+                              blob_col       BLOB,
+                              mediumblob_col MEDIUMBLOB,
+                              longblob_col   LONGBLOB,
+
+                              -- Enum and Set
+                              enum_col       ENUM('active','inactive','pending'),
+                              set_col SET('a','b','c'),
+
+                              -- JSON
+                              json_col       JSON,
+
+                              -- Spatial types
+                              point_col      POINT,
+                              linestring_col LINESTRING,
+                              polygon_col    POLYGON
+                          );"""
+
+FILL_MYSQL_TABLE_SQL = """
+                        INSERT INTO all_types_demo (tinyint_col,
+                                                    smallint_col,
+                                                    mediumint_col,
+                                                    int_col,
+                                                    bigint_col,
+                                                    decimal_col,
+                                                    float_col,
+                                                    double_col,
+                                                    bit_col,
+                                                    boolean_col,
+                                                    date_col,
+                                                    time_col,
+                                                    datetime_col,
+                                                    timestamp_col,
+                                                    year_col,
+                                                    char_col,
+                                                    varchar_col,
+                                                    binary_col,
+                                                    varbinary_col,
+                                                    tinytext_col,
+                                                    text_col,
+                                                    mediumtext_col,
+                                                    longtext_col,
+                                                    tinyblob_col,
+                                                    blob_col,
+                                                    mediumblob_col,
+                                                    longblob_col,
+                                                    enum_col,
+                                                    set_col,
+                                                    json_col,
+                                                    point_col,
+                                                    linestring_col,
+                                                    polygon_col)
+                        VALUES (1,
+                                100,
+                                1000,
+                                10000,
+                                10000000000,
+                                12345.67,
+                                3.14,
+                                3.1415926535,
+                                b'1010',
+                                TRUE,
+                                '2025-01-01',
+                                '12:30:00',
+                                '2025-01-01 12:30:00',
+                                '2026-02-26 15:12:55',
+                                2025,
+                                'ABCDE',
+                                'Hello MySQL',
+                                'ABCD',
+                                'varbin',
+                                'tiny text',
+                                'regular text column',
+                                'medium text column',
+                                'long text column',
+                                UNHEX('AA'),
+                                UNHEX('DEADBEEF'),
+                                UNHEX('DEADBEEFDEADBEEF'),
+                                UNHEX('DEADBEEFDEADBEEFDEADBEEF'),
+                                'active',
+                                'a,b',
+                                JSON_OBJECT('key', 'value'),
+                                ST_GeomFromText('POINT(1 2)'),
+                                ST_GeomFromText('LINESTRING(0 0, 1 1, 2 2)'),
+                                ST_GeomFromText('POLYGON((0 0, 0 5, 5 5, 5 0, 0 0))'));"""

databao-0.3.3.dev3/e2e-tests/src/databases/postgres_utils.py

@@ -0,0 +1,209 @@
+from dataclasses import dataclass
+
+import psycopg2
+from testcontainers.postgres import PostgresContainer
+
+
+@dataclass(frozen=True)
+class PostgresDB:
+    datasource_name: str
+    datasource_type = "postgres"
+    host: str | None = None
+    port: int | None = None
+    database: str | None = None
+    user: str | None = None
+    password: str | None = None
+    check_connection: bool = False
+    check_connection_succeed: bool = True
+
+    @classmethod
+    def get_database(cls, container: PostgresContainer):
+        return cls(
+            datasource_name="my_postgres",
+            host=container.get_container_host_ip(),
+            port=int(container.get_exposed_port(container.port)),
+            database=container.dbname,
+            user=container.username,
+            password=container.password,
+            check_connection=True,
+        )
+
+    def add_table(self, create_table_sql: str, fill_table_sql: str):
+        conn = psycopg2.connect(host=self.host, port=self.port, user=self.user, password=self.password, dbname=self.database)
+        try:
+            with conn.cursor() as cur:
+                cur.execute(create_table_sql)
+                cur.execute(fill_table_sql)
+                conn.commit()
+        finally:
+            conn.close()
+
+
+CREATE_POSTGRES_TABLE_SQL = """
+                            CREATE TABLE all_types_demo
+                            (
+                                -- Numeric types
+                                id              SERIAL PRIMARY KEY,
+                                small_int_col   SMALLINT,
+                                int_col         INTEGER,
+                                big_int_col     BIGINT,
+                                numeric_col     NUMERIC(10, 2),
+                                real_col        REAL,
+                                double_col      DOUBLE PRECISION,
+                                money_col       MONEY,
+
+                                -- Character types
+                                char_col        CHAR(5),
+                                varchar_col     VARCHAR(50),
+                                text_col        TEXT,
+
+                                -- Boolean
+                                boolean_col     BOOLEAN,
+
+                                -- Date & time
+                                date_col        DATE,
+                                time_col        TIME,
+                                timetz_col      TIME WITH TIME ZONE,
+                                timestamp_col   TIMESTAMP,
+                                timestamptz_col TIMESTAMP WITH TIME ZONE,
+                                interval_col INTERVAL,
+
+                                -- JSON
+                                json_col        JSON,
+                                jsonb_col       JSONB,
+
+                                -- Arrays
+                                int_array_col   INTEGER[],
+                                text_array_col  TEXT[],
+
+                                -- Binary
+                                bytea_col       BYTEA,
+
+                                -- Enum
+                                status_col      TEXT CHECK (status_col IN ('active', 'inactive', 'pending')),
+
+                                -- Network
+                                inet_col        INET,
+                                cidr_col        CIDR,
+                                macaddr_col     MACADDR,
+
+                                -- Geometric
+                                point_col       POINT,
+                                circle_col      CIRCLE,
+
+                                -- Bit string
+                                bit_col         BIT(4),
+                                bitvarying_col  BIT VARYING(8),
+
+                                -- XML
+                                xml_col         XML
+                            );"""
+
+FILL_POSTGRES_TABLE_SQL = """
+                          INSERT INTO all_types_demo (small_int_col,
+                                                      int_col,
+                                                      big_int_col,
+                                                      numeric_col,
+                                                      real_col,
+                                                      double_col,
+                                                      money_col,
+                                                      char_col,
+                                                      varchar_col,
+                                                      text_col,
+                                                      boolean_col,
+                                                      date_col,
+                                                      time_col,
+                                                      timetz_col,
+                                                      timestamp_col,
+                                                      timestamptz_col,
+                                                      interval_col,
+                                                      json_col,
+                                                      jsonb_col,
+                                                      int_array_col,
+                                                      text_array_col,
+                                                      bytea_col,
+                                                      status_col,
+                                                      inet_col,
+                                                      cidr_col,
+                                                      macaddr_col,
+                                                      point_col,
+                                                      circle_col,
+                                                      bit_col,
+                                                      bitvarying_col,
+                                                      xml_col)
+                          VALUES (10,
+                                  1000,
+                                  10000000000,
+                                  12345.67,
+                                  3.14,
+                                  3.1415926535,
+                                  99.99,
+                                  'ABCDE',
+                                  'Hello PostgreSQL',
+                                  'This is a long text column',
+                                  TRUE,
+                                  '2025-01-01',
+                                  '12:30:00',
+                                  '12:30:00+02',
+                                  '2025-01-01 12:30:00',
+                                  '2025-01-01 12:30:00+02',
+                                  '2 days 3 hours',
+                                  '{"key": "value"}',
+                                  '{"key": "value"}',
+                                  ARRAY[1, 2, 3],
+                                  ARRAY['a', 'b', 'c'],
+                                  decode('DEADBEEF', 'hex'),
+                                  'active',
+                                  '192.168.1.1',
+                                  '192.168.0.0/24',
+                                  '08:00:2b:01:02:03',
+                                  POINT(1, 2),
+                                  CIRCLE(POINT(1, 2), 5),
+                                  B'1010',
+                                  B'101010',
+                                  '<root><child>value</child></root>');"""
+
+CREATE_POSTGRES_PARTITIONED_TABLE_SQL = """
+                                        CREATE TABLE sales
+                                        (
+                                            id           SERIAL,
+                                            sale_date    DATE NOT NULL,
+                                            amount       NUMERIC(10, 2),
+                                            product_name VARCHAR(100),
+                                            region       VARCHAR(50)
+                                        ) PARTITION BY RANGE (sale_date);
+
+                                        CREATE TABLE sales_2024_q1 PARTITION OF sales
+                                            FOR VALUES FROM
+                                        (
+                                            '2024-01-01'
+                                        ) TO
+                                        (
+                                            '2024-04-01'
+                                        );
+
+                                        CREATE TABLE sales_2024_q2 PARTITION OF sales
+                                            FOR VALUES FROM
+                                        (
+                                            '2024-04-01'
+                                        ) TO
+                                        (
+                                            '2024-07-01'
+                                        );
+
+                                        CREATE TABLE sales_2024_q3 PARTITION OF sales
+                                            FOR VALUES FROM
+                                        (
+                                            '2024-07-01'
+                                        ) TO
+                                        (
+                                            '2024-10-01'
+                                        );
+                                        """
+
+FILL_POSTGRES_PARTITIONED_TABLE_SQL = """
+                                      INSERT INTO sales (sale_date, amount, product_name, region)
+                                      VALUES ('2024-01-15', 150.00, 'Product A', 'North'),
+                                             ('2024-05-20', 250.50, 'Product B', 'South'),
+                                             ('2024-08-10', 175.75, 'Product C', 'East');
+                                      """

databao-0.3.3.dev3/e2e-tests/src/databases/sqlite_utils.py

@@ -0,0 +1,99 @@
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class SqliteDB:
+    datasource_name: str | None = "test sqlite conn"
+    datasource_type: str = "sqlite"
+    database_path: Path | None = None
+    check_connection: bool = False
+    check_connection_succeed: bool = True
+
+    @classmethod
+    def get_database(cls, db_file: Path):
+        return cls(database_path=db_file, check_connection=True)
+
+    def add_table(self, create_table_sql: str, fill_table_sql: str):
+        conn = sqlite3.connect(database=str(self.database_path))
+        try:
+            with conn:
+                conn.execute(create_table_sql)
+                conn.execute(fill_table_sql)
+        finally:
+            conn.close()
+
+
+CREATE_SQLITE_TABLE_SQL = """
+                           CREATE TABLE all_types_demo
+                           (
+                               -- Primary key
+                               id           INTEGER PRIMARY KEY AUTOINCREMENT,
+
+                               -- INTEGER affinity
+                               int_col      INTEGER,
+                               bigint_col   BIGINT,
+                               smallint_col SMALLINT,
+                               tinyint_col  TINYINT,
+                               boolean_col  BOOLEAN,
+
+                               -- REAL affinity
+                               real_col     REAL,
+                               double_col DOUBLE,
+                               float_col    FLOAT,
+                               numeric_col  NUMERIC,
+                               decimal_col  DECIMAL(10, 2),
+
+                               -- TEXT affinity
+                               char_col     CHAR(5),
+                               varchar_col  VARCHAR(50),
+                               text_col     TEXT,
+                               clob_col     CLOB,
+                               date_col     DATE,
+                               datetime_col DATETIME,
+
+                               -- BLOB affinity
+                               blob_col     BLOB,
+
+                               -- No type (NONE affinity)
+                               no_type_col
+                           );"""
+
+FILL_SQLITE_TABLE_SQL = """
+                         INSERT INTO all_types_demo (int_col,
+                                                     bigint_col,
+                                                     smallint_col,
+                                                     tinyint_col,
+                                                     boolean_col,
+                                                     real_col,
+                                                     double_col,
+                                                     float_col,
+                                                     numeric_col,
+                                                     decimal_col,
+                                                     char_col,
+                                                     varchar_col,
+                                                     text_col,
+                                                     clob_col,
+                                                     date_col,
+                                                     datetime_col,
+                                                     blob_col,
+                                                     no_type_col)
+                         VALUES (42,
+                                 1234567890123,
+                                 123,
+                                 1,
+                                 1, -- SQLite stores boolean as INTEGER
+                                 3.14,
+                                 3.1415926535,
+                                 2.718,
+                                 999.99,
+                                 12345.67,
+                                 'ABCDE',
+                                 'Hello SQLite',
+                                 'This is a text column',
+                                 'This is a CLOB column',
+                                 '2025-01-01',
+                                 '2025-01-01 12:30:00',
+                                 X'DEADBEEF', -- BLOB value
+                                 'No declared type value');"""

{databao-0.3.3.dev2 → databao-0.3.3.dev3}/e2e-tests/tests/domains/test_duckdb.py

@@ -2,7 +2,7 @@ from pathlib import Path
 
 import allure
 import pytest
-from databases.duckdb_utils import DuckdbDB
+from databases.duckdb_utils import CREATE_DUCKDB_TABLE_SQL, FILL_DUCKDB_TABLE_SQL, DuckdbDB
 from project_utils import execute_build, execute_init
 from utils.path_utils import get_datasource_result
 from utils.yaml_compare import assert_introspections_equal
@@ -17,7 +17,8 @@ def temp_duckdb_file(tmp_path: Path):
 @allure.title("Test databao build with DuckDB")
 @allure.description("Initialize a project with DuckDB and build it, then compare results with expected introspection.")
 def test_databao_build_duckdb(project_folder: Path, temp_duckdb_file: Path):
-    db = DuckdbDB.prepare_database(temp_duckdb_file)
+    db = DuckdbDB.get_database(temp_duckdb_file)
+    db.add_table(CREATE_DUCKDB_TABLE_SQL, FILL_DUCKDB_TABLE_SQL)
     execute_init(project_folder, db)
     execute_build(project_folder)
     assert_introspections_equal(get_datasource_result(project_folder, db.datasource_name), "duckdb_introspections.yaml")

{databao-0.3.3.dev2 → databao-0.3.3.dev3}/e2e-tests/tests/domains/test_mysql.py

@@ -2,7 +2,7 @@ from pathlib import Path
 
 import allure
 import pytest
-from databases.mysql_utils import MysqlDB
+from databases.mysql_utils import CREATE_MYSQL_TABLE_SQL, FILL_MYSQL_TABLE_SQL, MysqlDB
 from project_utils import execute_build, execute_init
 from testcontainers.mysql import MySqlContainer
 from utils.path_utils import get_datasource_result
@@ -20,7 +20,8 @@ def mysql_container():
 @allure.title("Test databao build with MySQL")
 @allure.description("Initialize a project with MySQL and build it, then compare results with expected introspection.")
 def test_databao_build_mysql(project_folder: Path, mysql_container: MySqlContainer):
-    db = MysqlDB.prepare_database(mysql_container)
+    db = MysqlDB.get_database(mysql_container)
+    db.add_table(CREATE_MYSQL_TABLE_SQL, FILL_MYSQL_TABLE_SQL)
     execute_init(project_folder, db)
     execute_build(project_folder)
     assert_introspections_equal(get_datasource_result(project_folder, db.datasource_name), "mysql_introspections.yaml")