tablemaster 2.1.2__tar.gz → 2.1.3__tar.gz

{tablemaster-2.1.2 → tablemaster-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tablemaster
-Version: 2.1.2
+Version: 2.1.3
 Summary: tablemaster is a Python toolkit for moving and managing tabular data across databases, Feishu/Lark, Google Sheets, and local files with one consistent API.
 Author-email: Livid <livid.su@gmail.com>
 Project-URL: Homepage, https://github.com/ilivid/tablemaster

{tablemaster-2.1.2 → tablemaster-2.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tablemaster"
-version = "2.1.2"
+version = "2.1.3"
 description = "tablemaster is a Python toolkit for moving and managing tabular data across databases, Feishu/Lark, Google Sheets, and local files with one consistent API."
 readme = "README.md"
 requires-python = ">=3.9"

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster/feishu.py

@@ -178,10 +178,10 @@ def fs_write_df(sheet_address, df, feishu_cfg, loc='A1', clear_sheet=True):
             if clear_resp.json().get('code') == 0:
                 logger.info('sheet cleared')
             else:
-                raise RuntimeError(f"failed to clear sheet: {clear_resp.json().get('msg')}")
+                raise RuntimeError(f"failed to clear sheet: {clear_resp.json().get('msg')}")
         except Exception as e:
-            logger.exception('failed to clear sheet: %s', e)
-            raise
+            logger.exception('failed to clear sheet: %s', e)
+            raise
     
     # 处理 DataFrame 数据类型
     df_copy = df.copy()
@@ -306,7 +306,7 @@ def fs_write_base(sheet_address, df, feishu_cfg, clear_table=False):
     existing_fields = _get_bitable_fields(app_token, table_id, header)
     
     if not existing_fields:
-        raise ValueError('could not fetch table fields or table has no fields')
+        raise ValueError('could not fetch table fields or table has no fields')
     
     logger.info('table has %s fields', len(existing_fields))
     
@@ -323,7 +323,7 @@ def fs_write_base(sheet_address, df, feishu_cfg, clear_table=False):
             logger.warning('skip column: %s', field)
     
     if not valid_fields:
-        raise ValueError('no valid fields to write, all dataframe columns are missing in bitable')
+        raise ValueError('no valid fields to write, all dataframe columns are missing in bitable')
     
     logger.info('will write %s valid fields', len(valid_fields))
     
@@ -359,9 +359,9 @@ def fs_write_base(sheet_address, df, feishu_cfg, clear_table=False):
                     _request_with_retry("post", delete_url, headers=header, json_data=delete_data)
                 logger.info('deleted %s records', len(record_ids))
 
-        except Exception as e:
-            logger.exception('failed to clear table: %s', e)
-            raise
+        except Exception as e:
+            logger.exception('failed to clear table: %s', e)
+            raise
     
     # 处理 DataFrame - 只保留有效字段
     df_copy = df[list(valid_fields)].copy()
@@ -444,7 +444,7 @@ def fs_write_base(sheet_address, df, feishu_cfg, clear_table=False):
                     str_val = str(value)
                     if str_val and str_val != 'None' and str_val != 'nan':
                         fields[col] = str_val
-                except Exception:
+                except Exception:
                     if col not in skipped_cols:
                         skipped_cols.add(col)
                     continue
@@ -457,7 +457,7 @@ def fs_write_base(sheet_address, df, feishu_cfg, clear_table=False):
     # 批量写入（每次最多500条）
     batch_size = 500
     all_responses = []
-    failed_batches = []
+    failed_batches = []
     
     for i in range(0, len(records), batch_size):
         batch = records[i:i + batch_size]
@@ -474,11 +474,11 @@ def fs_write_base(sheet_address, df, feishu_cfg, clear_table=False):
                 logger.info('batch %s wrote %s records', i // batch_size + 1, len(batch))
             else:
                 logger.error('failed to write batch: %s', response.get('msg', 'Unknown error'))
-                failed_batches.append((i // batch_size + 1, response.get('msg', 'Unknown error')))
+                failed_batches.append((i // batch_size + 1, response.get('msg', 'Unknown error')))
                 
         except Exception as e:
             logger.exception('failed to write batch: %s', e)
-            failed_batches.append((i // batch_size + 1, str(e)))
+            failed_batches.append((i // batch_size + 1, str(e)))
     
     logger.info('write summary total records: %s', len(records))
     logger.info('write summary fields written: %s', len(valid_fields))
@@ -486,8 +486,8 @@ def fs_write_base(sheet_address, df, feishu_cfg, clear_table=False):
         logger.info('write summary fields skipped: %s', len(missing_fields))
         for field in sorted(missing_fields):
             logger.info('skip field: %s', field)
-    if failed_batches:
-        raise RuntimeError(f'bitable write failed for {len(failed_batches)} batch(es): {failed_batches}')
+    if failed_batches:
+        raise RuntimeError(f'bitable write failed for {len(failed_batches)} batch(es): {failed_batches}')
     logger.info('data is written')
     
     return all_responses

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster/gspread.py

@@ -1,15 +1,15 @@
-import gspread
-import pandas as pd
+import gspread
+import pandas as pd
 import re
 import warnings
 import logging
 from functools import lru_cache
 
 logger = logging.getLogger(__name__)
-
-def _is_google_sheet_id(s):
-    return len(s) > 40 and ' ' not in s
-
+
+def _is_google_sheet_id(s):
+    return len(s) > 40 and ' ' not in s
+
 
 def _is_cell_loc(value):
     return isinstance(value, str) and re.match(r'^[A-Za-z]+[1-9]\d*$', value.strip()) is not None
@@ -49,37 +49,37 @@ def gs_read_df(address, cfg=None, service_account_path=None):
     logger.info('reading google sheets')
     sa_path = _resolve_service_account_path(cfg, service_account_path)
     gc = _get_gspread_client(sa_path)
-
+
     spreadsheet_identifier = address[0]
     worksheet_name = address[1]
-
-    try:
-        if _is_google_sheet_id(spreadsheet_identifier):
+
+    try:
+        if _is_google_sheet_id(spreadsheet_identifier):
             logger.info('opening sheet by ID: %s', spreadsheet_identifier)
-            sh = gc.open_by_key(spreadsheet_identifier)
-        else:
+            sh = gc.open_by_key(spreadsheet_identifier)
+        else:
             logger.info('opening sheet by name: %s', spreadsheet_identifier)
-            sh = gc.open(spreadsheet_identifier)
-        
-        wks = sh.worksheet(worksheet_name)
-        df = pd.DataFrame(wks.get_all_records())
+            sh = gc.open(spreadsheet_identifier)
+        
+        wks = sh.worksheet(worksheet_name)
+        df = pd.DataFrame(wks.get_all_records())
         logger.info('google sheets read success')
         logger.debug('google sheets preview: %s', df.head())
-        return df
-
-    except gspread.exceptions.SpreadsheetNotFound:
+        return df
+
+    except gspread.exceptions.SpreadsheetNotFound:
         message = f"spreadsheet '{spreadsheet_identifier}' not found"
         logger.error(message)
         raise ValueError(message)
-    except gspread.exceptions.WorksheetNotFound:
+    except gspread.exceptions.WorksheetNotFound:
         message = f"worksheet '{worksheet_name}' not found in spreadsheet"
         logger.error(message)
         raise ValueError(message)
-    except Exception as e:
+    except Exception as e:
         logger.exception('an unexpected error occurred: %s', e)
         raise
-
-
+
+
 def gs_write_df(address, df, cfg=None, loc='A1', service_account_path=None):
     if isinstance(cfg, str) and _is_cell_loc(cfg):
         _warn_deprecated('Passing loc as the third positional argument is deprecated; use keyword loc=...')
@@ -91,44 +91,44 @@ def gs_write_df(address, df, cfg=None, loc='A1', service_account_path=None):
     logger.info('writing google sheets')
     sa_path = _resolve_service_account_path(cfg, service_account_path)
     gc = _get_gspread_client(sa_path)
-
+
     spreadsheet_identifier = address[0]
     worksheet_name = address[1]
 
-    is_id = _is_google_sheet_id(spreadsheet_identifier)
-
-    try:
-        if is_id:
+    is_id = _is_google_sheet_id(spreadsheet_identifier)
+
+    try:
+        if is_id:
             logger.info('opening sheet by ID: %s', spreadsheet_identifier)
-            sh = gc.open_by_key(spreadsheet_identifier)
-        else:
+            sh = gc.open_by_key(spreadsheet_identifier)
+        else:
             logger.info('opening sheet by name: %s', spreadsheet_identifier)
-            sh = gc.open(spreadsheet_identifier)
-
-    except gspread.exceptions.SpreadsheetNotFound:
-        if is_id:
+            sh = gc.open(spreadsheet_identifier)
+
+    except gspread.exceptions.SpreadsheetNotFound:
+        if is_id:
             message = f"spreadsheet ID '{spreadsheet_identifier}' not found, cannot create with specific ID"
             logger.error(message)
             raise ValueError(message)
-        else:
+        else:
             logger.info("spreadsheet '%s' not found, creating one", spreadsheet_identifier)
-            sh = gc.create(spreadsheet_identifier)
-
-    try:
-        wks = sh.worksheet(worksheet_name)
-    except gspread.exceptions.WorksheetNotFound:
+            sh = gc.create(spreadsheet_identifier)
+
+    try:
+        wks = sh.worksheet(worksheet_name)
+    except gspread.exceptions.WorksheetNotFound:
         logger.info('worksheet "%s" not found, creating one', worksheet_name)
-        wks = sh.add_worksheet(title=worksheet_name, rows="100", cols="20")
+        wks = sh.add_worksheet(title=worksheet_name, rows="100", cols="20")
 
-    try:
-        wks.clear()
+    try:
+        wks.clear()
         df_copy = df.copy()
         non_float_int_columns = df_copy.select_dtypes(exclude=['float64', 'int64']).columns
-        for col in non_float_int_columns:
+        for col in non_float_int_columns:
             df_copy[col] = df_copy[col].astype(str)
         wks.update(loc, ([df_copy.columns.values.tolist()] + df_copy.values.tolist()))
-
+
         logger.info('data is written')
-    except Exception as e:
+    except Exception as e:
         logger.exception('failed to update worksheet: %s', e)
         raise

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster/schema/dialects/base.py

@@ -90,3 +90,22 @@ class BaseDialect(ABC):
     @abstractmethod
     def gen_drop_index(self, table: str, index_name: str, schema_name: str | None = None) -> str:
         pass
+
+    @abstractmethod
+    def gen_drop_primary_key(
+        self,
+        table: str,
+        primary_key_name: str | None = None,
+        schema_name: str | None = None,
+    ) -> str:
+        pass
+
+    @abstractmethod
+    def gen_add_primary_key(
+        self,
+        table: str,
+        columns: list[str],
+        primary_key_name: str | None = None,
+        schema_name: str | None = None,
+    ) -> str:
+        pass

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster/schema/dialects/mysql.py

@@ -57,7 +57,9 @@ class MySQLDialect(BaseDialect):
         results: list[ActualTable] = []
         for name in names:
             columns = []
-            pk_cols = set(inspector.get_pk_constraint(name, schema=database).get('constrained_columns') or [])
+            pk_constraint = inspector.get_pk_constraint(name, schema=database) or {}
+            pk_column_list = list(pk_constraint.get('constrained_columns') or [])
+            pk_cols = set(pk_column_list)
             for col in inspector.get_columns(name, schema=database):
                 columns.append(
                     ActualColumn(
@@ -83,7 +85,16 @@ class MySQLDialect(BaseDialect):
                 table_comment = (inspector.get_table_comment(name, schema=database) or {}).get('text')
             except Exception:
                 table_comment = None
-            results.append(ActualTable(table=name, columns=columns, indexes=indexes, comment=table_comment))
+            results.append(
+                ActualTable(
+                    table=name,
+                    columns=columns,
+                    indexes=indexes,
+                    comment=table_comment,
+                    primary_key_columns=pk_column_list,
+                    primary_key_name=pk_constraint.get('name'),
+                )
+            )
         return results
 
     def _qualified_table(self, table: str, schema_name: str | None = None) -> str:
@@ -190,3 +201,21 @@ class MySQLDialect(BaseDialect):
 
     def gen_drop_index(self, table: str, index_name: str, schema_name: str | None = None) -> str:
         return f'DROP INDEX {_quote(index_name)} ON {self._qualified_table(table, schema_name)}'
+
+    def gen_drop_primary_key(
+        self,
+        table: str,
+        primary_key_name: str | None = None,
+        schema_name: str | None = None,
+    ) -> str:
+        return f'ALTER TABLE {self._qualified_table(table, schema_name)} DROP PRIMARY KEY'
+
+    def gen_add_primary_key(
+        self,
+        table: str,
+        columns: list[str],
+        primary_key_name: str | None = None,
+        schema_name: str | None = None,
+    ) -> str:
+        cols_sql = ', '.join(_quote(c) for c in columns)
+        return f'ALTER TABLE {self._qualified_table(table, schema_name)} ADD PRIMARY KEY ({cols_sql})'

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster/schema/dialects/postgresql.py

@@ -60,7 +60,9 @@ class PostgreSQLDialect(BaseDialect):
         results: list[ActualTable] = []
         for name in names:
             columns = []
-            pk_cols = set(inspector.get_pk_constraint(name, schema=schema).get('constrained_columns') or [])
+            pk_constraint = inspector.get_pk_constraint(name, schema=schema) or {}
+            pk_column_list = list(pk_constraint.get('constrained_columns') or [])
+            pk_cols = set(pk_column_list)
             for col in inspector.get_columns(name, schema=schema):
                 columns.append(
                     ActualColumn(
@@ -87,7 +89,15 @@ class PostgreSQLDialect(BaseDialect):
             except Exception:
                 table_comment = None
             results.append(
-                ActualTable(table=name, columns=columns, indexes=indexes, comment=table_comment, schema_name=schema)
+                ActualTable(
+                    table=name,
+                    columns=columns,
+                    indexes=indexes,
+                    comment=table_comment,
+                    schema_name=schema,
+                    primary_key_columns=pk_column_list,
+                    primary_key_name=pk_constraint.get('name'),
+                )
             )
         return results
 
@@ -189,3 +199,29 @@ class PostgreSQLDialect(BaseDialect):
     def gen_drop_index(self, table: str, index_name: str, schema_name: str | None = None) -> str:
         schema = schema_name or 'public'
         return f'DROP INDEX {_quote(schema)}.{_quote(index_name)}'
+
+    def gen_drop_primary_key(
+        self,
+        table: str,
+        primary_key_name: str | None = None,
+        schema_name: str | None = None,
+    ) -> str:
+        constraint_name = primary_key_name or f'{table}_pkey'
+        return (
+            f'ALTER TABLE {self._qualified_table(table, schema_name)} '
+            f'DROP CONSTRAINT {_quote(constraint_name)}'
+        )
+
+    def gen_add_primary_key(
+        self,
+        table: str,
+        columns: list[str],
+        primary_key_name: str | None = None,
+        schema_name: str | None = None,
+    ) -> str:
+        constraint_name = primary_key_name or f'{table}_pkey'
+        cols_sql = ', '.join(_quote(c) for c in columns)
+        return (
+            f'ALTER TABLE {self._qualified_table(table, schema_name)} '
+            f'ADD CONSTRAINT {_quote(constraint_name)} PRIMARY KEY ({cols_sql})'
+        )

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster/schema/diff.py

@@ -94,6 +94,23 @@ def generate_plan(
 
         desired_cols = {c.name: c for c in table_def.columns}
         actual_cols = {c.name: c for c in current.columns}
+        desired_pk = [c.name for c in table_def.columns if c.primary_key]
+        actual_pk = list(current.primary_key_columns or [c.name for c in current.columns if c.primary_key])
+        primary_key_changed = desired_pk != actual_pk
+
+        if primary_key_changed and actual_pk:
+            plan.actions.append(
+                _action(
+                    'DROP_PRIMARY_KEY',
+                    table_name,
+                    ddl=dialect.gen_drop_primary_key(
+                        table_name,
+                        primary_key_name=current.primary_key_name,
+                        schema_name=table_def.schema_name,
+                    ),
+                    detail={'old': actual_pk, 'new': desired_pk},
+                )
+            )
 
         for col_name, desired_col in desired_cols.items():
             actual_col = actual_cols.get(col_name)
@@ -126,7 +143,8 @@ def generate_plan(
                     )
                 )
 
-            if bool(desired_col.nullable) != bool(actual_col.nullable):
+            desired_nullable = bool(desired_col.nullable) and not bool(desired_col.primary_key)
+            if desired_nullable != bool(actual_col.nullable):
                 plan.actions.append(
                     _action(
                         'ALTER_COLUMN_NULLABLE',
@@ -135,11 +153,11 @@ def generate_plan(
                         ddl=dialect.gen_alter_column_nullable(
                             table_name,
                             col_name,
-                            desired_col.nullable,
+                            desired_nullable,
                             col_type=dialect.map_type(desired_col.type),
                             schema_name=table_def.schema_name,
                         ),
-                        detail={'old': actual_col.nullable, 'new': desired_col.nullable},
+                        detail={'old': actual_col.nullable, 'new': desired_nullable},
                     )
                 )
 
@@ -190,6 +208,21 @@ def generate_plan(
                     )
                 )
 
+        if primary_key_changed and desired_pk:
+            plan.actions.append(
+                _action(
+                    'ADD_PRIMARY_KEY',
+                    table_name,
+                    ddl=dialect.gen_add_primary_key(
+                        table_name,
+                        desired_pk,
+                        primary_key_name=current.primary_key_name,
+                        schema_name=table_def.schema_name,
+                    ),
+                    detail={'old': actual_pk, 'new': desired_pk},
+                )
+            )
+
         desired_indexes = {idx.name: idx for idx in table_def.indexes}
         actual_indexes = {idx.name: idx for idx in current.indexes}
 

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster/schema/models.py

@@ -50,6 +50,8 @@ class ActualTable:
     indexes: list[IndexDef]
     comment: Optional[str] = None
     schema_name: Optional[str] = None
+    primary_key_columns: list[str] = field(default_factory=list)
+    primary_key_name: Optional[str] = None
 
 
 @dataclass

{tablemaster-2.1.2 → tablemaster-2.1.3}/tablemaster.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tablemaster
-Version: 2.1.2
+Version: 2.1.3
 Summary: tablemaster is a Python toolkit for moving and managing tabular data across databases, Feishu/Lark, Google Sheets, and local files with one consistent API.
 Author-email: Livid <livid.su@gmail.com>
 Project-URL: Homepage, https://github.com/ilivid/tablemaster

{tablemaster-2.1.2 → tablemaster-2.1.3}/tests/test_schema_core.py

@@ -3,6 +3,7 @@ from tempfile import TemporaryDirectory
 import unittest
 
 from tablemaster.schema.dialects.mysql import MySQLDialect
+from tablemaster.schema.dialects.postgresql import PostgreSQLDialect
 from tablemaster.schema.diff import generate_plan
 from tablemaster.schema.loader import load_schema_definitions
 from tablemaster.schema.models import ActualColumn, ActualTable
@@ -132,6 +133,117 @@ class SchemaCoreTests(unittest.TestCase):
             self.assertEqual('订单:主表', loaded[0].comment)
             self.assertEqual('主键:业务单号', loaded[0].columns[0].comment)
 
+    def test_diff_rebuilds_composite_primary_key_for_postgresql(self):
+        with TemporaryDirectory() as td:
+            root = Path(td)
+            schema_dir = root / 'schema' / 'mydb'
+            schema_dir.mkdir(parents=True, exist_ok=True)
+            (schema_dir / 'orders.yaml').write_text(
+                '\n'.join(
+                    [
+                        'table: orders',
+                        'columns:',
+                        '  - name: amazon_order_id',
+                        '    type: VARCHAR(50)',
+                        '    primary_key: true',
+                        '    nullable: false',
+                        '  - name: amazon_order_item_code',
+                        '    type: VARCHAR(50)',
+                        '    primary_key: true',
+                        '    nullable: false',
+                    ]
+                ),
+                encoding='utf-8',
+            )
+            desired = load_schema_definitions(connection='mydb', root_dir=root / 'schema')
+            actual = [
+                ActualTable(
+                    table='orders',
+                    columns=[
+                        ActualColumn(
+                            name='amazon_order_id',
+                            type='VARCHAR(50)',
+                            nullable=False,
+                            default=None,
+                            comment=None,
+                            primary_key=True,
+                        ),
+                        ActualColumn(
+                            name='amazon_order_item_code',
+                            type='VARCHAR(50)',
+                            nullable=False,
+                            default=None,
+                            comment=None,
+                            primary_key=False,
+                        ),
+                    ],
+                    indexes=[],
+                    primary_key_columns=['amazon_order_id'],
+                    primary_key_name='orders_pkey',
+                )
+            ]
+            plan = generate_plan('mydb', desired, actual, PostgreSQLDialect())
+            actions = [a.action for a in plan.actions]
+            self.assertIn('DROP_PRIMARY_KEY', actions)
+            self.assertIn('ADD_PRIMARY_KEY', actions)
+            self.assertTrue(
+                any(
+                    'DROP CONSTRAINT "orders_pkey"' in a.ddl
+                    for a in plan.actions
+                    if a.action == 'DROP_PRIMARY_KEY'
+                )
+            )
+            self.assertTrue(
+                any(
+                    'PRIMARY KEY ("amazon_order_id", "amazon_order_item_code")' in a.ddl
+                    for a in plan.actions
+                    if a.action == 'ADD_PRIMARY_KEY'
+                )
+            )
+
+    def test_diff_primary_key_implies_not_null(self):
+        with TemporaryDirectory() as td:
+            root = Path(td)
+            schema_dir = root / 'schema' / 'mydb'
+            schema_dir.mkdir(parents=True, exist_ok=True)
+            (schema_dir / 'orders.yaml').write_text(
+                '\n'.join(
+                    [
+                        'table: orders',
+                        'columns:',
+                        '  - name: id',
+                        '    type: BIGINT',
+                        '    primary_key: true',
+                    ]
+                ),
+                encoding='utf-8',
+            )
+            desired = load_schema_definitions(connection='mydb', root_dir=root / 'schema')
+            actual = [
+                ActualTable(
+                    table='orders',
+                    columns=[
+                        ActualColumn(
+                            name='id',
+                            type='BIGINT',
+                            nullable=True,
+                            default=None,
+                            comment=None,
+                            primary_key=False,
+                        )
+                    ],
+                    indexes=[],
+                )
+            ]
+            plan = generate_plan('mydb', desired, actual, MySQLDialect())
+            self.assertTrue(
+                any(
+                    a.action == 'ALTER_COLUMN_NULLABLE' and a.column == 'id' and 'NOT NULL' in a.ddl
+                    for a in plan.actions
+                )
+            )
+            self.assertTrue(any(a.action == 'ADD_PRIMARY_KEY' for a in plan.actions))
+
 
 if __name__ == '__main__':
     unittest.main()