dbextractors 2.2.0__tar.gz → 2.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. {dbextractors-2.2.0 → dbextractors-2.3.0}/ARCHITECTURE.md +2 -0
  2. {dbextractors-2.2.0 → dbextractors-2.3.0}/CHANGELOG.md +143 -0
  3. {dbextractors-2.2.0 → dbextractors-2.3.0}/PKG-INFO +65 -12
  4. {dbextractors-2.2.0 → dbextractors-2.3.0}/README.md +64 -11
  5. {dbextractors-2.2.0 → dbextractors-2.3.0}/docs/legacy-compat.md +29 -2
  6. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/__init__.py +1 -1
  7. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/config.py +13 -5
  8. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/__init__.py +1 -0
  9. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/base.py +2 -1
  10. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/incremental.py +7 -11
  11. dbextractors-2.3.0/src/dbextractors/core/strategies/key_sync.py +223 -0
  12. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/target_pg.py +247 -10
  13. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/dialects/base.py +47 -0
  14. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/dialects/mssql.py +17 -9
  15. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/dialects/mysql.py +34 -0
  16. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/dialects/postgres.py +10 -0
  17. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/entrypoint.py +10 -3
  18. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors.egg-info/PKG-INFO +65 -12
  19. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors.egg-info/SOURCES.txt +3 -0
  20. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_config.py +27 -0
  21. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_entrypoint.py +17 -1
  22. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/test_mysql.py +72 -5
  23. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/test_source_db.py +162 -34
  24. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fakes.py +2 -1
  25. dbextractors-2.3.0/tests/strategies/test_incremental_window_sql.py +162 -0
  26. dbextractors-2.3.0/tests/strategies/test_key_sync.py +329 -0
  27. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_legacy_target_all_strategies.py +7 -1
  28. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_partitioning_db.py +35 -0
  29. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/target/test_target_pg.py +113 -0
  30. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/target/test_target_pg_db.py +48 -0
  31. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/test_skeleton.py +1 -0
  32. {dbextractors-2.2.0 → dbextractors-2.3.0}/CODE_OF_CONDUCT.md +0 -0
  33. {dbextractors-2.2.0 → dbextractors-2.3.0}/CONTRIBUTING.md +0 -0
  34. {dbextractors-2.2.0 → dbextractors-2.3.0}/LICENSE +0 -0
  35. {dbextractors-2.2.0 → dbextractors-2.3.0}/MANIFEST.in +0 -0
  36. {dbextractors-2.2.0 → dbextractors-2.3.0}/NOTICE +0 -0
  37. {dbextractors-2.2.0 → dbextractors-2.3.0}/SECURITY.md +0 -0
  38. {dbextractors-2.2.0 → dbextractors-2.3.0}/docs/data/mage_clean_name.py.txt +0 -0
  39. {dbextractors-2.2.0 → dbextractors-2.3.0}/docs/data/mage_sql_reserved_words.json +0 -0
  40. {dbextractors-2.2.0 → dbextractors-2.3.0}/docs/golden-test.md +0 -0
  41. {dbextractors-2.2.0 → dbextractors-2.3.0}/docs/mage-loader-block.md +0 -0
  42. {dbextractors-2.2.0 → dbextractors-2.3.0}/docs/partitioning-existing-table.md +0 -0
  43. {dbextractors-2.2.0 → dbextractors-2.3.0}/pyproject.toml +0 -0
  44. {dbextractors-2.2.0 → dbextractors-2.3.0}/setup.cfg +0 -0
  45. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/__init__.py +0 -0
  46. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/_reserved_words.py +0 -0
  47. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/coerce.py +0 -0
  48. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/hashing.py +0 -0
  49. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/logging.py +0 -0
  50. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/naming.py +0 -0
  51. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/partitioning.py +0 -0
  52. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/reading.py +0 -0
  53. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/retry.py +0 -0
  54. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/secrets.py +0 -0
  55. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/status.py +0 -0
  56. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/abra_watermark.py +0 -0
  57. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/full.py +0 -0
  58. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/full_by_source.py +0 -0
  59. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/hash_diff.py +0 -0
  60. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/id_watermark.py +0 -0
  61. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/strategies/parent_incremental.py +0 -0
  62. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/target_conn.py +0 -0
  63. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/core/tunnel.py +0 -0
  64. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/dialects/__init__.py +0 -0
  65. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/dialects/firebird.py +0 -0
  66. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/__init__.py +0 -0
  67. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/cli.py +0 -0
  68. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/compare.py +0 -0
  69. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/deviations.py +0 -0
  70. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/introspect.py +0 -0
  71. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/model.py +0 -0
  72. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/progress.py +0 -0
  73. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/report.py +0 -0
  74. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/runners.py +0 -0
  75. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/scratch.py +0 -0
  76. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/session.py +0 -0
  77. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/golden/sqlgen.py +0 -0
  78. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors/py.typed +0 -0
  79. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors.egg-info/dependency_links.txt +0 -0
  80. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors.egg-info/entry_points.txt +0 -0
  81. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors.egg-info/requires.txt +0 -0
  82. {dbextractors-2.2.0 → dbextractors-2.3.0}/src/dbextractors.egg-info/top_level.txt +0 -0
  83. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/coerce/test_characterization_coerce.py +0 -0
  84. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/coerce/test_int64_precision.py +0 -0
  85. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/coerce/test_strict_integer_precision.py +0 -0
  86. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/conftest.py +0 -0
  87. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_credential_leaks.py +0 -0
  88. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_logging.py +0 -0
  89. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_partitioning.py +0 -0
  90. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_reading.py +0 -0
  91. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_required_columns.py +0 -0
  92. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_retry.py +0 -0
  93. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_secrets.py +0 -0
  94. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_status.py +0 -0
  95. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_target_conn.py +0 -0
  96. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/core/test_tunnel.py +0 -0
  97. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/conftest.py +0 -0
  98. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/source_db.py +0 -0
  99. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/test_abra_counter.py +0 -0
  100. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/test_firebird.py +0 -0
  101. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/test_mssql.py +0 -0
  102. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/test_postgres.py +0 -0
  103. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/dialects/test_postgres_db.py +0 -0
  104. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-my-ld/compute_hash_columns.json +0 -0
  105. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-my-ld/create_column_mapping.json +0 -0
  106. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-my-ld/wait_for_port.json +0 -0
  107. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-my-ld/with_retry.json +0 -0
  108. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/_clean.json +0 -0
  109. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/_fmt_int_for_csv.json +0 -0
  110. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/_is_pd_na.json +0 -0
  111. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/_normalize_carriage_returns.json +0 -0
  112. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/_pk_to_str.json +0 -0
  113. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/add_hash_and_timestamp.json +0 -0
  114. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/align_df_columns_to_db.json +0 -0
  115. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/compute_hash_columns.json +0 -0
  116. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/create_column_mapping.json +0 -0
  117. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/decode_bytes_columns.json +0 -0
  118. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/drop_mage_cols.json +0 -0
  119. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/ensure_private_key_permissions.json +0 -0
  120. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/find_free_port.json +0 -0
  121. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/fix_column_values.json +0 -0
  122. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/fix_invalid_dates.json +0 -0
  123. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/normalize_private_key_contents.json +0 -0
  124. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/prepare_export_df.json +0 -0
  125. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/replace_pdna_with_none.json +0 -0
  126. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/sanitize_integer_columns.json +0 -0
  127. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/to_int_or_na.json +0 -0
  128. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/to_int_or_none.json +0 -0
  129. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/to_jsonb.json +0 -0
  130. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/to_str_or_none.json +0 -0
  131. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/update_stats.json +0 -0
  132. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/wait_for_port.json +0 -0
  133. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-mysql/with_retry.json +0 -0
  134. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg/_pk_to_str.json +0 -0
  135. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg/compute_hash_columns.json +0 -0
  136. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg/create_column_mapping.json +0 -0
  137. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg/fix_column_values.json +0 -0
  138. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg/wait_for_port.json +0 -0
  139. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg/with_retry.json +0 -0
  140. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg-ld/compute_hash_columns.json +0 -0
  141. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg-ld/create_column_mapping.json +0 -0
  142. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg-ld/quote_ident.json +0 -0
  143. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg-ld/wait_for_port.json +0 -0
  144. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/A-pg-ld/with_retry.json +0 -0
  145. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-fbird/create_column_mapping.json +0 -0
  146. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-fbird/fix_column_values.json +0 -0
  147. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-fbird/update_stats.json +0 -0
  148. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-fbird/wait_for_port.json +0 -0
  149. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-fbird/with_retry.json +0 -0
  150. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/_decode_value.json +0 -0
  151. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/_pk_to_str.json +0 -0
  152. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/compute_hash_columns.json +0 -0
  153. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/create_column_mapping.json +0 -0
  154. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/fix_column_values.json +0 -0
  155. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/update_stats.json +0 -0
  156. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/wait_for_port.json +0 -0
  157. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mssql/with_retry.json +0 -0
  158. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-my-uni/compute_hash_columns.json +0 -0
  159. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-my-uni/create_column_mapping.json +0 -0
  160. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-my-uni/wait_for_port.json +0 -0
  161. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-my-uni/with_retry.json +0 -0
  162. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/_clean.json +0 -0
  163. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/_pk_to_str.json +0 -0
  164. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/compute_hash_columns.json +0 -0
  165. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/create_column_mapping.json +0 -0
  166. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/fix_column_values.json +0 -0
  167. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/update_stats.json +0 -0
  168. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/wait_for_port.json +0 -0
  169. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-mysql/with_retry.json +0 -0
  170. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/_pk_to_str.json +0 -0
  171. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/compute_hash_columns.json +0 -0
  172. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/create_column_mapping.json +0 -0
  173. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/fix_column_values.json +0 -0
  174. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/to_bool.json +0 -0
  175. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/to_date_str.json +0 -0
  176. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/to_datetime_str.json +0 -0
  177. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/to_jsonb.json +0 -0
  178. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/to_time_str.json +0 -0
  179. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/wait_for_port.json +0 -0
  180. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg/with_retry.json +0 -0
  181. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg-ld/compute_hash_columns.json +0 -0
  182. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg-ld/create_column_mapping.json +0 -0
  183. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg-ld/wait_for_port.json +0 -0
  184. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/B-pg-ld/with_retry.json +0 -0
  185. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/_pk_to_str.json +0 -0
  186. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/compute_hash_columns.json +0 -0
  187. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/create_column_mapping.json +0 -0
  188. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/fix_column_values.json +0 -0
  189. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/is_truthy.json +0 -0
  190. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/update_stats.json +0 -0
  191. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/wait_for_port.json +0 -0
  192. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mssql/with_retry.json +0 -0
  193. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-uni/compute_hash_columns.json +0 -0
  194. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-uni/create_column_mapping.json +0 -0
  195. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-uni/wait_for_port.json +0 -0
  196. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-uni/with_retry.json +0 -0
  197. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-v2/_pk_to_str.json +0 -0
  198. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-v2/compute_hash_columns.json +0 -0
  199. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-v2/create_column_mapping.json +0 -0
  200. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-v2/fix_column_values.json +0 -0
  201. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-v2/update_stats.json +0 -0
  202. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-v2/wait_for_port.json +0 -0
  203. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-my-v2/with_retry.json +0 -0
  204. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mysql/_pk_to_str.json +0 -0
  205. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mysql/compute_hash_columns.json +0 -0
  206. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mysql/create_column_mapping.json +0 -0
  207. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mysql/fix_column_values.json +0 -0
  208. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mysql/update_stats.json +0 -0
  209. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mysql/wait_for_port.json +0 -0
  210. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/C-mysql/with_retry.json +0 -0
  211. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/fixtures/oracle/index.json +0 -0
  212. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/conftest.py +0 -0
  213. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_cli.py +0 -0
  214. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_comparator.py +0 -0
  215. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_deviations.py +0 -0
  216. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_golden_credential_leaks.py +0 -0
  217. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_partitioning_golden.py +0 -0
  218. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_perturb.py +0 -0
  219. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_progress.py +0 -0
  220. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_safety.py +0 -0
  221. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_session.py +0 -0
  222. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/golden/test_sqlgen_and_report.py +0 -0
  223. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/hashing/test_golden_hashes.py +0 -0
  224. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/hashing/test_hashing.py +0 -0
  225. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/naming/test_naming.py +0 -0
  226. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/naming/test_reserved_words.py +0 -0
  227. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/oracle_store.py +0 -0
  228. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/reference_oracle.py +0 -0
  229. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/conftest.py +0 -0
  230. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_abra_watermark.py +0 -0
  231. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_full.py +0 -0
  232. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_full_by_source.py +0 -0
  233. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_hash_diff.py +0 -0
  234. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_id_watermark.py +0 -0
  235. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_incremental.py +0 -0
  236. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_integer_precision_wiring.py +0 -0
  237. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_nchar_conversion_wiring.py +0 -0
  238. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_parent_incremental.py +0 -0
  239. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_parent_incremental_sql.py +0 -0
  240. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/strategies/test_source_gains_column.py +0 -0
  241. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/target/conftest.py +0 -0
  242. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/target/test_fingerprint_contract.py +0 -0
  243. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/target/test_target_timestamp_type.py +0 -0
  244. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/target_pin.py +0 -0
  245. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/test_e2e_run.py +0 -0
  246. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/test_oracle_store.py +0 -0
  247. {dbextractors-2.2.0 → dbextractors-2.3.0}/tests/test_reference_oracle.py +0 -0
@@ -43,6 +43,7 @@ src/dbextractors/
43
43
  abra_watermark.py advance by the counter in an ABRA ERP record ID
44
44
  full_by_source.py partitioning of the target by source (MSSQL only so far)
45
45
  parent_incremental.py window taken from a parent table (Firebird only so far)
46
+ key_sync.py deletions only: compare primary keys, fetch no rows
46
47
 
47
48
  dialects/
48
49
  base.py SourceDialect ABC
@@ -156,6 +157,7 @@ Notes from the existing code:
156
157
  | `abra_watermark` | ABRA ERP tables with no change timestamp | — |
157
158
  | `full_by_source` | one target table fed from several sources | 16 |
158
159
  | `parent_incremental` | window taken from a parent table (Firebird) | 24 |
160
+ | `key_sync` | deletions only, next to a loader that keeps content current | — |
159
161
 
160
162
  `hash_diff` is both the dominant and the most expensive one — today it is
161
163
  **O(source rows, not changed rows)**, ~15,000 rows/s. Optimising this single strategy
@@ -7,6 +7,149 @@ Every release is tagged, and the tag is what a deployment pins in `requirements.
7
7
  **Every change carries a note on what it breaks** — roughly 670 tables depend on
8
8
  this package.
9
9
 
10
+ ## [2.3.0]
11
+
12
+ ### Added
13
+
14
+ - **`DBX_TARGET_COPY_TIMEOUT` — an upper bound for one `COPY … FROM STDIN` into
15
+ the target**, as a PostgreSQL duration (`300000` = ms, `300s`, `5min`). Seen in
16
+ production: a COPY on one connection slowed down to ~10 kB/s, every batch of a
17
+ full load took 15–16 minutes and the run hit the orchestrator's limit after five
18
+ hours, while a fresh connection loaded the same table at full speed. With the
19
+ bound the COPY raises `target_pg.CopyTimeoutError`, the load rolls back and the
20
+ orchestrator's retry starts on a new connection. Two halves: the client stops
21
+ feeding `copy_expert` once the bound is up (psycopg2 reads the server's answer
22
+ only after the last `read()`, so a server cancel alone surfaced only after all
23
+ data had been sent), and `SET LOCAL statement_timeout` covers the server's own
24
+ share; it is put back to the default right after the COPY, so the swap and the
25
+ rest of the transaction stay unbounded. `SET LOCAL`, not `SET` or a DSN option,
26
+ because the target sits behind pgbouncer in transaction mode. All five COPY
27
+ paths in `target_pg` go through the new `target_pg.copy_in`. An invalid value
28
+ raises `TargetError` before the first statement.
29
+
30
+ **Breaks:** nothing while the variable is unset. With it set, the only new
31
+ statements are the two `SET LOCAL` around each COPY; on an autocommit connection
32
+ only the client half applies.
33
+
34
+ - **`load_method: key_sync` — deletion reconciliation that reads only the primary
35
+ key.** For a table whose content is kept current by `incremental` or
36
+ `id_watermark` and whose deletions are not tracked. The source gets one
37
+ streamed `SELECT <pk> FROM <table> ORDER BY <pk>`; the keys are `COPY`ed into a
38
+ temporary table and `mark_deleted_in_source` (the step `hash_diff` ends with)
39
+ sets `_deleted_in_source = TRUE` and `_timestamp` on target rows whose key left
40
+ the source, and flips it back to `FALSE` for keys that returned. No row is read
41
+ from the target or fetched from the source; a source key the target lacks is
42
+ only counted (`new_in_source` in the phase metrics, next to `source_keys`,
43
+ `target_rows`, `deleted_marked`, `revived`). A missing or empty target returns a
44
+ `fallback_reason` and does nothing, a surrogate key raises `StrategyError`, and
45
+ a failed scan or an empty key set against a non-empty target raises instead of
46
+ marking (`empty_rows_ok` allows the latter). New helpers:
47
+ `target_pg.SourceKeySnapshot`, `target_pg.mark_deleted_counts`.
48
+ - `SOURCE_DB.time_zone` (MySQL/MariaDB), see Changed.
49
+
50
+ ### Fixed
51
+
52
+ - **The incremental window no longer depends on a MySQL-only literal outside
53
+ MySQL.** The window expression is now built by the dialect
54
+ (`SourceDialect.incremental_window_expr`): only a dialect with a zero date
55
+ (MySQL/MariaDB) wraps the modification column in `NULLIF(col, '0000-00-00
56
+ 00:00:00')`; PostgreSQL, MSSQL and Firebird get a plain
57
+ `COALESCE(updated, created)`. On PostgreSQL that literal is rejected whatever
58
+ the data holds (`date/time field value out of range` /
59
+ `invalid input syntax for type timestamp`). The legacy
60
+ `incremental_date_column` window had a second PostgreSQL-only failure,
61
+ `COALESCE(col, 0)` (`COALESCE types timestamp … and integer cannot be matched`);
62
+ PostgreSQL now renders the equivalent `col >= cutoff`, since `NULL >= x` is
63
+ already not true. MySQL, MSSQL and Firebird windows are character-for-character
64
+ what they were. The cutoff itself (`days_back`, `incremental_lookback_hours`,
65
+ the deep cutoff) is computed before the dialect is involved and is untouched.
66
+
67
+ ### Changed
68
+
69
+ - **MySQL/MariaDB sessions are pinned to UTC.** Every new source connection now
70
+ runs `SET time_zone = '+00:00'` after the existing network timeouts, unless
71
+ `SOURCE_DB.time_zone` gives another offset or zone name (validated, since it is
72
+ interpolated into the statement; default `+00:00`). A MySQL `TIMESTAMP` is
73
+ stored as UTC and rendered in the session zone, so two servers configured with
74
+ different zones used to return different text for the same instant.
75
+ `SourceDialect.session_statements(time_zone)` is the new hook; other dialects
76
+ return their `session_sql` unchanged.
77
+
78
+ ### Breaks
79
+
80
+ - **`TIMESTAMP` columns read from a MySQL/MariaDB server whose session zone is
81
+ not UTC now arrive in UTC.** The values written to the target for those columns
82
+ change by the server's offset (a `Europe/Prague` server: one or two hours
83
+ earlier). `DATETIME` columns are stored as written and do not move, and a
84
+ server already running in UTC is unaffected. `row_hash` computed by the source
85
+ (`SHA2` over the cast text) changes with it for a table with such a column, so
86
+ under `load_method: hash` every row of that table looks changed on the first
87
+ run — run it with `forced_full_load`, or set `SOURCE_DB.time_zone` to the
88
+ server's own zone to keep the old values. `incremental` windows are compared
89
+ against date literals and shift by the same offset, which a window measured in
90
+ days does not notice.
91
+ - `load_method: key_sync` is a new value; nothing existing changes. A deployment
92
+ that sets it on 2.2.x fails with `Unknown load_method`, so pin 2.3.0 before
93
+ configuring it.
94
+ - `forced_full_load` rewrites `key_sync` to `full`, as it does every other method.
95
+ - The PostgreSQL legacy window (`incremental_date_column`) is rendered
96
+ differently (see Fixed); it returns the same rows, and previously could not run
97
+ at all.
98
+
99
+ ## [2.2.1]
100
+
101
+ ### Fixed
102
+
103
+ - **An upsert into a partitioned target no longer scans every partition.**
104
+ `replace_by_key` deleted with `t.pk = s.pk` alone against a staging table that
105
+ had no statistics, so the planner picked a hash join over a seq scan of the
106
+ whole target. On a 61-partition production table one 5 M-row `id_watermark`
107
+ checkpoint ran over 40 minutes spilling to disk — and with checkpoints that
108
+ repeats every 5 M rows. The staging table is now `ANALYZE`d and the delete is
109
+ bounded by the staging key range (`BETWEEN MIN AND MAX`, taken from staging in
110
+ the target's collation), which turns each partition into an index range scan.
111
+ Locally on 12 partitions × 3 M rows: 1.17 s → 0.2 ms. An empty staging table
112
+ deletes nothing.
113
+ - **A resume by a `VARBINARY` key no longer reads nothing and reports success.**
114
+ mysql-connector returns such a key as `bytes`, and `sql_literal` rendered it as
115
+ the text `'b''6ac9…'''`, which sorts above every key starting with a digit. After
116
+ a dropped connection the resumed query `WHERE id > 'b''…'''` returned no rows and
117
+ the run ended as a success with the rest of the delta unread (production,
118
+ 2026-10-12: 5 M of 42.7 M rows). MySQL now renders bytes as a hex literal
119
+ (`X'…'`), which compares byte-wise like the column; the other dialects raise
120
+ `TypeError` instead of rendering a wrong literal. The watermark itself comes from
121
+ the target and was not affected, so the next run picked up the rest.
122
+
123
+ ### Changed
124
+
125
+ - **MSSQL `NVARCHAR`, `NCHAR` and `NTEXT` arrive whole with pymssql 2.4.4 —
126
+ the driver changed, not this package.** pymssql 2.4.4 (2026-10-09, *"Fixed
127
+ unicode encoding problem #854 & #1000"*) builds its Linux wheels with
128
+ `glibc-gconv-extra` (GNU libiconv on musl), so the bundled FreeTDS can
129
+ convert to the `cp1250` connection charset instead of falling back to latin-1.
130
+ Measured against the `dbo.unicode_edge` fixture with the key off:
131
+ `'příliš žluťoučký kůň'` arrives whole instead of as `'p'`, and a character
132
+ CP1250 cannot express (`ø`, Cyrillic, emoji) becomes `?` instead of cutting
133
+ the string or arriving as a different letter (`ø` → `ř`). 2.4.2 and 2.4.3 still
134
+ truncate. The legacy single-byte `VARCHAR`/`CHAR`/`TEXT` and
135
+ `convert_nchar_to_varchar` read identically on both. The dependency range
136
+ `pymssql>=2.3.13,<3` admits 2.4.4, so a fresh install gets it.
137
+
138
+ The live tests in `tests/dialects/test_source_db.py` now assert whichever
139
+ behaviour the installed pymssql has, instead of pinning the truncation.
140
+
141
+ ### Breaks
142
+
143
+ - **What lands in N-typed MSSQL columns depends on the pymssql that gets
144
+ installed.** A deployment that resolves pymssql 2.4.4 or newer writes the whole
145
+ text where it used to write the truncated one, for every table — with or
146
+ without `convert_nchar_to_varchar`. `row_hash` is computed by the server from
147
+ the raw `NVARCHAR` and does not move, so under `load_method: hash` existing rows
148
+ are not rewritten and keep their truncated text next to new rows that are
149
+ whole; run a table with `forced_full_load` to make its history consistent. An
150
+ N-typed primary key changes its value along with the text. Pin
151
+ `pymssql<2.4.4` in the deployment's requirements to keep the old contents.
152
+
10
153
  ## [2.2.0]
11
154
 
12
155
  ### Fixed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dbextractors
3
- Version: 2.2.0
3
+ Version: 2.3.0
4
4
  Summary: Table extraction from MySQL, MSSQL, PostgreSQL and Firebird into PostgreSQL
5
5
  Author-email: Robert Junek <robert@bisuperhero.cz>
6
6
  License: Apache-2.0
@@ -244,6 +244,11 @@ config = {
244
244
  'password': os.environ['SOURCE_DB_PASSWORD'], # a secret store, not the source file
245
245
  'database': 'source_database',
246
246
  'charset': 'utf8mb4',
247
+ 'time_zone': '+00:00', # MySQL/MariaDB only: the session time zone. Default
248
+ # '+00:00' (UTC), so TIMESTAMP columns read the same
249
+ # whatever zone the server is set to. An offset
250
+ # ('+02:00') or a zone name ('Europe/Prague').
251
+ # DATETIME columns are not affected.
247
252
 
248
253
  'host': '10.0.0.1', # direct connection
249
254
  'port': 3306, # empty = the dialect's default port
@@ -265,6 +270,7 @@ config = {
265
270
  'LOAD_SETTINGS': {
266
271
  'load_method': 'hash', # full | full_by_source | incremental
267
272
  # hash (= hash_diff) | id_watermark | abra_watermark
273
+ # key_sync (deletions only, reads just the PK)
268
274
  # parent_incremental is inferred from 'incremental_parent_*'
269
275
  # An unknown value raises — no quiet fallback to full load.
270
276
  'primary_column': 'id', # PK, case-sensitive (alias: 'primary_key_column')
@@ -305,7 +311,8 @@ config = {
305
311
  # behaviour. See "Large integers and NULL".
306
312
  'convert_nchar_to_varchar': False, # MSSQL only: read NVARCHAR/NCHAR/NTEXT through a
307
313
  # server-side CONVERT so the text arrives whole.
308
- # Off = the inherited truncation. Turning it on
314
+ # Off = the inherited truncation with pymssql
315
+ # < 2.4.4 (whole from 2.4.4). Turning it on
309
316
  # means reloading that table — see "MSSQL and
310
317
  # NVARCHAR text".
311
318
  'read_retry_attempts': None, # override core.reading.DEFAULT_ATTEMPTS for this
@@ -359,6 +366,7 @@ is the most expensive possible outcome of a typo in YAML.
359
366
  | `id_watermark` | append-only, increasing PK | an increasing `primary_column` | appends rows above `MAX(pk)` in the target | only the rows above the watermark |
360
367
  | `abra_watermark` | ABRA ERP tables with no change timestamp | the ABRA record ID as `primary_column`, `abra_id_suffixes` | upserts records whose ID counter is above the watermark | a scan on the source, but only the new rows travel |
361
368
  | `parent_incremental` | child tables with no change date of their own | a parent table with a date column, and a key into it; Firebird only today | deletes the window and reloads it, in one transaction | only the window |
369
+ | `key_sync` | deletions only, next to a method that keeps the content current (`incremental`, `id_watermark`) | `primary_column`; no surrogate key; the target must exist | flips `_deleted_in_source` for keys that left the source (and back for ones that returned); loads no rows | the source's primary keys, nothing else |
362
370
 
363
371
  Two methods are not selected by name:
364
372
 
@@ -645,6 +653,39 @@ Two things will surprise you, and both are deliberate parity with the predecesso
645
653
  A missing target, and a parent that is not in the target, both fall back to a full
646
654
  load.
647
655
 
656
+ ### `key_sync`
657
+
658
+ Reconciles **deletions only**, for a table whose content another pipeline already
659
+ keeps current. `incremental` never sees a row that was deleted, and `id_watermark`
660
+ never marks one; `hash_diff` would, but only by hashing every column of every row.
661
+ `key_sync` reads **just the primary key** — one streamed `SELECT <pk> FROM <table>
662
+ ORDER BY <pk>`, in `batch_size` batches — and nothing else leaves the source.
663
+
664
+ The keys are `COPY`ed into a temporary PostgreSQL table, and the same two `UPDATE`
665
+ statements `hash_diff` ends with do the rest, on the keys alone (no target row is
666
+ read out):
667
+
668
+ - a target row whose key is no longer in the source gets `_deleted_in_source =
669
+ TRUE` and a fresh `_timestamp`;
670
+ - a row already marked that is back in the source is set to `FALSE` again;
671
+ - every other row is left exactly as it is.
672
+
673
+ It **never fetches a row**. A source key the target lacks is only counted
674
+ (`new_in_source` in the phase metrics, with `source_keys`, `target_rows`,
675
+ `deleted_marked` and `revived`); loading it is the other method's job. Run it as
676
+ its own pipeline on a slower schedule than the loader, against the same target.
677
+
678
+ It needs `primary_column`, and the target table has to exist. A missing or empty
679
+ target is **not** turned into a full load, as `incremental` does: the run does
680
+ nothing and says why in `fallback_reason`, because a multi-million row reload is a
681
+ surprising result of a deletion check. A **surrogate key is not supported** and
682
+ raises: the key is an expression over several columns, so it cannot be read on its
683
+ own. It also refuses to mark from an incomplete view of the source: a failed scan
684
+ raises rather than falling back, and a scan that returns no keys while the target
685
+ has rows raises unless `empty_rows_ok` is set. `where_clause` narrows the key scan,
686
+ so a row the filter hides counts as deleted — as it does under `hash_diff`.
687
+ `forced_full_load` rewrites it to `full`, like every other non-`full` method.
688
+
648
689
  ### What every method does the same
649
690
 
650
691
  **Fallback to a full load is reported, never silent.** Whenever one of the
@@ -802,16 +843,28 @@ no check placed here can see.
802
843
  ## MSSQL and `NVARCHAR` text
803
844
 
804
845
  MSSQL is read over a `cp1250` connection, inherited character for character from
805
- both predecessor extractors, and pymssql and FreeTDS disagree about what that
806
- means: the bytes are produced as latin-1 and decoded as cp1250. An `NVARCHAR`,
807
- `NCHAR` or `NTEXT` value is therefore **cut** at the first character latin-1 cannot
808
- express — `'příliš žluťoučký kůň'` arrives as `'p'` — and a character latin-1 *can*
809
- express comes back as a different one (`ø` as `ř`). ASCII is unaffected, and so are
810
- the single-byte `VARCHAR`, `CHAR` and `TEXT`, which are read correctly **because**
811
- of that charset and are the majority of a CP1250-collated source.
812
-
813
- No charset value gets both right; seven were measured. What does is converting on
814
- the server instead:
846
+ both predecessor extractors. **What that does to the N-types depends on the
847
+ pymssql release.**
848
+
849
+ **pymssql 2.4.4 and newer:** `NVARCHAR`, `NCHAR` and `NTEXT` arrive whole. Their
850
+ Linux wheels ship the iconv converters FreeTDS needs, so the text really is
851
+ converted to CP1250; a character CP1250 cannot express (`ø`, Cyrillic, emoji)
852
+ becomes `?` and the rest of the value stays.
853
+
854
+ **pymssql before 2.4.4** (2.4.2 and 2.4.3 measured): the wheel's FreeTDS has no
855
+ CP1250 converter and falls back to latin-1, while pymssql decodes the bytes as
856
+ cp1250. An N-typed value is therefore **cut** at the first character latin-1
857
+ cannot express — `'příliš žluťoučký kůň'` arrives as `'p'` — and a character
858
+ latin-1 *can* express comes back as a different one (`ø` as `ř`).
859
+
860
+ The single-byte `VARCHAR`, `CHAR` and `TEXT` are read correctly on both, **because**
861
+ of that charset, and are the majority of a CP1250-collated source. Upgrading
862
+ pymssql changes what lands in the N-typed columns; see the
863
+ [changelog](CHANGELOG.md) for what that does to existing rows, and pin
864
+ `pymssql<2.4.4` to keep the old contents.
865
+
866
+ With an older pymssql no charset value gets both right; seven were measured. What
867
+ does is converting on the server instead:
815
868
 
816
869
  ```yaml
817
870
  LOAD_SETTINGS:
@@ -196,6 +196,11 @@ config = {
196
196
  'password': os.environ['SOURCE_DB_PASSWORD'], # a secret store, not the source file
197
197
  'database': 'source_database',
198
198
  'charset': 'utf8mb4',
199
+ 'time_zone': '+00:00', # MySQL/MariaDB only: the session time zone. Default
200
+ # '+00:00' (UTC), so TIMESTAMP columns read the same
201
+ # whatever zone the server is set to. An offset
202
+ # ('+02:00') or a zone name ('Europe/Prague').
203
+ # DATETIME columns are not affected.
199
204
 
200
205
  'host': '10.0.0.1', # direct connection
201
206
  'port': 3306, # empty = the dialect's default port
@@ -217,6 +222,7 @@ config = {
217
222
  'LOAD_SETTINGS': {
218
223
  'load_method': 'hash', # full | full_by_source | incremental
219
224
  # hash (= hash_diff) | id_watermark | abra_watermark
225
+ # key_sync (deletions only, reads just the PK)
220
226
  # parent_incremental is inferred from 'incremental_parent_*'
221
227
  # An unknown value raises — no quiet fallback to full load.
222
228
  'primary_column': 'id', # PK, case-sensitive (alias: 'primary_key_column')
@@ -257,7 +263,8 @@ config = {
257
263
  # behaviour. See "Large integers and NULL".
258
264
  'convert_nchar_to_varchar': False, # MSSQL only: read NVARCHAR/NCHAR/NTEXT through a
259
265
  # server-side CONVERT so the text arrives whole.
260
- # Off = the inherited truncation. Turning it on
266
+ # Off = the inherited truncation with pymssql
267
+ # < 2.4.4 (whole from 2.4.4). Turning it on
261
268
  # means reloading that table — see "MSSQL and
262
269
  # NVARCHAR text".
263
270
  'read_retry_attempts': None, # override core.reading.DEFAULT_ATTEMPTS for this
@@ -311,6 +318,7 @@ is the most expensive possible outcome of a typo in YAML.
311
318
  | `id_watermark` | append-only, increasing PK | an increasing `primary_column` | appends rows above `MAX(pk)` in the target | only the rows above the watermark |
312
319
  | `abra_watermark` | ABRA ERP tables with no change timestamp | the ABRA record ID as `primary_column`, `abra_id_suffixes` | upserts records whose ID counter is above the watermark | a scan on the source, but only the new rows travel |
313
320
  | `parent_incremental` | child tables with no change date of their own | a parent table with a date column, and a key into it; Firebird only today | deletes the window and reloads it, in one transaction | only the window |
321
+ | `key_sync` | deletions only, next to a method that keeps the content current (`incremental`, `id_watermark`) | `primary_column`; no surrogate key; the target must exist | flips `_deleted_in_source` for keys that left the source (and back for ones that returned); loads no rows | the source's primary keys, nothing else |
314
322
 
315
323
  Two methods are not selected by name:
316
324
 
@@ -597,6 +605,39 @@ Two things will surprise you, and both are deliberate parity with the predecesso
597
605
  A missing target, and a parent that is not in the target, both fall back to a full
598
606
  load.
599
607
 
608
+ ### `key_sync`
609
+
610
+ Reconciles **deletions only**, for a table whose content another pipeline already
611
+ keeps current. `incremental` never sees a row that was deleted, and `id_watermark`
612
+ never marks one; `hash_diff` would, but only by hashing every column of every row.
613
+ `key_sync` reads **just the primary key** — one streamed `SELECT <pk> FROM <table>
614
+ ORDER BY <pk>`, in `batch_size` batches — and nothing else leaves the source.
615
+
616
+ The keys are `COPY`ed into a temporary PostgreSQL table, and the same two `UPDATE`
617
+ statements `hash_diff` ends with do the rest, on the keys alone (no target row is
618
+ read out):
619
+
620
+ - a target row whose key is no longer in the source gets `_deleted_in_source =
621
+ TRUE` and a fresh `_timestamp`;
622
+ - a row already marked that is back in the source is set to `FALSE` again;
623
+ - every other row is left exactly as it is.
624
+
625
+ It **never fetches a row**. A source key the target lacks is only counted
626
+ (`new_in_source` in the phase metrics, with `source_keys`, `target_rows`,
627
+ `deleted_marked` and `revived`); loading it is the other method's job. Run it as
628
+ its own pipeline on a slower schedule than the loader, against the same target.
629
+
630
+ It needs `primary_column`, and the target table has to exist. A missing or empty
631
+ target is **not** turned into a full load, as `incremental` does: the run does
632
+ nothing and says why in `fallback_reason`, because a multi-million row reload is a
633
+ surprising result of a deletion check. A **surrogate key is not supported** and
634
+ raises: the key is an expression over several columns, so it cannot be read on its
635
+ own. It also refuses to mark from an incomplete view of the source: a failed scan
636
+ raises rather than falling back, and a scan that returns no keys while the target
637
+ has rows raises unless `empty_rows_ok` is set. `where_clause` narrows the key scan,
638
+ so a row the filter hides counts as deleted — as it does under `hash_diff`.
639
+ `forced_full_load` rewrites it to `full`, like every other non-`full` method.
640
+
600
641
  ### What every method does the same
601
642
 
602
643
  **Fallback to a full load is reported, never silent.** Whenever one of the
@@ -754,16 +795,28 @@ no check placed here can see.
754
795
  ## MSSQL and `NVARCHAR` text
755
796
 
756
797
  MSSQL is read over a `cp1250` connection, inherited character for character from
757
- both predecessor extractors, and pymssql and FreeTDS disagree about what that
758
- means: the bytes are produced as latin-1 and decoded as cp1250. An `NVARCHAR`,
759
- `NCHAR` or `NTEXT` value is therefore **cut** at the first character latin-1 cannot
760
- express — `'příliš žluťoučký kůň'` arrives as `'p'` — and a character latin-1 *can*
761
- express comes back as a different one (`ø` as `ř`). ASCII is unaffected, and so are
762
- the single-byte `VARCHAR`, `CHAR` and `TEXT`, which are read correctly **because**
763
- of that charset and are the majority of a CP1250-collated source.
764
-
765
- No charset value gets both right; seven were measured. What does is converting on
766
- the server instead:
798
+ both predecessor extractors. **What that does to the N-types depends on the
799
+ pymssql release.**
800
+
801
+ **pymssql 2.4.4 and newer:** `NVARCHAR`, `NCHAR` and `NTEXT` arrive whole. Their
802
+ Linux wheels ship the iconv converters FreeTDS needs, so the text really is
803
+ converted to CP1250; a character CP1250 cannot express (`ø`, Cyrillic, emoji)
804
+ becomes `?` and the rest of the value stays.
805
+
806
+ **pymssql before 2.4.4** (2.4.2 and 2.4.3 measured): the wheel's FreeTDS has no
807
+ CP1250 converter and falls back to latin-1, while pymssql decodes the bytes as
808
+ cp1250. An N-typed value is therefore **cut** at the first character latin-1
809
+ cannot express — `'příliš žluťoučký kůň'` arrives as `'p'` — and a character
810
+ latin-1 *can* express comes back as a different one (`ø` as `ř`).
811
+
812
+ The single-byte `VARCHAR`, `CHAR` and `TEXT` are read correctly on both, **because**
813
+ of that charset, and are the majority of a CP1250-collated source. Upgrading
814
+ pymssql changes what lands in the N-typed columns; see the
815
+ [changelog](CHANGELOG.md) for what that does to existing rows, and pin
816
+ `pymssql<2.4.4` to keep the old contents.
817
+
818
+ With an older pymssql no charset value gets both right; seven were measured. What
819
+ does is converting on the server instead:
767
820
 
768
821
  ```yaml
769
822
  LOAD_SETTINGS:
@@ -154,6 +154,32 @@ closing the gap means changing that test on purpose.
154
154
 
155
155
  ## MSSQL: `NVARCHAR` text arrives truncated
156
156
 
157
+ **Fixed by pymssql 2.4.4 — the driver, not this package.** Everything in this
158
+ section describes pymssql **before 2.4.4** (2.3.13, 2.4.2 and 2.4.3 measured).
159
+ pymssql 2.4.4 (2026-10-09, changelog *"Fixed unicode encoding problem #854 &
160
+ #1000"*) changed no Python or Cython code; it builds the Linux wheels with
161
+ `glibc-gconv-extra` (GNU libiconv on musl), so the statically linked FreeTDS has
162
+ the CP1250 converter it was missing and stops falling back to latin-1. Measured
163
+ against the same fixture with that release:
164
+
165
+ | the value contains | key off, pymssql ≥ 2.4.4 | key on (any pymssql) |
166
+ |---|---|---|
167
+ | ASCII only | intact | intact |
168
+ | CP1250 text (`ř`, `š`, `ž`) | intact | intact |
169
+ | `ø` (latin-1, not CP1250) | `?` | folded onto `o` |
170
+ | a character CP1250 cannot express (Cyrillic, emoji) | `?`, rest kept | `?`, rest kept |
171
+ | single-byte `VARCHAR`/`CHAR`/`TEXT` | intact | intact — never wrapped |
172
+
173
+ So with 2.4.4 the key is no longer needed for Czech text. What it costs to get
174
+ there is the same as switching the key on: the contents of every N-typed column
175
+ change while `row_hash` does not, so a `load_method: hash` table needs one
176
+ `forced_full_load` to repair its history — but this time for **every** table
177
+ with N-typed text, triggered by whatever pymssql the deployment resolves
178
+ (`pymssql>=2.3.13,<3` admits 2.4.4). Pin `pymssql<2.4.4` in the deployment's
179
+ requirements to keep the old contents until those reloads are planned. The
180
+ fix is in the published wheels; a pymssql built from source links the system
181
+ FreeTDS and iconv and may behave either way regardless of its version.
182
+
157
183
  **Symptom.** A Czech `NVARCHAR` value in an MSSQL source lands in the target as
158
184
  its first character. `'příliš žluťoučký kůň'` becomes `'p'`. There is no error,
159
185
  no warning and no replacement character; the row count matches and the column is
@@ -164,7 +190,7 @@ correct UTF-16 (`LEN()` returns 20, `CONVERT(…, VARBINARY(…), …)` shows th
164
190
  string). The loss happens on the way out.
165
191
 
166
192
  **Cause.** The connection is opened with `charset='cp1250'`
167
- (`MSSQLDialect.connect_args`). With pymssql 2.3.13 that combination is
193
+ (`MSSQLDialect.connect_args`). With pymssql up to 2.4.3 that combination is
168
194
  internally inconsistent: the driver converts the server's UCS-2 to the
169
195
  single-byte charset as **latin-1**, and pymssql then decodes those bytes with
170
196
  the real **cp1250** codec. Two consequences follow from the one mismatch:
@@ -253,7 +279,8 @@ character rather than lossy, and costs two connections per table; it can be adde
253
279
  as another key if the `?` ever matters.
254
280
 
255
281
  Pinned by `tests/dialects/test_source_db.py` under "the MSSQL client charset" and
256
- "convert_nchar_to_varchar, the way out", against the `dbo.unicode_edge` fixture in
282
+ "convert_nchar_to_varchar, the way out" — for both sides of the 2.4.4 boundary,
283
+ choosing the expectation by the installed pymssql version — against the `dbo.unicode_edge` fixture in
257
284
  `docker/seed/mssql/001_types.sql`. Which columns get wrapped is settled in
258
285
  `tests/dialects/test_mssql.py`, and the wiring from `LOAD_SETTINGS` to the SELECT
259
286
  in `tests/strategies/test_nchar_conversion_wiring.py`.
@@ -26,4 +26,4 @@ logging.getLogger("dbextractors").addHandler(logging.NullHandler())
26
26
 
27
27
  __all__ = ["__version__", "run"]
28
28
 
29
- __version__ = "2.2.0"
29
+ __version__ = "2.3.0"
@@ -110,6 +110,7 @@ SOURCE_DB_KEYS: frozenset[str] = frozenset(
110
110
  "port",
111
111
  "database",
112
112
  "charset",
113
+ "time_zone",
113
114
  "ssh_address_or_host",
114
115
  "ssh_port",
115
116
  "ssh_username",
@@ -346,11 +347,12 @@ class LoadSettingsConfig:
346
347
  #: Read MSSQL's national-character columns through a server-side
347
348
  #: ``CONVERT(VARCHAR(MAX), …)``.
348
349
  #:
349
- #: MSSQL is read over a ``cp1250`` connection, which pymssql and FreeTDS
350
- #: disagree about: the bytes are produced as latin-1 and decoded as cp1250,
351
- #: so an ``NVARCHAR``/``NCHAR``/``NTEXT`` value is **cut** at the first
352
- #: character latin-1 cannot express and a character latin-1 *can* express
353
- #: comes back as a different one. With this key on, the server converts those
350
+ #: MSSQL is read over a ``cp1250`` connection, which a pymssql before 2.4.4
351
+ #: and its FreeTDS disagree about: the bytes are produced as latin-1 and
352
+ #: decoded as cp1250, so an ``NVARCHAR``/``NCHAR``/``NTEXT`` value is **cut**
353
+ #: at the first character latin-1 cannot express and a character latin-1
354
+ #: *can* express comes back as a different one (2.4.4 converts to CP1250 and
355
+ #: no longer needs the key). With this key on, the server converts those
354
356
  #: columns before they reach the wire, so they arrive whole; see
355
357
  #: `dialects.mssql.MSSQLDialect.render_nchar_convert` and
356
358
  #: docs/legacy-compat.md.
@@ -430,6 +432,10 @@ class SourceDbConfig:
430
432
  port: int | None = None
431
433
  database: str | None = None
432
434
  charset: str | None = None
435
+ #: Session time zone for the source connection. ``None`` = the dialect's own
436
+ #: default (MySQL: ``+00:00``, see `MySQLDialect.session_statements`); other
437
+ #: dialects ignore it.
438
+ time_zone: str | None = None
433
439
  ssh_address_or_host: str | None = None
434
440
  ssh_port: int = 22
435
441
  ssh_username: str | None = None
@@ -687,6 +693,8 @@ def parse(config: dict) -> ParsedConfig:
687
693
  port=source_db_cfg.get("port", legacy_conn.get("port")),
688
694
  database=source_db_cfg.get("database", legacy_conn.get("database")),
689
695
  charset=source_db_cfg.get("charset"),
696
+ # `or None`: an empty value means "not filled in", not "the empty zone".
697
+ time_zone=str(source_db_cfg.get("time_zone") or "").strip() or None,
690
698
  ssh_address_or_host=source_db_cfg.get(
691
699
  "ssh_address_or_host", ssh_defaults.get("ssh_address_or_host")
692
700
  ),
@@ -11,6 +11,7 @@ How many tables each one serves in production today:
11
11
  | ``abra_watermark`` | ABRA ERP tables with no change timestamp: new records by the ID counter | — |
12
12
  | ``full_by_source`` | one target table assembled from several sources | 16 |
13
13
  | ``parent_incremental`` | window derived from a parent table | 24 |
14
+ | ``key_sync`` | deletions only: compare primary keys, flag what vanished, fetch nothing | — |
14
15
 
15
16
  ``hash_diff`` is both the dominant and the most expensive strategy — optimising
16
17
  this one touches ~530 tables.
@@ -181,7 +181,7 @@ class LoadStrategy(ABC):
181
181
  """One way of getting a table from the source into the target."""
182
182
 
183
183
  #: 'full' | 'hash_diff' | 'incremental' | 'id_watermark' | 'abra_watermark' |
184
- #: 'full_by_source' | 'parent_incremental'
184
+ #: 'full_by_source' | 'parent_incremental' | 'key_sync'
185
185
  name: str
186
186
  #: Dialect features without which the strategy makes no sense.
187
187
  #: Checked in `validate()`, i.e. before the first read from the source.
@@ -352,6 +352,7 @@ STRATEGIES: Dict[str, str] = {
352
352
  "abra_watermark": "abra_watermark:AbraWatermarkStrategy",
353
353
  "full_by_source": "full_by_source:FullBySourceStrategy",
354
354
  "parent_incremental": "parent_incremental:ParentIncrementalStrategy",
355
+ "key_sync": "key_sync:KeySyncStrategy",
355
356
  }.items()
356
357
  }
357
358
 
@@ -209,7 +209,6 @@ class IncrementalStrategy(LoadStrategy):
209
209
  assert updated_raw is not None # guaranteed by validate()
210
210
  updated = ctx.dialect.quote_ident(updated_raw)
211
211
  created = _created_column(ctx)
212
- sentinel = getattr(ctx.dialect, "zero_datetime_literal", None)
213
212
 
214
213
  if _uses_legacy_date_column_window(ctx):
215
214
  # Literally the predecessor's parent condition: `OR` across both
@@ -217,21 +216,18 @@ class IncrementalStrategy(LoadStrategy):
217
216
  # condition.
218
217
  value = ctx.dialect.sql_literal(ctx.dialect.cutoff_value(cutoff))
219
218
  columns = [updated] + ([ctx.dialect.quote_ident(created)] if created else [])
220
- parts = [f"COALESCE({c}, 0) >= {value}" for c in columns]
219
+ parts = [ctx.dialect.incremental_at_least(c, value) for c in columns]
221
220
  clause = "(" + " OR ".join(parts) + ")"
222
221
  if ctx.where:
223
222
  clause = f"({clause}) AND ({ctx.where})"
224
223
  return clause
225
224
 
226
- if created:
227
- created_q = ctx.dialect.quote_ident(created)
228
- expr = (
229
- f"COALESCE(NULLIF({updated}, {sentinel!r}), {created_q})"
230
- if sentinel
231
- else f"COALESCE({updated}, {created_q})"
232
- )
233
- else:
234
- expr = updated
225
+ # The dialect decides how the two columns combine (a zero date exists only
226
+ # on MySQL), so a literal that is invalid for another dialect never reaches
227
+ # its source.
228
+ expr = ctx.dialect.incremental_window_expr(
229
+ updated, ctx.dialect.quote_ident(created) if created else None
230
+ )
235
231
 
236
232
  clause = f"{expr} >= '{cutoff.strftime('%Y-%m-%d')} 00:00:00'"
237
233
  if ctx.where: