execsql2 2.24.0__py3-none-any.whl → 2.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. execsql/api.py +169 -160
  2. execsql/cli/__init__.py +2 -0
  3. execsql/cli/commands/lint.py +2 -10
  4. execsql/cli/commands/run.py +35 -27
  5. execsql/cli/dsn.py +11 -4
  6. execsql/cli/lint.py +271 -28
  7. execsql/cli/options.py +1 -1
  8. execsql/cli/run.py +56 -19
  9. execsql/config.py +5 -19
  10. execsql/data/execsql.conf.template +14 -4
  11. execsql/db/access.py +29 -8
  12. execsql/db/base.py +40 -8
  13. execsql/db/dsn.py +13 -4
  14. execsql/db/duckdb.py +3 -1
  15. execsql/db/factory.py +11 -2
  16. execsql/db/firebird.py +21 -7
  17. execsql/db/mysql.py +37 -16
  18. execsql/db/oracle.py +49 -16
  19. execsql/db/postgres.py +30 -6
  20. execsql/db/sqlite.py +7 -9
  21. execsql/db/sqlserver.py +25 -8
  22. execsql/exporters/delimited.py +87 -28
  23. execsql/exporters/xlsx.py +24 -9
  24. execsql/exporters/zip.py +1 -2
  25. execsql/format.py +29 -2
  26. execsql/gui/desktop.py +1 -3
  27. execsql/gui/tui.py +2 -2
  28. execsql/importers/base.py +35 -1
  29. execsql/importers/csv.py +3 -1
  30. execsql/manifest.py +33 -22
  31. execsql/metacommands/conditions.py +6 -7
  32. execsql/metacommands/data.py +4 -1
  33. execsql/metacommands/debug.py +2 -4
  34. execsql/metacommands/io_fileops.py +62 -37
  35. execsql/metacommands/io_import.py +0 -1
  36. execsql/metacommands/prompt.py +2 -0
  37. execsql/metacommands/upsert.py +5 -2
  38. execsql/models.py +3 -2
  39. execsql/parser.py +14 -5
  40. execsql/plugins.py +24 -0
  41. execsql/script/engine.py +40 -11
  42. execsql/script/executor.py +9 -5
  43. execsql/script/parser.py +167 -118
  44. execsql/state.py +23 -9
  45. execsql/state.pyi +8 -0
  46. execsql/types.py +22 -13
  47. execsql/utils/auth.py +169 -24
  48. execsql/utils/datetime.py +55 -17
  49. execsql/utils/errors.py +71 -9
  50. execsql/utils/fileio.py +404 -117
  51. execsql/utils/gui.py +3 -0
  52. execsql/utils/mail.py +37 -12
  53. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/METADATA +18 -12
  54. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/RECORD +72 -72
  55. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/README.md +0 -0
  56. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  57. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
  58. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
  59. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_compare.sql +0 -0
  60. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
  61. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
  62. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
  63. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
  64. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_upsert.sql +0 -0
  65. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/script_template.sql +0 -0
  66. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
  67. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
  68. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
  69. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/WHEEL +0 -0
  70. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/entry_points.txt +0 -0
  71. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/licenses/LICENSE.txt +0 -0
  72. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/licenses/NOTICE +0 -0
execsql/db/oracle.py CHANGED
@@ -15,14 +15,42 @@ from execsql.db.base import Database
15
15
  from execsql.db.tiers import SupportTier
16
16
  from execsql.exceptions import ErrInfo
17
17
  from execsql.utils.errors import exception_desc, fatal_error
18
- from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
18
+ from execsql.utils.auth import (
19
+ begin_login,
20
+ clear_stored_password,
21
+ get_password,
22
+ is_login_failure,
23
+ password_from_keyring,
24
+ remember_password,
25
+ )
19
26
  import execsql.state as _state
20
27
 
21
28
  __all__ = ["OracleDatabase"]
22
29
 
23
30
 
31
+ def _oracle_driver() -> Any:
32
+ """The Oracle driver module: python-oracledb, or else the legacy cx_Oracle.
33
+
34
+ ``execsql2[oracle]`` installs oracledb. cx_Oracle, the driver it replaced
35
+ (same ``makedsn`` / ``connect`` calls), is used only when it is the one
36
+ installed. ``None`` if neither is.
37
+ """
38
+ try:
39
+ import oracledb
40
+
41
+ return oracledb
42
+ except ImportError:
43
+ pass
44
+ try:
45
+ import cx_Oracle
46
+
47
+ return cx_Oracle
48
+ except ImportError:
49
+ return None
50
+
51
+
24
52
  class OracleDatabase(Database):
25
- """Oracle adapter using the cx_Oracle (python-oracledb) driver."""
53
+ """Oracle adapter using the python-oracledb driver."""
26
54
 
27
55
  #: Oracle commits every DDL statement at once, together with everything
28
56
  #: before it, including CREATE GLOBAL TEMPORARY TABLE.
@@ -46,11 +74,9 @@ class OracleDatabase(Database):
46
74
  encoding: str | None = "UTF8",
47
75
  password: str | None = None,
48
76
  ) -> None:
49
- try:
50
- import cx_Oracle # noqa: F401
51
- except Exception:
77
+ if _oracle_driver() is None:
52
78
  fatal_error(
53
- "The cx-Oracle module is required to connect to Oracle. See https://pypi.org/project/cx-Oracle/",
79
+ 'The oracledb module is required to connect to Oracle. Install it with: pip install "execsql2[oracle]"',
54
80
  )
55
81
  from execsql.types import dbt_oracle
56
82
 
@@ -64,7 +90,6 @@ class OracleDatabase(Database):
64
90
  self.type = dbt_oracle
65
91
  self.password = password
66
92
  self.encoding = encoding or "UTF8"
67
- self.encode_commands = False
68
93
  self.paramstr = ":1"
69
94
  self.open_db()
70
95
  self.password = None # Clear cleartext password after successful connection
@@ -77,39 +102,45 @@ class OracleDatabase(Database):
77
102
 
78
103
  def open_db(self) -> None:
79
104
  """Open a connection to the Oracle database."""
80
- import cx_Oracle
105
+ driver = _oracle_driver()
81
106
 
82
107
  def db_conn(db: OracleDatabase, db_name: str):
83
- dsn = cx_Oracle.makedsn(db.server_name, db.port, service_name=db_name)
108
+ dsn = driver.makedsn(db.server_name, db.port, service_name=db_name)
84
109
  if db.user and db.password:
85
- return cx_Oracle.connect(user=db.user, password=db.password, dsn=dsn)
110
+ return driver.connect(user=db.user, password=db.password, dsn=dsn)
86
111
  else:
87
- return cx_Oracle.connect(dsn=dsn)
112
+ return driver.connect(dsn=dsn)
88
113
 
89
114
  if self.conn is None:
90
115
  try:
116
+ begin_login()
91
117
  if self.user and self.need_passwd and not self.password:
92
118
  self.password = get_password(
93
119
  "Oracle",
94
120
  self.db_name,
95
121
  self.user,
96
122
  server_name=self.server_name,
123
+ port=self.port,
97
124
  )
98
125
  try:
99
126
  self.conn = db_conn(self, self.db_name)
100
- except Exception:
101
- if not password_from_keyring():
127
+ except Exception as e:
128
+ # Only a rejected password means the stored one is stale;
129
+ # a timeout or a missing database leaves it alone.
130
+ if not (password_from_keyring() and is_login_failure(e)):
102
131
  raise
103
- clear_stored_password("Oracle", self.db_name, self.user, self.server_name)
132
+ clear_stored_password("Oracle", self.db_name, self.user, self.server_name, port=self.port)
104
133
  self.password = get_password(
105
134
  "Oracle",
106
135
  self.db_name,
107
136
  self.user,
108
137
  server_name=self.server_name,
138
+ port=self.port,
109
139
  skip_keyring=True,
110
140
  other_msg="(stored credential failed — enter current password)",
111
141
  )
112
142
  self.conn = db_conn(self, self.db_name)
143
+ remember_password("Oracle", self.db_name, self.user, self.server_name, port=self.port)
113
144
  except SystemExit:
114
145
  # If the user canceled the password prompt.
115
146
  raise
@@ -153,6 +184,9 @@ class OracleDatabase(Database):
153
184
 
154
185
  raise DatabaseNotImplementedError(self.name(), "schema_exists")
155
186
 
187
+ def current_schema(self) -> str | None:
188
+ return self._query_current_schema("select sys_context('USERENV', 'CURRENT_SCHEMA') from dual")
189
+
156
190
  def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
157
191
  """Return True if the named table exists in the Oracle database."""
158
192
  # Oracle folds unquoted identifiers to uppercase; normalise so a lookup
@@ -288,8 +322,7 @@ class OracleDatabase(Database):
288
322
  return len(rows) > 0
289
323
 
290
324
  def drop_table(self, tablename: str) -> None:
291
- """Drop the named table with cascade constraints."""
292
- tablename = self.type.quoted(tablename)
325
+ """Drop *tablename* (schema-qualified and quoted by the caller) with cascade constraints."""
293
326
  self.execute(f"drop table {tablename} cascade constraints")
294
327
 
295
328
  def paramsubs(self, paramcount: int) -> str:
execsql/db/postgres.py CHANGED
@@ -16,7 +16,14 @@ from typing import Any
16
16
  from execsql.db.base import Database
17
17
  from execsql.exceptions import ErrInfo
18
18
  from execsql.utils.errors import exception_desc, fatal_error
19
- from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
19
+ from execsql.utils.auth import (
20
+ begin_login,
21
+ clear_stored_password,
22
+ get_password,
23
+ is_login_failure,
24
+ password_from_keyring,
25
+ remember_password,
26
+ )
20
27
  from execsql.utils.strings import encodings_match
21
28
  import execsql.state as _state
22
29
 
@@ -60,7 +67,6 @@ class PostgresDatabase(Database):
60
67
  self.type = dbt_postgres
61
68
  self.password = password
62
69
  self.new_db = new_db
63
- self.encode_commands = False
64
70
  self.paramstr = "%s"
65
71
  self.connect_timeout = connect_timeout
66
72
  self.open_db()
@@ -123,31 +129,37 @@ class PostgresDatabase(Database):
123
129
 
124
130
  if self.conn is None:
125
131
  try:
132
+ begin_login()
126
133
  if self.user and self.need_passwd and not self.password:
127
134
  self.password = get_password(
128
135
  "PostgreSQL",
129
136
  self.db_name,
130
137
  self.user,
131
138
  server_name=self.server_name,
139
+ port=self.port,
132
140
  )
133
141
  if self.new_db:
134
142
  create_db(self)
135
143
  try:
136
144
  self.conn = db_conn(self, self.db_name)
137
- except (ErrInfo, Exception):
138
- if not password_from_keyring():
145
+ except (ErrInfo, Exception) as e:
146
+ # Only a rejected password means the stored one is stale;
147
+ # a timeout or a missing database leaves it alone.
148
+ if not (password_from_keyring() and is_login_failure(e)):
139
149
  raise
140
150
  # Stored credential is stale — clear it and re-prompt.
141
- clear_stored_password("PostgreSQL", self.db_name, self.user, self.server_name)
151
+ clear_stored_password("PostgreSQL", self.db_name, self.user, self.server_name, port=self.port)
142
152
  self.password = get_password(
143
153
  "PostgreSQL",
144
154
  self.db_name,
145
155
  self.user,
146
156
  server_name=self.server_name,
157
+ port=self.port,
147
158
  skip_keyring=True,
148
159
  other_msg="(stored credential failed — enter current password)",
149
160
  )
150
161
  self.conn = db_conn(self, self.db_name)
162
+ remember_password("PostgreSQL", self.db_name, self.user, self.server_name, port=self.port)
151
163
  except SystemExit:
152
164
  # If the user canceled the password prompt.
153
165
  raise
@@ -178,6 +190,9 @@ class PostgresDatabase(Database):
178
190
  rows = curs.fetchall()
179
191
  return len(rows) > 0
180
192
 
193
+ # table_exists() without a schema looks only where CREATE TABLE puts a table: it searches the temp schema and the search path.
194
+ unqualified_lookup_is_scoped = True
195
+
181
196
  def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
182
197
  """Return True if the named table exists in the PostgreSQL database."""
183
198
  if schema_name is not None:
@@ -336,6 +351,8 @@ class PostgresDatabase(Database):
336
351
  and not _state.conf.trim_strings
337
352
  and not _state.conf.replace_newlines
338
353
  ):
354
+ from execsql.exporters.delimited import blank_lines_as_empty_rows
355
+
339
356
  # Use Postgres' COPY FROM method via psycopg3's cursor.copy() context manager.
340
357
  rf = csv_file_obj.open("rt")
341
358
  if skipheader:
@@ -364,7 +381,14 @@ class PostgresDatabase(Database):
364
381
  with self._cursor() as curs:
365
382
  try:
366
383
  with curs.copy(copy_cmd) as copy:
367
- while chunk := rf.read(_state.conf.import_buffer):
384
+ # COPY rejects a blank line; the row path reads it as an empty row.
385
+ for chunk in blank_lines_as_empty_rows(
386
+ rf,
387
+ delim,
388
+ csv_file_obj.quotechar,
389
+ len(csv_file_cols),
390
+ _state.conf.import_buffer,
391
+ ):
368
392
  copy.write(chunk)
369
393
  except ErrInfo:
370
394
  raise
execsql/db/sqlite.py CHANGED
@@ -45,7 +45,6 @@ class SQLiteDatabase(Database):
45
45
  encoding="UTF-8",
46
46
  )
47
47
  self.type = dbt_sqlite
48
- self.encode_commands = False
49
48
  self.timeout = timeout
50
49
  self.open_db()
51
50
 
@@ -95,6 +94,9 @@ class SQLiteDatabase(Database):
95
94
  self.rollback()
96
95
  raise
97
96
 
97
+ # table_exists() without a schema looks only where CREATE TABLE puts a table: one namespace.
98
+ unqualified_lookup_is_scoped = True
99
+
98
100
  def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
99
101
  """Return True if the named table exists in the SQLite database."""
100
102
  # SQLite stores names as-created; match case-insensitively so a lookup
@@ -170,8 +172,7 @@ class SQLiteDatabase(Database):
170
172
  return False
171
173
 
172
174
  def drop_table(self, tablename: str) -> None:
173
- """Drop the named table from the SQLite database if it exists."""
174
- tablename = self.type.quoted(tablename)
175
+ """Drop *tablename* (schema-qualified and quoted by the caller) if it exists."""
175
176
  self.execute(f"drop table if exists {tablename};")
176
177
 
177
178
  def populate_table(
@@ -237,14 +238,11 @@ class SQLiteDatabase(Database):
237
238
 
238
239
  last_line: list[Any] = []
239
240
  try:
240
- for datalineno, line in enumerate(rowsource):
241
+ for line in rowsource:
241
242
  # Skip empty rows.
242
243
  if not (len(line) == 1 and line[0] is None):
243
- if len(line) < len(columns):
244
- raise ErrInfo(
245
- type="error",
246
- other_msg=f"Too few values on data line {datalineno} of input.",
247
- )
244
+ # Trailing values left off the line are NULL.
245
+ line = list(line) + [None] * (len(ts_colnames) - len(line))
248
246
  if _state.conf.trim_strings or _state.conf.replace_newlines or not _state.conf.empty_strings:
249
247
  for i in range(len(line)):
250
248
  if line[i] is not None and isinstance(line[i], _state.stringtypes):
execsql/db/sqlserver.py CHANGED
@@ -12,7 +12,14 @@ import re
12
12
  from execsql.db.base import Database
13
13
  from execsql.exceptions import ErrInfo
14
14
  from execsql.utils.errors import fatal_error
15
- from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
15
+ from execsql.utils.auth import (
16
+ begin_login,
17
+ clear_stored_password,
18
+ get_password,
19
+ is_login_failure,
20
+ password_from_keyring,
21
+ remember_password,
22
+ )
16
23
  import execsql.state as _state
17
24
 
18
25
  __all__ = ["SqlServerDatabase"]
@@ -47,11 +54,13 @@ class SqlServerDatabase(Database):
47
54
  self.type = dbt_sqlserver
48
55
  self.password = password
49
56
  self.encoding = encoding or "latin1" # Default on installation of SQL Server
50
- self.encode_commands = True
51
57
  self.paramstr = "?"
52
58
  self.open_db()
53
59
  self.password = None # Clear cleartext password after successful connection
54
60
 
61
+ def current_schema(self) -> str | None:
62
+ return self._query_current_schema("select schema_name()")
63
+
55
64
  def __repr__(self) -> str:
56
65
  return (
57
66
  f"SqlServerDatabase({self.server_name!r}, {self.db_name!r}, {self.user!r}, "
@@ -75,12 +84,14 @@ class SqlServerDatabase(Database):
75
84
  import pyodbc
76
85
 
77
86
  if self.conn is None:
87
+ begin_login()
78
88
  if self.user and self.need_passwd and not self.password:
79
89
  self.password = get_password(
80
90
  "SQL Server",
81
91
  self.db_name,
82
92
  self.user,
83
93
  server_name=self.server_name,
94
+ port=self.port,
84
95
  )
85
96
  # Use pyodbc to connect. Try different driver versions from newest to oldest.
86
97
  ssdrivers = (
@@ -94,6 +105,8 @@ class SqlServerDatabase(Database):
94
105
  "SQL Server",
95
106
  )
96
107
 
108
+ errors: list[BaseException] = []
109
+
97
110
  def _try_drivers():
98
111
  for drv in ssdrivers:
99
112
  if self.user:
@@ -117,7 +130,8 @@ class SqlServerDatabase(Database):
117
130
  # default; pyodbc treats `timeout` as the login-and-
118
131
  # query timeout on the connection.
119
132
  self.conn = pyodbc.connect(connstr, timeout=30)
120
- except Exception:
133
+ except Exception as e:
134
+ errors.append(e)
121
135
  if _state.exec_log is not None:
122
136
  _state.exec_log.log_status_info(
123
137
  f"Could not connect using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
@@ -130,18 +144,22 @@ class SqlServerDatabase(Database):
130
144
  return True
131
145
  return False
132
146
 
133
- if not _try_drivers() and password_from_keyring():
134
- # Stored credential is stale — clear it and re-prompt.
135
- clear_stored_password("SQL Server", self.db_name, self.user, self.server_name)
147
+ # Only a rejected password means the stored one is stale; a
148
+ # timeout or a missing database leaves it alone.
149
+ if not _try_drivers() and password_from_keyring() and any(is_login_failure(e) for e in errors):
150
+ clear_stored_password("SQL Server", self.db_name, self.user, self.server_name, port=self.port)
136
151
  self.password = get_password(
137
152
  "SQL Server",
138
153
  self.db_name,
139
154
  self.user,
140
155
  server_name=self.server_name,
156
+ port=self.port,
141
157
  skip_keyring=True,
142
158
  other_msg="(stored credential failed — enter current password)",
143
159
  )
144
160
  _try_drivers()
161
+ if self.conn:
162
+ remember_password("SQL Server", self.db_name, self.user, self.server_name, port=self.port)
145
163
 
146
164
  if not self.conn:
147
165
  raise ErrInfo(
@@ -193,9 +211,8 @@ class SqlServerDatabase(Database):
193
211
  return len(rows) > 0
194
212
 
195
213
  def drop_table(self, tablename: str) -> None:
196
- """Drop the named table from the SQL Server database."""
214
+ """Drop *tablename*, schema-qualified and quoted by the caller."""
197
215
  # SQL Server and Firebird will throw an error if there are foreign keys to the table.
198
- tablename = self.type.quoted(tablename)
199
216
  self.execute(f"drop table {tablename};")
200
217
 
201
218
  def import_entire_file(
@@ -28,7 +28,14 @@ from execsql.utils.errors import exception_desc
28
28
  from execsql.utils.fileio import filewriter_close
29
29
  from execsql.utils.strings import clean_words, dedup_words, fold_words
30
30
 
31
- __all__ = ["LineDelimiter", "CsvFile", "CsvWriter", "DelimitedWriter", "write_delimited_file"]
31
+ __all__ = [
32
+ "LineDelimiter",
33
+ "CsvFile",
34
+ "CsvWriter",
35
+ "DelimitedWriter",
36
+ "blank_lines_as_empty_rows",
37
+ "write_delimited_file",
38
+ ]
32
39
 
33
40
  _SPACE_DELIM_RX = re.compile(r" +")
34
41
 
@@ -618,6 +625,9 @@ class CsvFile(EncodedFile):
618
625
  prev_state = state
619
626
  state = exec_vector[state]()
620
627
  end()
628
+ # An empty result is either a blank line or the end of the file; the
629
+ # reader needs to know which.
630
+ self._at_eof = c == ""
621
631
  if len(self.parse_errors) > 0:
622
632
  raise ErrInfo("error", other_msg=", ".join(self.parse_errors))
623
633
  return elements
@@ -653,48 +663,63 @@ class CsvFile(EncodedFile):
653
663
  doublequote=True,
654
664
  strict=False,
655
665
  )
656
- for elements in csv_reader:
657
- if len(elements) == 0:
658
- break
659
- # Normalize empty strings to None for parity with the slow reader.
660
- normalized_elements = [e if e != "" else None for e in elements]
661
- if conf.del_empty_cols and len(self.blank_cols) > 0:
662
- blanks = copy.copy(self.blank_cols)
663
- while len(blanks) > 0:
664
- b = blanks.pop()
665
- del normalized_elements[b]
666
- yield normalized_elements
666
+ # Normalize empty strings to None for parity with the slow reader.
667
+ raw = ([e if e != "" else None for e in elements] for elements in csv_reader)
668
+ yield from self._rows(raw, conf)
667
669
  finally:
668
670
  f.close()
669
671
 
670
672
  def _slow_reader(self, conf: Any) -> Any:
671
673
  """Read using the character-at-a-time state machine (fallback for non-standard formats)."""
672
674
  f = self.openclean("rt")
673
- line_no = 0
674
- try:
675
+
676
+ def raw() -> Any:
677
+ line_no = 0
675
678
  while True:
676
679
  line_no += 1
677
680
  try:
678
681
  elements = self.read_and_parse_line(f)
679
682
  except ErrInfo as e:
680
683
  raise ErrInfo("error", other_msg=f"{e.other} on line {line_no}.") from e
681
- except:
682
- raise
683
- if len(elements) > 0:
684
- if conf.del_empty_cols and len(self.blank_cols) > 0:
685
- blanks = copy.copy(self.blank_cols)
686
- while len(blanks) > 0:
687
- b = blanks.pop()
688
- del elements[b]
689
- yield elements
690
- else:
691
- break
684
+ if not elements and self._at_eof:
685
+ return
686
+ yield elements
687
+
688
+ try:
689
+ yield from self._rows(raw(), conf)
692
690
  finally:
693
691
  f.close()
694
692
 
695
- def writer(self, append: bool = False) -> CsvWriter:
696
- """Return a :class:`CsvWriter` configured with this file's format settings."""
697
- return CsvWriter(self.filename, self.encoding, self.delimiter, self.quotechar, self.escapechar, append)
693
+ def _rows(self, raw: Any, conf: Any) -> Any:
694
+ """Yield parsed rows, turning a blank line inside the data into an empty row.
695
+
696
+ A blank line (no fields at all) is an all-``None`` row as wide as the
697
+ header, so ``CONFIG EMPTY_ROWS`` decides whether it is loaded, as it
698
+ does for a row of empty fields. Blank lines with no data after them
699
+ end the file, as they always have, so a trailing newline does not add
700
+ an empty row.
701
+ """
702
+ width = None
703
+ pending_blank = 0
704
+ for elements in raw:
705
+ if not elements:
706
+ if width is not None:
707
+ pending_blank += 1
708
+ continue
709
+ if width is None:
710
+ width = len(elements)
711
+ for _ in range(pending_blank):
712
+ yield self._without_blank_cols([None] * width, conf)
713
+ pending_blank = 0
714
+ yield self._without_blank_cols(elements, conf)
715
+
716
+ def _without_blank_cols(self, elements: list, conf: Any) -> list:
717
+ if conf.del_empty_cols and len(self.blank_cols) > 0:
718
+ blanks = copy.copy(self.blank_cols)
719
+ while len(blanks) > 0:
720
+ b = blanks.pop()
721
+ del elements[b]
722
+ return elements
698
723
 
699
724
  def _colhdrs(self, inf: Any) -> list[str]:
700
725
  conf = _state.conf
@@ -764,6 +789,40 @@ class CsvFile(EncodedFile):
764
789
  return self.table_data.create_table(database_type, schemaname, tablename, pretty)
765
790
 
766
791
 
792
+ def blank_lines_as_empty_rows(f: Any, delimiter: str, quotechar: str | None, columns: int, chunk_size: int) -> Any:
793
+ """Yield the text of *f* in chunks of about *chunk_size*, with each blank line inside the data made an empty row.
794
+
795
+ For a reader of the raw file, such as PostgreSQL's COPY, which rejects a
796
+ blank line. The line becomes *columns* empty fields (``,,`` for three),
797
+ matching what the row-by-row reader loads; blank lines at the end of the
798
+ file are dropped, as there. A blank line inside a quoted value is part
799
+ of that value and is kept: quote characters are counted, and a doubled
800
+ quote (the only escape a raw CSV reader is given) counts twice.
801
+ """
802
+ empty_row = delimiter * (columns - 1) + "\n"
803
+ in_quotes = False
804
+ pending_blank = 0
805
+ buf: list[str] = []
806
+ size = 0
807
+ for line in f:
808
+ if not in_quotes and line in ("\n", "\r\n"):
809
+ pending_blank += 1
810
+ continue
811
+ if pending_blank:
812
+ buf.append(empty_row * pending_blank)
813
+ size += len(empty_row) * pending_blank
814
+ pending_blank = 0
815
+ if quotechar and line.count(quotechar) % 2:
816
+ in_quotes = not in_quotes
817
+ buf.append(line)
818
+ size += len(line)
819
+ if size >= chunk_size:
820
+ yield "".join(buf)
821
+ buf, size = [], 0
822
+ if buf:
823
+ yield "".join(buf)
824
+
825
+
767
826
  def write_delimited_file(
768
827
  outfile: str,
769
828
  filefmt: str,
execsql/exporters/xlsx.py CHANGED
@@ -38,7 +38,9 @@ def _require_openpyxl() -> Any:
38
38
 
39
39
  return openpyxl
40
40
  except ImportError:
41
- fatal_error("The openpyxl library is needed to write Excel (.xlsx) spreadsheets (install execsql2[excel]).")
41
+ fatal_error(
42
+ "The openpyxl library is needed to write Excel (.xlsx) spreadsheets (install the formats extra, execsql2[formats]).",
43
+ )
42
44
 
43
45
 
44
46
  def _cell_value(item: Any) -> Any:
@@ -72,6 +74,19 @@ def _cell_value(item: Any) -> Any:
72
74
  return str(item)
73
75
 
74
76
 
77
+ def _write_cell(ws: Any, row: int, column: int, value: Any) -> Any:
78
+ """Write *value* to a cell as data, never as a formula, and return the cell.
79
+
80
+ openpyxl stores any string that starts with ``=`` as a formula, so text
81
+ from the database such as ``=1+1`` or ``=HYPERLINK(...)`` would be
82
+ evaluated when the workbook is opened. Such cells are stored as text.
83
+ """
84
+ cell = ws.cell(row=row, column=column, value=value)
85
+ if cell.data_type == "f":
86
+ cell.data_type = "s"
87
+ return cell
88
+
89
+
75
90
  # ---------------------------------------------------------------------------
76
91
  # Public API
77
92
  # ---------------------------------------------------------------------------
@@ -145,7 +160,7 @@ def write_query_to_xlsx(
145
160
  ("datasheet_name", "created_on", "created_by", "description", "source"),
146
161
  start=1,
147
162
  ):
148
- cell = inv_ws.cell(row=1, column=col_idx, value=hdr)
163
+ cell = _write_cell(inv_ws, 1, col_idx, hdr)
149
164
  cell.font = bold_font
150
165
  else:
151
166
  inv_ws = wb[inventory_name]
@@ -158,13 +173,13 @@ def write_query_to_xlsx(
158
173
 
159
174
  # Header row
160
175
  for col_idx, hdr in enumerate(hdrs, start=1):
161
- cell = ws.cell(row=1, column=col_idx, value=str(hdr))
176
+ cell = _write_cell(ws, 1, col_idx, str(hdr))
162
177
  cell.font = bold_font
163
178
 
164
179
  # Data rows
165
180
  for row_idx, row in enumerate(rows, start=2):
166
181
  for col_idx, item in enumerate(row, start=1):
167
- ws.cell(row=row_idx, column=col_idx, value=_cell_value(item))
182
+ _write_cell(ws, row_idx, col_idx, _cell_value(item))
168
183
 
169
184
  # ------------------------------------------------------------------
170
185
  # Update inventory sheet
@@ -185,7 +200,7 @@ def write_query_to_xlsx(
185
200
  ),
186
201
  start=1,
187
202
  ):
188
- inv_ws.cell(row=next_row, column=col_idx, value=value)
203
+ _write_cell(inv_ws, next_row, col_idx, value)
189
204
 
190
205
  wb.save(outfile)
191
206
  wb.close()
@@ -256,7 +271,7 @@ def write_queries_to_xlsx(
256
271
  ("datasheet_name", "created_on", "created_by", "description", "source"),
257
272
  start=1,
258
273
  ):
259
- cell = inv_ws.cell(row=1, column=col_idx, value=hdr)
274
+ cell = _write_cell(inv_ws, 1, col_idx, hdr)
260
275
  cell.font = bold_font
261
276
  else:
262
277
  inv_ws = wb[inventory_name]
@@ -297,12 +312,12 @@ def write_queries_to_xlsx(
297
312
  ws = wb.create_sheet(sheet_name)
298
313
 
299
314
  for col_idx, hdr in enumerate(hdrs, start=1):
300
- cell = ws.cell(row=1, column=col_idx, value=str(hdr))
315
+ cell = _write_cell(ws, 1, col_idx, str(hdr))
301
316
  cell.font = bold_font
302
317
 
303
318
  for row_idx, row in enumerate(rows, start=2):
304
319
  for col_idx, item in enumerate(row, start=1):
305
- ws.cell(row=row_idx, column=col_idx, value=_cell_value(item))
320
+ _write_cell(ws, row_idx, col_idx, _cell_value(item))
306
321
 
307
322
  # Determine per-sheet description.
308
323
  if desc is None:
@@ -326,7 +341,7 @@ def write_queries_to_xlsx(
326
341
  ),
327
342
  start=1,
328
343
  ):
329
- inv_ws.cell(row=next_row, column=col_idx, value=value)
344
+ _write_cell(inv_ws, next_row, col_idx, value)
330
345
 
331
346
  if tee and outfile.lower() != "stdout":
332
347
  prettyprint_query(select_stmt, db, "stdout", False, desc=d)
execsql/exporters/zip.py CHANGED
@@ -54,8 +54,7 @@ class WriteableZipfile:
54
54
  date_time=time.localtime(time.time())[:6],
55
55
  )
56
56
  self.current_zinfo.compress_type = self.zf.compression
57
- if sys.version_info.major >= 3 and sys.version_info.minor >= 7:
58
- self.current_zinfo._compresslevel = self.zf.compresslevel # type: ignore[attr-defined]
57
+ self.current_zinfo._compresslevel = self.zf.compresslevel # type: ignore[attr-defined]
59
58
  # See https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip
60
59
  self.current_zinfo.external_attr = 0o100755 << 16 # ?rw-rw-rw-
61
60
  if sys.platform.startswith("win"):