excel-list-transform 0.8.5__tar.gz → 1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. excel_list_transform-0.8.5/README_pypi.md → excel_list_transform-1.0/PKG-INFO +183 -149
  2. excel_list_transform-1.0/README.md +116 -0
  3. excel_list_transform-0.8.5/PKG-INFO → excel_list_transform-1.0/README_pypi.md +156 -180
  4. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/PKG-INFO +167 -164
  5. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/SOURCES.txt +3 -6
  6. excel_list_transform-1.0/excel_list_transform.egg-info/requires.txt +9 -0
  7. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/pyproject.toml +3 -3
  8. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/setup.py +10 -14
  9. excel_list_transform-1.0/src/excel_list_transform/__init__.py +3 -0
  10. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/__main__.py +2 -2
  11. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/check_indata_common.py +1 -1
  12. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/commontypes.py +3 -15
  13. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/config_enums.py +1 -16
  14. excel_list_transform-1.0/src/excel_list_transform/config_excel_list_transform.py +552 -0
  15. excel_list_transform-1.0/src/excel_list_transform/config_match.py +22 -0
  16. excel_list_transform-1.0/src/excel_list_transform/config_read_old.py +128 -0
  17. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/config_xls_list_transf_name.py +23 -19
  18. excel_list_transform-1.0/src/excel_list_transform/config_xls_list_transf_num.py +84 -0
  19. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/file_extension.py +1 -1
  20. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/file_must_exist.py +1 -1
  21. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/generate_cfg.py +117 -94
  22. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/generate_txt.py +142 -75
  23. excel_list_transform-1.0/src/excel_list_transform/handle_empty_column.py +17 -0
  24. excel_list_transform-1.0/src/excel_list_transform/handle_tableio.py +131 -0
  25. excel_list_transform-1.0/src/excel_list_transform/migrate_cfg.py +22 -0
  26. excel_list_transform-1.0/src/excel_list_transform/migrate_cfg_warn_hook.py +21 -0
  27. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/num_named_conversion.py +2 -25
  28. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/recode.py +4 -6
  29. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/rewrite_value.py +8 -10
  30. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/row_split_merge.py +16 -32
  31. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/str_to_enum.py +1 -1
  32. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_cmd.py +33 -23
  33. excel_list_transform-1.0/src/excel_list_transform/transform_func.py +41 -0
  34. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_func_common.py +13 -16
  35. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_func_named.py +16 -34
  36. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_func_num.py +15 -29
  37. excel_list_transform-1.0/src/excel_list_transform/version_information.py +32 -0
  38. excel_list_transform-0.8.5/README.md +0 -85
  39. excel_list_transform-0.8.5/excel_list_transform.egg-info/requires.txt +0 -13
  40. excel_list_transform-0.8.5/src/excel_list_transform/__init__.py +0 -3
  41. excel_list_transform-0.8.5/src/excel_list_transform/assert_dict_equal.py +0 -57
  42. excel_list_transform-0.8.5/src/excel_list_transform/config.py +0 -572
  43. excel_list_transform-0.8.5/src/excel_list_transform/config_auto_change_hook.py +0 -59
  44. excel_list_transform-0.8.5/src/excel_list_transform/config_excel_list_transform.py +0 -391
  45. excel_list_transform-0.8.5/src/excel_list_transform/config_factory.py +0 -101
  46. excel_list_transform-0.8.5/src/excel_list_transform/config_xls_list_transf_num.py +0 -77
  47. excel_list_transform-0.8.5/src/excel_list_transform/handle_csv.py +0 -76
  48. excel_list_transform-0.8.5/src/excel_list_transform/handle_empty_column.py +0 -28
  49. excel_list_transform-0.8.5/src/excel_list_transform/handle_excel.py +0 -234
  50. excel_list_transform-0.8.5/src/excel_list_transform/migrate_cfg.py +0 -33
  51. excel_list_transform-0.8.5/src/excel_list_transform/migrate_cfg_warn_hook.py +0 -32
  52. excel_list_transform-0.8.5/src/excel_list_transform/transform_func.py +0 -50
  53. excel_list_transform-0.8.5/src/excel_list_transform/version_information.py +0 -210
  54. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/LICENSE.txt +0 -0
  55. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/dependency_links.txt +0 -0
  56. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/top_level.txt +0 -0
  57. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/setup.cfg +0 -0
  58. {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/py.typed +0 -0
@@ -1,31 +1,75 @@
1
+ Metadata-Version: 2.4
2
+ Name: excel-list-transform
3
+ Version: 1.0
4
+ Summary: Transform table data with configurable row and column changes.
5
+ Author: Tom Björkholm
6
+ Author-email: Tom Björkholm <klausuler_linnet0q@icloud.com>
7
+ License-Expression: MIT
8
+ Project-URL: Source code, https://bitbucket.org/tom-bjorkholm/excel-list-transform
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Operating System :: OS Independent
11
+ Requires-Python: >=3.13
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE.txt
14
+ Requires-Dist: tableio-cfg-json>=0.4
15
+ Requires-Dist: tableio>=1.0
16
+ Requires-Dist: config-as-json>=1.2
17
+ Requires-Dist: versionreporter>=0.2
18
+ Requires-Dist: argcomplete>=3.6.3
19
+ Requires-Dist: pip>=26.1
20
+ Requires-Dist: setuptools>=82.0.1
21
+ Requires-Dist: build>=1.5.0
22
+ Requires-Dist: wheel>=0.47.0
23
+ Dynamic: author
24
+ Dynamic: license-file
25
+ Dynamic: requires-dist
26
+ Dynamic: requires-python
27
+
1
28
  # excel-list-transform
2
29
 
3
30
  ## Background
4
31
 
5
- This python application was born out of an experience at sail racing events. At the start of the events we received an excel list with participants (from a registration web) to enter into the scoring software and into tool for online notice board. The information was present in the excel file, but the columns were all wrong. To avoid the stressful manual rework of the information in the excel list this application was created.
32
+ This python application was born out of an experience at sail racing events.
33
+ At the start of the events we received an excel list with participants (from
34
+ a registration web) to enter into the scoring software and into tool for
35
+ online notice board. The information was present in the excel file, but the
36
+ columns were all wrong. To avoid the stressful manual rework of the
37
+ information in the excel list this application was created.
6
38
 
7
39
  ## What it does
8
40
 
9
41
  This small python application:
10
42
 
11
- * reads data (that is a list with columns) from an excel file or from a comma separate values (CSV) file.
12
- * split rows in the list (creating separate rows when a cell contains something that can be split to several values, see below)
13
- * merge rows (if specified columns have the same value in several rows the rows are merged to one row)
14
- * splits columns in the list (like creating "first name" and "last name" columns from "name" column)
15
- * merges columns in the list (like creating "name" column from "first name" and "last name" columns)
16
- * removes columns in the list
17
- * reorders columns in the list
18
- * renames columns in the list
19
- * inserts columns in the list
20
- * rewrites columns in the list (like transforming telephone numbers from local/national format to international format)
21
- * writes the resulting data (that is a list with columns) to an excel file or to a comma separate values (CSV) file.
22
-
23
- How this is done is governed by a configuration file. The application can create a number of example configuration files with accompanying description text files.
43
+ - reads data (that is a list with columns) from an excel file, an ODS file,
44
+ or from a comma separated values (CSV) file.
45
+ - split rows in the list (creating separate rows when a cell contains
46
+ something that can be split to several values, see below)
47
+ - merge rows (if specified columns have the same value in several rows
48
+ the rows are merged to one row)
49
+ - splits columns in the list (like creating "first name" and "last name"
50
+ columns from "name" column)
51
+ - merges columns in the list (like creating "name" column from "first name"
52
+ and "last name" columns)
53
+ - removes columns in the list
54
+ - reorders columns in the list
55
+ - renames columns in the list
56
+ - inserts columns in the list
57
+ - rewrites columns in the list (like transforming telephone numbers from
58
+ local/national format to international format)
59
+ - writes the resulting data (that is a list with columns) to an excel file,
60
+ an ODS spreadsheet file, a comma separated values (CSV) file, or
61
+ any of a number of other file formats.
62
+
63
+ How this is done is governed by a configuration file. The application can
64
+ create a number of example configuration files with accompanying description
65
+ text files.
24
66
 
25
67
  ## Installing it
26
68
 
27
- If you want to use it, install it using pip. A precondition is that you have Python installed on you computer. See version table below for information on needed Python version.
28
- Python can be downloaded from [https://www.python.org/downloads/](https://www.python.org/downloads/).
69
+ If you want to use it, install it using pip. A precondition is that you have
70
+ Python installed on your computer. See version table below for information
71
+ on needed Python version. Python can be downloaded
72
+ from [https://www.python.org/downloads/](https://www.python.org/downloads/).
29
73
 
30
74
  ### Installing on mac and Linux
31
75
 
@@ -39,31 +83,6 @@ pip3 install --upgrade excel-list-transform
39
83
  pip install --upgrade excel-list-transform
40
84
  ````
41
85
 
42
- ## Version history
43
-
44
- | Version | Date | Python version | Description |
45
- |---------|-------------|-----------------|-------------------------------------|
46
- | 0.5 | 10 Sep 2024 | 3.10.5 or newer | First released version |
47
- | 0.6 | 22 Sep 2024 | 3.12.6 or newer | Configured encoding for CSV |
48
- | 0.6.2 | 28 Sep 2024 | 3.12.6 or newer | Fix name in test report |
49
- | 0.7.0 | 03 Nov 2024 | 3.12.6 or newer | Option to strip space in excel |
50
- | 0.7.1 | 15 Dec 2024 | 3.12.6 or newer | Refactor tests |
51
- | 0.7.3 | 06 Jan 2025 | 3.12.6 or newer | Example config for Sailwave to RRS |
52
- | 0.7.5 | 21 Mar 2025 | 3.12.6 or newer | Example config for Sailarena to RRS |
53
- | 0.7.6 | 21 Mar 2025 | 3.13.2 or newer | Adapted to Python 3.13.2 |
54
- | 0.7.7 | 09 Apr 2025 | 3.13.3 or newer | refactor phone number rewrite cfg |
55
- | 0.7.9 | 09 Jun 2025 | 3.10.x | Backport 0.7.12 to Python 3.10 |
56
- | 0.7.10 | 09 Jun 2025 | 3.11.x | Backport 0.7.12 to Python 3.11 |
57
- | 0.7.11 | 09 Jun 2025 | 3.12.x | Backport 0.7.12 to Python 3.12 |
58
- | 0.7.12 | 09 Jun 2025 | 3.13.3 or newer | Add version sub-command |
59
- | 0.7.13 | 11 Jun 2025 | 3.13 or newer | Relax Python version requirement |
60
- | 0.8.0 | 23 Jul 2025 | 3.10.x | Backport 0.8.3 to Python 3.10 |
61
- | 0.8.1 | 23 Jul 2025 | 3.11.x | Backport 0.8.3 to Python 3.11 |
62
- | 0.8.2 | 23 Jul 2025 | 3.12.x | Backport 0.8.3 to Python 3.12 |
63
- | 0.8.3 | 23 Jul 2025 | 3.13 or newer | Add split-row and merge-row |
64
- | 0.8.4 | 23 Nov 2025 | 3.12.x | Backport 0.8.5 to Python 3.12 |
65
- | 0.8.5 | 23 Nov 2025 | 3.13 or newer | Adapted to Python 3.14 |
66
-
67
86
  ## Running the application
68
87
 
69
88
  ### Running the application on mac and Linux
@@ -94,43 +113,68 @@ python -m excel_list_transform migrate-cfg -i old.cfg -o new.cfg
94
113
 
95
114
  ## Suggested way to get started
96
115
 
97
- 1. Use the "cfg-example" sub-command to generate a few example configuration (.cfg) files with description (.txt) files.
98
- 2. Read the example configuration (.cfg) files and the accompanying description (.txt) files.
116
+ 1. Use the "cfg-example" sub-command to generate a few example configuration
117
+ (.cfg) files with description (.txt) files.
118
+ 2. Read the example configuration (.cfg) files and the accompanying
119
+ description (.txt) files.
99
120
  3. Find an example that is close to what you want to achieve.
100
121
  4. Modify that configuration file to achieve what you want to achieve.
101
- 5. Use the "transform" sub-command to read your data and output it transformed or reorganized according to your modified configuration file.
102
- 6. Read the produced output. If necessary go back to step 4 and adjust how the data is transformed.
122
+ 5. Use the "transform" sub-command to read your data and output it transformed
123
+ or reorganized according to your modified configuration file.
124
+ 6. Read the produced output. If necessary go back to step 4 and adjust how the
125
+ data is transformed.
103
126
 
104
127
  ### Example configuration files
105
128
 
106
- When using the "cfg-example" sub-command to generate an example configuration file (say example.cfg) a text file describing the configuration and the syntax of the configuration file is also generated. If the example configuration file is named example.cfg, then the text file descriging the configuration is named example.txt.
129
+ When using the "cfg-example" sub-command to generate an example configuration
130
+ file (say example.cfg) a text file describing the configuration and the syntax
131
+ of the configuration file is also generated. If the example configuration
132
+ file is named example.cfg, then the text file describing the configuration
133
+ is named example.txt.
107
134
 
108
- You can generate several example configuration files each with an accompanying text file descriping it.
135
+ You can generate several example configuration files each with an accompanying
136
+ text file describing it.
109
137
 
110
- Read the text file describing the configuration file while looking at the configuration file to understand the syntax and the possible options.
138
+ Read the text file describing the configuration file while looking at the
139
+ configuration file to understand the syntax and the possible options.
111
140
 
112
141
  ## Performance
113
142
 
114
- This application is written for the moderate amounts of data when registering participants for the majority of sports events. If you have millions of rows this application is not for you.
115
-
116
- With an input file consisting of 20 columns producing an output file of 11 columns I have measured the following performance on a MacBook Air (laptop) from 2020:
117
-
118
- * 1 000 rows processed in less than 0.5 seconds
119
- * 10 000 rows processed in less than 4 seconds
120
- * 80 000 rows processed in less than 30 seconds
121
- * 120 000 rows processed in 40 seconds
122
-
123
- Naturally your performance will be different based on computer hardware, operating system and Python version. Generally it should be reasonably fast for less than 10 000 rows, painfully slow but somewhat usable up to 100 000 rows and probably so slow that it is unusable for more than 100 000 rows.
143
+ This application is written for the moderate amounts of data when
144
+ registering participants for the majority of sports events, or
145
+ processing scrum backlogs in excel. If you have millions of rows
146
+ this application is not for you.
147
+
148
+ With an input file consisting of 20 columns producing an output file of
149
+ 11 columns I have measured the following performance:
150
+
151
+ | number of rows | on MacBook Air M4 |
152
+ |----------------|-----------------------|
153
+ | 100 | less than 0.2 seconds |
154
+ | 1000 | less than 0.3 seconds |
155
+ | 10 000 | less than 2.5 seconds |
156
+ | 20 000 | less than 5 seconds |
157
+ | 40 000 | less than 11 seconds |
158
+ | 80 000 | less than 23 seconds |
159
+ | 120 000 | less than 34 seconds |
160
+
161
+ Naturally your performance will be different based on computer hardware,
162
+ operating system and Python version. Generally it should be reasonably
163
+ fast for less than 10 000 rows, painfully slow but somewhat usable up
164
+ to 100 000 rows and probably so slow that it is unusable for more
165
+ than 100 000 rows.
124
166
 
125
167
  ## Description of how to write/change the configuration file
126
168
 
127
- The configuration file is in JSON syntax. [https://en.wikipedia.org/wiki/JSON](https://en.wikipedia.org/wiki/JSON)
169
+ The configuration file is in JSON syntax.
170
+ [https://en.wikipedia.org/wiki/JSON](https://en.wikipedia.org/wiki/JSON)
128
171
 
129
172
  The keywords and the nesting is important. The order of keywords
130
173
  have no significance (the examples use alphabetical order).
131
174
  Indentation and line breaks have no significance.
132
175
 
133
- The encoding for the configuration file must be UTF-8. (US-ACII is a subset of UTF-8.)
176
+ The encoding for the configuration file must be UTF-8. (US-ASCII is a
177
+ subset of UTF-8.)
134
178
 
135
179
  It is recommended that you let the command generate a configuration
136
180
  file and then edit that file to match your needs. It is NOT recommended
@@ -141,14 +185,15 @@ that the user writes the configuration file from scratch.
141
185
  The keyword **column_ref** is used to tell if the configuration
142
186
  references columns by name (value *"BY_NAME"*) or by number
143
187
  (value *"BY_NUMBER"*). The syntax of parts of the configuration file
144
- is slightly different depending on the how columns are referenced.
188
+ is slightly different depending on how columns are referenced.
145
189
  Thus, you first have to decide how you want to reference columns.
146
190
 
147
191
  Generally it is easier to write a correct configuration using *"BY_NAME"*,
148
- so *"BY_NAME"* is recommended when possible. *"BY_NAME"* imposes a few restrictions:
192
+ so *"BY_NAME"* is recommended when possible. *"BY_NAME"* imposes a few
193
+ restrictions:
149
194
 
150
- * each column need to have a unique text at the first line to use as column name.
151
- * the column names need to be stable and known.
195
+ - each column needs to have a unique text at the first line to use as column name.
196
+ - the column names need to be stable and known.
152
197
 
153
198
  If you cannot meet these restrictions you will have to reference the
154
199
  columns by number *"BY_NUMBER"*. When referencing the columns by number
@@ -160,49 +205,37 @@ right. The leftmost column is number 0.
160
205
 
161
206
  ## Type of input and output file
162
207
 
163
- The type of input file to read is determined by **"in_type"**.
164
- The type of output file to write is determined by **"out_type"**.
165
- **"in_type"** and **"out_type"** can have values *"CSV"* or *"EXCEL"*.
166
-
167
- Excel files can be read and written using three libraries.
168
- **"in_excel_library"** and **"out_excel_library"** can have values
169
- *"OPENPYXL"*, *"XLSXWRITER"* or *"PYLIGHTXL"*. These are different
170
- third party libraries that can read/write excel. My experience
171
- is that *"PYLIGHTXL"* most often is able to read and write excel
172
- files correctly. If you have trouble reading/writing your
173
- particular excel file, please try different combinations of
174
- these libraries. **"in_excel_library"** is always needed in
175
- the configuration file but is not used if the input is CSV.
176
- **"out_excel_library"** is always needed in the configuration file
177
- but is not used if the output is CSV.
178
-
179
- Currently *"PYLIGHTXL"* appears to be best to write files for
180
- Microsoft Excel to open without complaints. But *"OPENPYXL"*
181
- appears to be best to write files to be imported by
182
- [https://www.racingrulesofsailing.org](https://www.racingrulesofsailing.org).
183
- When newer versions arrive this might change...
184
-
185
- Comma separated values files (CSV files) may differ slightly
186
- depening on the programs used to read/write them and the locale
187
- used. **"in_csv_dialect"** and **"out_csv_dialect"** changes how CSV files
188
- are read and written. They are always needed in the configuration
189
- file, but are not used if the input and output are excel files.
190
-
191
- Comma separated values files might have different encoding for
192
- the text in the file. [https://en.wikipedia.org/wiki/Character_encoding](https://en.wikipedia.org/wiki/Character_encoding)
193
- This is specified with **"in_csv_encoding"** and **"out_csv_encoding"**.
194
- Unless you know that you need another encoding leave these as
195
- in the generated example configuration. (In version 0.5 these
196
- configuration parameters are missing. To be compatible with 0.5
197
- configuration files "in_csv_encoding" defaults to "utf_8_sig"
198
- and "out_csv_encoding" defaults to "utf-8" if missing in the
199
- configuration file.)
208
+ The type of input file to read is configured in the **"input_table"**
209
+ section. The type of output file to write is configured in the
210
+ **"output_table"** section. These sections use the TableIO JSON
211
+ configuration syntax.
212
+
213
+ The most important keyword in each table section is **"format_name"**.
214
+ Generated examples commonly use *"Excel"* or *"CSV"*, but TableIO may
215
+ support more formats. In most cases you should leave **"implementation"**
216
+ missing or set to *null*. Then TableIO chooses a suitable implementation.
217
+ If you know that you need a specific implementation, you can add it
218
+ manually.
219
+
220
+ CSV files may differ slightly depending on the programs used to read/write
221
+ them and the locale used. CSV-specific settings are configured in the
222
+ nested **"csv"** section inside **"input_table"** or **"output_table"**.
223
+ The **"character_encoding"** keyword in the table section controls the
224
+ text encoding.
225
+ [https://en.wikipedia.org/wiki/Character_encoding](https://en.wikipedia.org/wiki/Character_encoding)
226
+
227
+ Output tables can request extra presentation features. Set
228
+ **"output_borders"** to *true* to request table borders. Set
229
+ **"output_filtered_table"** to *true* to request a filtered table area.
230
+ These features are ignorable requests. TableIO will prefer an
231
+ implementation that can provide them, but the transform can still write
232
+ the output if the selected file format cannot represent them.
200
233
 
201
234
  ## Extra spaces in excel input files
202
235
 
203
236
  When viewing an excel file in excel it is very hard to notice if
204
- some string value in a cell has trailing white space. These trailing
205
- trailing spaces can make the further processing of a file difficult
237
+ some string value in a cell has trailing whitespace. These trailing
238
+ spaces can make the further processing of a file difficult
206
239
  as the strings in the file are not what you thought they are.
207
240
  (This especially is a problem if you have trailing spaces in a
208
241
  cell on the first line, and refer to columns by their names.)
@@ -212,7 +245,7 @@ to strip off leading and trailing whitespace from all columns
212
245
  values read from the first line of the excel input file.
213
246
  The configuration **"in_excel_values_strip"** can be set to true,
214
247
  to strip off leading and trailing whitespace from all columns
215
- values read from from the other lines (not the first line) of
248
+ values read from the other lines (not the first line) of
216
249
  the excel input file. (In version 0.6.2 and earlier these are
217
250
  missing. To be compatible with version 0.6.2 and earlier
218
251
  **"in_excel_col_name_strip"** and **"in_excel_values_strip"**
@@ -220,10 +253,10 @@ defaults to false if missing in the configuration file.)
220
253
 
221
254
  ## Column manipulation
222
255
 
223
- A number or records starting with "s" and a number describe
256
+ A number of records starting with "s" and a number describe
224
257
  column manipulation to be done. These manipulations are done
225
- in order of the number: s1_(something) before s2_(something),
226
- before s3_(something). As the "s" number records might add, remove
258
+ in order of the number: s01_(something) before s02_(something),
259
+ before s03_(something). As the "s" number records might add, remove
227
260
  or rename columns, it is important to keep track of the order that
228
261
  they are applied.
229
262
 
@@ -235,12 +268,12 @@ referenced by number.
235
268
 
236
269
  The first operation is to split rows based on column values.
237
270
  This operation is configured using the **"s01_split_rows"** record,
238
- that have an array of splits to be done. Each row split has the
271
+ that has an array of splits to be done. Each row split has the
239
272
  following keys: **"column"**, **"separators"** and **"not_separators"**.
240
273
 
241
274
  The **"column"** keyword is used to identify the column that is
242
275
  split into several rows. This is a column number in the case of
243
- *"BY_NUMBER"*, and a column name/title in the sace of *"BY_NAME"*.
276
+ *"BY_NUMBER"*, and a column name/title in the case of *"BY_NAME"*.
244
277
 
245
278
  New rows will be created so that the parts of the identified
246
279
  column will be put in that column only one part per row.
@@ -252,7 +285,7 @@ these strings are present in the identified column it is seen
252
285
  as the separator between the parts of the column value that go
253
286
  into different rows.
254
287
 
255
- **"not_separators"** take as arguement a list of strings. These strings
288
+ **"not_separators"** take as argument a list of strings. These strings
256
289
  are not regarded as separators even if they include the strings
257
290
  of one or more separator. (For instance ";" could be a separator,
258
291
  but using **"not_separators"** the string "\\;" could be seen as not
@@ -262,14 +295,14 @@ a separator.)
262
295
 
263
296
  The next operation is to merge rows based on column values.
264
297
  This is the opposite operation to the splitting of rows.
265
- Row mergins is configured using the **"s02_merge_rows"** record,
266
- that have an array of merges to be done. Each row merge have
298
+ Row merging is configured using the **"s02_merge_rows"** record,
299
+ that has an array of merges to be done. Each row merge has
267
300
  the following keys: **"columns"** and **"separator"**.
268
301
 
269
302
  The **"columns"** keyword is used to identify the columns that need
270
303
  to have identical values to merge two or more rows. The **"columns"**
271
- keyword take a list of columns. These are column numbers in the case of
272
- *"BY_NUMBER"*, and a column names/titles in the sace of *"BY_NAME"*.
304
+ keyword takes a list of columns. These are column numbers in the case of
305
+ *"BY_NUMBER"*, and column names/titles in the case of *"BY_NAME"*.
273
306
 
274
307
  For each column that has the same value for all rows merged, that
275
308
  value will be in the merged row. When rows being merged have different
@@ -289,7 +322,7 @@ for customers in the following format:
289
322
  | Brewery | Beer | Smith + Bush |
290
323
  | Dairy | Milk | Jones |
291
324
 
292
- But for distrubuting it to customers you would like to have the
325
+ But for distributing it to customers you would like to have the
293
326
  list in the following format:
294
327
 
295
328
  | To | What | From |
@@ -299,27 +332,27 @@ list in the following format:
299
332
  | Bush | Beer | Brewery |
300
333
 
301
334
  To do this transformation we use the **"s01_split_rows"** to split
302
- rows base on the "To" column using " + " as separator.
335
+ rows based on the "To" column using " + " as separator.
303
336
  Then we use **"s02_merge_rows"** to merge the rows that have
304
337
  identical values in the "To" column.
305
338
 
306
339
  ### "s03_split_columns"
307
340
 
308
- The first operation that is done is splitting of columns.
309
- The key **"s03_split_columns"** have an array of splits to be
341
+ The first column operation that is done is splitting of columns.
342
+ The key **"s03_split_columns"** has an array of splits to be
310
343
  done. (When the list of splits has more than one split,
311
344
  the least confusion is to split columns to the right before
312
345
  columns to the left. Named references also helps to avoid
313
346
  confusion.)
314
347
 
315
348
  Each split has the following keys: **"column"**, **"separator"** and
316
- **"where"**. In the case of number column references the there is
349
+ **"where"**. In the case of number column references there is
317
350
  also the key **"store_single"**. In the case of named column references
318
351
  there is instead the key **"right_name"**.
319
352
 
320
- **"column"** take as argument the column to be split. This is a
353
+ **"column"** takes as argument the column to be split. This is a
321
354
  column number in the case of *"BY_NUMBER"*, and a column name/title
322
- in the sace of *"BY_NAME"*.
355
+ in the case of *"BY_NAME"*.
323
356
 
324
357
  **"separator"** is a string of characters that indicate the position
325
358
  where the column shall be split. (For instance a single space
@@ -328,7 +361,7 @@ if splitting between two words.)
328
361
  **"where"** can either have the value *"RIGHTMOST"* or the value
329
362
  *"LEFTMOST"*. This is used if the separator string is present
330
363
  more than once in the column value to split. The split is then
331
- done at the leftmost or rightmost occurence according to the
364
+ done at the leftmost or rightmost occurrence according to the
332
365
  value of **"where"**.
333
366
 
334
367
  **"right_name"** is only used if column references are *"BY_NAME"*.
@@ -348,11 +381,11 @@ always stored in the column with the original name.
348
381
 
349
382
  **"s04_remove_columns"** is only used with column references *"BY_NUMBER"*.
350
383
  The value of **"s04_remove_columns"** is a list of column numbers to
351
- remove. (For columns references *"BY_NAME"* see **"s10_column_order"**.)
384
+ remove. (For column references *"BY_NAME"* see **"s10_column_order"**.)
352
385
 
353
386
  ### "s05_merge_columns"
354
387
 
355
- The key **"s05_merge_columns"** have an array of merges to be done.
388
+ The key **"s05_merge_columns"** has an array of merges to be done.
356
389
  Each merge have the keys **"columns"** and **"separator"**.
357
390
 
358
391
  **"columns"** have a list of column references. If *"BY_NAME"* the
@@ -368,16 +401,16 @@ the column values being merged.
368
401
  The key **"s06_place_columns_first"** has a value that is a list of the
369
402
  column numbers to be placed first in order. This step re-orders the
370
403
  columns.
371
- (For columns references *"BY_NAME"* see **"s10_column_order"**.)
404
+ (For column references *"BY_NAME"* see **"s10_column_order"**.)
372
405
 
373
406
  ### "s07_rename_columns"
374
407
 
375
408
  The key **"s07_rename_columns"** has a value that is a list of column rename
376
409
  operations. Each column rename operation has the keys **"column"** and **"name"**.
377
410
 
378
- **"column"** is the number/name of the column before renameing. This is a
411
+ **"column"** is the number/name of the column before renaming. This is a
379
412
  column number in the case of *"BY_NUMBER"*, and a column name/title
380
- in the sace of *"BY_NAME"*.
413
+ in the case of *"BY_NAME"*.
381
414
 
382
415
  **"name"** is the new name/title of the column identified by **"column"**.
383
416
 
@@ -412,7 +445,7 @@ keys. **"kind"** can have the values: *"STRIP"*, *"REMOVECHARS"*,
412
445
  **"kind"** value *"STRIP"* shall have the keys: **"kind"**, **"case"**, **"chars"**
413
446
  and **"column"**. This case means that the characters in **"chars"** shall
414
447
  be stripped off the beginning and end of the column value. If key
415
- **"chars"** is empty string, then white space is stripped off.
448
+ **"chars"** is empty string, then whitespace is stripped off.
416
449
 
417
450
  **"kind"** value *"REMOVECHARS"* shall have the keys: **"kind"**, **"case"**,
418
451
  **"chars"** and **"column"**. This case means that the characters in **"chars"**
@@ -430,7 +463,7 @@ and if a sub-string of the column value equals this string that part is
430
463
  replaced with the value of key **"to"**.
431
464
 
432
465
  **"column"** identifies the column to rewrite. This is a column number in the
433
- case of *"BY_NUMBER"*, and a column name/title in the sace of *"BY_NAME"*.
466
+ case of *"BY_NUMBER"*, and a column name/title in the case of *"BY_NAME"*.
434
467
 
435
468
  **"chars"** is a list of characters written as a string. These characters
436
469
  are removed or stripped off the beginning/end depending on the **"kind"**
@@ -440,8 +473,8 @@ value.
440
473
  value of **"from"** shall be case sensitive or not. Possible values are
441
474
  *"MATCH_CASE"* and *"IGNORE_CASE"*.
442
475
 
443
- **"from"** specifies what the part of the column value shall be to substituted
444
- when matching. The from part in the substitute from something to something.
476
+ **"from"** specifies what part of the column value shall be substituted
477
+ when matching.
445
478
 
446
479
  **"to"** specifies the string that substitution will replace **"from"** with.
447
480
 
@@ -454,22 +487,21 @@ Columns not mentioned in **"s10_column_order"** will not be output,
454
487
  and will thus be removed.
455
488
  (For *"BY_NUMBER"* see **"s04_remove_columns"** and **"s06_place_columns_first"**.)
456
489
 
457
- ## Backward compatibility of configuration keywords
490
+ ## Utility for recoding text files
458
491
 
459
- Old keywords according to the table below are automatically renamed to the new keywords, when reading the configuration file.
492
+ When working with text based files like comma separated values (CSV)
493
+ the file may have an incorrect character encoding, and even worse
494
+ you may not know what character encoding was used to create the file.
495
+ As a small tool for these situations the excel-list-transform includes
496
+ a small recode utility. Using this you can by trial and error recode
497
+ from different encodings until you find which encoding works. (This
498
+ may be the way to figure out which character encoding to write into
499
+ a configuration file.)
460
500
 
461
- | Version 0.7.13 and lower | Version 0.8 and higher |
462
- |--------------------------|-------------------------|
463
- | - | s01_split_rows |
464
- | - | s02_merge_rows |
465
- | s1_split_columns | s03_split_columns |
466
- | s2_remove_columns | s04_remove_columns |
467
- | s3_merge_columns | s05_merge_columns |
468
- | s4_place_columns_first | s06_place_columns_first |
469
- | s5_rename_columns | s07_rename_columns |
470
- | s6_insert_columns | s08_insert_columns |
471
- | s7_rewrite_columns | s09_rewrite_columns |
472
- | s8_column_order | s10_column_order |
501
+ ```python
502
+ python3 -m excel_list_transform.recode --help
503
+ python3 -m excel_list_transform.recode -i a.txt -o b.txt --from cp1250 --to utf-8
504
+ ```
473
505
 
474
506
  ## Source code
475
507
 
@@ -477,7 +509,9 @@ Source code and tests are available at [https://bitbucket.org/tom-bjorkholm/exce
477
509
 
478
510
  ## Test summary
479
511
 
480
- * Test result: 4119 passed in 10s
481
- * No Flake8 warnings.
482
- * No mypy errors found.
483
- * 0.8.5 built and tested using python version: Python 3.14.0
512
+ - Test result: 1368 passed in 65s (0:01:05)
513
+ - No flake8 warnings.
514
+ - No mypy errors found.
515
+ - No python layout warnings.
516
+ - Built version(s): 1.0
517
+ - Build and test using Python 3.14.5
@@ -0,0 +1,116 @@
1
+ # excel-list-transform
2
+
3
+ ## Background
4
+
5
+ This python application was born out of an experience at sail racing events. At the start of the events we received an excel list with participants (from a registration web) to enter into the scoring software and into tool for online notice board. The information was present in the excel file, but the columns were all wrong. To avoid the stressful manual rework of the information in the excel list this application was created.
6
+
7
+ ## Using it
8
+
9
+ If you want to use it install it using pip from [https://pypi.org/project/excel-list-transform](https://pypi.org/project/excel-list-transform).
10
+ There is no need to download anything from Bitbucket to use the application.
11
+
12
+ ### Installing on mac and Linux
13
+
14
+ ````sh
15
+ pip3 install --upgrade excel-list-transform
16
+ ````
17
+
18
+ ### Installing on Microsoft Windows
19
+
20
+ ````sh
21
+ pip install --upgrade excel-list-transform
22
+ ````
23
+
24
+ ### Information for use
25
+
26
+ Please see [https://pypi.org/project/excel-list-transform](https://pypi.org/project/excel-list-transform) or please see README_pypi.md
27
+
28
+ ## What it does
29
+
30
+ This small python application:
31
+
32
+ - reads data (that is a list with columns) from an excel file, an ODS file,
33
+ or from a comma separated values (CSV) file.
34
+ - splits columns in the list (like creating "first name" and "last name"
35
+ columns from "name" column)
36
+ - merges columns in the list (like creating "name" column from "first name"
37
+ and "last name" columns)
38
+ - removes columns in the list
39
+ - reorders columns in the list
40
+ - renames columns in the list
41
+ - inserts columns in the list
42
+ - rewrites columns in the list (like transforming telephone numbers from
43
+ local/national format to international format)
44
+ - writes the resulting data (that is a list with columns) to an excel file,
45
+ an ODS spreadsheet file, a comma separated values (CSV) file, or
46
+ any of a number of other file formats.
47
+
48
+ How this is done is governed by a configuration file. The application can create a number of example configuration files with accompanying description text files.
49
+
50
+ ## Cloning
51
+
52
+ This repository uses submodules. Clone it with:
53
+
54
+ ```sh
55
+ git clone --recurse-submodules \
56
+ git@bitbucket.org:tom-bjorkholm/excel-list-transform.git
57
+ ```
58
+
59
+ If you already cloned without submodules, initialize them with:
60
+
61
+ ```sh
62
+ git submodule update --init --recursive
63
+ ```
64
+
65
+ To update the checked-out submodule revisions:
66
+
67
+ ```sh
68
+ git submodule update --remote --merge
69
+ ```
70
+
71
+ ## Supported Python versions
72
+
73
+ - Package runtime baseline: Python 3.13 or newer
74
+ - Maintainer validation target: Python 3.13, and 3.14
75
+ - Main day-to-day development: usually the newest supported Python version
76
+
77
+ ## Development workflow
78
+
79
+ On macOS and Linux, the normal workflow is:
80
+
81
+ 1. Run `./run_setup_build_environment.py` once after cloning or when the
82
+ build environment needs to be recreated.
83
+ 2. Run `./run_build.py` for the normal build-and-test cycle.
84
+ 3. Run `./run_clean_build.py` before review or release work that needs a
85
+ completely fresh build.
86
+
87
+ The helper scripts are:
88
+
89
+ - `run_setup_build_environment.py`
90
+ Create or refresh the build environment.
91
+ - `run_build.py`
92
+ Build the package and run the configured checks in the project virtual
93
+ environment.
94
+ - `run_clean.py`
95
+ Remove files generated by the build system.
96
+ - `run_clean_build.py`
97
+ Perform a clean build from scratch. This is especially useful because some
98
+ duplicate-code diagnostics only appear on a clean build.
99
+ - `run_pypi_build.py`
100
+ Create the distribution artifacts intended for PyPI publishing.
101
+
102
+ The standard verification suite includes pytest, pylint, flake8, and mypy.
103
+ After a build, the generated reports can be browsed through
104
+ `reports/index.html`.
105
+
106
+ After running `run_build.py` or `run_clean_build.py`, you can manually test
107
+ the built and installed application in the virtual environment `./venv`.
108
+
109
+ ## Test summary
110
+
111
+ - Test result: 1368 passed in 65s (0:01:05)
112
+ - No flake8 warnings.
113
+ - No mypy errors found.
114
+ - No python layout warnings.
115
+ - Built version(s): 1.0
116
+ - Build and test using Python 3.14.5