excel-list-transform 0.8.5__tar.gz → 1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- excel_list_transform-0.8.5/README_pypi.md → excel_list_transform-1.0/PKG-INFO +183 -149
- excel_list_transform-1.0/README.md +116 -0
- excel_list_transform-0.8.5/PKG-INFO → excel_list_transform-1.0/README_pypi.md +156 -180
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/PKG-INFO +167 -164
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/SOURCES.txt +3 -6
- excel_list_transform-1.0/excel_list_transform.egg-info/requires.txt +9 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/pyproject.toml +3 -3
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/setup.py +10 -14
- excel_list_transform-1.0/src/excel_list_transform/__init__.py +3 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/__main__.py +2 -2
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/check_indata_common.py +1 -1
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/commontypes.py +3 -15
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/config_enums.py +1 -16
- excel_list_transform-1.0/src/excel_list_transform/config_excel_list_transform.py +552 -0
- excel_list_transform-1.0/src/excel_list_transform/config_match.py +22 -0
- excel_list_transform-1.0/src/excel_list_transform/config_read_old.py +128 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/config_xls_list_transf_name.py +23 -19
- excel_list_transform-1.0/src/excel_list_transform/config_xls_list_transf_num.py +84 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/file_extension.py +1 -1
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/file_must_exist.py +1 -1
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/generate_cfg.py +117 -94
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/generate_txt.py +142 -75
- excel_list_transform-1.0/src/excel_list_transform/handle_empty_column.py +17 -0
- excel_list_transform-1.0/src/excel_list_transform/handle_tableio.py +131 -0
- excel_list_transform-1.0/src/excel_list_transform/migrate_cfg.py +22 -0
- excel_list_transform-1.0/src/excel_list_transform/migrate_cfg_warn_hook.py +21 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/num_named_conversion.py +2 -25
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/recode.py +4 -6
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/rewrite_value.py +8 -10
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/row_split_merge.py +16 -32
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/str_to_enum.py +1 -1
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_cmd.py +33 -23
- excel_list_transform-1.0/src/excel_list_transform/transform_func.py +41 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_func_common.py +13 -16
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_func_named.py +16 -34
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/transform_func_num.py +15 -29
- excel_list_transform-1.0/src/excel_list_transform/version_information.py +32 -0
- excel_list_transform-0.8.5/README.md +0 -85
- excel_list_transform-0.8.5/excel_list_transform.egg-info/requires.txt +0 -13
- excel_list_transform-0.8.5/src/excel_list_transform/__init__.py +0 -3
- excel_list_transform-0.8.5/src/excel_list_transform/assert_dict_equal.py +0 -57
- excel_list_transform-0.8.5/src/excel_list_transform/config.py +0 -572
- excel_list_transform-0.8.5/src/excel_list_transform/config_auto_change_hook.py +0 -59
- excel_list_transform-0.8.5/src/excel_list_transform/config_excel_list_transform.py +0 -391
- excel_list_transform-0.8.5/src/excel_list_transform/config_factory.py +0 -101
- excel_list_transform-0.8.5/src/excel_list_transform/config_xls_list_transf_num.py +0 -77
- excel_list_transform-0.8.5/src/excel_list_transform/handle_csv.py +0 -76
- excel_list_transform-0.8.5/src/excel_list_transform/handle_empty_column.py +0 -28
- excel_list_transform-0.8.5/src/excel_list_transform/handle_excel.py +0 -234
- excel_list_transform-0.8.5/src/excel_list_transform/migrate_cfg.py +0 -33
- excel_list_transform-0.8.5/src/excel_list_transform/migrate_cfg_warn_hook.py +0 -32
- excel_list_transform-0.8.5/src/excel_list_transform/transform_func.py +0 -50
- excel_list_transform-0.8.5/src/excel_list_transform/version_information.py +0 -210
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/LICENSE.txt +0 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/dependency_links.txt +0 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/excel_list_transform.egg-info/top_level.txt +0 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/setup.cfg +0 -0
- {excel_list_transform-0.8.5 → excel_list_transform-1.0}/src/excel_list_transform/py.typed +0 -0
|
@@ -1,31 +1,75 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: excel-list-transform
|
|
3
|
+
Version: 1.0
|
|
4
|
+
Summary: Transform table data with configurable row and column changes.
|
|
5
|
+
Author: Tom Björkholm
|
|
6
|
+
Author-email: Tom Björkholm <klausuler_linnet0q@icloud.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Source code, https://bitbucket.org/tom-bjorkholm/excel-list-transform
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.13
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE.txt
|
|
14
|
+
Requires-Dist: tableio-cfg-json>=0.4
|
|
15
|
+
Requires-Dist: tableio>=1.0
|
|
16
|
+
Requires-Dist: config-as-json>=1.2
|
|
17
|
+
Requires-Dist: versionreporter>=0.2
|
|
18
|
+
Requires-Dist: argcomplete>=3.6.3
|
|
19
|
+
Requires-Dist: pip>=26.1
|
|
20
|
+
Requires-Dist: setuptools>=82.0.1
|
|
21
|
+
Requires-Dist: build>=1.5.0
|
|
22
|
+
Requires-Dist: wheel>=0.47.0
|
|
23
|
+
Dynamic: author
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
Dynamic: requires-dist
|
|
26
|
+
Dynamic: requires-python
|
|
27
|
+
|
|
1
28
|
# excel-list-transform
|
|
2
29
|
|
|
3
30
|
## Background
|
|
4
31
|
|
|
5
|
-
This python application was born out of an experience at sail racing events.
|
|
32
|
+
This python application was born out of an experience at sail racing events.
|
|
33
|
+
At the start of the events we received an excel list with participants (from
|
|
34
|
+
a registration web) to enter into the scoring software and into tool for
|
|
35
|
+
online notice board. The information was present in the excel file, but the
|
|
36
|
+
columns were all wrong. To avoid the stressful manual rework of the
|
|
37
|
+
information in the excel list this application was created.
|
|
6
38
|
|
|
7
39
|
## What it does
|
|
8
40
|
|
|
9
41
|
This small python application:
|
|
10
42
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
43
|
+
- reads data (that is a list with columns) from an excel file, an ODS file,
|
|
44
|
+
or from a comma separated values (CSV) file.
|
|
45
|
+
- split rows in the list (creating separate rows when a cell contains
|
|
46
|
+
something that can be split to several values, see below)
|
|
47
|
+
- merge rows (if specified columns have the same value in several rows
|
|
48
|
+
the rows are merged to one row)
|
|
49
|
+
- splits columns in the list (like creating "first name" and "last name"
|
|
50
|
+
columns from "name" column)
|
|
51
|
+
- merges columns in the list (like creating "name" column from "first name"
|
|
52
|
+
and "last name" columns)
|
|
53
|
+
- removes columns in the list
|
|
54
|
+
- reorders columns in the list
|
|
55
|
+
- renames columns in the list
|
|
56
|
+
- inserts columns in the list
|
|
57
|
+
- rewrites columns in the list (like transforming telephone numbers from
|
|
58
|
+
local/national format to international format)
|
|
59
|
+
- writes the resulting data (that is a list with columns) to an excel file,
|
|
60
|
+
an ODS spreadsheet file, a comma separated values (CSV) file, or
|
|
61
|
+
any of a number of other file formats.
|
|
62
|
+
|
|
63
|
+
How this is done is governed by a configuration file. The application can
|
|
64
|
+
create a number of example configuration files with accompanying description
|
|
65
|
+
text files.
|
|
24
66
|
|
|
25
67
|
## Installing it
|
|
26
68
|
|
|
27
|
-
If you want to use it, install it using pip. A precondition is that you have
|
|
28
|
-
Python
|
|
69
|
+
If you want to use it, install it using pip. A precondition is that you have
|
|
70
|
+
Python installed on your computer. See version table below for information
|
|
71
|
+
on needed Python version. Python can be downloaded
|
|
72
|
+
from [https://www.python.org/downloads/](https://www.python.org/downloads/).
|
|
29
73
|
|
|
30
74
|
### Installing on mac and Linux
|
|
31
75
|
|
|
@@ -39,31 +83,6 @@ pip3 install --upgrade excel-list-transform
|
|
|
39
83
|
pip install --upgrade excel-list-transform
|
|
40
84
|
````
|
|
41
85
|
|
|
42
|
-
## Version history
|
|
43
|
-
|
|
44
|
-
| Version | Date | Python version | Description |
|
|
45
|
-
|---------|-------------|-----------------|-------------------------------------|
|
|
46
|
-
| 0.5 | 10 Sep 2024 | 3.10.5 or newer | First released version |
|
|
47
|
-
| 0.6 | 22 Sep 2024 | 3.12.6 or newer | Configured encoding for CSV |
|
|
48
|
-
| 0.6.2 | 28 Sep 2024 | 3.12.6 or newer | Fix name in test report |
|
|
49
|
-
| 0.7.0 | 03 Nov 2024 | 3.12.6 or newer | Option to strip space in excel |
|
|
50
|
-
| 0.7.1 | 15 Dec 2024 | 3.12.6 or newer | Refactor tests |
|
|
51
|
-
| 0.7.3 | 06 Jan 2025 | 3.12.6 or newer | Example config for Sailwave to RRS |
|
|
52
|
-
| 0.7.5 | 21 Mar 2025 | 3.12.6 or newer | Example config for Sailarena to RRS |
|
|
53
|
-
| 0.7.6 | 21 Mar 2025 | 3.13.2 or newer | Adapted to Python 3.13.2 |
|
|
54
|
-
| 0.7.7 | 09 Apr 2025 | 3.13.3 or newer | refactor phone number rewrite cfg |
|
|
55
|
-
| 0.7.9 | 09 Jun 2025 | 3.10.x | Backport 0.7.12 to Python 3.10 |
|
|
56
|
-
| 0.7.10 | 09 Jun 2025 | 3.11.x | Backport 0.7.12 to Python 3.11 |
|
|
57
|
-
| 0.7.11 | 09 Jun 2025 | 3.12.x | Backport 0.7.12 to Python 3.12 |
|
|
58
|
-
| 0.7.12 | 09 Jun 2025 | 3.13.3 or newer | Add version sub-command |
|
|
59
|
-
| 0.7.13 | 11 Jun 2025 | 3.13 or newer | Relax Python version requirement |
|
|
60
|
-
| 0.8.0 | 23 Jul 2025 | 3.10.x | Backport 0.8.3 to Python 3.10 |
|
|
61
|
-
| 0.8.1 | 23 Jul 2025 | 3.11.x | Backport 0.8.3 to Python 3.11 |
|
|
62
|
-
| 0.8.2 | 23 Jul 2025 | 3.12.x | Backport 0.8.3 to Python 3.12 |
|
|
63
|
-
| 0.8.3 | 23 Jul 2025 | 3.13 or newer | Add split-row and merge-row |
|
|
64
|
-
| 0.8.4 | 23 Nov 2025 | 3.12.x | Backport 0.8.5 to Python 3.12 |
|
|
65
|
-
| 0.8.5 | 23 Nov 2025 | 3.13 or newer | Adapted to Python 3.14 |
|
|
66
|
-
|
|
67
86
|
## Running the application
|
|
68
87
|
|
|
69
88
|
### Running the application on mac and Linux
|
|
@@ -94,43 +113,68 @@ python -m excel_list_transform migrate-cfg -i old.cfg -o new.cfg
|
|
|
94
113
|
|
|
95
114
|
## Suggested way to get started
|
|
96
115
|
|
|
97
|
-
1. Use the "cfg-example" sub-command to generate a few example configuration
|
|
98
|
-
|
|
116
|
+
1. Use the "cfg-example" sub-command to generate a few example configuration
|
|
117
|
+
(.cfg) files with description (.txt) files.
|
|
118
|
+
2. Read the example configuration (.cfg) files and the accompanying
|
|
119
|
+
description (.txt) files.
|
|
99
120
|
3. Find an example that is close to what you want to achieve.
|
|
100
121
|
4. Modify that configuration file to achieve what you want to achieve.
|
|
101
|
-
5. Use the "transform" sub-command to read your data and output it transformed
|
|
102
|
-
|
|
122
|
+
5. Use the "transform" sub-command to read your data and output it transformed
|
|
123
|
+
or reorganized according to your modified configuration file.
|
|
124
|
+
6. Read the produced output. If necessary go back to step 4 and adjust how the
|
|
125
|
+
data is transformed.
|
|
103
126
|
|
|
104
127
|
### Example configuration files
|
|
105
128
|
|
|
106
|
-
When using the "cfg-example" sub-command to generate an example configuration
|
|
129
|
+
When using the "cfg-example" sub-command to generate an example configuration
|
|
130
|
+
file (say example.cfg) a text file describing the configuration and the syntax
|
|
131
|
+
of the configuration file is also generated. If the example configuration
|
|
132
|
+
file is named example.cfg, then the text file describing the configuration
|
|
133
|
+
is named example.txt.
|
|
107
134
|
|
|
108
|
-
You can generate several example configuration files each with an accompanying
|
|
135
|
+
You can generate several example configuration files each with an accompanying
|
|
136
|
+
text file describing it.
|
|
109
137
|
|
|
110
|
-
Read the text file describing the configuration file while looking at the
|
|
138
|
+
Read the text file describing the configuration file while looking at the
|
|
139
|
+
configuration file to understand the syntax and the possible options.
|
|
111
140
|
|
|
112
141
|
## Performance
|
|
113
142
|
|
|
114
|
-
This application is written for the moderate amounts of data when
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
143
|
+
This application is written for the moderate amounts of data when
|
|
144
|
+
registering participants for the majority of sports events, or
|
|
145
|
+
processing scrum backlogs in excel. If you have millions of rows
|
|
146
|
+
this application is not for you.
|
|
147
|
+
|
|
148
|
+
With an input file consisting of 20 columns producing an output file of
|
|
149
|
+
11 columns I have measured the following performance:
|
|
150
|
+
|
|
151
|
+
| number of rows | on MacBook Air M4 |
|
|
152
|
+
|----------------|-----------------------|
|
|
153
|
+
| 100 | less than 0.2 seconds |
|
|
154
|
+
| 1000 | less than 0.3 seconds |
|
|
155
|
+
| 10 000 | less than 2.5 seconds |
|
|
156
|
+
| 20 000 | less than 5 seconds |
|
|
157
|
+
| 40 000 | less than 11 seconds |
|
|
158
|
+
| 80 000 | less than 23 seconds |
|
|
159
|
+
| 120 000 | less than 34 seconds |
|
|
160
|
+
|
|
161
|
+
Naturally your performance will be different based on computer hardware,
|
|
162
|
+
operating system and Python version. Generally it should be reasonably
|
|
163
|
+
fast for less than 10 000 rows, painfully slow but somewhat usable up
|
|
164
|
+
to 100 000 rows and probably so slow that it is unusable for more
|
|
165
|
+
than 100 000 rows.
|
|
124
166
|
|
|
125
167
|
## Description of how to write/change the configuration file
|
|
126
168
|
|
|
127
|
-
The configuration file is in JSON syntax.
|
|
169
|
+
The configuration file is in JSON syntax.
|
|
170
|
+
[https://en.wikipedia.org/wiki/JSON](https://en.wikipedia.org/wiki/JSON)
|
|
128
171
|
|
|
129
172
|
The keywords and the nesting is important. The order of keywords
|
|
130
173
|
have no significance (the examples use alphabetical order).
|
|
131
174
|
Indentation and line breaks have no significance.
|
|
132
175
|
|
|
133
|
-
The encoding for the configuration file must be UTF-8. (US-
|
|
176
|
+
The encoding for the configuration file must be UTF-8. (US-ASCII is a
|
|
177
|
+
subset of UTF-8.)
|
|
134
178
|
|
|
135
179
|
It is recommended that you let the command generate a configuration
|
|
136
180
|
file and then edit that file to match your needs. It is NOT recommended
|
|
@@ -141,14 +185,15 @@ that the user writes the configuration file from scratch.
|
|
|
141
185
|
The keyword **column_ref** is used to tell if the configuration
|
|
142
186
|
references columns by name (value *"BY_NAME"*) or by number
|
|
143
187
|
(value *"BY_NUMBER"*). The syntax of parts of the configuration file
|
|
144
|
-
is slightly different depending on
|
|
188
|
+
is slightly different depending on how columns are referenced.
|
|
145
189
|
Thus, you first have to decide how you want to reference columns.
|
|
146
190
|
|
|
147
191
|
Generally it is easier to write a correct configuration using *"BY_NAME"*,
|
|
148
|
-
so *"BY_NAME"* is recommended when possible. *"BY_NAME"* imposes a few
|
|
192
|
+
so *"BY_NAME"* is recommended when possible. *"BY_NAME"* imposes a few
|
|
193
|
+
restrictions:
|
|
149
194
|
|
|
150
|
-
|
|
151
|
-
|
|
195
|
+
- each column needs to have a unique text at the first line to use as column name.
|
|
196
|
+
- the column names need to be stable and known.
|
|
152
197
|
|
|
153
198
|
If you cannot meet these restrictions you will have to reference the
|
|
154
199
|
columns by number *"BY_NUMBER"*. When referencing the columns by number
|
|
@@ -160,49 +205,37 @@ right. The leftmost column is number 0.
|
|
|
160
205
|
|
|
161
206
|
## Type of input and output file
|
|
162
207
|
|
|
163
|
-
The type of input file to read is
|
|
164
|
-
The type of output file to write is
|
|
165
|
-
**"
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
are read and written. They are always needed in the configuration
|
|
189
|
-
file, but are not used if the input and output are excel files.
|
|
190
|
-
|
|
191
|
-
Comma separated values files might have different encoding for
|
|
192
|
-
the text in the file. [https://en.wikipedia.org/wiki/Character_encoding](https://en.wikipedia.org/wiki/Character_encoding)
|
|
193
|
-
This is specified with **"in_csv_encoding"** and **"out_csv_encoding"**.
|
|
194
|
-
Unless you know that you need another encoding leave these as
|
|
195
|
-
in the generated example configuration. (In version 0.5 these
|
|
196
|
-
configuration parameters are missing. To be compatible with 0.5
|
|
197
|
-
configuration files "in_csv_encoding" defaults to "utf_8_sig"
|
|
198
|
-
and "out_csv_encoding" defaults to "utf-8" if missing in the
|
|
199
|
-
configuration file.)
|
|
208
|
+
The type of input file to read is configured in the **"input_table"**
|
|
209
|
+
section. The type of output file to write is configured in the
|
|
210
|
+
**"output_table"** section. These sections use the TableIO JSON
|
|
211
|
+
configuration syntax.
|
|
212
|
+
|
|
213
|
+
The most important keyword in each table section is **"format_name"**.
|
|
214
|
+
Generated examples commonly use *"Excel"* or *"CSV"*, but TableIO may
|
|
215
|
+
support more formats. In most cases you should leave **"implementation"**
|
|
216
|
+
missing or set to *null*. Then TableIO chooses a suitable implementation.
|
|
217
|
+
If you know that you need a specific implementation, you can add it
|
|
218
|
+
manually.
|
|
219
|
+
|
|
220
|
+
CSV files may differ slightly depending on the programs used to read/write
|
|
221
|
+
them and the locale used. CSV-specific settings are configured in the
|
|
222
|
+
nested **"csv"** section inside **"input_table"** or **"output_table"**.
|
|
223
|
+
The **"character_encoding"** keyword in the table section controls the
|
|
224
|
+
text encoding.
|
|
225
|
+
[https://en.wikipedia.org/wiki/Character_encoding](https://en.wikipedia.org/wiki/Character_encoding)
|
|
226
|
+
|
|
227
|
+
Output tables can request extra presentation features. Set
|
|
228
|
+
**"output_borders"** to *true* to request table borders. Set
|
|
229
|
+
**"output_filtered_table"** to *true* to request a filtered table area.
|
|
230
|
+
These features are ignorable requests. TableIO will prefer an
|
|
231
|
+
implementation that can provide them, but the transform can still write
|
|
232
|
+
the output if the selected file format cannot represent them.
|
|
200
233
|
|
|
201
234
|
## Extra spaces in excel input files
|
|
202
235
|
|
|
203
236
|
When viewing an excel file in excel it is very hard to notice if
|
|
204
|
-
some string value in a cell has trailing
|
|
205
|
-
|
|
237
|
+
some string value in a cell has trailing whitespace. These trailing
|
|
238
|
+
spaces can make the further processing of a file difficult
|
|
206
239
|
as the strings in the file are not what you thought they are.
|
|
207
240
|
(This especially is a problem if you have trailing spaces in a
|
|
208
241
|
cell on the first line, and refer to columns by their names.)
|
|
@@ -212,7 +245,7 @@ to strip off leading and trailing whitespace from all columns
|
|
|
212
245
|
values read from the first line of the excel input file.
|
|
213
246
|
The configuration **"in_excel_values_strip"** can be set to true,
|
|
214
247
|
to strip off leading and trailing whitespace from all columns
|
|
215
|
-
values read from
|
|
248
|
+
values read from the other lines (not the first line) of
|
|
216
249
|
the excel input file. (In version 0.6.2 and earlier these are
|
|
217
250
|
missing. To be compatible with version 0.6.2 and earlier
|
|
218
251
|
**"in_excel_col_name_strip"** and **"in_excel_values_strip"**
|
|
@@ -220,10 +253,10 @@ defaults to false if missing in the configuration file.)
|
|
|
220
253
|
|
|
221
254
|
## Column manipulation
|
|
222
255
|
|
|
223
|
-
A number
|
|
256
|
+
A number of records starting with "s" and a number describe
|
|
224
257
|
column manipulation to be done. These manipulations are done
|
|
225
|
-
in order of the number:
|
|
226
|
-
before
|
|
258
|
+
in order of the number: s01_(something) before s02_(something),
|
|
259
|
+
before s03_(something). As the "s" number records might add, remove
|
|
227
260
|
or rename columns, it is important to keep track of the order that
|
|
228
261
|
they are applied.
|
|
229
262
|
|
|
@@ -235,12 +268,12 @@ referenced by number.
|
|
|
235
268
|
|
|
236
269
|
The first operation is to split rows based on column values.
|
|
237
270
|
This operation is configured using the **"s01_split_rows"** record,
|
|
238
|
-
that
|
|
271
|
+
that has an array of splits to be done. Each row split has the
|
|
239
272
|
following keys: **"column"**, **"separators"** and **"not_separators"**.
|
|
240
273
|
|
|
241
274
|
The **"column"** keyword is used to identify the column that is
|
|
242
275
|
split into several rows. This is a column number in the case of
|
|
243
|
-
*"BY_NUMBER"*, and a column name/title in the
|
|
276
|
+
*"BY_NUMBER"*, and a column name/title in the case of *"BY_NAME"*.
|
|
244
277
|
|
|
245
278
|
New rows will be created so that the parts of the identified
|
|
246
279
|
column will be put in that column only one part per row.
|
|
@@ -252,7 +285,7 @@ these strings are present in the identified column it is seen
|
|
|
252
285
|
as the separator between the parts of the column value that go
|
|
253
286
|
into different rows.
|
|
254
287
|
|
|
255
|
-
**"not_separators"** take as
|
|
288
|
+
**"not_separators"** take as argument a list of strings. These strings
|
|
256
289
|
are not regarded as separators even if they include the strings
|
|
257
290
|
of one or more separator. (For instance ";" could be a separator,
|
|
258
291
|
but using **"not_separators"** the string "\\;" could be seen as not
|
|
@@ -262,14 +295,14 @@ a separator.)
|
|
|
262
295
|
|
|
263
296
|
The next operation is to merge rows based on column values.
|
|
264
297
|
This is the opposite operation to the splitting of rows.
|
|
265
|
-
Row
|
|
266
|
-
that
|
|
298
|
+
Row merging is configured using the **"s02_merge_rows"** record,
|
|
299
|
+
that has an array of merges to be done. Each row merge has
|
|
267
300
|
the following keys: **"columns"** and **"separator"**.
|
|
268
301
|
|
|
269
302
|
The **"columns"** keyword is used to identify the columns that need
|
|
270
303
|
to have identical values to merge two or more rows. The **"columns"**
|
|
271
|
-
keyword
|
|
272
|
-
*"BY_NUMBER"*, and
|
|
304
|
+
keyword takes a list of columns. These are column numbers in the case of
|
|
305
|
+
*"BY_NUMBER"*, and column names/titles in the case of *"BY_NAME"*.
|
|
273
306
|
|
|
274
307
|
For each column that has the same value for all rows merged, that
|
|
275
308
|
value will be in the merged row. When rows being merged have different
|
|
@@ -289,7 +322,7 @@ for customers in the following format:
|
|
|
289
322
|
| Brewery | Beer | Smith + Bush |
|
|
290
323
|
| Dairy | Milk | Jones |
|
|
291
324
|
|
|
292
|
-
But for
|
|
325
|
+
But for distributing it to customers you would like to have the
|
|
293
326
|
list in the following format:
|
|
294
327
|
|
|
295
328
|
| To | What | From |
|
|
@@ -299,27 +332,27 @@ list in the following format:
|
|
|
299
332
|
| Bush | Beer | Brewery |
|
|
300
333
|
|
|
301
334
|
To do this transformation we use the **"s01_split_rows"** to split
|
|
302
|
-
rows
|
|
335
|
+
rows based on the "To" column using " + " as separator.
|
|
303
336
|
Then we use **"s02_merge_rows"** to merge the rows that have
|
|
304
337
|
identical values in the "To" column.
|
|
305
338
|
|
|
306
339
|
### "s03_split_columns"
|
|
307
340
|
|
|
308
|
-
The first operation that is done is splitting of columns.
|
|
309
|
-
The key **"s03_split_columns"**
|
|
341
|
+
The first column operation that is done is splitting of columns.
|
|
342
|
+
The key **"s03_split_columns"** has an array of splits to be
|
|
310
343
|
done. (When the list of splits has more than one split,
|
|
311
344
|
the least confusion is to split columns to the right before
|
|
312
345
|
columns to the left. Named references also helps to avoid
|
|
313
346
|
confusion.)
|
|
314
347
|
|
|
315
348
|
Each split has the following keys: **"column"**, **"separator"** and
|
|
316
|
-
**"where"**. In the case of number column references
|
|
349
|
+
**"where"**. In the case of number column references there is
|
|
317
350
|
also the key **"store_single"**. In the case of named column references
|
|
318
351
|
there is instead the key **"right_name"**.
|
|
319
352
|
|
|
320
|
-
**"column"**
|
|
353
|
+
**"column"** takes as argument the column to be split. This is a
|
|
321
354
|
column number in the case of *"BY_NUMBER"*, and a column name/title
|
|
322
|
-
in the
|
|
355
|
+
in the case of *"BY_NAME"*.
|
|
323
356
|
|
|
324
357
|
**"separator"** is a string of characters that indicate the position
|
|
325
358
|
where the column shall be split. (For instance a single space
|
|
@@ -328,7 +361,7 @@ if splitting between two words.)
|
|
|
328
361
|
**"where"** can either have the value *"RIGHTMOST"* or the value
|
|
329
362
|
*"LEFTMOST"*. This is used if the separator string is present
|
|
330
363
|
more than once in the column value to split. The split is then
|
|
331
|
-
done at the leftmost or rightmost
|
|
364
|
+
done at the leftmost or rightmost occurrence according to the
|
|
332
365
|
value of **"where"**.
|
|
333
366
|
|
|
334
367
|
**"right_name"** is only used if column references are *"BY_NAME"*.
|
|
@@ -348,11 +381,11 @@ always stored in the column with the original name.
|
|
|
348
381
|
|
|
349
382
|
**"s04_remove_columns"** is only used with column references *"BY_NUMBER"*.
|
|
350
383
|
The value of **"s04_remove_columns"** is a list of column numbers to
|
|
351
|
-
remove. (For
|
|
384
|
+
remove. (For column references *"BY_NAME"* see **"s10_column_order"**.)
|
|
352
385
|
|
|
353
386
|
### "s05_merge_columns"
|
|
354
387
|
|
|
355
|
-
The key **"s05_merge_columns"**
|
|
388
|
+
The key **"s05_merge_columns"** has an array of merges to be done.
|
|
356
389
|
Each merge have the keys **"columns"** and **"separator"**.
|
|
357
390
|
|
|
358
391
|
**"columns"** have a list of column references. If *"BY_NAME"* the
|
|
@@ -368,16 +401,16 @@ the column values being merged.
|
|
|
368
401
|
The key **"s06_place_columns_first"** has a value that is a list of the
|
|
369
402
|
column numbers to be placed first in order. This step re-orders the
|
|
370
403
|
columns.
|
|
371
|
-
(For
|
|
404
|
+
(For column references *"BY_NAME"* see **"s10_column_order"**.)
|
|
372
405
|
|
|
373
406
|
### "s07_rename_columns"
|
|
374
407
|
|
|
375
408
|
The key **"s07_rename_columns"** has a value that is a list of column rename
|
|
376
409
|
operations. Each column rename operation has the keys **"column"** and **"name"**.
|
|
377
410
|
|
|
378
|
-
**"column"** is the number/name of the column before
|
|
411
|
+
**"column"** is the number/name of the column before renaming. This is a
|
|
379
412
|
column number in the case of *"BY_NUMBER"*, and a column name/title
|
|
380
|
-
in the
|
|
413
|
+
in the case of *"BY_NAME"*.
|
|
381
414
|
|
|
382
415
|
**"name"** is the new name/title of the column identified by **"column"**.
|
|
383
416
|
|
|
@@ -412,7 +445,7 @@ keys. **"kind"** can have the values: *"STRIP"*, *"REMOVECHARS"*,
|
|
|
412
445
|
**"kind"** value *"STRIP"* shall have the keys: **"kind"**, **"case"**, **"chars"**
|
|
413
446
|
and **"column"**. This case means that the characters in **"chars"** shall
|
|
414
447
|
be stripped off the beginning and end of the column value. If key
|
|
415
|
-
**"chars"** is empty string, then
|
|
448
|
+
**"chars"** is empty string, then whitespace is stripped off.
|
|
416
449
|
|
|
417
450
|
**"kind"** value *"REMOVECHARS"* shall have the keys: **"kind"**, **"case"**,
|
|
418
451
|
**"chars"** and **"column"**. This case means that the characters in **"chars"**
|
|
@@ -430,7 +463,7 @@ and if a sub-string of the column value equals this string that part is
|
|
|
430
463
|
replaced with the value of key **"to"**.
|
|
431
464
|
|
|
432
465
|
**"column"** identifies the column to rewrite. This is a column number in the
|
|
433
|
-
case of *"BY_NUMBER"*, and a column name/title in the
|
|
466
|
+
case of *"BY_NUMBER"*, and a column name/title in the case of *"BY_NAME"*.
|
|
434
467
|
|
|
435
468
|
**"chars"** is a list of characters written as a string. These characters
|
|
436
469
|
are removed or stripped off the beginning/end depending on the **"kind"**
|
|
@@ -440,8 +473,8 @@ value.
|
|
|
440
473
|
value of **"from"** shall be case sensitive or not. Possible values are
|
|
441
474
|
*"MATCH_CASE"* and *"IGNORE_CASE"*.
|
|
442
475
|
|
|
443
|
-
**"from"** specifies what
|
|
444
|
-
when matching.
|
|
476
|
+
**"from"** specifies what part of the column value shall be substituted
|
|
477
|
+
when matching.
|
|
445
478
|
|
|
446
479
|
**"to"** specifies the string that substitution will replace **"from"** with.
|
|
447
480
|
|
|
@@ -454,22 +487,21 @@ Columns not mentioned in **"s10_column_order"** will not be output,
|
|
|
454
487
|
and will thus be removed.
|
|
455
488
|
(For *"BY_NUMBER"* see **"s04_remove_columns"** and **"s06_place_columns_first"**.)
|
|
456
489
|
|
|
457
|
-
##
|
|
490
|
+
## Utility for recoding text files
|
|
458
491
|
|
|
459
|
-
|
|
492
|
+
When working with text based files like comma separated values (CSV)
|
|
493
|
+
the file may have an incorrect character encoding, and even worse
|
|
494
|
+
you may not know what character encoding was used to create the file.
|
|
495
|
+
As a small tool for these situations the excel-list-transform includes
|
|
496
|
+
a small recode utility. Using this you can by trial and error recode
|
|
497
|
+
from different encodings until you find which encoding works. (This
|
|
498
|
+
may be the way to figure out which character encoding to write into
|
|
499
|
+
a configuration file.)
|
|
460
500
|
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
| s1_split_columns | s03_split_columns |
|
|
466
|
-
| s2_remove_columns | s04_remove_columns |
|
|
467
|
-
| s3_merge_columns | s05_merge_columns |
|
|
468
|
-
| s4_place_columns_first | s06_place_columns_first |
|
|
469
|
-
| s5_rename_columns | s07_rename_columns |
|
|
470
|
-
| s6_insert_columns | s08_insert_columns |
|
|
471
|
-
| s7_rewrite_columns | s09_rewrite_columns |
|
|
472
|
-
| s8_column_order | s10_column_order |
|
|
501
|
+
```python
|
|
502
|
+
python3 -m excel_list_transform.recode --help
|
|
503
|
+
python3 -m excel_list_transform.recode -i a.txt -o b.txt --from cp1250 --to utf-8
|
|
504
|
+
```
|
|
473
505
|
|
|
474
506
|
## Source code
|
|
475
507
|
|
|
@@ -477,7 +509,9 @@ Source code and tests are available at [https://bitbucket.org/tom-bjorkholm/exce
|
|
|
477
509
|
|
|
478
510
|
## Test summary
|
|
479
511
|
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
512
|
+
- Test result: 1368 passed in 65s (0:01:05)
|
|
513
|
+
- No flake8 warnings.
|
|
514
|
+
- No mypy errors found.
|
|
515
|
+
- No python layout warnings.
|
|
516
|
+
- Built version(s): 1.0
|
|
517
|
+
- Build and test using Python 3.14.5
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# excel-list-transform
|
|
2
|
+
|
|
3
|
+
## Background
|
|
4
|
+
|
|
5
|
+
This python application was born out of an experience at sail racing events. At the start of the events we received an excel list with participants (from a registration web) to enter into the scoring software and into tool for online notice board. The information was present in the excel file, but the columns were all wrong. To avoid the stressful manual rework of the information in the excel list this application was created.
|
|
6
|
+
|
|
7
|
+
## Using it
|
|
8
|
+
|
|
9
|
+
If you want to use it install it using pip from [https://pypi.org/project/excel-list-transform](https://pypi.org/project/excel-list-transform).
|
|
10
|
+
There is no need to download anything from Bitbucket to use the application.
|
|
11
|
+
|
|
12
|
+
### Installing on mac and Linux
|
|
13
|
+
|
|
14
|
+
````sh
|
|
15
|
+
pip3 install --upgrade excel-list-transform
|
|
16
|
+
````
|
|
17
|
+
|
|
18
|
+
### Installing on Microsoft Windows
|
|
19
|
+
|
|
20
|
+
````sh
|
|
21
|
+
pip install --upgrade excel-list-transform
|
|
22
|
+
````
|
|
23
|
+
|
|
24
|
+
### Information for use
|
|
25
|
+
|
|
26
|
+
Please see [https://pypi.org/project/excel-list-transform](https://pypi.org/project/excel-list-transform) or please see README_pypi.md
|
|
27
|
+
|
|
28
|
+
## What it does
|
|
29
|
+
|
|
30
|
+
This small python application:
|
|
31
|
+
|
|
32
|
+
- reads data (that is a list with columns) from an excel file, an ODS file,
|
|
33
|
+
or from a comma separated values (CSV) file.
|
|
34
|
+
- splits columns in the list (like creating "first name" and "last name"
|
|
35
|
+
columns from "name" column)
|
|
36
|
+
- merges columns in the list (like creating "name" column from "first name"
|
|
37
|
+
and "last name" columns)
|
|
38
|
+
- removes columns in the list
|
|
39
|
+
- reorders columns in the list
|
|
40
|
+
- renames columns in the list
|
|
41
|
+
- inserts columns in the list
|
|
42
|
+
- rewrites columns in the list (like transforming telephone numbers from
|
|
43
|
+
local/national format to international format)
|
|
44
|
+
- writes the resulting data (that is a list with columns) to an excel file,
|
|
45
|
+
an ODS spreadsheet file, a comma separated values (CSV) file, or
|
|
46
|
+
any of a number of other file formats.
|
|
47
|
+
|
|
48
|
+
How this is done is governed by a configuration file. The application can create a number of example configuration files with accompanying description text files.
|
|
49
|
+
|
|
50
|
+
## Cloning
|
|
51
|
+
|
|
52
|
+
This repository uses submodules. Clone it with:
|
|
53
|
+
|
|
54
|
+
```sh
|
|
55
|
+
git clone --recurse-submodules \
|
|
56
|
+
git@bitbucket.org:tom-bjorkholm/excel-list-transform.git
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
If you already cloned without submodules, initialize them with:
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
git submodule update --init --recursive
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
To update the checked-out submodule revisions:
|
|
66
|
+
|
|
67
|
+
```sh
|
|
68
|
+
git submodule update --remote --merge
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Supported Python versions
|
|
72
|
+
|
|
73
|
+
- Package runtime baseline: Python 3.13 or newer
|
|
74
|
+
- Maintainer validation target: Python 3.13, and 3.14
|
|
75
|
+
- Main day-to-day development: usually the newest supported Python version
|
|
76
|
+
|
|
77
|
+
## Development workflow
|
|
78
|
+
|
|
79
|
+
On macOS and Linux, the normal workflow is:
|
|
80
|
+
|
|
81
|
+
1. Run `./run_setup_build_environment.py` once after cloning or when the
|
|
82
|
+
build environment needs to be recreated.
|
|
83
|
+
2. Run `./run_build.py` for the normal build-and-test cycle.
|
|
84
|
+
3. Run `./run_clean_build.py` before review or release work that needs a
|
|
85
|
+
completely fresh build.
|
|
86
|
+
|
|
87
|
+
The helper scripts are:
|
|
88
|
+
|
|
89
|
+
- `run_setup_build_environment.py`
|
|
90
|
+
Create or refresh the build environment.
|
|
91
|
+
- `run_build.py`
|
|
92
|
+
Build the package and run the configured checks in the project virtual
|
|
93
|
+
environment.
|
|
94
|
+
- `run_clean.py`
|
|
95
|
+
Remove files generated by the build system.
|
|
96
|
+
- `run_clean_build.py`
|
|
97
|
+
Perform a clean build from scratch. This is especially useful because some
|
|
98
|
+
duplicate-code diagnostics only appear on a clean build.
|
|
99
|
+
- `run_pypi_build.py`
|
|
100
|
+
Create the distribution artifacts intended for PyPI publishing.
|
|
101
|
+
|
|
102
|
+
The standard verification suite includes pytest, pylint, flake8, and mypy.
|
|
103
|
+
After a build, the generated reports can be browsed through
|
|
104
|
+
`reports/index.html`.
|
|
105
|
+
|
|
106
|
+
After running `run_build.py` or `run_clean_build.py`, you can manually test
|
|
107
|
+
the built and installed application in the virtual environment `./venv`.
|
|
108
|
+
|
|
109
|
+
## Test summary
|
|
110
|
+
|
|
111
|
+
- Test result: 1368 passed in 65s (0:01:05)
|
|
112
|
+
- No flake8 warnings.
|
|
113
|
+
- No mypy errors found.
|
|
114
|
+
- No python layout warnings.
|
|
115
|
+
- Built version(s): 1.0
|
|
116
|
+
- Build and test using Python 3.14.5
|