excel-list-transform 0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- excel_list_transform-0.5/LICENSE.txt +22 -0
- excel_list_transform-0.5/PKG-INFO +114 -0
- excel_list_transform-0.5/README.md +76 -0
- excel_list_transform-0.5/README_pypi.md +91 -0
- excel_list_transform-0.5/excel_list_transform.egg-info/PKG-INFO +114 -0
- excel_list_transform-0.5/excel_list_transform.egg-info/SOURCES.txt +35 -0
- excel_list_transform-0.5/excel_list_transform.egg-info/dependency_links.txt +1 -0
- excel_list_transform-0.5/excel_list_transform.egg-info/requires.txt +9 -0
- excel_list_transform-0.5/excel_list_transform.egg-info/top_level.txt +1 -0
- excel_list_transform-0.5/pyproject.toml +22 -0
- excel_list_transform-0.5/setup.cfg +4 -0
- excel_list_transform-0.5/setup.py +26 -0
- excel_list_transform-0.5/src/excel_list_transform/__init__.py +1 -0
- excel_list_transform-0.5/src/excel_list_transform/__main__.py +11 -0
- excel_list_transform-0.5/src/excel_list_transform/check_indata_common.py +39 -0
- excel_list_transform-0.5/src/excel_list_transform/commontypes.py +59 -0
- excel_list_transform-0.5/src/excel_list_transform/config.py +358 -0
- excel_list_transform-0.5/src/excel_list_transform/config_enums.py +53 -0
- excel_list_transform-0.5/src/excel_list_transform/config_excel_list_transform.py +278 -0
- excel_list_transform-0.5/src/excel_list_transform/config_factory.py +95 -0
- excel_list_transform-0.5/src/excel_list_transform/config_xls_list_refmt_name.py +40 -0
- excel_list_transform-0.5/src/excel_list_transform/config_xls_list_refmt_num.py +70 -0
- excel_list_transform-0.5/src/excel_list_transform/file_extension.py +30 -0
- excel_list_transform-0.5/src/excel_list_transform/file_must_exist.py +22 -0
- excel_list_transform-0.5/src/excel_list_transform/generate_cfg.py +374 -0
- excel_list_transform-0.5/src/excel_list_transform/generate_txt.py +259 -0
- excel_list_transform-0.5/src/excel_list_transform/handle_csv.py +73 -0
- excel_list_transform-0.5/src/excel_list_transform/handle_empty_column.py +28 -0
- excel_list_transform-0.5/src/excel_list_transform/handle_excel.py +161 -0
- excel_list_transform-0.5/src/excel_list_transform/num_named_conversion.py +60 -0
- excel_list_transform-0.5/src/excel_list_transform/rewrite_value.py +123 -0
- excel_list_transform-0.5/src/excel_list_transform/str_to_enum.py +39 -0
- excel_list_transform-0.5/src/excel_list_transform/transform_cmd.py +146 -0
- excel_list_transform-0.5/src/excel_list_transform/transform_func.py +50 -0
- excel_list_transform-0.5/src/excel_list_transform/transform_func_common.py +249 -0
- excel_list_transform-0.5/src/excel_list_transform/transform_func_named.py +136 -0
- excel_list_transform-0.5/src/excel_list_transform/transform_func_num.py +157 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Tom Björkholm
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: excel-list-transform
|
|
3
|
+
Version: 0.5
|
|
4
|
+
Summary: Transform a list in excel or CSV
|
|
5
|
+
Author: Tom Björkholm
|
|
6
|
+
Author-email: Tom Björkholm <klausuler_linnet0q@icloud.com>
|
|
7
|
+
Project-URL: Source code, https://bitbucket.org/tom-bjorkholm/excel-list-transform
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.10.5
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE.txt
|
|
14
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
15
|
+
Requires-Dist: types-openpyxl>=3.1.5.20240822
|
|
16
|
+
Requires-Dist: pylightxl>=1.61
|
|
17
|
+
Requires-Dist: XlsxWriter>=3.2.0
|
|
18
|
+
Requires-Dist: pip>=23.2
|
|
19
|
+
Requires-Dist: Pillow>=10.0.1
|
|
20
|
+
Requires-Dist: setuptools>=74.0.0
|
|
21
|
+
Requires-Dist: build>=1.2.1
|
|
22
|
+
Requires-Dist: wheel>=0.44.0
|
|
23
|
+
|
|
24
|
+
# excel-list-transform
|
|
25
|
+
|
|
26
|
+
## Background
|
|
27
|
+
|
|
28
|
+
This python application was born out of an experience at sail racing events. At the start of the events we received an excel list with participants (from a registration web) to enter into the scoring software and into tool for online notice board. The information was present in the excel file, but the columns were all wrong. To avoid the stressful manual rework of the information in the excel list this application was created.
|
|
29
|
+
|
|
30
|
+
## What it does
|
|
31
|
+
|
|
32
|
+
This small python application:
|
|
33
|
+
* reads data (that is a list with columns) from an excel file or from a comma separate values (CSV) file.
|
|
34
|
+
* splits columns in the list (like creating "first name" and "last name" columns from "name" column)
|
|
35
|
+
* merges columns in the list (like creating "name" column from "first name" and "last name" columns)
|
|
36
|
+
* removes columns in the list
|
|
37
|
+
* reorders columns in the list
|
|
38
|
+
* renames columns in the list
|
|
39
|
+
* inserts columns in the list
|
|
40
|
+
* rewrites columns in the list (like transforming telephone numbers from local/national format to international format)
|
|
41
|
+
* writest the resulting data (that is a list with columns) to an excel file or to a comma separate values (CSV) file.
|
|
42
|
+
|
|
43
|
+
How this is done is governed by a configuration file. The application can create a number of example configuration files with accompanying description text files.
|
|
44
|
+
|
|
45
|
+
## Installing it
|
|
46
|
+
|
|
47
|
+
If you want to use it install it using pip. A precondition is that you have Python 3.10.5 or newer installed on you computer. Python can be downloaded from [https://www.python.org/downloads/](https://www.python.org/downloads/).
|
|
48
|
+
|
|
49
|
+
### Installing on mac and Linux
|
|
50
|
+
|
|
51
|
+
````
|
|
52
|
+
pip3 install excel-list-transform
|
|
53
|
+
````
|
|
54
|
+
|
|
55
|
+
### Installing on Microsoft Windows
|
|
56
|
+
|
|
57
|
+
````
|
|
58
|
+
pip install excel-list-transform
|
|
59
|
+
````
|
|
60
|
+
|
|
61
|
+
## Running the application
|
|
62
|
+
|
|
63
|
+
### Running the application on mac and Linux
|
|
64
|
+
|
|
65
|
+
````
|
|
66
|
+
python3 -m excel_list_transform --help
|
|
67
|
+
python3 -m excel_list_transform example --help
|
|
68
|
+
python3 -m excel_list_transform transform --help
|
|
69
|
+
python3 -m excel_list_transform example -k forms_to_rrs -r by_name -o example.cfg
|
|
70
|
+
python3 -m excel_list_transform transform -c example.cfg -i input.xlsx -o output.xlsx
|
|
71
|
+
````
|
|
72
|
+
|
|
73
|
+
### Running the application on Microsoft Windows
|
|
74
|
+
|
|
75
|
+
````
|
|
76
|
+
python -m excel_list_transform --help
|
|
77
|
+
python -m excel_list_transform example --help
|
|
78
|
+
python -m excel_list_transform transform --help
|
|
79
|
+
python -m excel_list_transform example -k forms_to_rrs -r by_name -o example.cfg
|
|
80
|
+
python -m excel_list_transform transform -c example.cfg -i input.xlsx -o output.xlsx
|
|
81
|
+
````
|
|
82
|
+
|
|
83
|
+
## Suggested way to get started
|
|
84
|
+
|
|
85
|
+
1. Use the "example" sub-command to generate a few example configuration (.cfg) files with description (.txt) files.
|
|
86
|
+
2. Read the example configuration (.cfg) files and the accompanying description (.txt) files.
|
|
87
|
+
3. Find an example that is close to what you want to achieve.
|
|
88
|
+
4. Modify that configuration file to achieve what you want to achieve.
|
|
89
|
+
5. Use the "transform" sub-command to read your data and output it transformed or reorganized according to your modified configuration file.
|
|
90
|
+
6. Read the produced output. If necessary go back to step 4 and adjust how the data is transformed.
|
|
91
|
+
|
|
92
|
+
### Description of configuration file
|
|
93
|
+
|
|
94
|
+
When using the "example" sub-command to generate an example configuration file (say example.cfg) a text file describing the configuration and the syntax of the configuration file is also generated. If the example configuration file is named example.cfg, then the text file descriging the configuration is named example.txt.
|
|
95
|
+
|
|
96
|
+
You can generate several example configuration files each with an accompanying text file descriping it.
|
|
97
|
+
|
|
98
|
+
Read the text file describing the configuration file while looking at the configuration file to understand the syntax and the possible options.
|
|
99
|
+
|
|
100
|
+
## Performance
|
|
101
|
+
|
|
102
|
+
This application is written for the moderate amounts of data when registering participants for the majority of sports events. If you have millions of rows this application is not for you.
|
|
103
|
+
|
|
104
|
+
With an input file consisting of 20 columns producing an output file of 11 columns I have measured the following performance on a MacBook Air (laptop) from 2020:
|
|
105
|
+
* 1 000 rows processed in less than 0.5 seconds
|
|
106
|
+
* 10 000 rows processed in less than 4 seconds
|
|
107
|
+
* 80 000 rows processed in less than 30 seconds
|
|
108
|
+
* 120 000 rows processed in 40 seconds
|
|
109
|
+
|
|
110
|
+
Naturally your performance will be different based on computer hardware, operating system and Python version. Generally it should be reasonably fast for less than 10 000 rows, painfully slow but somewhat usable up to 100 000 rows and probably so slow that it is unusable for more than 100 000 rows.
|
|
111
|
+
|
|
112
|
+
## Source code
|
|
113
|
+
|
|
114
|
+
Source code and tests are available at [https://bitbucket.org/tom-bjorkholm/excel-list-transform](https://bitbucket.org/tom-bjorkholm/excel-list-transform).
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# excel-list-transform
|
|
2
|
+
|
|
3
|
+
## Background
|
|
4
|
+
|
|
5
|
+
This python application was born out of an experience at sail racing events. At the start of the events we received an excel list with participants (from a registration web) to enter into the scoring software and into tool for online notice board. The information was present in the excel file, but the columns were all wrong. To avoid the stressful manual rework of the information in the excel list this application was created.
|
|
6
|
+
|
|
7
|
+
## Using it
|
|
8
|
+
|
|
9
|
+
If you want to use it install it using pip from [https://pypi.org/project/excel-list-transform](https://pypi.org/project/excel-list-transform). There is no need download anything from Bitbucket to use the application.
|
|
10
|
+
|
|
11
|
+
### Installing on mac and Linux
|
|
12
|
+
|
|
13
|
+
````
|
|
14
|
+
pip3 install excel-list-transform
|
|
15
|
+
````
|
|
16
|
+
|
|
17
|
+
### Installing on Microsoft Windows
|
|
18
|
+
|
|
19
|
+
````
|
|
20
|
+
pip install excel-list-transform
|
|
21
|
+
````
|
|
22
|
+
|
|
23
|
+
### Information for use
|
|
24
|
+
|
|
25
|
+
Please see [https://pypi.org/project/excel-list-transform](https://pypi.org/project/excel-list-transform) or please see README_pypi.md
|
|
26
|
+
|
|
27
|
+
## What it does
|
|
28
|
+
|
|
29
|
+
This small python application:
|
|
30
|
+
* reads data (that is a list with columns) from an excel file or from a comma separate values (CSV) file.
|
|
31
|
+
* splits columns in the list (like creating "first name" and "last name" columns from "name" column)
|
|
32
|
+
* merges columns in the list (like creating "name" column from "first name" and "last name" columns)
|
|
33
|
+
* removes columns in the list
|
|
34
|
+
* reorders columns in the list
|
|
35
|
+
* renames columns in the list
|
|
36
|
+
* inserts columns in the list
|
|
37
|
+
* rewrites columns in the list (like transforming telephone numbers from local/national format to international format)
|
|
38
|
+
* writest the resulting data (that is a list with columns) to an excel file or to a comma separate values (CSV) file.
|
|
39
|
+
|
|
40
|
+
How this is done is governed by a configuration file. The application can create a number of example configuration files with accompanying description text files.
|
|
41
|
+
|
|
42
|
+
## For developers
|
|
43
|
+
|
|
44
|
+
### Needed environment
|
|
45
|
+
|
|
46
|
+
#### OS
|
|
47
|
+
|
|
48
|
+
For running the script and running the test suite you need a mac or a Linux computer. Even if the resulting application can be installed and used on Windows, the scripts for building and testing is only implemented for mac and Linux.
|
|
49
|
+
|
|
50
|
+
#### Python version
|
|
51
|
+
|
|
52
|
+
The tests and the script for running the tests, coverage, mypy etc. requires Python version 3.12.5 or newer.
|
|
53
|
+
|
|
54
|
+
#### Zsh
|
|
55
|
+
|
|
56
|
+
The scripts are all zsh. zsh is available by default on modern macs. zsh can easily be installed on Linux (on Ubuntu: sudo apt install zsh).
|
|
57
|
+
|
|
58
|
+
### Internal APIs not quaranteed
|
|
59
|
+
|
|
60
|
+
The internal APIs in this package are not guaranteed to be stable. They can change without warning between versions.
|
|
61
|
+
|
|
62
|
+
### Building application
|
|
63
|
+
|
|
64
|
+
There are 3 scripts for building the application
|
|
65
|
+
* setup_build_environment.zsh
|
|
66
|
+
Run this script first to get the environment set up for building
|
|
67
|
+
* doBuild.zsh
|
|
68
|
+
Run this script to build an installation package (.whl) and to run the tests on it in a venv (virtual environment).
|
|
69
|
+
* clean.zsh
|
|
70
|
+
Deletes all files that was produced by the build to start over from a clean state.
|
|
71
|
+
|
|
72
|
+
The "testing" includes pytest, pylint, flake8 and mypy.
|
|
73
|
+
|
|
74
|
+
After running doBuild.zsh you can open reports/index.htm to see all test reports.
|
|
75
|
+
|
|
76
|
+
After running doBuild.zsh you can do manual test of the built and installed application in the virtual environment ./venv
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# excel-list-transform
|
|
2
|
+
|
|
3
|
+
## Background
|
|
4
|
+
|
|
5
|
+
This python application was born out of an experience at sail racing events. At the start of the events we received an excel list with participants (from a registration web) to enter into the scoring software and into tool for online notice board. The information was present in the excel file, but the columns were all wrong. To avoid the stressful manual rework of the information in the excel list this application was created.
|
|
6
|
+
|
|
7
|
+
## What it does
|
|
8
|
+
|
|
9
|
+
This small python application:
|
|
10
|
+
* reads data (that is a list with columns) from an excel file or from a comma separate values (CSV) file.
|
|
11
|
+
* splits columns in the list (like creating "first name" and "last name" columns from "name" column)
|
|
12
|
+
* merges columns in the list (like creating "name" column from "first name" and "last name" columns)
|
|
13
|
+
* removes columns in the list
|
|
14
|
+
* reorders columns in the list
|
|
15
|
+
* renames columns in the list
|
|
16
|
+
* inserts columns in the list
|
|
17
|
+
* rewrites columns in the list (like transforming telephone numbers from local/national format to international format)
|
|
18
|
+
* writest the resulting data (that is a list with columns) to an excel file or to a comma separate values (CSV) file.
|
|
19
|
+
|
|
20
|
+
How this is done is governed by a configuration file. The application can create a number of example configuration files with accompanying description text files.
|
|
21
|
+
|
|
22
|
+
## Installing it
|
|
23
|
+
|
|
24
|
+
If you want to use it install it using pip. A precondition is that you have Python 3.10.5 or newer installed on you computer. Python can be downloaded from [https://www.python.org/downloads/](https://www.python.org/downloads/).
|
|
25
|
+
|
|
26
|
+
### Installing on mac and Linux
|
|
27
|
+
|
|
28
|
+
````
|
|
29
|
+
pip3 install excel-list-transform
|
|
30
|
+
````
|
|
31
|
+
|
|
32
|
+
### Installing on Microsoft Windows
|
|
33
|
+
|
|
34
|
+
````
|
|
35
|
+
pip install excel-list-transform
|
|
36
|
+
````
|
|
37
|
+
|
|
38
|
+
## Running the application
|
|
39
|
+
|
|
40
|
+
### Running the application on mac and Linux
|
|
41
|
+
|
|
42
|
+
````
|
|
43
|
+
python3 -m excel_list_transform --help
|
|
44
|
+
python3 -m excel_list_transform example --help
|
|
45
|
+
python3 -m excel_list_transform transform --help
|
|
46
|
+
python3 -m excel_list_transform example -k forms_to_rrs -r by_name -o example.cfg
|
|
47
|
+
python3 -m excel_list_transform transform -c example.cfg -i input.xlsx -o output.xlsx
|
|
48
|
+
````
|
|
49
|
+
|
|
50
|
+
### Running the application on Microsoft Windows
|
|
51
|
+
|
|
52
|
+
````
|
|
53
|
+
python -m excel_list_transform --help
|
|
54
|
+
python -m excel_list_transform example --help
|
|
55
|
+
python -m excel_list_transform transform --help
|
|
56
|
+
python -m excel_list_transform example -k forms_to_rrs -r by_name -o example.cfg
|
|
57
|
+
python -m excel_list_transform transform -c example.cfg -i input.xlsx -o output.xlsx
|
|
58
|
+
````
|
|
59
|
+
|
|
60
|
+
## Suggested way to get started
|
|
61
|
+
|
|
62
|
+
1. Use the "example" sub-command to generate a few example configuration (.cfg) files with description (.txt) files.
|
|
63
|
+
2. Read the example configuration (.cfg) files and the accompanying description (.txt) files.
|
|
64
|
+
3. Find an example that is close to what you want to achieve.
|
|
65
|
+
4. Modify that configuration file to achieve what you want to achieve.
|
|
66
|
+
5. Use the "transform" sub-command to read your data and output it transformed or reorganized according to your modified configuration file.
|
|
67
|
+
6. Read the produced output. If necessary go back to step 4 and adjust how the data is transformed.
|
|
68
|
+
|
|
69
|
+
### Description of configuration file
|
|
70
|
+
|
|
71
|
+
When using the "example" sub-command to generate an example configuration file (say example.cfg) a text file describing the configuration and the syntax of the configuration file is also generated. If the example configuration file is named example.cfg, then the text file descriging the configuration is named example.txt.
|
|
72
|
+
|
|
73
|
+
You can generate several example configuration files each with an accompanying text file descriping it.
|
|
74
|
+
|
|
75
|
+
Read the text file describing the configuration file while looking at the configuration file to understand the syntax and the possible options.
|
|
76
|
+
|
|
77
|
+
## Performance
|
|
78
|
+
|
|
79
|
+
This application is written for the moderate amounts of data when registering participants for the majority of sports events. If you have millions of rows this application is not for you.
|
|
80
|
+
|
|
81
|
+
With an input file consisting of 20 columns producing an output file of 11 columns I have measured the following performance on a MacBook Air (laptop) from 2020:
|
|
82
|
+
* 1 000 rows processed in less than 0.5 seconds
|
|
83
|
+
* 10 000 rows processed in less than 4 seconds
|
|
84
|
+
* 80 000 rows processed in less than 30 seconds
|
|
85
|
+
* 120 000 rows processed in 40 seconds
|
|
86
|
+
|
|
87
|
+
Naturally your performance will be different based on computer hardware, operating system and Python version. Generally it should be reasonably fast for less than 10 000 rows, painfully slow but somewhat usable up to 100 000 rows and probably so slow that it is unusable for more than 100 000 rows.
|
|
88
|
+
|
|
89
|
+
## Source code
|
|
90
|
+
|
|
91
|
+
Source code and tests are available at [https://bitbucket.org/tom-bjorkholm/excel-list-transform](https://bitbucket.org/tom-bjorkholm/excel-list-transform).
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: excel-list-transform
|
|
3
|
+
Version: 0.5
|
|
4
|
+
Summary: Transform a list in excel or CSV
|
|
5
|
+
Author: Tom Björkholm
|
|
6
|
+
Author-email: Tom Björkholm <klausuler_linnet0q@icloud.com>
|
|
7
|
+
Project-URL: Source code, https://bitbucket.org/tom-bjorkholm/excel-list-transform
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.10.5
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE.txt
|
|
14
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
15
|
+
Requires-Dist: types-openpyxl>=3.1.5.20240822
|
|
16
|
+
Requires-Dist: pylightxl>=1.61
|
|
17
|
+
Requires-Dist: XlsxWriter>=3.2.0
|
|
18
|
+
Requires-Dist: pip>=23.2
|
|
19
|
+
Requires-Dist: Pillow>=10.0.1
|
|
20
|
+
Requires-Dist: setuptools>=74.0.0
|
|
21
|
+
Requires-Dist: build>=1.2.1
|
|
22
|
+
Requires-Dist: wheel>=0.44.0
|
|
23
|
+
|
|
24
|
+
# excel-list-transform
|
|
25
|
+
|
|
26
|
+
## Background
|
|
27
|
+
|
|
28
|
+
This python application was born out of an experience at sail racing events. At the start of the events we received an excel list with participants (from a registration web) to enter into the scoring software and into tool for online notice board. The information was present in the excel file, but the columns were all wrong. To avoid the stressful manual rework of the information in the excel list this application was created.
|
|
29
|
+
|
|
30
|
+
## What it does
|
|
31
|
+
|
|
32
|
+
This small python application:
|
|
33
|
+
* reads data (that is a list with columns) from an excel file or from a comma separate values (CSV) file.
|
|
34
|
+
* splits columns in the list (like creating "first name" and "last name" columns from "name" column)
|
|
35
|
+
* merges columns in the list (like creating "name" column from "first name" and "last name" columns)
|
|
36
|
+
* removes columns in the list
|
|
37
|
+
* reorders columns in the list
|
|
38
|
+
* renames columns in the list
|
|
39
|
+
* inserts columns in the list
|
|
40
|
+
* rewrites columns in the list (like transforming telephone numbers from local/national format to international format)
|
|
41
|
+
* writest the resulting data (that is a list with columns) to an excel file or to a comma separate values (CSV) file.
|
|
42
|
+
|
|
43
|
+
How this is done is governed by a configuration file. The application can create a number of example configuration files with accompanying description text files.
|
|
44
|
+
|
|
45
|
+
## Installing it
|
|
46
|
+
|
|
47
|
+
If you want to use it install it using pip. A precondition is that you have Python 3.10.5 or newer installed on you computer. Python can be downloaded from [https://www.python.org/downloads/](https://www.python.org/downloads/).
|
|
48
|
+
|
|
49
|
+
### Installing on mac and Linux
|
|
50
|
+
|
|
51
|
+
````
|
|
52
|
+
pip3 install excel-list-transform
|
|
53
|
+
````
|
|
54
|
+
|
|
55
|
+
### Installing on Microsoft Windows
|
|
56
|
+
|
|
57
|
+
````
|
|
58
|
+
pip install excel-list-transform
|
|
59
|
+
````
|
|
60
|
+
|
|
61
|
+
## Running the application
|
|
62
|
+
|
|
63
|
+
### Running the application on mac and Linux
|
|
64
|
+
|
|
65
|
+
````
|
|
66
|
+
python3 -m excel_list_transform --help
|
|
67
|
+
python3 -m excel_list_transform example --help
|
|
68
|
+
python3 -m excel_list_transform transform --help
|
|
69
|
+
python3 -m excel_list_transform example -k forms_to_rrs -r by_name -o example.cfg
|
|
70
|
+
python3 -m excel_list_transform transform -c example.cfg -i input.xlsx -o output.xlsx
|
|
71
|
+
````
|
|
72
|
+
|
|
73
|
+
### Running the application on Microsoft Windows
|
|
74
|
+
|
|
75
|
+
````
|
|
76
|
+
python -m excel_list_transform --help
|
|
77
|
+
python -m excel_list_transform example --help
|
|
78
|
+
python -m excel_list_transform transform --help
|
|
79
|
+
python -m excel_list_transform example -k forms_to_rrs -r by_name -o example.cfg
|
|
80
|
+
python -m excel_list_transform transform -c example.cfg -i input.xlsx -o output.xlsx
|
|
81
|
+
````
|
|
82
|
+
|
|
83
|
+
## Suggested way to get started
|
|
84
|
+
|
|
85
|
+
1. Use the "example" sub-command to generate a few example configuration (.cfg) files with description (.txt) files.
|
|
86
|
+
2. Read the example configuration (.cfg) files and the accompanying description (.txt) files.
|
|
87
|
+
3. Find an example that is close to what you want to achieve.
|
|
88
|
+
4. Modify that configuration file to achieve what you want to achieve.
|
|
89
|
+
5. Use the "transform" sub-command to read your data and output it transformed or reorganized according to your modified configuration file.
|
|
90
|
+
6. Read the produced output. If necessary go back to step 4 and adjust how the data is transformed.
|
|
91
|
+
|
|
92
|
+
### Description of configuration file
|
|
93
|
+
|
|
94
|
+
When using the "example" sub-command to generate an example configuration file (say example.cfg) a text file describing the configuration and the syntax of the configuration file is also generated. If the example configuration file is named example.cfg, then the text file descriging the configuration is named example.txt.
|
|
95
|
+
|
|
96
|
+
You can generate several example configuration files each with an accompanying text file descriping it.
|
|
97
|
+
|
|
98
|
+
Read the text file describing the configuration file while looking at the configuration file to understand the syntax and the possible options.
|
|
99
|
+
|
|
100
|
+
## Performance
|
|
101
|
+
|
|
102
|
+
This application is written for the moderate amounts of data when registering participants for the majority of sports events. If you have millions of rows this application is not for you.
|
|
103
|
+
|
|
104
|
+
With an input file consisting of 20 columns producing an output file of 11 columns I have measured the following performance on a MacBook Air (laptop) from 2020:
|
|
105
|
+
* 1 000 rows processed in less than 0.5 seconds
|
|
106
|
+
* 10 000 rows processed in less than 4 seconds
|
|
107
|
+
* 80 000 rows processed in less than 30 seconds
|
|
108
|
+
* 120 000 rows processed in 40 seconds
|
|
109
|
+
|
|
110
|
+
Naturally your performance will be different based on computer hardware, operating system and Python version. Generally it should be reasonably fast for less than 10 000 rows, painfully slow but somewhat usable up to 100 000 rows and probably so slow that it is unusable for more than 100 000 rows.
|
|
111
|
+
|
|
112
|
+
## Source code
|
|
113
|
+
|
|
114
|
+
Source code and tests are available at [https://bitbucket.org/tom-bjorkholm/excel-list-transform](https://bitbucket.org/tom-bjorkholm/excel-list-transform).
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
LICENSE.txt
|
|
2
|
+
README.md
|
|
3
|
+
README_pypi.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
setup.py
|
|
6
|
+
excel_list_transform.egg-info/PKG-INFO
|
|
7
|
+
excel_list_transform.egg-info/SOURCES.txt
|
|
8
|
+
excel_list_transform.egg-info/dependency_links.txt
|
|
9
|
+
excel_list_transform.egg-info/requires.txt
|
|
10
|
+
excel_list_transform.egg-info/top_level.txt
|
|
11
|
+
src/excel_list_transform/__init__.py
|
|
12
|
+
src/excel_list_transform/__main__.py
|
|
13
|
+
src/excel_list_transform/check_indata_common.py
|
|
14
|
+
src/excel_list_transform/commontypes.py
|
|
15
|
+
src/excel_list_transform/config.py
|
|
16
|
+
src/excel_list_transform/config_enums.py
|
|
17
|
+
src/excel_list_transform/config_excel_list_transform.py
|
|
18
|
+
src/excel_list_transform/config_factory.py
|
|
19
|
+
src/excel_list_transform/config_xls_list_refmt_name.py
|
|
20
|
+
src/excel_list_transform/config_xls_list_refmt_num.py
|
|
21
|
+
src/excel_list_transform/file_extension.py
|
|
22
|
+
src/excel_list_transform/file_must_exist.py
|
|
23
|
+
src/excel_list_transform/generate_cfg.py
|
|
24
|
+
src/excel_list_transform/generate_txt.py
|
|
25
|
+
src/excel_list_transform/handle_csv.py
|
|
26
|
+
src/excel_list_transform/handle_empty_column.py
|
|
27
|
+
src/excel_list_transform/handle_excel.py
|
|
28
|
+
src/excel_list_transform/num_named_conversion.py
|
|
29
|
+
src/excel_list_transform/rewrite_value.py
|
|
30
|
+
src/excel_list_transform/str_to_enum.py
|
|
31
|
+
src/excel_list_transform/transform_cmd.py
|
|
32
|
+
src/excel_list_transform/transform_func.py
|
|
33
|
+
src/excel_list_transform/transform_func_common.py
|
|
34
|
+
src/excel_list_transform/transform_func_named.py
|
|
35
|
+
src/excel_list_transform/transform_func_num.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
excel_list_transform
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "excel-list-transform"
|
|
7
|
+
version = "0.5"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="Tom Björkholm", email="klausuler_linnet0q@icloud.com" },
|
|
10
|
+
]
|
|
11
|
+
description = "Transform a list in excel or CSV"
|
|
12
|
+
readme = "README_pypi.md"
|
|
13
|
+
requires-python = ">=3.10.5"
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
]
|
|
19
|
+
dynamic = ["dependencies"]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
"Source code" = "https://bitbucket.org/tom-bjorkholm/excel-list-transform"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Setup file specifying build of apo_tools.whl with APO tools library."""
|
|
3
|
+
|
|
4
|
+
from setuptools import setup
|
|
5
|
+
|
|
6
|
+
setup(
|
|
7
|
+
name='excel-list-transform',
|
|
8
|
+
version='0.5',
|
|
9
|
+
description='Transform a list in excel or CSV.',
|
|
10
|
+
author='Tom Björkholm',
|
|
11
|
+
author_email='klausuler_linnet0q@icloud.com',
|
|
12
|
+
python_requires='>=3.10.5',
|
|
13
|
+
packages=['excel_list_transform'],
|
|
14
|
+
package_dir={'excel_list_transform': 'src/excel_list_transform'},
|
|
15
|
+
install_requires=[
|
|
16
|
+
'openpyxl >= 3.1.5',
|
|
17
|
+
'types-openpyxl >= 3.1.5.20240822',
|
|
18
|
+
'pylightxl >= 1.61',
|
|
19
|
+
'XlsxWriter >= 3.2.0',
|
|
20
|
+
'pip >= 23.2',
|
|
21
|
+
'Pillow >= 10.0.1',
|
|
22
|
+
'setuptools >= 74.0.0',
|
|
23
|
+
'build >= 1.2.1',
|
|
24
|
+
'wheel>=0.44.0'
|
|
25
|
+
]
|
|
26
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Take an excel file and and change columns before saving as excel or CSV."""
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Take an excel file and and change columns before saving as excel or CSV."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from excel_list_transform.transform_cmd import transform_cmd # pragma: no cover # noqa: E501
|
|
9
|
+
|
|
10
|
+
if __name__ == '__main__': # pragma: no cover
|
|
11
|
+
transform_cmd()
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Function for common indata checks.."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
from typing import Callable, Optional
|
|
10
|
+
from excel_list_transform.commontypes import Data
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def check_indata_common(indata: Data,
|
|
14
|
+
fix_indata_empty_rows: Callable[[Data], None]) \
|
|
15
|
+
-> None:
|
|
16
|
+
"""Check that the indata is well formed common part."""
|
|
17
|
+
assert isinstance(indata, list), 'Internal error. ' + \
|
|
18
|
+
f'Expected list of rows but got {type(indata).__name__}'
|
|
19
|
+
if len(indata) < 1:
|
|
20
|
+
msg = 'No rows in input data'
|
|
21
|
+
print(msg, file=sys.stderr)
|
|
22
|
+
sys.exit(1)
|
|
23
|
+
fix_indata_empty_rows(indata)
|
|
24
|
+
if len(indata) < 1:
|
|
25
|
+
msg = 'No columns in input data'
|
|
26
|
+
print(msg, file=sys.stderr)
|
|
27
|
+
sys.exit(1)
|
|
28
|
+
num_cols: Optional[int] = None
|
|
29
|
+
for row in indata:
|
|
30
|
+
rowlen: int = len(row)
|
|
31
|
+
if num_cols is None:
|
|
32
|
+
num_cols = rowlen
|
|
33
|
+
else:
|
|
34
|
+
assert num_cols is not None
|
|
35
|
+
if num_cols != rowlen:
|
|
36
|
+
msg = 'Number of columns different between lines. '
|
|
37
|
+
msg += f'Found {num_cols} and {rowlen}. Aborting.'
|
|
38
|
+
print(msg, file=sys.stderr)
|
|
39
|
+
sys.exit(1)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Common type definitions used for type hints."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from typing import TypeAlias, Optional, cast, Sequence, Mapping, TypeVar, \
|
|
9
|
+
Any
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
# imports needed by mypy, but not by python:
|
|
12
|
+
from typing import Union, List, Dict # pylint: disable=unused-import,ungrouped-imports # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# types used to describe input and output data
|
|
16
|
+
Value: TypeAlias = Optional[str | int | float | datetime]
|
|
17
|
+
NumRow: TypeAlias = list[Value]
|
|
18
|
+
NumRowSeq: TypeAlias = Sequence[Value]
|
|
19
|
+
NameRow: TypeAlias = dict[str, Value]
|
|
20
|
+
NameRowMap: TypeAlias = Mapping[str, Value]
|
|
21
|
+
NumData: TypeAlias = list[NumRow]
|
|
22
|
+
NumDataSeq: TypeAlias = list[NumRowSeq]
|
|
23
|
+
NameData: TypeAlias = list[NameRow]
|
|
24
|
+
NameDataMap: TypeAlias = Sequence[NameRowMap]
|
|
25
|
+
Data = TypeVar('Data', NumData, NameData)
|
|
26
|
+
DataCov = TypeVar('DataCov', NumDataSeq, NameDataMap)
|
|
27
|
+
|
|
28
|
+
JsonType: TypeAlias = \
|
|
29
|
+
'Union[None, int, str, bool, List[JsonType], Dict[str, JsonType]]'
|
|
30
|
+
|
|
31
|
+
# helper functions
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def num_row_to_str_list(row: NumRowSeq) -> list[str]:
|
|
35
|
+
"""Convert NumRow to list of str."""
|
|
36
|
+
ret: list[str] = []
|
|
37
|
+
for i in row:
|
|
38
|
+
if isinstance(i, str):
|
|
39
|
+
ret.append(i)
|
|
40
|
+
elif i is None:
|
|
41
|
+
raise TypeError('Found None when expecting str.')
|
|
42
|
+
else:
|
|
43
|
+
ret.append(str(i))
|
|
44
|
+
return ret
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def str_list_to_num_row(row: list[str]) -> NumRow:
|
|
48
|
+
"""Convert list of str to NumRow."""
|
|
49
|
+
return cast(NumRow, row)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
T = TypeVar('T')
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def get_checked_type(value: Optional[Any], istype: type[T]) -> T:
|
|
56
|
+
"""Get value narrowed to be of type istype."""
|
|
57
|
+
assert value is not None
|
|
58
|
+
assert isinstance(value, istype)
|
|
59
|
+
return value
|