rewe-ebon-parser 0.0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,162 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # poetry
98
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102
+ #poetry.lock
103
+
104
+ # pdm
105
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106
+ #pdm.lock
107
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108
+ # in version control.
109
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
110
+ .pdm.toml
111
+ .pdm-python
112
+ .pdm-build/
113
+
114
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
115
+ __pypackages__/
116
+
117
+ # Celery stuff
118
+ celerybeat-schedule
119
+ celerybeat.pid
120
+
121
+ # SageMath parsed files
122
+ *.sage.py
123
+
124
+ # Environments
125
+ .env
126
+ .venv
127
+ env/
128
+ venv/
129
+ ENV/
130
+ env.bak/
131
+ venv.bak/
132
+
133
+ # Spyder project settings
134
+ .spyderproject
135
+ .spyproject
136
+
137
+ # Rope project settings
138
+ .ropeproject
139
+
140
+ # mkdocs documentation
141
+ /site
142
+
143
+ # mypy
144
+ .mypy_cache/
145
+ .dmypy.json
146
+ dmypy.json
147
+
148
+ # Pyre type checker
149
+ .pyre/
150
+
151
+ # pytype static type analyzer
152
+ .pytype/
153
+
154
+ # Cython debug symbols
155
+ cython_debug/
156
+
157
+ # PyCharm
158
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
159
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
160
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
161
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
162
+ #.idea/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2024 Egor Kotov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,134 @@
1
+ Metadata-Version: 2.3
2
+ Name: rewe-ebon-parser
3
+ Version: 0.0.5
4
+ Summary: Parse and convert REWE eBons (digital receipts) to JSON.
5
+ Project-URL: Homepage, https://github.com/e-kotov/rewe-ebon-parser
6
+ Project-URL: Issues, https://github.com/e-kotov/rewe-ebon-parser/issues
7
+ Author-email: Egor Kotov <kotov.egor@gmail.com>
8
+ License-File: LICENSE
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Programming Language :: Python :: 3
12
+ Requires-Python: >=3.6
13
+ Requires-Dist: pdfplumber
14
+ Requires-Dist: pytz
15
+ Requires-Dist: tqdm
16
+ Provides-Extra: dev
17
+ Requires-Dist: pytest; extra == 'dev'
18
+ Requires-Dist: pytest-asyncio; extra == 'dev'
19
+ Description-Content-Type: text/markdown
20
+
21
+
22
+ # REWE eBon Parser
23
+
24
+ The REWE eBon Parser is a Python package designed to parse REWE eBons (receipts) from PDF files and convert them into structured JSON format. The package also provides functionality to output raw text extracted from the PDFs for debugging purposes. This project is a re-write of the the [`rewe-ebon-parser`](https://github.com/webD97/rewe-ebon-parser) TypeScript library, example PDFs are borrowed from the same library.
25
+
26
+ ## Features
27
+
28
+ - Parse individual PDF files or entire folders containing PDF files.
29
+ - Output parsed data as JSON.
30
+ - Extract and output raw text from PDF files (bascially, the output of the underlying `pdfplumber`).
31
+ - Concurrent processing of multiple PDF files with adjustable threading.
32
+ - Detailed logging of processing results in CSV format.
33
+
34
+ ## Installation
35
+
36
+ You can install the package using pip:
37
+
38
+ ```bash
39
+ pip install rewe-ebon-parser
40
+ ```
41
+
42
+ ## Usage
43
+
44
+ *You can find PDF receipt files to test on in the `examples/eBons` folder in this repo borrowed from [`rewe-ebon-parser`](https://github.com/webD97/rewe-ebon-parser).*
45
+
46
+ ### Command Line Interface (CLI)
47
+
48
+ #### Parse a Single PDF File and save to JSON
49
+
50
+ ```bash
51
+ rewe-ebon-parser [--file] <input_pdf_path> [output_json_path]
52
+ ```
53
+
54
+ Example:
55
+
56
+ ```bash
57
+ rewe-ebon-parser examples/eBons/1.pdf
58
+ ```
59
+
60
+ #### Parsing Multiple PDF Files in a Folder
61
+
62
+ ```bash
63
+ rewe-ebon-parser [--folder] <input_folder> [output_folder] [--nthreads <number_of_threads>]
64
+ ```
65
+
66
+ Example:
67
+
68
+ ```bash
69
+ rewe-ebon-parser examples/eBons/
70
+ ```
71
+
72
+ #### Optional Arguments
73
+
74
+ - `--file`: Explicitly specify if the input and output paths are files.
75
+ - `--folder`: Explicitly specify if the input and output paths are folders.
76
+ - `--nthreads`: Number of concurrent threads to use for processing files.
77
+ - `--rawtext-file`: Output raw text extracted from the PDF files to .txt files (mostly for debugging).
78
+ - `--rawtext-stdout`: Print raw text extracted from the PDF files to the console (mostly for debugging).
79
+ - `--version`: show module version.
80
+ - `-h`, `--help`: show help.
81
+
82
+ #### Auto-detection Mode
83
+
84
+ If neither `--file` nor `--folder` is specified, the script will automatically detect if the input path is a file or a folder and process accordingly.
85
+
86
+ #### Output
87
+
88
+ - If `output_json_path` is not specified for a single file, the output will be saved in the same directory as the input file with a `.json` extension.
89
+ - If `output_folder` is not specified for a folder, a subfolder named `rewe_json_out` will be created in the input folder, and the output JSON files will be saved there.
90
+
91
+ #### Logging
92
+
93
+ A detailed log of processing results will be saved in the output folder as `processing_log.csv`, containing information on which files were successfully processed and which failed, along with error messages if any.
94
+
95
+
96
+ ### Use as a Python module in your own Python code
97
+
98
+ #### Direct use on files
99
+
100
+ ```python
101
+ from rewe_ebon_parser.parse import parse_pdf_ebon
102
+
103
+ parse_pdf_ebon("examples/eBons/1.pdf")
104
+ ```
105
+
106
+ #### Passing a data_buffer: bytes
107
+
108
+ ```python
109
+ from rewe_ebon_parser.parse import parse_ebon
110
+
111
+ # here the function is once again getting the data from a file,
112
+ # but input can come from anywhere
113
+ def process_pdf(pdf_path):
114
+ with open(pdf_path, 'rb') as f:
115
+ data = f.read()
116
+ result = parse_ebon(data)
117
+ return result
118
+
119
+ process_pdf("examples/eBons/1.pdf")
120
+ ```
121
+
122
+
123
+ ## License
124
+
125
+ This project is licensed under the MIT License. For details see [LICENSE](LICENSE) file.
126
+
127
+
128
+ ## Caveats
129
+
130
+ So far the module reliably parses the items, but sometimes fails on PAYBACK points, as these are often presented differently in REWE receipts.
131
+
132
+ ## Future Work
133
+
134
+ - Dump all shopping items into a single CSV file with purchase dates.
@@ -0,0 +1,114 @@
1
+
2
+ # REWE eBon Parser
3
+
4
+ The REWE eBon Parser is a Python package designed to parse REWE eBons (receipts) from PDF files and convert them into structured JSON format. The package also provides functionality to output raw text extracted from the PDFs for debugging purposes. This project is a re-write of the the [`rewe-ebon-parser`](https://github.com/webD97/rewe-ebon-parser) TypeScript library, example PDFs are borrowed from the same library.
5
+
6
+ ## Features
7
+
8
+ - Parse individual PDF files or entire folders containing PDF files.
9
+ - Output parsed data as JSON.
10
+ - Extract and output raw text from PDF files (bascially, the output of the underlying `pdfplumber`).
11
+ - Concurrent processing of multiple PDF files with adjustable threading.
12
+ - Detailed logging of processing results in CSV format.
13
+
14
+ ## Installation
15
+
16
+ You can install the package using pip:
17
+
18
+ ```bash
19
+ pip install rewe-ebon-parser
20
+ ```
21
+
22
+ ## Usage
23
+
24
+ *You can find PDF receipt files to test on in the `examples/eBons` folder in this repo borrowed from [`rewe-ebon-parser`](https://github.com/webD97/rewe-ebon-parser).*
25
+
26
+ ### Command Line Interface (CLI)
27
+
28
+ #### Parse a Single PDF File and save to JSON
29
+
30
+ ```bash
31
+ rewe-ebon-parser [--file] <input_pdf_path> [output_json_path]
32
+ ```
33
+
34
+ Example:
35
+
36
+ ```bash
37
+ rewe-ebon-parser examples/eBons/1.pdf
38
+ ```
39
+
40
+ #### Parsing Multiple PDF Files in a Folder
41
+
42
+ ```bash
43
+ rewe-ebon-parser [--folder] <input_folder> [output_folder] [--nthreads <number_of_threads>]
44
+ ```
45
+
46
+ Example:
47
+
48
+ ```bash
49
+ rewe-ebon-parser examples/eBons/
50
+ ```
51
+
52
+ #### Optional Arguments
53
+
54
+ - `--file`: Explicitly specify if the input and output paths are files.
55
+ - `--folder`: Explicitly specify if the input and output paths are folders.
56
+ - `--nthreads`: Number of concurrent threads to use for processing files.
57
+ - `--rawtext-file`: Output raw text extracted from the PDF files to .txt files (mostly for debugging).
58
+ - `--rawtext-stdout`: Print raw text extracted from the PDF files to the console (mostly for debugging).
59
+ - `--version`: show module version.
60
+ - `-h`, `--help`: show help.
61
+
62
+ #### Auto-detection Mode
63
+
64
+ If neither `--file` nor `--folder` is specified, the script will automatically detect if the input path is a file or a folder and process accordingly.
65
+
66
+ #### Output
67
+
68
+ - If `output_json_path` is not specified for a single file, the output will be saved in the same directory as the input file with a `.json` extension.
69
+ - If `output_folder` is not specified for a folder, a subfolder named `rewe_json_out` will be created in the input folder, and the output JSON files will be saved there.
70
+
71
+ #### Logging
72
+
73
+ A detailed log of processing results will be saved in the output folder as `processing_log.csv`, containing information on which files were successfully processed and which failed, along with error messages if any.
74
+
75
+
76
+ ### Use as a Python module in your own Python code
77
+
78
+ #### Direct use on files
79
+
80
+ ```python
81
+ from rewe_ebon_parser.parse import parse_pdf_ebon
82
+
83
+ parse_pdf_ebon("examples/eBons/1.pdf")
84
+ ```
85
+
86
+ #### Passing a data_buffer: bytes
87
+
88
+ ```python
89
+ from rewe_ebon_parser.parse import parse_ebon
90
+
91
+ # here the function is once again getting the data from a file,
92
+ # but input can come from anywhere
93
+ def process_pdf(pdf_path):
94
+ with open(pdf_path, 'rb') as f:
95
+ data = f.read()
96
+ result = parse_ebon(data)
97
+ return result
98
+
99
+ process_pdf("examples/eBons/1.pdf")
100
+ ```
101
+
102
+
103
+ ## License
104
+
105
+ This project is licensed under the MIT License. For details see [LICENSE](LICENSE) file.
106
+
107
+
108
+ ## Caveats
109
+
110
+ So far the module reliably parses the items, but sometimes fails on PAYBACK points, as these are often presented differently in REWE receipts.
111
+
112
+ ## Future Work
113
+
114
+ - Dump all shopping items into a single CSV file with purchase dates.
@@ -0,0 +1,51 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "rewe-ebon-parser"
7
+ version = "0.0.5"
8
+ authors = [
9
+ { name="Egor Kotov", email="kotov.egor@gmail.com" },
10
+ ]
11
+ description = "Parse and convert REWE eBons (digital receipts) to JSON."
12
+ readme = "README.md"
13
+ requires-python = ">=3.6"
14
+ classifiers = [
15
+ "Programming Language :: Python :: 3",
16
+ "License :: OSI Approved :: MIT License",
17
+ "Operating System :: OS Independent",
18
+ ]
19
+
20
+ dependencies = [
21
+ "pdfplumber",
22
+ "pytz",
23
+ "tqdm"
24
+ ]
25
+
26
+ [project.optional-dependencies]
27
+ dev = [
28
+ "pytest",
29
+ "pytest-asyncio"
30
+ ]
31
+
32
+ [project.scripts]
33
+ rewe-ebon-parser = "rewe_ebon_parser.cli:main"
34
+
35
+ [project.urls]
36
+ Homepage = "https://github.com/e-kotov/rewe-ebon-parser"
37
+ Issues = "https://github.com/e-kotov/rewe-ebon-parser/issues"
38
+
39
+ [tool.hatch.build]
40
+ exclude = [
41
+ "**/.*",
42
+ "docs",
43
+ "tests",
44
+ "examples"
45
+ ]
46
+
47
+ [tool.hatch.build.targets.wheel]
48
+ only-include = ["src/rewe_ebon_parser"]
49
+
50
+ [tool.hatch.build.targets.wheel.sources]
51
+ "src" = ""
@@ -0,0 +1,5 @@
1
+ # src/rewe_ebon_parser/__init__.py
2
+
3
+ __version__ = "0.0.5"
4
+
5
+ from .parse import parse_ebon, parse_pdf_ebon
@@ -0,0 +1,177 @@
1
+ # classes.py
2
+ import math
3
+ from datetime import datetime
4
+ from typing import List, Optional
5
+ import pytz
6
+
7
+ class TaxCategory:
8
+ A = 'A'
9
+ B = 'B'
10
+
11
+ class MarketAddress:
12
+ def __init__(self, street: str, zip: str, city: str):
13
+ self.street = street
14
+ self.zip = zip
15
+ self.city = city
16
+
17
+ def to_dict(self):
18
+ return {
19
+ 'street': self.street,
20
+ 'zip': self.zip,
21
+ 'city': self.city
22
+ }
23
+
24
+ class ReceiptItem:
25
+ def __init__(self, tax_category: str, name: str, sub_total: float, payback_qualified: bool, amount: float, unit: Optional[str] = None, price_per_unit: Optional[float] = None):
26
+ self.tax_category = tax_category
27
+ self.name = name
28
+ self.sub_total = sub_total
29
+ self.payback_qualified = payback_qualified
30
+ self.amount = amount
31
+ self.unit = unit
32
+ self.price_per_unit = price_per_unit
33
+
34
+ def to_dict(self):
35
+ data = {
36
+ 'taxCategory': self.tax_category,
37
+ 'name': self.name,
38
+ 'amount': self.amount,
39
+ 'subTotal': self.sub_total,
40
+ 'paybackQualified': self.payback_qualified
41
+ }
42
+ if self.unit is not None:
43
+ data['unit'] = self.unit
44
+ if self.price_per_unit is not None:
45
+ data['pricePerUnit'] = self.price_per_unit
46
+ return data
47
+
48
+ class Payment:
49
+ def __init__(self, type: str, value: float):
50
+ self.type = type
51
+ self.value = value
52
+
53
+ def to_dict(self):
54
+ return {
55
+ 'type': self.type,
56
+ 'value': self.value
57
+ }
58
+
59
+ class TaxDetailsEntry:
60
+ def __init__(self, tax_percent: float, net: float, tax: float, gross: float):
61
+ self.tax_percent = tax_percent
62
+ self.net = net
63
+ self.tax = tax
64
+ self.gross = gross
65
+
66
+ def to_dict(self):
67
+ data = {
68
+ 'net': self.net,
69
+ 'tax': self.tax,
70
+ 'gross': self.gross
71
+ }
72
+ if not math.isnan(self.tax_percent):
73
+ data['taxPercent'] = self.tax_percent
74
+ return data
75
+
76
+ class TaxDetails:
77
+ def __init__(self, total: TaxDetailsEntry, A: Optional[TaxDetailsEntry] = None, B: Optional[TaxDetailsEntry] = None):
78
+ self.total = total
79
+ self.A = A
80
+ self.B = B
81
+
82
+ def to_dict(self):
83
+ data = {
84
+ 'total': self.total.to_dict()
85
+ }
86
+ if self.A is not None:
87
+ data['A'] = self.A.to_dict()
88
+ if self.B is not None:
89
+ data['B'] = self.B.to_dict()
90
+ return data
91
+
92
+ class PaybackCoupon:
93
+ def __init__(self, name: str, points: int):
94
+ self.name = name
95
+ self.points = points
96
+
97
+ def to_dict(self):
98
+ return {
99
+ 'name': self.name,
100
+ 'points': self.points
101
+ }
102
+
103
+ class PaybackData:
104
+ def __init__(self, card: str, points_before: float, earned_points: int, used_coupons: List[PaybackCoupon], used_rewe_credit: Optional[float], new_rewe_credit: Optional[float], payback_revenue: float):
105
+ self.card = card
106
+ self.points_before = points_before
107
+ self.earned_points = earned_points
108
+ self.used_coupons = used_coupons
109
+ self.used_rewe_credit = used_rewe_credit
110
+ self.new_rewe_credit = new_rewe_credit
111
+ self.payback_revenue = payback_revenue
112
+
113
+ @property
114
+ def base_points(self):
115
+ return self.earned_points - self.coupon_points
116
+
117
+ @property
118
+ def coupon_points(self):
119
+ return sum(coupon.points for coupon in self.used_coupons)
120
+
121
+ @property
122
+ def qualified_revenue(self):
123
+ return self.payback_revenue
124
+
125
+ def to_dict(self):
126
+ data = {
127
+ 'card': self.card,
128
+ 'pointsBefore': self.points_before,
129
+ 'earnedPoints': self.earned_points,
130
+ 'basePoints': self.base_points,
131
+ 'couponPoints': self.coupon_points,
132
+ 'qualifiedRevenue': self.qualified_revenue,
133
+ 'usedCoupons': [coupon.to_dict() for coupon in self.used_coupons]
134
+ }
135
+ if self.used_rewe_credit is not None:
136
+ data['usedREWECredit'] = self.used_rewe_credit
137
+ if self.new_rewe_credit is not None:
138
+ data['newREWECredit'] = self.new_rewe_credit
139
+ return data
140
+
141
+ class Receipt:
142
+ def __init__(self, date: datetime, market: str, market_address: Optional[MarketAddress], cashier: str, checkout: str, vatin: str, items: List[ReceiptItem], total: float, given: List[Payment], change: Optional[float], payout: Optional[float], payback: Optional[PaybackData], tax_details: TaxDetails):
143
+ self.date = date
144
+ self.market = market
145
+ self.market_address = market_address
146
+ self.cashier = cashier
147
+ self.checkout = checkout
148
+ self.vatin = vatin
149
+ self.items = items
150
+ self.total = total
151
+ self.given = given
152
+ self.change = change
153
+ self.payout = payout
154
+ self.payback = payback
155
+ self.tax_details = tax_details
156
+
157
+ def to_dict(self):
158
+ data = {
159
+ 'datetime_local': self.date.isoformat(timespec='seconds'),
160
+ 'datetime_utc': self.date.astimezone(pytz.utc).isoformat(),
161
+ 'market': self.market,
162
+ 'marketAddress': self.market_address.to_dict() if self.market_address else None,
163
+ 'cashier': self.cashier,
164
+ 'checkout': self.checkout,
165
+ 'vatin': self.vatin,
166
+ 'items': [item.to_dict() for item in self.items],
167
+ 'total': self.total,
168
+ 'given': [payment.to_dict() for payment in self.given],
169
+ 'taxDetails': self.tax_details.to_dict()
170
+ }
171
+ if self.change is not None:
172
+ data['change'] = self.change
173
+ if self.payback is not None:
174
+ data['payback'] = self.payback.to_dict()
175
+ if self.payout is not None:
176
+ data['payout'] = self.payout
177
+ return data
@@ -0,0 +1,75 @@
1
+ # src/rewe_ebon_parser/cli.py
2
+ import sys
3
+ from pathlib import Path
4
+ import argparse
5
+ from .output import process_pdf, process_folder
6
+ from . import __version__
7
+
8
+ def main():
9
+ parser = argparse.ArgumentParser(description="Parse REWE eBons from PDF to JSON.")
10
+ parser.add_argument("input_path", type=str, nargs='?', help="Path to input PDF file or folder containing PDF files.")
11
+ parser.add_argument("output_path", type=str, nargs='?', default=None, help="Path to output JSON file or folder for JSON files.")
12
+ parser.add_argument("--file", action="store_true", help="Specify if the input and output paths are files.")
13
+ parser.add_argument("--folder", action="store_true", help="Specify if the input and output paths are folders.")
14
+ parser.add_argument("--nthreads", type=int, default=None, help="Number of concurrent threads to use for processing files. Defaults to maximum available cpu cores.")
15
+ parser.add_argument("--rawtext-file", action="store_true", help="Output raw text extracted from the PDF files to .txt files.")
16
+ parser.add_argument("--rawtext-stdout", action="store_true", help="Print raw text extracted from the PDF files to the console.")
17
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}", help="Show the version number and exit.")
18
+
19
+ args = parser.parse_args()
20
+
21
+ # Print help if no arguments are provided
22
+ if len(sys.argv) == 1:
23
+ parser.print_help(sys.stderr)
24
+ sys.exit(1)
25
+
26
+ input_path = Path(args.input_path) if args.input_path else None
27
+ output_path = Path(args.output_path) if args.output_path else None
28
+ max_workers = args.nthreads
29
+ rawtext_file = args.rawtext_file
30
+ rawtext_stdout = args.rawtext_stdout
31
+
32
+ if args.file:
33
+ if not output_path:
34
+ output_path = input_path.with_suffix('.json')
35
+ if input_path.is_file() and (output_path.is_file() or not output_path.exists()):
36
+ process_pdf(input_path, output_path, rawtext_file, rawtext_stdout)
37
+ else:
38
+ print("Error: Input and output paths must be files when using --file.")
39
+ sys.exit(1)
40
+ elif args.folder:
41
+ if not output_path:
42
+ output_path = input_path / 'rewe_json_out'
43
+ if input_path.is_dir() and (output_path.is_dir() or not output_path.exists()):
44
+ process_folder(input_path, output_path, max_workers, rawtext_file, rawtext_stdout)
45
+ else:
46
+ print("Error: Input and output paths must be directories when using --folder.")
47
+ sys.exit(1)
48
+ else:
49
+ # Auto-detection mode
50
+ if input_path:
51
+ if input_path.is_dir():
52
+ if not output_path:
53
+ output_path = input_path / 'rewe_json_out'
54
+ if output_path.is_dir() or not output_path.exists():
55
+ process_folder(input_path, output_path, max_workers, rawtext_file, rawtext_stdout)
56
+ else:
57
+ print("Error: Output path should be a directory when the input path is a directory.")
58
+ sys.exit(1)
59
+ elif input_path.is_file():
60
+ if not output_path:
61
+ output_path = input_path.with_suffix('.json')
62
+ if output_path.is_file() or not output_path.exists():
63
+ process_pdf(input_path, output_path, rawtext_file, rawtext_stdout)
64
+ else:
65
+ print("Error: Output path should be a file when the input path is a file.")
66
+ sys.exit(1)
67
+ else:
68
+ print("Error: Invalid input or output path.")
69
+ sys.exit(1)
70
+ else:
71
+ print("Error: No input path provided.")
72
+ sys.exit(1)
73
+
74
+ if __name__ == '__main__':
75
+ main()
@@ -0,0 +1,83 @@
1
+ # output.py
2
+ import json
3
+ import time
4
+ import csv
5
+ from pathlib import Path
6
+ from concurrent.futures import ProcessPoolExecutor, as_completed
7
+ from tqdm import tqdm
8
+ from .parse import extract_raw_text, parse_ebon
9
+
10
+ def process_pdf(pdf_path, output_path=None, rawtext_file=False, rawtext_stdout=False):
11
+ output_path = Path(output_path) if output_path else None
12
+ if output_path:
13
+ output_path.parent.mkdir(parents=True, exist_ok=True) # Ensure the output directory exists
14
+
15
+ try:
16
+ with open(pdf_path, 'rb') as f:
17
+ data = f.read()
18
+
19
+ if rawtext_file or rawtext_stdout:
20
+ raw_text = extract_raw_text(data)
21
+ if rawtext_file:
22
+ rawtext_path = output_path.with_suffix('.txt') if output_path else pdf_path.with_suffix('.txt')
23
+ rawtext_path.write_text(raw_text, encoding='utf-8')
24
+ if rawtext_stdout:
25
+ print(raw_text)
26
+ return
27
+
28
+ result = parse_ebon(data)
29
+
30
+ if result and output_path:
31
+ with open(output_path, 'w', encoding='utf-8') as json_file:
32
+ json.dump(result, json_file, default=str, indent=2, ensure_ascii=False)
33
+ except Exception as e:
34
+ print(f"Failed to process {pdf_path}: {e}")
35
+ raise
36
+
37
+ def process_folder(input_folder, output_folder=None, max_workers=None, rawtext_file=False, rawtext_stdout=False):
38
+ if output_folder:
39
+ output_folder.mkdir(parents=True, exist_ok=True)
40
+
41
+ start_time = time.time()
42
+ success_count = 0
43
+ failure_count = 0
44
+ log_entries = []
45
+
46
+ pdf_files = list(input_folder.glob("*.pdf"))
47
+ total_files = len(pdf_files)
48
+
49
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
50
+ futures = {}
51
+ for pdf_file in pdf_files:
52
+ output_file = (output_folder / (pdf_file.stem + ".json")) if output_folder else None
53
+ future = executor.submit(process_pdf, pdf_file, output_file, rawtext_file, rawtext_stdout)
54
+ futures[future] = pdf_file
55
+
56
+ with tqdm(total=total_files, desc="Processing PDFs", unit="file") as pbar:
57
+ for future in as_completed(futures):
58
+ pdf_file = futures[future]
59
+ try:
60
+ future.result()
61
+ log_entries.append((pdf_file.name, "Success", ""))
62
+ success_count += 1
63
+ except Exception as exc:
64
+ log_entries.append((pdf_file.name, "Failure", str(exc)))
65
+ failure_count += 1
66
+ pbar.update(1)
67
+
68
+ end_time = time.time()
69
+ elapsed_time = end_time - start_time
70
+
71
+ # Print summary
72
+ print(f"Processed {success_count + failure_count} files in {elapsed_time:.2f} seconds.")
73
+ print(f"Successfully processed: {success_count}")
74
+ print(f"Failed to process: {failure_count}")
75
+
76
+ # Save log to CSV
77
+ if output_folder:
78
+ log_file_path = output_folder / 'processing_log.csv'
79
+ with open(log_file_path, 'w', newline='', encoding='utf-8') as csvfile:
80
+ log_writer = csv.writer(csvfile)
81
+ log_writer.writerow(["File Name", "Status", "Error Message"])
82
+ log_writer.writerows(log_entries)
83
+
@@ -0,0 +1,272 @@
1
+ # parse.py
2
+ import re
3
+ from datetime import datetime
4
+ from typing import List, Optional
5
+ import pdfplumber
6
+ import io
7
+ import math
8
+ import pytz
9
+ from collections import OrderedDict
10
+ from .classes import *
11
+
12
+ def extract_raw_text(data_buffer: bytes) -> str:
13
+ raw_text = ''
14
+ with pdfplumber.open(io.BytesIO(data_buffer)) as pdf:
15
+ for page in pdf.pages:
16
+ raw_text += page.extract_text()
17
+ return raw_text
18
+
19
+ def parse_ebon(data_buffer: bytes) -> dict:
20
+ data_text = extract_raw_text(data_buffer)
21
+
22
+ lines = list(filter(None, map(str.strip, data_text.replace(' ', ' ').split('\n'))))
23
+
24
+ date = None
25
+ market = '?'
26
+ market_address = None
27
+ cashier = '?'
28
+ checkout = '?'
29
+ uid = '?'
30
+ items = []
31
+ total = float('nan')
32
+ given = []
33
+ change = float('nan')
34
+ payout = float('nan')
35
+ payback_points_before = float('nan')
36
+ payback_points = float('nan')
37
+ payback_revenue = float('nan')
38
+ payback_card_number = '?'
39
+ payback_coupons = []
40
+ used_rewe_credit = float('nan')
41
+ new_rewe_credit = float('nan')
42
+ tax_details_total = TaxDetailsEntry(float('nan'), float('nan'), float('nan'), float('nan'))
43
+ tax_details_A = None
44
+ tax_details_B = None
45
+
46
+ # Improved address extraction logic
47
+ address_match = re.search(r'[\s\*]*([a-zäöüß \d.,-]+?)\s*[\s\*]*(\d{5})\s*([a-zäöüß \d.,-]+)[\s\*]*', data_text, re.IGNORECASE)
48
+ if address_match:
49
+ market_address = MarketAddress(
50
+ street=address_match.group(1).replace(' ', ' ').replace(',', '').strip(),
51
+ zip=address_match.group(2),
52
+ city=address_match.group(3).strip()
53
+ )
54
+
55
+ for line in lines:
56
+ item_hit = re.match(r'([0-9A-Za-zäöüÄÖÜß &%.!+,\-]*) (-?\d*,\d\d) ([AB]) ?(\*?)', line)
57
+ if item_hit:
58
+ item = item_hit.group(1)
59
+ price = float(item_hit.group(2).replace(',', '.'))
60
+ category = item_hit.group(3)
61
+ payback_qualified = not item_hit.group(4) and price > 0
62
+
63
+ items.append(ReceiptItem(
64
+ tax_category=category,
65
+ name=item,
66
+ sub_total=price,
67
+ payback_qualified=payback_qualified,
68
+ amount=1
69
+ ))
70
+ continue
71
+
72
+ menge_hit = re.match(r'(.*) (.*) x (.*).*', line)
73
+ if menge_hit:
74
+ amount = float(menge_hit.group(1).replace(',', '.'))
75
+ unit = menge_hit.group(2)
76
+ price_per_unit_str = re.search(r'(\d+,\d+)', menge_hit.group(3))
77
+ price_per_unit = float(price_per_unit_str.group(1).replace(',', '.')) if price_per_unit_str else None
78
+ if unit == 'Stk':
79
+ amount = int(amount)
80
+ if items:
81
+ items[-1].amount = amount
82
+ items[-1].unit = unit
83
+ items[-1].price_per_unit = price_per_unit
84
+ continue
85
+
86
+ total_hit = re.match(r'SUMME EUR (-?\d*,\d\d)', line)
87
+ if total_hit:
88
+ total = float(total_hit.group(1).replace(',', '.'))
89
+ continue
90
+
91
+ gegeben_hit = re.match(r'Geg\.(.*) EUR ([0-9,]*)', line)
92
+ if gegeben_hit:
93
+ given.append(Payment(
94
+ type=gegeben_hit.group(1).strip(),
95
+ value=float(gegeben_hit.group(2).replace(',', '.'))
96
+ ))
97
+ continue
98
+
99
+ return_hit = re.match(r'Rückgeld BAR EUR ([0-9,]*)', line)
100
+ if return_hit:
101
+ change = float(return_hit.group(1).replace(',', '.'))
102
+ continue
103
+
104
+ payout_match = re.match(r'AUSZAHLUNG EUR ([0-9,]*)', line)
105
+ if payout_match:
106
+ payout = float(payout_match.group(1).replace(',', '.'))
107
+ continue
108
+
109
+ timestamp_hit = re.match(r'(\d{2})\.(\d{2})\.(\d{4}) (\d{2}):(\d{2}) Bon-Nr\.:.*', line)
110
+ if timestamp_hit:
111
+ date = datetime(
112
+ year=int(timestamp_hit.group(3)),
113
+ month=int(timestamp_hit.group(2)),
114
+ day=int(timestamp_hit.group(1)),
115
+ hour=int(timestamp_hit.group(4)),
116
+ minute=int(timestamp_hit.group(5)),
117
+ second=0
118
+ )
119
+ local_tz = pytz.timezone('Europe/Berlin')
120
+ date = local_tz.localize(date)
121
+ continue
122
+
123
+ markt_match = re.match(r'Markt:(.*) Kasse:(.*) Bed\.:(.*)', line)
124
+ if markt_match:
125
+ market = markt_match.group(1).strip()
126
+ checkout = markt_match.group(2).strip()
127
+ cashier = markt_match.group(3).strip()
128
+ continue
129
+
130
+ uid_match = re.match(r'UID Nr.: (.*)', line)
131
+ if uid_match:
132
+ uid = uid_match.group(1).strip()
133
+ continue
134
+
135
+ payback_info_match = re.match(r'PAYBACK Karten-Nr\.: ([0-9#]*)Punkte vor dem Einkauf: ([0-9.,]*) Punkte', line)
136
+ if payback_info_match:
137
+ payback_card_number = payback_info_match.group(1)
138
+ payback_points_before = int(payback_info_match.group(2).replace('.', '').replace(',', ''))
139
+ continue
140
+
141
+ payback_points_match = re.match(r'Sie erhalten (\d*) PAYBACK Punkte? auf|Mit diesem Einkauf gesammelt: (\d*) Punkte?', line)
142
+ if payback_points_match:
143
+ match = next(group for group in payback_points_match.groups() if group is not None)
144
+ payback_points = int(match)
145
+ continue
146
+
147
+ payback_revenue_match = re.match(r'einen PAYBACK Umsatz von (.*) EUR!', line)
148
+ if payback_revenue_match:
149
+ payback_revenue = float(payback_revenue_match.group(1).replace(',', '.'))
150
+ continue
151
+
152
+ payback_points_before_match = re.match(r'Punktestand vor Einkauf: ([0-9.]*)|Punkte vor dem Einkauf: ([0-9.]*)', line)
153
+ if payback_points_before_match:
154
+ match = next(group for group in payback_points_before_match.groups() if group is not None)
155
+ payback_points_before = int(match.replace('.', ''))
156
+ continue
157
+
158
+ payback_card_number_match = re.match(r'PAYBACK Karten-Nr\.: ([0-9#]*)', line)
159
+ if payback_card_number_match:
160
+ payback_card_number = payback_card_number_match.group(1)
161
+ continue
162
+
163
+ payback_coupon_match = re.match(r'(.*) ([0-9.]*) Punkte?', line)
164
+ if payback_coupon_match:
165
+ payback_coupons.append(PaybackCoupon(
166
+ name=payback_coupon_match.group(1),
167
+ points=int(payback_coupon_match.group(2).replace('.', ''))
168
+ ))
169
+ continue
170
+
171
+ tax_details_match = re.match(r'([AB])= ([0-9,]*)% ([0-9,]*) ([0-9,]*) ([0-9,]*)', line)
172
+ if tax_details_match:
173
+ category = tax_details_match.group(1)
174
+ tax_details_entry = TaxDetailsEntry(
175
+ tax_percent=float(tax_details_match.group(2).replace(',', '.')),
176
+ net=float(tax_details_match.group(3).replace(',', '.')),
177
+ tax=float(tax_details_match.group(4).replace(',', '.')),
178
+ gross=float(tax_details_match.group(5).replace(',', '.'))
179
+ )
180
+ if category == 'A':
181
+ tax_details_A = tax_details_entry
182
+ elif category == 'B':
183
+ tax_details_B = tax_details_entry
184
+ continue
185
+
186
+ total_tax_match = re.match(r'Gesamtbetrag ([0-9,]*) ([0-9,]*) ([0-9,]*)', line)
187
+ if total_tax_match:
188
+ tax_details_total = TaxDetailsEntry(
189
+ tax_percent=float('nan'),
190
+ net=float(total_tax_match.group(1).replace(',', '.')),
191
+ tax=float(total_tax_match.group(2).replace(',', '.')),
192
+ gross=float(total_tax_match.group(3).replace(',', '.'))
193
+ )
194
+ continue
195
+
196
+ used_rewe_credit_match = re.match(r'Eingesetztes REWE Guthaben: ([0-9,]*) EUR', line)
197
+ if used_rewe_credit_match:
198
+ used_rewe_credit = float(used_rewe_credit_match.group(1).replace(',', '.'))
199
+
200
+ new_rewe_credit_match = re.match(r'Neues REWE Guthaben: ([0-9,]*) EUR', line)
201
+ if new_rewe_credit_match:
202
+ new_rewe_credit = float(new_rewe_credit_match.group(1).replace(',', '.'))
203
+
204
+ if date is None:
205
+ raise ValueError("Date not found in the receipt")
206
+
207
+ real_total_in_cents = sum(item.sub_total * 100 for item in items)
208
+ total_in_cents = total * 100
209
+
210
+ if round(real_total_in_cents, 2) != round(total_in_cents, 2):
211
+ raise ValueError(f"Something went wrong when parsing the eBon: The eBon states a total sum of {total_in_cents} but the parser only found items worth {real_total_in_cents}.")
212
+
213
+ qualified_revenue = payback_revenue if not math.isnan(payback_revenue) else sum(item.sub_total for item in items if item.payback_qualified or item.sub_total < 0)
214
+
215
+ receipt = Receipt(
216
+ date=date,
217
+ market=market,
218
+ market_address=market_address,
219
+ cashier=cashier,
220
+ checkout=checkout,
221
+ vatin=uid,
222
+ items=items,
223
+ total=total,
224
+ given=given,
225
+ change=change if not math.isnan(change) else None,
226
+ payout=payout if not math.isnan(payout) else None,
227
+ payback=PaybackData(
228
+ card=payback_card_number,
229
+ points_before=payback_points_before,
230
+ earned_points=payback_points,
231
+ used_coupons=payback_coupons,
232
+ used_rewe_credit=used_rewe_credit if not math.isnan(used_rewe_credit) else None,
233
+ new_rewe_credit=new_rewe_credit if not math.isnan(new_rewe_credit) else None,
234
+ payback_revenue=qualified_revenue
235
+ ) if payback_card_number != '?' else None,
236
+ tax_details=TaxDetails(
237
+ total=tax_details_total,
238
+ A=tax_details_A,
239
+ B=tax_details_B
240
+ )
241
+ )
242
+
243
+ # Create an ordered dictionary to ensure specific order of keys
244
+ receipt_dict = receipt.to_dict()
245
+ ordered_receipt_dict = OrderedDict([
246
+ ('datetime_local', receipt_dict.get('datetime_local')),
247
+ ('datetime_utc', receipt_dict.get('datetime_utc')),
248
+ ('market', receipt_dict.get('market')),
249
+ ('marketAddress', receipt_dict.get('marketAddress')),
250
+ ('cashier', receipt_dict.get('cashier')),
251
+ ('checkout', receipt_dict.get('checkout')),
252
+ ('vatin', receipt_dict.get('vatin')),
253
+ ('items', receipt_dict.get('items')),
254
+ ('total', receipt_dict.get('total')),
255
+ ('given', receipt_dict.get('given')),
256
+ ('change', receipt_dict.get('change')),
257
+ ('payout', receipt_dict.get('payout')),
258
+ ('payback', receipt_dict.get('payback')),
259
+ ('taxDetails', receipt_dict.get('taxDetails')),
260
+ ])
261
+
262
+ # Remove None values to clean up the dictionary
263
+ ordered_receipt_dict = {k: v for k, v in ordered_receipt_dict.items() if v is not None}
264
+
265
+ return ordered_receipt_dict
266
+
267
+ def parse_pdf_ebon(pdf_path: str) -> dict:
268
+ with open(pdf_path, 'rb') as f:
269
+ data = f.read()
270
+ result = parse_ebon(data)
271
+ return result
272
+