il-supermarket-parser 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- il_supermarket_parser-0.0.1/MANIFEST.in +2 -0
- il_supermarket_parser-0.0.1/PKG-INFO +104 -0
- il_supermarket_parser-0.0.1/README.md +77 -0
- il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/PKG-INFO +104 -0
- il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/SOURCES.txt +46 -0
- il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/dependency_links.txt +1 -0
- il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/requires.txt +8 -0
- il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/top_level.txt +1 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/__init__.py +3 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/conf/__init__.py +0 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/__init__.py +3 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/base.py +61 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/xml_dataframe_parser.py +100 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/xml_dataframe_subroot_praser.py +77 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/__init__.py +3 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/base.py +105 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/big_id.py +64 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/branches.py +25 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/main.py +37 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/multiprocess_pharser.py +71 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/normlizers/base.py +49 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parser_factroy.py +74 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/__init__.py +11 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/bareket.py +40 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/confix.py +45 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/het_cohen.py +27 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/mahsani_a_shuk.py +17 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/other.py +170 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/salach_dabach.py +22 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/shufersal.py +15 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/super_pharm.py +35 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/tiv_taam.py +45 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/victory.py +15 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/raw_parsing_pipeline.py +155 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/__init__.py +23 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/data_loader.py +132 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/database.py +119 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/dataframe_utils.py +35 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/diff.py +39 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/logger.py +44 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/multi_prcoessing.py +145 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/test_utils.py +68 -0
- il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/xml_utils.py +130 -0
- il_supermarket_parser-0.0.1/requirements-dev.txt +3 -0
- il_supermarket_parser-0.0.1/requirements.txt +5 -0
- il_supermarket_parser-0.0.1/setup.cfg +7 -0
- il_supermarket_parser-0.0.1/setup.py +52 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: il-supermarket-parser
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: python package that process the data dumped by the israeli supermarket
|
|
5
|
+
Home-page: https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers
|
|
6
|
+
Author: Sefi Erlich
|
|
7
|
+
Author-email: erlichsefi@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Keywords: israel,israeli,scraper,supermarket
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Software Development :: Build Tools
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.4
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.5
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: pandas==2.2.2
|
|
20
|
+
Requires-Dist: lxml==5.2.1
|
|
21
|
+
Requires-Dist: pymongo==4.6.3
|
|
22
|
+
Requires-Dist: il-supermarket-scraper>=0.4.6
|
|
23
|
+
Requires-Dist: tqdm==4.66
|
|
24
|
+
Provides-Extra: test
|
|
25
|
+
Requires-Dist: pytest; extra == "test"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
Israel Supermarket Parsers: Clients to parser the data published by the supermarkets.
|
|
29
|
+
=======================================
|
|
30
|
+
This is a parser for ALL the supermarket chains listed in the GOV.IL site.
|
|
31
|
+
|
|
32
|
+
שקיפות מחירים (השוואת מחירים) - https://www.gov.il/he/departments/legalInfo/cpfta_prices_regulations
|
|
33
|
+
הגדרת הקבצים שהיו צריכים להיות זמינים באתרים - https://www.nevo.co.il/law_html/law01/501_131.htm
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml)
|
|
37
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml)
|
|
38
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml)
|
|
39
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml)
|
|
40
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
## 🤗 Want to support my work?
|
|
44
|
+
<p align="center">
|
|
45
|
+
<a href="https://buymeacoffee.com/erlichsefi" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;">
|
|
46
|
+
</a>
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
Got a question?
|
|
50
|
+
---------------
|
|
51
|
+
|
|
52
|
+
You can email me at erlichsefi@gmail.com
|
|
53
|
+
|
|
54
|
+
If you think you've found a bug:
|
|
55
|
+
|
|
56
|
+
- Create issue in [issue tracker](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/issues) to see if
|
|
57
|
+
it's already been reported
|
|
58
|
+
- Please consider solving the issue by yourself and creating a pull request.
|
|
59
|
+
|
|
60
|
+
What is il_supermarket_parsers?
|
|
61
|
+
-------------
|
|
62
|
+
|
|
63
|
+
A simple access layer to the data the supermarkets publish.
|
|
64
|
+
You can download the data using https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.
|
|
65
|
+
|
|
66
|
+
You only need to run the following code to parse all the data currently shared by the supermarkets and was downloaded using the package above.
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from il_supermarket_parsers import ConvertingTask
|
|
70
|
+
|
|
71
|
+
scraper = ConvertingTask(data_folder="dumps")
|
|
72
|
+
scraper.run()
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
Quick start
|
|
77
|
+
-----------
|
|
78
|
+
|
|
79
|
+
il_supermarket_parsers can be installed using pip:
|
|
80
|
+
|
|
81
|
+
python3 pip install il_supermarket_parsers
|
|
82
|
+
|
|
83
|
+
If you want to run the latest version of the code, you can install it from the
|
|
84
|
+
repo directly:
|
|
85
|
+
|
|
86
|
+
python3 -m pip install -U git+https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.git
|
|
87
|
+
# or if you don't have 'git' installed
|
|
88
|
+
python3 -m pip install -U https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/main
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
Contributing
|
|
92
|
+
------------
|
|
93
|
+
|
|
94
|
+
Help in testing, development, documentation and other tasks is
|
|
95
|
+
highly appreciated and useful to the project. There are tasks for
|
|
96
|
+
contributors of all experience levels.
|
|
97
|
+
|
|
98
|
+
If you need help getting started, don't hesitate to contact me.
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
Development status
|
|
102
|
+
------------------
|
|
103
|
+
|
|
104
|
+
IL SuperMarket Parser is beta software, as far as i see devlopment stoped until new issues will be found.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Israel Supermarket Parsers: Clients to parser the data published by the supermarkets.
|
|
2
|
+
=======================================
|
|
3
|
+
This is a parser for ALL the supermarket chains listed in the GOV.IL site.
|
|
4
|
+
|
|
5
|
+
שקיפות מחירים (השוואת מחירים) - https://www.gov.il/he/departments/legalInfo/cpfta_prices_regulations
|
|
6
|
+
הגדרת הקבצים שהיו צריכים להיות זמינים באתרים - https://www.nevo.co.il/law_html/law01/501_131.htm
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml)
|
|
10
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml)
|
|
11
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml)
|
|
12
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml)
|
|
13
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## 🤗 Want to support my work?
|
|
17
|
+
<p align="center">
|
|
18
|
+
<a href="https://buymeacoffee.com/erlichsefi" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;">
|
|
19
|
+
</a>
|
|
20
|
+
</p>
|
|
21
|
+
|
|
22
|
+
Got a question?
|
|
23
|
+
---------------
|
|
24
|
+
|
|
25
|
+
You can email me at erlichsefi@gmail.com
|
|
26
|
+
|
|
27
|
+
If you think you've found a bug:
|
|
28
|
+
|
|
29
|
+
- Create issue in [issue tracker](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/issues) to see if
|
|
30
|
+
it's already been reported
|
|
31
|
+
- Please consider solving the issue by yourself and creating a pull request.
|
|
32
|
+
|
|
33
|
+
What is il_supermarket_parsers?
|
|
34
|
+
-------------
|
|
35
|
+
|
|
36
|
+
A simple access layer to the data the supermarkets publish.
|
|
37
|
+
You can download the data using https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.
|
|
38
|
+
|
|
39
|
+
You only need to run the following code to parse all the data currently shared by the supermarkets and was downloaded using the package above.
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from il_supermarket_parsers import ConvertingTask
|
|
43
|
+
|
|
44
|
+
scraper = ConvertingTask(data_folder="dumps")
|
|
45
|
+
scraper.run()
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
Quick start
|
|
50
|
+
-----------
|
|
51
|
+
|
|
52
|
+
il_supermarket_parsers can be installed using pip:
|
|
53
|
+
|
|
54
|
+
python3 pip install il_supermarket_parsers
|
|
55
|
+
|
|
56
|
+
If you want to run the latest version of the code, you can install it from the
|
|
57
|
+
repo directly:
|
|
58
|
+
|
|
59
|
+
python3 -m pip install -U git+https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.git
|
|
60
|
+
# or if you don't have 'git' installed
|
|
61
|
+
python3 -m pip install -U https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/main
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
Contributing
|
|
65
|
+
------------
|
|
66
|
+
|
|
67
|
+
Help in testing, development, documentation and other tasks is
|
|
68
|
+
highly appreciated and useful to the project. There are tasks for
|
|
69
|
+
contributors of all experience levels.
|
|
70
|
+
|
|
71
|
+
If you need help getting started, don't hesitate to contact me.
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
Development status
|
|
75
|
+
------------------
|
|
76
|
+
|
|
77
|
+
IL SuperMarket Parser is beta software, as far as i see devlopment stoped until new issues will be found.
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: il-supermarket-parser
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: python package that process the data dumped by the israeli supermarket
|
|
5
|
+
Home-page: https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers
|
|
6
|
+
Author: Sefi Erlich
|
|
7
|
+
Author-email: erlichsefi@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Keywords: israel,israeli,scraper,supermarket
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Software Development :: Build Tools
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.4
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.5
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Requires-Dist: pandas==2.2.2
|
|
20
|
+
Requires-Dist: lxml==5.2.1
|
|
21
|
+
Requires-Dist: pymongo==4.6.3
|
|
22
|
+
Requires-Dist: il-supermarket-scraper>=0.4.6
|
|
23
|
+
Requires-Dist: tqdm==4.66
|
|
24
|
+
Provides-Extra: test
|
|
25
|
+
Requires-Dist: pytest; extra == "test"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
Israel Supermarket Parsers: Clients to parser the data published by the supermarkets.
|
|
29
|
+
=======================================
|
|
30
|
+
This is a parser for ALL the supermarket chains listed in the GOV.IL site.
|
|
31
|
+
|
|
32
|
+
שקיפות מחירים (השוואת מחירים) - https://www.gov.il/he/departments/legalInfo/cpfta_prices_regulations
|
|
33
|
+
הגדרת הקבצים שהיו צריכים להיות זמינים באתרים - https://www.nevo.co.il/law_html/law01/501_131.htm
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml)
|
|
37
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml)
|
|
38
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml)
|
|
39
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml)
|
|
40
|
+
[](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
## 🤗 Want to support my work?
|
|
44
|
+
<p align="center">
|
|
45
|
+
<a href="https://buymeacoffee.com/erlichsefi" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;">
|
|
46
|
+
</a>
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
Got a question?
|
|
50
|
+
---------------
|
|
51
|
+
|
|
52
|
+
You can email me at erlichsefi@gmail.com
|
|
53
|
+
|
|
54
|
+
If you think you've found a bug:
|
|
55
|
+
|
|
56
|
+
- Create issue in [issue tracker](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/issues) to see if
|
|
57
|
+
it's already been reported
|
|
58
|
+
- Please consider solving the issue by yourself and creating a pull request.
|
|
59
|
+
|
|
60
|
+
What is il_supermarket_parsers?
|
|
61
|
+
-------------
|
|
62
|
+
|
|
63
|
+
A simple access layer to the data the supermarkets publish.
|
|
64
|
+
You can download the data using https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.
|
|
65
|
+
|
|
66
|
+
You only need to run the following code to parse all the data currently shared by the supermarkets and was downloaded using the package above.
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from il_supermarket_parsers import ConvertingTask
|
|
70
|
+
|
|
71
|
+
scraper = ConvertingTask(data_folder="dumps")
|
|
72
|
+
scraper.run()
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
Quick start
|
|
77
|
+
-----------
|
|
78
|
+
|
|
79
|
+
il_supermarket_parsers can be installed using pip:
|
|
80
|
+
|
|
81
|
+
python3 pip install il_supermarket_parsers
|
|
82
|
+
|
|
83
|
+
If you want to run the latest version of the code, you can install it from the
|
|
84
|
+
repo directly:
|
|
85
|
+
|
|
86
|
+
python3 -m pip install -U git+https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.git
|
|
87
|
+
# or if you don't have 'git' installed
|
|
88
|
+
python3 -m pip install -U https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/main
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
Contributing
|
|
92
|
+
------------
|
|
93
|
+
|
|
94
|
+
Help in testing, development, documentation and other tasks is
|
|
95
|
+
highly appreciated and useful to the project. There are tasks for
|
|
96
|
+
contributors of all experience levels.
|
|
97
|
+
|
|
98
|
+
If you need help getting started, don't hesitate to contact me.
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
Development status
|
|
102
|
+
------------------
|
|
103
|
+
|
|
104
|
+
IL SuperMarket Parser is beta software, as far as i see devlopment stoped until new issues will be found.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
MANIFEST.in
|
|
2
|
+
README.md
|
|
3
|
+
requirements-dev.txt
|
|
4
|
+
requirements.txt
|
|
5
|
+
setup.cfg
|
|
6
|
+
setup.py
|
|
7
|
+
il_supermarket_parser.egg-info/PKG-INFO
|
|
8
|
+
il_supermarket_parser.egg-info/SOURCES.txt
|
|
9
|
+
il_supermarket_parser.egg-info/dependency_links.txt
|
|
10
|
+
il_supermarket_parser.egg-info/requires.txt
|
|
11
|
+
il_supermarket_parser.egg-info/top_level.txt
|
|
12
|
+
il_supermarket_parsers/__init__.py
|
|
13
|
+
il_supermarket_parsers/main.py
|
|
14
|
+
il_supermarket_parsers/multiprocess_pharser.py
|
|
15
|
+
il_supermarket_parsers/parser_factroy.py
|
|
16
|
+
il_supermarket_parsers/raw_parsing_pipeline.py
|
|
17
|
+
il_supermarket_parsers/conf/__init__.py
|
|
18
|
+
il_supermarket_parsers/documents/__init__.py
|
|
19
|
+
il_supermarket_parsers/documents/base.py
|
|
20
|
+
il_supermarket_parsers/documents/xml_dataframe_parser.py
|
|
21
|
+
il_supermarket_parsers/documents/xml_dataframe_subroot_praser.py
|
|
22
|
+
il_supermarket_parsers/engines/__init__.py
|
|
23
|
+
il_supermarket_parsers/engines/base.py
|
|
24
|
+
il_supermarket_parsers/engines/big_id.py
|
|
25
|
+
il_supermarket_parsers/engines/branches.py
|
|
26
|
+
il_supermarket_parsers/normlizers/base.py
|
|
27
|
+
il_supermarket_parsers/parsers/__init__.py
|
|
28
|
+
il_supermarket_parsers/parsers/bareket.py
|
|
29
|
+
il_supermarket_parsers/parsers/confix.py
|
|
30
|
+
il_supermarket_parsers/parsers/het_cohen.py
|
|
31
|
+
il_supermarket_parsers/parsers/mahsani_a_shuk.py
|
|
32
|
+
il_supermarket_parsers/parsers/other.py
|
|
33
|
+
il_supermarket_parsers/parsers/salach_dabach.py
|
|
34
|
+
il_supermarket_parsers/parsers/shufersal.py
|
|
35
|
+
il_supermarket_parsers/parsers/super_pharm.py
|
|
36
|
+
il_supermarket_parsers/parsers/tiv_taam.py
|
|
37
|
+
il_supermarket_parsers/parsers/victory.py
|
|
38
|
+
il_supermarket_parsers/utils/__init__.py
|
|
39
|
+
il_supermarket_parsers/utils/data_loader.py
|
|
40
|
+
il_supermarket_parsers/utils/database.py
|
|
41
|
+
il_supermarket_parsers/utils/dataframe_utils.py
|
|
42
|
+
il_supermarket_parsers/utils/diff.py
|
|
43
|
+
il_supermarket_parsers/utils/logger.py
|
|
44
|
+
il_supermarket_parsers/utils/multi_prcoessing.py
|
|
45
|
+
il_supermarket_parsers/utils/test_utils.py
|
|
46
|
+
il_supermarket_parsers/utils/xml_utils.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
il_supermarket_parsers
|
|
File without changes
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import List
|
|
3
|
+
import os
|
|
4
|
+
from il_supermarket_parsers.utils import build_value, get_root
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class XmlBaseConverter(ABC):
|
|
8
|
+
"""parser the xml docuement"""
|
|
9
|
+
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
list_key: List[str],
|
|
13
|
+
id_field: str,
|
|
14
|
+
roots=None,
|
|
15
|
+
ignore_column=None,
|
|
16
|
+
**additional_constant,
|
|
17
|
+
):
|
|
18
|
+
self.list_key = list_key
|
|
19
|
+
self.roots = roots
|
|
20
|
+
self.id_field = id_field
|
|
21
|
+
self.ignore_column = ignore_column if ignore_column else []
|
|
22
|
+
self.additional_constant = additional_constant
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def validate_succussful_extraction(
|
|
26
|
+
self, data, source_file, ignore_missing_columns=None
|
|
27
|
+
):
|
|
28
|
+
"""validate column requested"""
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def reduce_size(self, data):
|
|
32
|
+
"""reduce the size"""
|
|
33
|
+
|
|
34
|
+
def build_value(self, name, no_content):
|
|
35
|
+
"""get the value"""
|
|
36
|
+
return build_value(name, self.additional_constant, no_content=no_content)
|
|
37
|
+
|
|
38
|
+
def convert(self, found_store, file_name, **kwarg):
|
|
39
|
+
"""parse file to data frame"""
|
|
40
|
+
source_file = os.path.join(found_store, file_name)
|
|
41
|
+
root, root_store = get_root(source_file, self.list_key, self.roots)
|
|
42
|
+
|
|
43
|
+
data = self._phrse(
|
|
44
|
+
root,
|
|
45
|
+
found_store,
|
|
46
|
+
file_name,
|
|
47
|
+
root_store,
|
|
48
|
+
**kwarg,
|
|
49
|
+
)
|
|
50
|
+
return self.reduce_size(data)
|
|
51
|
+
|
|
52
|
+
@abstractmethod
|
|
53
|
+
def _phrse(
|
|
54
|
+
self,
|
|
55
|
+
root,
|
|
56
|
+
found_folder,
|
|
57
|
+
file_name,
|
|
58
|
+
root_store,
|
|
59
|
+
**kwarg,
|
|
60
|
+
):
|
|
61
|
+
pass
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from il_supermarket_parsers.utils import (
|
|
3
|
+
count_tag_in_xml,
|
|
4
|
+
collect_unique_keys_from_xml,
|
|
5
|
+
collect_unique_columns_from_nested_json,
|
|
6
|
+
)
|
|
7
|
+
from .base import XmlBaseConverter
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class XmlDataFrameConverter(XmlBaseConverter):
|
|
11
|
+
"""parser the xml docuement"""
|
|
12
|
+
|
|
13
|
+
def reduce_size(self, data):
|
|
14
|
+
for col in data.columns:
|
|
15
|
+
data[col] = data[col].mask(data[col] == data[col].shift())
|
|
16
|
+
return data
|
|
17
|
+
|
|
18
|
+
def validate_succussful_extraction(
|
|
19
|
+
self, data, source_file, ignore_missing_columns=None
|
|
20
|
+
):
|
|
21
|
+
# if there is an empty file
|
|
22
|
+
# we expected it to return none
|
|
23
|
+
tag_count = count_tag_in_xml(source_file, self.id_field)
|
|
24
|
+
|
|
25
|
+
if self.roots and tag_count > 0:
|
|
26
|
+
for root in self.roots:
|
|
27
|
+
if root not in data.columns:
|
|
28
|
+
raise ValueError(
|
|
29
|
+
f"parse error for file {source_file},"
|
|
30
|
+
f"columns {root} missing from {data.columns}"
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
if self.id_field not in data.columns:
|
|
34
|
+
raise ValueError(
|
|
35
|
+
f"parse error for file {source_file}, "
|
|
36
|
+
f"id {self.id_field} missing from {data.columns}"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
if data.shape[0] != tag_count:
|
|
40
|
+
raise ValueError(
|
|
41
|
+
f"for file {source_file}, missing data,"
|
|
42
|
+
f"data shape {data.shape} tag count is {tag_count}"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
ignore_list = self.ignore_column
|
|
46
|
+
if ignore_missing_columns:
|
|
47
|
+
ignore_list = ignore_list + ignore_missing_columns
|
|
48
|
+
keys_not_used = (
|
|
49
|
+
set(collect_unique_keys_from_xml(source_file))
|
|
50
|
+
- collect_unique_columns_from_nested_json(data)
|
|
51
|
+
- set(ignore_list)
|
|
52
|
+
)
|
|
53
|
+
if len(keys_not_used) > 0:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"for file {source_file}, there is data we didn't get {keys_not_used}"
|
|
56
|
+
)
|
|
57
|
+
assert "found_folder" in data.columns
|
|
58
|
+
assert "file_name" in data.columns
|
|
59
|
+
|
|
60
|
+
def list_single_entry(self, elem, found_folder, file_name, **sub_root_store):
|
|
61
|
+
"""build a single row"""
|
|
62
|
+
values = {
|
|
63
|
+
"found_folder": found_folder,
|
|
64
|
+
"file_name": file_name,
|
|
65
|
+
**sub_root_store,
|
|
66
|
+
}
|
|
67
|
+
for name in list(elem):
|
|
68
|
+
tag = name.tag
|
|
69
|
+
value = self.build_value(name, no_content="")
|
|
70
|
+
|
|
71
|
+
values[tag] = value
|
|
72
|
+
return values.copy()
|
|
73
|
+
|
|
74
|
+
def _phrse(
|
|
75
|
+
self,
|
|
76
|
+
root,
|
|
77
|
+
found_folder,
|
|
78
|
+
file_name,
|
|
79
|
+
root_store,
|
|
80
|
+
**kwarg,
|
|
81
|
+
):
|
|
82
|
+
rows = []
|
|
83
|
+
columns = [self.id_field, "found_folder", "file_name"] + (
|
|
84
|
+
self.roots if self.roots else []
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
if root is None:
|
|
88
|
+
return pd.DataFrame(columns=columns)
|
|
89
|
+
|
|
90
|
+
elements = list(root)
|
|
91
|
+
if len(root) == 0:
|
|
92
|
+
|
|
93
|
+
return pd.DataFrame(columns=columns)
|
|
94
|
+
|
|
95
|
+
for elem in elements:
|
|
96
|
+
rows.append(
|
|
97
|
+
self.list_single_entry(elem, found_folder, file_name, **root_store)
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
return pd.DataFrame(rows)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from .xml_dataframe_parser import XmlDataFrameConverter
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class SubRootedXmlDataFrameConverter(XmlDataFrameConverter):
|
|
6
|
+
"""parser the xml docuement with extra indentations"""
|
|
7
|
+
|
|
8
|
+
def __init__(
|
|
9
|
+
self,
|
|
10
|
+
list_key,
|
|
11
|
+
id_field,
|
|
12
|
+
roots=None,
|
|
13
|
+
sub_roots=None,
|
|
14
|
+
list_sub_key="",
|
|
15
|
+
ignore_column=None,
|
|
16
|
+
**additional_constant,
|
|
17
|
+
):
|
|
18
|
+
super().__init__(
|
|
19
|
+
list_key=list_key,
|
|
20
|
+
id_field=id_field,
|
|
21
|
+
roots=roots,
|
|
22
|
+
ignore_column=ignore_column,
|
|
23
|
+
additional_constant=additional_constant,
|
|
24
|
+
)
|
|
25
|
+
self.sub_roots = sub_roots if sub_roots else []
|
|
26
|
+
self.list_sub_key = list_sub_key
|
|
27
|
+
|
|
28
|
+
def validate_succussful_extraction(
|
|
29
|
+
self, data, source_file, ignore_missing_columns=None
|
|
30
|
+
):
|
|
31
|
+
"""validation"""
|
|
32
|
+
super().validate_succussful_extraction(
|
|
33
|
+
data, source_file, ignore_missing_columns=ignore_missing_columns
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
# if the user asked to include the headers
|
|
37
|
+
if self.sub_roots:
|
|
38
|
+
for root in self.sub_roots:
|
|
39
|
+
if root not in data.columns:
|
|
40
|
+
raise ValueError(
|
|
41
|
+
f"parse error for file {source_file}, "
|
|
42
|
+
f"columns {root} missing from {data.columns}"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
def _phrse(
|
|
46
|
+
self,
|
|
47
|
+
root,
|
|
48
|
+
found_folder,
|
|
49
|
+
file_name,
|
|
50
|
+
root_store,
|
|
51
|
+
**_,
|
|
52
|
+
):
|
|
53
|
+
"""parse file to data frame"""
|
|
54
|
+
|
|
55
|
+
rows = []
|
|
56
|
+
|
|
57
|
+
if root is None or len(root) == 0:
|
|
58
|
+
return pd.DataFrame(
|
|
59
|
+
columns=self.sub_roots
|
|
60
|
+
+ [self.id_field, "found_folder", "file_name"]
|
|
61
|
+
+ (self.roots if self.roots else [])
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
for sub_elem in list(root):
|
|
65
|
+
sub_root_store = root_store.copy()
|
|
66
|
+
|
|
67
|
+
for k in self.sub_roots:
|
|
68
|
+
sub_root_store[k] = sub_elem.find(k).text
|
|
69
|
+
|
|
70
|
+
for elem in sub_elem.find(self.list_sub_key):
|
|
71
|
+
rows.append(
|
|
72
|
+
self.list_single_entry(
|
|
73
|
+
elem, found_folder, file_name, **sub_root_store
|
|
74
|
+
)
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
return pd.DataFrame(rows)
|