il-supermarket-parser 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. il_supermarket_parser-0.0.1/MANIFEST.in +2 -0
  2. il_supermarket_parser-0.0.1/PKG-INFO +104 -0
  3. il_supermarket_parser-0.0.1/README.md +77 -0
  4. il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/PKG-INFO +104 -0
  5. il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/SOURCES.txt +46 -0
  6. il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/dependency_links.txt +1 -0
  7. il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/requires.txt +8 -0
  8. il_supermarket_parser-0.0.1/il_supermarket_parser.egg-info/top_level.txt +1 -0
  9. il_supermarket_parser-0.0.1/il_supermarket_parsers/__init__.py +3 -0
  10. il_supermarket_parser-0.0.1/il_supermarket_parsers/conf/__init__.py +0 -0
  11. il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/__init__.py +3 -0
  12. il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/base.py +61 -0
  13. il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/xml_dataframe_parser.py +100 -0
  14. il_supermarket_parser-0.0.1/il_supermarket_parsers/documents/xml_dataframe_subroot_praser.py +77 -0
  15. il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/__init__.py +3 -0
  16. il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/base.py +105 -0
  17. il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/big_id.py +64 -0
  18. il_supermarket_parser-0.0.1/il_supermarket_parsers/engines/branches.py +25 -0
  19. il_supermarket_parser-0.0.1/il_supermarket_parsers/main.py +37 -0
  20. il_supermarket_parser-0.0.1/il_supermarket_parsers/multiprocess_pharser.py +71 -0
  21. il_supermarket_parser-0.0.1/il_supermarket_parsers/normlizers/base.py +49 -0
  22. il_supermarket_parser-0.0.1/il_supermarket_parsers/parser_factroy.py +74 -0
  23. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/__init__.py +11 -0
  24. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/bareket.py +40 -0
  25. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/confix.py +45 -0
  26. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/het_cohen.py +27 -0
  27. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/mahsani_a_shuk.py +17 -0
  28. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/other.py +170 -0
  29. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/salach_dabach.py +22 -0
  30. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/shufersal.py +15 -0
  31. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/super_pharm.py +35 -0
  32. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/tiv_taam.py +45 -0
  33. il_supermarket_parser-0.0.1/il_supermarket_parsers/parsers/victory.py +15 -0
  34. il_supermarket_parser-0.0.1/il_supermarket_parsers/raw_parsing_pipeline.py +155 -0
  35. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/__init__.py +23 -0
  36. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/data_loader.py +132 -0
  37. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/database.py +119 -0
  38. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/dataframe_utils.py +35 -0
  39. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/diff.py +39 -0
  40. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/logger.py +44 -0
  41. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/multi_prcoessing.py +145 -0
  42. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/test_utils.py +68 -0
  43. il_supermarket_parser-0.0.1/il_supermarket_parsers/utils/xml_utils.py +130 -0
  44. il_supermarket_parser-0.0.1/requirements-dev.txt +3 -0
  45. il_supermarket_parser-0.0.1/requirements.txt +5 -0
  46. il_supermarket_parser-0.0.1/setup.cfg +7 -0
  47. il_supermarket_parser-0.0.1/setup.py +52 -0
@@ -0,0 +1,2 @@
1
+ include requirements.txt
2
+ include requirements-dev.txt
@@ -0,0 +1,104 @@
1
+ Metadata-Version: 2.1
2
+ Name: il-supermarket-parser
3
+ Version: 0.0.1
4
+ Summary: python package that process the data dumped by the israeli supermarket
5
+ Home-page: https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers
6
+ Author: Sefi Erlich
7
+ Author-email: erlichsefi@gmail.com
8
+ License: MIT
9
+ Keywords: israel,israeli,scraper,supermarket
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Topic :: Software Development :: Build Tools
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.4
16
+ Classifier: Programming Language :: Python :: 3.5
17
+ Classifier: Programming Language :: Python :: 3.6
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: pandas==2.2.2
20
+ Requires-Dist: lxml==5.2.1
21
+ Requires-Dist: pymongo==4.6.3
22
+ Requires-Dist: il-supermarket-scraper>=0.4.6
23
+ Requires-Dist: tqdm==4.66
24
+ Provides-Extra: test
25
+ Requires-Dist: pytest; extra == "test"
26
+
27
+
28
+ Israel Supermarket Parsers: Clients to parser the data published by the supermarkets.
29
+ =======================================
30
+ This is a parser for ALL the supermarket chains listed in the GOV.IL site.
31
+
32
+ שקיפות מחירים (השוואת מחירים) - https://www.gov.il/he/departments/legalInfo/cpfta_prices_regulations
33
+ הגדרת הקבצים שהיו צריכים להיות זמינים באתרים - https://www.nevo.co.il/law_html/law01/501_131.htm
34
+
35
+
36
+ [![Unit & Integration Tests](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml/badge.svg?event=push)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml)
37
+ [![CodeQL](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml)
38
+ [![Pylint](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml)
39
+ [![Publish Docker image](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml)
40
+ [![Upload Python Package](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml)
41
+
42
+
43
+ ## 🤗 Want to support my work?
44
+ <p align="center">
45
+ <a href="https://buymeacoffee.com/erlichsefi" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;">
46
+ </a>
47
+ </p>
48
+
49
+ Got a question?
50
+ ---------------
51
+
52
+ You can email me at erlichsefi@gmail.com
53
+
54
+ If you think you've found a bug:
55
+
56
+ - Create issue in [issue tracker](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/issues) to see if
57
+ it's already been reported
58
+ - Please consider solving the issue by yourself and creating a pull request.
59
+
60
+ What is il_supermarket_parsers?
61
+ -------------
62
+
63
+ A simple access layer to the data the supermarkets publish.
64
+ You can download the data using https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.
65
+
66
+ You only need to run the following code to parse all the data currently shared by the supermarkets and was downloaded using the package above.
67
+
68
+ ```python
69
+ from il_supermarket_parsers import ConvertingTask
70
+
71
+ scraper = ConvertingTask(data_folder="dumps")
72
+ scraper.run()
73
+ ```
74
+
75
+
76
+ Quick start
77
+ -----------
78
+
79
+ il_supermarket_parsers can be installed using pip:
80
+
81
+ python3 pip install il_supermarket_parsers
82
+
83
+ If you want to run the latest version of the code, you can install it from the
84
+ repo directly:
85
+
86
+ python3 -m pip install -U git+https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.git
87
+ # or if you don't have 'git' installed
88
+ python3 -m pip install -U https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/main
89
+
90
+
91
+ Contributing
92
+ ------------
93
+
94
+ Help in testing, development, documentation and other tasks is
95
+ highly appreciated and useful to the project. There are tasks for
96
+ contributors of all experience levels.
97
+
98
+ If you need help getting started, don't hesitate to contact me.
99
+
100
+
101
+ Development status
102
+ ------------------
103
+
104
+ IL SuperMarket Parser is beta software, as far as i see devlopment stoped until new issues will be found.
@@ -0,0 +1,77 @@
1
+ Israel Supermarket Parsers: Clients to parser the data published by the supermarkets.
2
+ =======================================
3
+ This is a parser for ALL the supermarket chains listed in the GOV.IL site.
4
+
5
+ שקיפות מחירים (השוואת מחירים) - https://www.gov.il/he/departments/legalInfo/cpfta_prices_regulations
6
+ הגדרת הקבצים שהיו צריכים להיות זמינים באתרים - https://www.nevo.co.il/law_html/law01/501_131.htm
7
+
8
+
9
+ [![Unit & Integration Tests](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml/badge.svg?event=push)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml)
10
+ [![CodeQL](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml)
11
+ [![Pylint](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml)
12
+ [![Publish Docker image](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml)
13
+ [![Upload Python Package](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml)
14
+
15
+
16
+ ## 🤗 Want to support my work?
17
+ <p align="center">
18
+ <a href="https://buymeacoffee.com/erlichsefi" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;">
19
+ </a>
20
+ </p>
21
+
22
+ Got a question?
23
+ ---------------
24
+
25
+ You can email me at erlichsefi@gmail.com
26
+
27
+ If you think you've found a bug:
28
+
29
+ - Create issue in [issue tracker](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/issues) to see if
30
+ it's already been reported
31
+ - Please consider solving the issue by yourself and creating a pull request.
32
+
33
+ What is il_supermarket_parsers?
34
+ -------------
35
+
36
+ A simple access layer to the data the supermarkets publish.
37
+ You can download the data using https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.
38
+
39
+ You only need to run the following code to parse all the data currently shared by the supermarkets and was downloaded using the package above.
40
+
41
+ ```python
42
+ from il_supermarket_parsers import ConvertingTask
43
+
44
+ scraper = ConvertingTask(data_folder="dumps")
45
+ scraper.run()
46
+ ```
47
+
48
+
49
+ Quick start
50
+ -----------
51
+
52
+ il_supermarket_parsers can be installed using pip:
53
+
54
+ python3 pip install il_supermarket_parsers
55
+
56
+ If you want to run the latest version of the code, you can install it from the
57
+ repo directly:
58
+
59
+ python3 -m pip install -U git+https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.git
60
+ # or if you don't have 'git' installed
61
+ python3 -m pip install -U https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/main
62
+
63
+
64
+ Contributing
65
+ ------------
66
+
67
+ Help in testing, development, documentation and other tasks is
68
+ highly appreciated and useful to the project. There are tasks for
69
+ contributors of all experience levels.
70
+
71
+ If you need help getting started, don't hesitate to contact me.
72
+
73
+
74
+ Development status
75
+ ------------------
76
+
77
+ IL SuperMarket Parser is beta software, as far as i see devlopment stoped until new issues will be found.
@@ -0,0 +1,104 @@
1
+ Metadata-Version: 2.1
2
+ Name: il-supermarket-parser
3
+ Version: 0.0.1
4
+ Summary: python package that process the data dumped by the israeli supermarket
5
+ Home-page: https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers
6
+ Author: Sefi Erlich
7
+ Author-email: erlichsefi@gmail.com
8
+ License: MIT
9
+ Keywords: israel,israeli,scraper,supermarket
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Topic :: Software Development :: Build Tools
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.4
16
+ Classifier: Programming Language :: Python :: 3.5
17
+ Classifier: Programming Language :: Python :: 3.6
18
+ Description-Content-Type: text/markdown
19
+ Requires-Dist: pandas==2.2.2
20
+ Requires-Dist: lxml==5.2.1
21
+ Requires-Dist: pymongo==4.6.3
22
+ Requires-Dist: il-supermarket-scraper>=0.4.6
23
+ Requires-Dist: tqdm==4.66
24
+ Provides-Extra: test
25
+ Requires-Dist: pytest; extra == "test"
26
+
27
+
28
+ Israel Supermarket Parsers: Clients to parser the data published by the supermarkets.
29
+ =======================================
30
+ This is a parser for ALL the supermarket chains listed in the GOV.IL site.
31
+
32
+ שקיפות מחירים (השוואת מחירים) - https://www.gov.il/he/departments/legalInfo/cpfta_prices_regulations
33
+ הגדרת הקבצים שהיו צריכים להיות זמינים באתרים - https://www.nevo.co.il/law_html/law01/501_131.htm
34
+
35
+
36
+ [![Unit & Integration Tests](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml/badge.svg?event=push)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/test-suite.yml)
37
+ [![CodeQL](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/codeql.yml)
38
+ [![Pylint](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/pylint.yml)
39
+ [![Publish Docker image](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/docker-publish.yml)
40
+ [![Upload Python Package](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml/badge.svg)](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/actions/workflows/python-publish.yml)
41
+
42
+
43
+ ## 🤗 Want to support my work?
44
+ <p align="center">
45
+ <a href="https://buymeacoffee.com/erlichsefi" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;">
46
+ </a>
47
+ </p>
48
+
49
+ Got a question?
50
+ ---------------
51
+
52
+ You can email me at erlichsefi@gmail.com
53
+
54
+ If you think you've found a bug:
55
+
56
+ - Create issue in [issue tracker](https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/issues) to see if
57
+ it's already been reported
58
+ - Please consider solving the issue by yourself and creating a pull request.
59
+
60
+ What is il_supermarket_parsers?
61
+ -------------
62
+
63
+ A simple access layer to the data the supermarkets publish.
64
+ You can download the data using https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.
65
+
66
+ You only need to run the following code to parse all the data currently shared by the supermarkets and was downloaded using the package above.
67
+
68
+ ```python
69
+ from il_supermarket_parsers import ConvertingTask
70
+
71
+ scraper = ConvertingTask(data_folder="dumps")
72
+ scraper.run()
73
+ ```
74
+
75
+
76
+ Quick start
77
+ -----------
78
+
79
+ il_supermarket_parsers can be installed using pip:
80
+
81
+ python3 pip install il_supermarket_parsers
82
+
83
+ If you want to run the latest version of the code, you can install it from the
84
+ repo directly:
85
+
86
+ python3 -m pip install -U git+https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers.git
87
+ # or if you don't have 'git' installed
88
+ python3 -m pip install -U https://github.com/OpenIsraeliSupermarkets/israeli-supermarket-parsers/main
89
+
90
+
91
+ Contributing
92
+ ------------
93
+
94
+ Help in testing, development, documentation and other tasks is
95
+ highly appreciated and useful to the project. There are tasks for
96
+ contributors of all experience levels.
97
+
98
+ If you need help getting started, don't hesitate to contact me.
99
+
100
+
101
+ Development status
102
+ ------------------
103
+
104
+ IL SuperMarket Parser is beta software, as far as i see devlopment stoped until new issues will be found.
@@ -0,0 +1,46 @@
1
+ MANIFEST.in
2
+ README.md
3
+ requirements-dev.txt
4
+ requirements.txt
5
+ setup.cfg
6
+ setup.py
7
+ il_supermarket_parser.egg-info/PKG-INFO
8
+ il_supermarket_parser.egg-info/SOURCES.txt
9
+ il_supermarket_parser.egg-info/dependency_links.txt
10
+ il_supermarket_parser.egg-info/requires.txt
11
+ il_supermarket_parser.egg-info/top_level.txt
12
+ il_supermarket_parsers/__init__.py
13
+ il_supermarket_parsers/main.py
14
+ il_supermarket_parsers/multiprocess_pharser.py
15
+ il_supermarket_parsers/parser_factroy.py
16
+ il_supermarket_parsers/raw_parsing_pipeline.py
17
+ il_supermarket_parsers/conf/__init__.py
18
+ il_supermarket_parsers/documents/__init__.py
19
+ il_supermarket_parsers/documents/base.py
20
+ il_supermarket_parsers/documents/xml_dataframe_parser.py
21
+ il_supermarket_parsers/documents/xml_dataframe_subroot_praser.py
22
+ il_supermarket_parsers/engines/__init__.py
23
+ il_supermarket_parsers/engines/base.py
24
+ il_supermarket_parsers/engines/big_id.py
25
+ il_supermarket_parsers/engines/branches.py
26
+ il_supermarket_parsers/normlizers/base.py
27
+ il_supermarket_parsers/parsers/__init__.py
28
+ il_supermarket_parsers/parsers/bareket.py
29
+ il_supermarket_parsers/parsers/confix.py
30
+ il_supermarket_parsers/parsers/het_cohen.py
31
+ il_supermarket_parsers/parsers/mahsani_a_shuk.py
32
+ il_supermarket_parsers/parsers/other.py
33
+ il_supermarket_parsers/parsers/salach_dabach.py
34
+ il_supermarket_parsers/parsers/shufersal.py
35
+ il_supermarket_parsers/parsers/super_pharm.py
36
+ il_supermarket_parsers/parsers/tiv_taam.py
37
+ il_supermarket_parsers/parsers/victory.py
38
+ il_supermarket_parsers/utils/__init__.py
39
+ il_supermarket_parsers/utils/data_loader.py
40
+ il_supermarket_parsers/utils/database.py
41
+ il_supermarket_parsers/utils/dataframe_utils.py
42
+ il_supermarket_parsers/utils/diff.py
43
+ il_supermarket_parsers/utils/logger.py
44
+ il_supermarket_parsers/utils/multi_prcoessing.py
45
+ il_supermarket_parsers/utils/test_utils.py
46
+ il_supermarket_parsers/utils/xml_utils.py
@@ -0,0 +1,8 @@
1
+ pandas==2.2.2
2
+ lxml==5.2.1
3
+ pymongo==4.6.3
4
+ il-supermarket-scraper>=0.4.6
5
+ tqdm==4.66
6
+
7
+ [test]
8
+ pytest
@@ -0,0 +1 @@
1
+ il_supermarket_parsers
@@ -0,0 +1,3 @@
1
+ from .main import ConvertingTask
2
+ from .parser_factroy import ParserFactory
3
+ from .utils import FileTypesFilters
@@ -0,0 +1,3 @@
1
+ from .base import XmlBaseConverter
2
+ from .xml_dataframe_parser import XmlDataFrameConverter
3
+ from .xml_dataframe_subroot_praser import SubRootedXmlDataFrameConverter
@@ -0,0 +1,61 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import List
3
+ import os
4
+ from il_supermarket_parsers.utils import build_value, get_root
5
+
6
+
7
+ class XmlBaseConverter(ABC):
8
+ """parser the xml docuement"""
9
+
10
+ def __init__(
11
+ self,
12
+ list_key: List[str],
13
+ id_field: str,
14
+ roots=None,
15
+ ignore_column=None,
16
+ **additional_constant,
17
+ ):
18
+ self.list_key = list_key
19
+ self.roots = roots
20
+ self.id_field = id_field
21
+ self.ignore_column = ignore_column if ignore_column else []
22
+ self.additional_constant = additional_constant
23
+
24
+ @abstractmethod
25
+ def validate_succussful_extraction(
26
+ self, data, source_file, ignore_missing_columns=None
27
+ ):
28
+ """validate column requested"""
29
+
30
+ @abstractmethod
31
+ def reduce_size(self, data):
32
+ """reduce the size"""
33
+
34
+ def build_value(self, name, no_content):
35
+ """get the value"""
36
+ return build_value(name, self.additional_constant, no_content=no_content)
37
+
38
+ def convert(self, found_store, file_name, **kwarg):
39
+ """parse file to data frame"""
40
+ source_file = os.path.join(found_store, file_name)
41
+ root, root_store = get_root(source_file, self.list_key, self.roots)
42
+
43
+ data = self._phrse(
44
+ root,
45
+ found_store,
46
+ file_name,
47
+ root_store,
48
+ **kwarg,
49
+ )
50
+ return self.reduce_size(data)
51
+
52
+ @abstractmethod
53
+ def _phrse(
54
+ self,
55
+ root,
56
+ found_folder,
57
+ file_name,
58
+ root_store,
59
+ **kwarg,
60
+ ):
61
+ pass
@@ -0,0 +1,100 @@
1
+ import pandas as pd
2
+ from il_supermarket_parsers.utils import (
3
+ count_tag_in_xml,
4
+ collect_unique_keys_from_xml,
5
+ collect_unique_columns_from_nested_json,
6
+ )
7
+ from .base import XmlBaseConverter
8
+
9
+
10
+ class XmlDataFrameConverter(XmlBaseConverter):
11
+ """parser the xml docuement"""
12
+
13
+ def reduce_size(self, data):
14
+ for col in data.columns:
15
+ data[col] = data[col].mask(data[col] == data[col].shift())
16
+ return data
17
+
18
+ def validate_succussful_extraction(
19
+ self, data, source_file, ignore_missing_columns=None
20
+ ):
21
+ # if there is an empty file
22
+ # we expected it to return none
23
+ tag_count = count_tag_in_xml(source_file, self.id_field)
24
+
25
+ if self.roots and tag_count > 0:
26
+ for root in self.roots:
27
+ if root not in data.columns:
28
+ raise ValueError(
29
+ f"parse error for file {source_file},"
30
+ f"columns {root} missing from {data.columns}"
31
+ )
32
+
33
+ if self.id_field not in data.columns:
34
+ raise ValueError(
35
+ f"parse error for file {source_file}, "
36
+ f"id {self.id_field} missing from {data.columns}"
37
+ )
38
+
39
+ if data.shape[0] != tag_count:
40
+ raise ValueError(
41
+ f"for file {source_file}, missing data,"
42
+ f"data shape {data.shape} tag count is {tag_count}"
43
+ )
44
+
45
+ ignore_list = self.ignore_column
46
+ if ignore_missing_columns:
47
+ ignore_list = ignore_list + ignore_missing_columns
48
+ keys_not_used = (
49
+ set(collect_unique_keys_from_xml(source_file))
50
+ - collect_unique_columns_from_nested_json(data)
51
+ - set(ignore_list)
52
+ )
53
+ if len(keys_not_used) > 0:
54
+ raise ValueError(
55
+ f"for file {source_file}, there is data we didn't get {keys_not_used}"
56
+ )
57
+ assert "found_folder" in data.columns
58
+ assert "file_name" in data.columns
59
+
60
+ def list_single_entry(self, elem, found_folder, file_name, **sub_root_store):
61
+ """build a single row"""
62
+ values = {
63
+ "found_folder": found_folder,
64
+ "file_name": file_name,
65
+ **sub_root_store,
66
+ }
67
+ for name in list(elem):
68
+ tag = name.tag
69
+ value = self.build_value(name, no_content="")
70
+
71
+ values[tag] = value
72
+ return values.copy()
73
+
74
+ def _phrse(
75
+ self,
76
+ root,
77
+ found_folder,
78
+ file_name,
79
+ root_store,
80
+ **kwarg,
81
+ ):
82
+ rows = []
83
+ columns = [self.id_field, "found_folder", "file_name"] + (
84
+ self.roots if self.roots else []
85
+ )
86
+
87
+ if root is None:
88
+ return pd.DataFrame(columns=columns)
89
+
90
+ elements = list(root)
91
+ if len(root) == 0:
92
+
93
+ return pd.DataFrame(columns=columns)
94
+
95
+ for elem in elements:
96
+ rows.append(
97
+ self.list_single_entry(elem, found_folder, file_name, **root_store)
98
+ )
99
+
100
+ return pd.DataFrame(rows)
@@ -0,0 +1,77 @@
1
+ import pandas as pd
2
+ from .xml_dataframe_parser import XmlDataFrameConverter
3
+
4
+
5
+ class SubRootedXmlDataFrameConverter(XmlDataFrameConverter):
6
+ """parser the xml docuement with extra indentations"""
7
+
8
+ def __init__(
9
+ self,
10
+ list_key,
11
+ id_field,
12
+ roots=None,
13
+ sub_roots=None,
14
+ list_sub_key="",
15
+ ignore_column=None,
16
+ **additional_constant,
17
+ ):
18
+ super().__init__(
19
+ list_key=list_key,
20
+ id_field=id_field,
21
+ roots=roots,
22
+ ignore_column=ignore_column,
23
+ additional_constant=additional_constant,
24
+ )
25
+ self.sub_roots = sub_roots if sub_roots else []
26
+ self.list_sub_key = list_sub_key
27
+
28
+ def validate_succussful_extraction(
29
+ self, data, source_file, ignore_missing_columns=None
30
+ ):
31
+ """validation"""
32
+ super().validate_succussful_extraction(
33
+ data, source_file, ignore_missing_columns=ignore_missing_columns
34
+ )
35
+
36
+ # if the user asked to include the headers
37
+ if self.sub_roots:
38
+ for root in self.sub_roots:
39
+ if root not in data.columns:
40
+ raise ValueError(
41
+ f"parse error for file {source_file}, "
42
+ f"columns {root} missing from {data.columns}"
43
+ )
44
+
45
+ def _phrse(
46
+ self,
47
+ root,
48
+ found_folder,
49
+ file_name,
50
+ root_store,
51
+ **_,
52
+ ):
53
+ """parse file to data frame"""
54
+
55
+ rows = []
56
+
57
+ if root is None or len(root) == 0:
58
+ return pd.DataFrame(
59
+ columns=self.sub_roots
60
+ + [self.id_field, "found_folder", "file_name"]
61
+ + (self.roots if self.roots else [])
62
+ )
63
+
64
+ for sub_elem in list(root):
65
+ sub_root_store = root_store.copy()
66
+
67
+ for k in self.sub_roots:
68
+ sub_root_store[k] = sub_elem.find(k).text
69
+
70
+ for elem in sub_elem.find(self.list_sub_key):
71
+ rows.append(
72
+ self.list_single_entry(
73
+ elem, found_folder, file_name, **sub_root_store
74
+ )
75
+ )
76
+
77
+ return pd.DataFrame(rows)
@@ -0,0 +1,3 @@
1
+ from .base import BaseFileConverter
2
+ from .big_id import BigIDFileConverter
3
+ from .branches import BigIdBranchesFileConverter