odoo-addon-base-import-async 16.0.1.2.0.1__py3-none-any.whl → 18.0.1.0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- odoo/addons/base_import_async/README.rst +57 -52
- odoo/addons/base_import_async/__manifest__.py +3 -3
- odoo/addons/base_import_async/data/queue_job_function_data.xml +0 -7
- odoo/addons/base_import_async/i18n/base_import_async.pot +5 -12
- odoo/addons/base_import_async/i18n/tr.po +6 -8
- odoo/addons/base_import_async/models/base_import_import.py +5 -3
- odoo/addons/base_import_async/models/queue_job.py +2 -4
- odoo/addons/base_import_async/readme/CONTRIBUTORS.md +25 -0
- odoo/addons/base_import_async/readme/CREDITS.md +2 -0
- odoo/addons/base_import_async/readme/DESCRIPTION.md +2 -0
- odoo/addons/base_import_async/readme/HISTORY.md +3 -0
- odoo/addons/base_import_async/readme/ROADMAP.md +4 -0
- odoo/addons/base_import_async/readme/USAGE.md +25 -0
- odoo/addons/base_import_async/static/description/index.html +32 -39
- odoo/addons/base_import_async/static/src/js/import_model.esm.js +26 -0
- odoo/addons/base_import_async/static/src/xml/{import.xml → import_data_sidepanel.xml} +9 -4
- odoo/addons/base_import_async/tests/__init__.py +1 -0
- odoo/addons/base_import_async/tests/test_base_import_import.py +99 -0
- {odoo_addon_base_import_async-16.0.1.2.0.1.dist-info → odoo_addon_base_import_async-18.0.1.0.0.4.dist-info}/METADATA +63 -58
- odoo_addon_base_import_async-18.0.1.0.0.4.dist-info/RECORD +27 -0
- {odoo_addon_base_import_async-16.0.1.2.0.1.dist-info → odoo_addon_base_import_async-18.0.1.0.0.4.dist-info}/WHEEL +1 -1
- odoo_addon_base_import_async-18.0.1.0.0.4.dist-info/top_level.txt +1 -0
- odoo/addons/base_import_async/readme/CONTRIBUTORS.rst +0 -17
- odoo/addons/base_import_async/readme/CREDITS.rst +0 -1
- odoo/addons/base_import_async/readme/DESCRIPTION.rst +0 -3
- odoo/addons/base_import_async/readme/HISTORY.rst +0 -4
- odoo/addons/base_import_async/readme/ROADMAP.rst +0 -4
- odoo/addons/base_import_async/readme/USAGE.rst +0 -29
- odoo/addons/base_import_async/static/src/js/import.js +0 -30
- odoo_addon_base_import_async-16.0.1.2.0.1.dist-info/RECORD +0 -25
- odoo_addon_base_import_async-16.0.1.2.0.1.dist-info/top_level.txt +0 -1
|
@@ -7,7 +7,7 @@ Asynchronous Import
|
|
|
7
7
|
!! This file is generated by oca-gen-addon-readme !!
|
|
8
8
|
!! changes will be overwritten. !!
|
|
9
9
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
10
|
-
!! source digest: sha256:
|
|
10
|
+
!! source digest: sha256:299c64b165f6348dfa062b8925a95235390daa6063964f21a5a1d0757cdf1614
|
|
11
11
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
12
12
|
|
|
13
13
|
.. |badge1| image:: https://img.shields.io/badge/maturity-Production%2FStable-green.png
|
|
@@ -17,20 +17,19 @@ Asynchronous Import
|
|
|
17
17
|
:target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
|
|
18
18
|
:alt: License: AGPL-3
|
|
19
19
|
.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github
|
|
20
|
-
:target: https://github.com/OCA/queue/tree/
|
|
20
|
+
:target: https://github.com/OCA/queue/tree/18.0/base_import_async
|
|
21
21
|
:alt: OCA/queue
|
|
22
22
|
.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
|
|
23
|
-
:target: https://translation.odoo-community.org/projects/queue-
|
|
23
|
+
:target: https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async
|
|
24
24
|
:alt: Translate me on Weblate
|
|
25
25
|
.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
|
|
26
|
-
:target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=
|
|
26
|
+
:target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0
|
|
27
27
|
:alt: Try me on Runboat
|
|
28
28
|
|
|
29
29
|
|badge1| |badge2| |badge3| |badge4| |badge5|
|
|
30
30
|
|
|
31
|
-
This module extends the standard CSV import functionality
|
|
32
|
-
|
|
33
|
-
framework.
|
|
31
|
+
This module extends the standard CSV import functionality to import
|
|
32
|
+
files in the background using the OCA/queue framework.
|
|
34
33
|
|
|
35
34
|
**Table of contents**
|
|
36
35
|
|
|
@@ -40,51 +39,47 @@ framework.
|
|
|
40
39
|
Usage
|
|
41
40
|
=====
|
|
42
41
|
|
|
43
|
-
The user is presented with a new checkbox in the import
|
|
44
|
-
|
|
45
|
-
job.
|
|
42
|
+
The user is presented with a new checkbox in the import screen. When
|
|
43
|
+
selected, the import is delayed in a background job.
|
|
46
44
|
|
|
47
|
-
This job in turn splits the CSV file in chunks of minimum
|
|
48
|
-
|
|
49
|
-
|
|
45
|
+
This job in turn splits the CSV file in chunks of minimum 100 lines (or
|
|
46
|
+
more to align with record boundaries). Each chunk is then imported in a
|
|
47
|
+
separate background job.
|
|
50
48
|
|
|
51
|
-
When an import fails, the job is marked as such and the
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
in synchronous mode since the chunks are small.
|
|
49
|
+
When an import fails, the job is marked as such and the user can read
|
|
50
|
+
the error in the job status. The CSV chunk being imported is stored as
|
|
51
|
+
an attachment to the job, making it easy to download it, fix it and run
|
|
52
|
+
a new import, possibly in synchronous mode since the chunks are small.
|
|
56
53
|
|
|
57
|
-
Any file that can be imported by the standard import mechanism
|
|
58
|
-
|
|
54
|
+
Any file that can be imported by the standard import mechanism can also
|
|
55
|
+
be imported in the background.
|
|
59
56
|
|
|
60
|
-
This module's scope is limited to making standard imports
|
|
61
|
-
|
|
62
|
-
automate ETL flows.
|
|
57
|
+
This module's scope is limited to making standard imports asynchronous.
|
|
58
|
+
It does not attempt to transform the data nor automate ETL flows.
|
|
63
59
|
|
|
64
60
|
Other modules may benefit from this infrastructure in the following way
|
|
65
61
|
(as illustrated in the test suite):
|
|
66
62
|
|
|
67
|
-
1. create an instance of
|
|
68
|
-
(
|
|
69
|
-
2. invoke the
|
|
70
|
-
|
|
71
|
-
`use_queue`, `chunk_size`).
|
|
63
|
+
1. create an instance of base_import.import and populate its fields
|
|
64
|
+
(res_model, file, file_name),
|
|
65
|
+
2. invoke the do method with appropriate options (header, encoding,
|
|
66
|
+
separator, quoting, use_queue, chunk_size).
|
|
72
67
|
|
|
73
68
|
Known issues / Roadmap
|
|
74
69
|
======================
|
|
75
70
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
71
|
+
- There is currently no user interface to control the chunk size, which
|
|
72
|
+
is currently 100 by default. Should this proves to be an issue, it is
|
|
73
|
+
easy to add an option to extend the import screen.
|
|
74
|
+
- Validation cannot be run in the background.
|
|
80
75
|
|
|
81
76
|
Changelog
|
|
82
77
|
=========
|
|
83
78
|
|
|
84
79
|
13.0.1.0.0 (2019-12-20)
|
|
85
|
-
|
|
80
|
+
-----------------------
|
|
86
81
|
|
|
87
|
-
|
|
82
|
+
- [MIGRATION] from 12.0 branched at rev. a7f8031
|
|
88
83
|
|
|
89
84
|
Bug Tracker
|
|
90
85
|
===========
|
|
@@ -92,7 +87,7 @@ Bug Tracker
|
|
|
92
87
|
Bugs are tracked on `GitHub Issues <https://github.com/OCA/queue/issues>`_.
|
|
93
88
|
In case of trouble, please check there if your issue has already been reported.
|
|
94
89
|
If you spotted it first, help us to smash it by providing a detailed and welcomed
|
|
95
|
-
`feedback <https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%
|
|
90
|
+
`feedback <https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
|
|
96
91
|
|
|
97
92
|
Do not contact contributors directly about support or help with technical issues.
|
|
98
93
|
|
|
@@ -100,39 +95,49 @@ Credits
|
|
|
100
95
|
=======
|
|
101
96
|
|
|
102
97
|
Authors
|
|
103
|
-
|
|
98
|
+
-------
|
|
104
99
|
|
|
105
100
|
* Akretion
|
|
106
101
|
* ACSONE SA/NV
|
|
107
102
|
|
|
108
103
|
Contributors
|
|
109
|
-
|
|
104
|
+
------------
|
|
110
105
|
|
|
111
106
|
Sébastien Beau (Akretion) authored the initial prototype.
|
|
112
107
|
|
|
113
108
|
Stéphane Bidoul (ACSONE) extended it to version 1.0 to support
|
|
114
|
-
multi-line records, store data to import as attachments
|
|
115
|
-
|
|
109
|
+
multi-line records, store data to import as attachments and let the user
|
|
110
|
+
control the asynchronous behaviour.
|
|
116
111
|
|
|
117
112
|
Other contributors include:
|
|
118
113
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
114
|
+
- Anthony Muschang (ACSONE)
|
|
115
|
+
|
|
116
|
+
- David Béal (Akretion)
|
|
117
|
+
|
|
118
|
+
- Jonathan Nemry (ACSONE)
|
|
119
|
+
|
|
120
|
+
- Laurent Mignon (ACSONE)
|
|
121
|
+
|
|
122
|
+
- Dennis Sluijk (Onestein)
|
|
123
|
+
|
|
124
|
+
- Guewen Baconnier (Camptocamp)
|
|
125
|
+
|
|
126
|
+
- `Trobz <https://trobz.com>`__:
|
|
127
|
+
|
|
128
|
+
- Dzung Tran <dungtd@trobz.com>
|
|
129
|
+
- Do Anh Duy <duyda@trobz.com>
|
|
130
|
+
|
|
131
|
+
- Daniel Duque (FactorLibre)
|
|
128
132
|
|
|
129
133
|
Other credits
|
|
130
|
-
|
|
134
|
+
-------------
|
|
131
135
|
|
|
132
|
-
The migration of this module from
|
|
136
|
+
The migration of this module from 17.0 to 18.0 was financially supported
|
|
137
|
+
by Camptocamp
|
|
133
138
|
|
|
134
139
|
Maintainers
|
|
135
|
-
|
|
140
|
+
-----------
|
|
136
141
|
|
|
137
142
|
This module is maintained by the OCA.
|
|
138
143
|
|
|
@@ -144,6 +149,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose
|
|
|
144
149
|
mission is to support the collaborative development of Odoo features and
|
|
145
150
|
promote its widespread use.
|
|
146
151
|
|
|
147
|
-
This module is part of the `OCA/queue <https://github.com/OCA/queue/tree/
|
|
152
|
+
This module is part of the `OCA/queue <https://github.com/OCA/queue/tree/18.0/base_import_async>`_ project on GitHub.
|
|
148
153
|
|
|
149
154
|
You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
{
|
|
6
6
|
"name": "Asynchronous Import",
|
|
7
7
|
"summary": "Import CSV files in the background",
|
|
8
|
-
"version": "
|
|
8
|
+
"version": "18.0.1.0.0",
|
|
9
9
|
"author": "Akretion, ACSONE SA/NV, Odoo Community Association (OCA)",
|
|
10
10
|
"license": "AGPL-3",
|
|
11
11
|
"website": "https://github.com/OCA/queue",
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
],
|
|
17
17
|
"assets": {
|
|
18
18
|
"web.assets_backend": [
|
|
19
|
-
"base_import_async/static/src/js/
|
|
20
|
-
"base_import_async/static/src/xml/
|
|
19
|
+
"base_import_async/static/src/js/import_model.esm.js",
|
|
20
|
+
"base_import_async/static/src/xml/import_data_sidepanel.xml",
|
|
21
21
|
],
|
|
22
22
|
},
|
|
23
23
|
"installable": True,
|
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
<odoo noupdate="1">
|
|
2
|
-
<record model="queue.job.channel" id="channel_base_import">
|
|
3
|
-
<field name="name">base_import</field>
|
|
4
|
-
<field name="parent_id" ref="queue_job.channel_root" />
|
|
5
|
-
</record>
|
|
6
|
-
|
|
7
2
|
<record id="job_function_base_import_import_split_file" model="queue.job.function">
|
|
8
3
|
<field name="model_id" ref="base_import.model_base_import_import" />
|
|
9
4
|
<field name="method">_split_file</field>
|
|
10
|
-
<field name="channel_id" ref="channel_base_import" />
|
|
11
5
|
<field
|
|
12
6
|
name="related_action"
|
|
13
7
|
eval='{"func_name": "_related_action_attachment"}'
|
|
@@ -19,7 +13,6 @@
|
|
|
19
13
|
>
|
|
20
14
|
<field name="model_id" ref="base_import.model_base_import_import" />
|
|
21
15
|
<field name="method">_import_one_chunk</field>
|
|
22
|
-
<field name="channel_id" ref="channel_base_import" />
|
|
23
16
|
<field
|
|
24
17
|
name="related_action"
|
|
25
18
|
eval='{"func_name": "_related_action_attachment"}'
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
#
|
|
5
5
|
msgid ""
|
|
6
6
|
msgstr ""
|
|
7
|
-
"Project-Id-Version: Odoo Server
|
|
7
|
+
"Project-Id-Version: Odoo Server 18.0\n"
|
|
8
8
|
"Report-Msgid-Bugs-To: \n"
|
|
9
9
|
"Last-Translator: \n"
|
|
10
10
|
"Language-Team: \n"
|
|
@@ -16,7 +16,6 @@ msgstr ""
|
|
|
16
16
|
#. module: base_import_async
|
|
17
17
|
#. odoo-python
|
|
18
18
|
#: code:addons/base_import_async/models/queue_job.py:0
|
|
19
|
-
#, python-format
|
|
20
19
|
msgid "Attachment"
|
|
21
20
|
msgstr ""
|
|
22
21
|
|
|
@@ -28,7 +27,6 @@ msgstr ""
|
|
|
28
27
|
#. module: base_import_async
|
|
29
28
|
#. odoo-python
|
|
30
29
|
#: code:addons/base_import_async/models/base_import_import.py:0
|
|
31
|
-
#, python-format
|
|
32
30
|
msgid ""
|
|
33
31
|
"Import %(model)s from file %(file_name)s - #%(chunk)s - lines %(from)s to "
|
|
34
32
|
"%(to)s"
|
|
@@ -37,14 +35,12 @@ msgstr ""
|
|
|
37
35
|
#. module: base_import_async
|
|
38
36
|
#. odoo-python
|
|
39
37
|
#: code:addons/base_import_async/models/base_import_import.py:0
|
|
40
|
-
#, python-format
|
|
41
38
|
msgid "Import %(model)s from file %(from_file)s"
|
|
42
39
|
msgstr ""
|
|
43
40
|
|
|
44
41
|
#. module: base_import_async
|
|
45
42
|
#. odoo-javascript
|
|
46
|
-
#: code:addons/base_import_async/static/src/xml/
|
|
47
|
-
#, python-format
|
|
43
|
+
#: code:addons/base_import_async/static/src/xml/import_data_sidepanel.xml:0
|
|
48
44
|
msgid "Import in the background"
|
|
49
45
|
msgstr ""
|
|
50
46
|
|
|
@@ -55,8 +51,7 @@ msgstr ""
|
|
|
55
51
|
|
|
56
52
|
#. module: base_import_async
|
|
57
53
|
#. odoo-javascript
|
|
58
|
-
#: code:addons/base_import_async/static/src/xml/
|
|
59
|
-
#, python-format
|
|
54
|
+
#: code:addons/base_import_async/static/src/xml/import_data_sidepanel.xml:0
|
|
60
55
|
msgid ""
|
|
61
56
|
"When checked, the import will be executed as a background job, after "
|
|
62
57
|
"splitting your file in small chunks that will be processed independently. "
|
|
@@ -65,14 +60,12 @@ msgstr ""
|
|
|
65
60
|
|
|
66
61
|
#. module: base_import_async
|
|
67
62
|
#. odoo-javascript
|
|
68
|
-
#: code:addons/base_import_async/static/src/js/
|
|
69
|
-
#, python-format
|
|
63
|
+
#: code:addons/base_import_async/static/src/js/import_model.esm.js:0
|
|
70
64
|
msgid "You can check the status of this job in menu 'Queue / Jobs'."
|
|
71
65
|
msgstr ""
|
|
72
66
|
|
|
73
67
|
#. module: base_import_async
|
|
74
68
|
#. odoo-javascript
|
|
75
|
-
#: code:addons/base_import_async/static/src/js/
|
|
76
|
-
#, python-format
|
|
69
|
+
#: code:addons/base_import_async/static/src/js/import_model.esm.js:0
|
|
77
70
|
msgid "Your request is being processed"
|
|
78
71
|
msgstr ""
|
|
@@ -7,15 +7,15 @@ msgstr ""
|
|
|
7
7
|
"Project-Id-Version: Odoo Server 15.0-20221029\n"
|
|
8
8
|
"Report-Msgid-Bugs-To: \n"
|
|
9
9
|
"POT-Creation-Date: 2022-11-24 07:21+0000\n"
|
|
10
|
-
"PO-Revision-Date:
|
|
11
|
-
"Last-Translator:
|
|
10
|
+
"PO-Revision-Date: 2022-11-24 10:25+0300\n"
|
|
11
|
+
"Last-Translator: \n"
|
|
12
12
|
"Language-Team: \n"
|
|
13
|
-
"Language:
|
|
13
|
+
"Language: tr_TR\n"
|
|
14
14
|
"MIME-Version: 1.0\n"
|
|
15
15
|
"Content-Type: text/plain; charset=UTF-8\n"
|
|
16
16
|
"Content-Transfer-Encoding: 8bit\n"
|
|
17
|
-
"Plural-Forms: nplurals=2; plural=n != 1;\n"
|
|
18
|
-
"X-Generator:
|
|
17
|
+
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
|
18
|
+
"X-Generator: Poedit 2.4.2\n"
|
|
19
19
|
|
|
20
20
|
#. module: base_import_async
|
|
21
21
|
#. odoo-python
|
|
@@ -37,15 +37,13 @@ msgid ""
|
|
|
37
37
|
"Import %(model)s from file %(file_name)s - #%(chunk)s - lines %(from)s to "
|
|
38
38
|
"%(to)s"
|
|
39
39
|
msgstr ""
|
|
40
|
-
"%(model)s dosyasını %(file_name)s dosyasından içe aktarın - #%(chunk)s - "
|
|
41
|
-
"%(from)s ile %(to)s arasındaki satırları"
|
|
42
40
|
|
|
43
41
|
#. module: base_import_async
|
|
44
42
|
#. odoo-python
|
|
45
43
|
#: code:addons/base_import_async/models/base_import_import.py:0
|
|
46
44
|
#, python-format
|
|
47
45
|
msgid "Import %(model)s from file %(from_file)s"
|
|
48
|
-
msgstr "
|
|
46
|
+
msgstr ""
|
|
49
47
|
|
|
50
48
|
#. module: base_import_async
|
|
51
49
|
#. odoo-javascript
|
|
@@ -12,6 +12,7 @@ from os.path import splitext
|
|
|
12
12
|
from odoo import _, api, models
|
|
13
13
|
from odoo.models import fix_import_export_id_paths
|
|
14
14
|
|
|
15
|
+
from odoo.addons.base_import.models.base_import import ImportValidationError
|
|
15
16
|
from odoo.addons.queue_job.exception import FailedJobError
|
|
16
17
|
|
|
17
18
|
# options defined in base_import/import.js
|
|
@@ -42,8 +43,8 @@ class BaseImportImport(models.TransientModel):
|
|
|
42
43
|
data, import_fields = self._convert_import_data(fields, options)
|
|
43
44
|
# Parse date and float field
|
|
44
45
|
data = self._parse_import_data(data, import_fields, options)
|
|
45
|
-
except ValueError as e:
|
|
46
|
-
return {"messages": [
|
|
46
|
+
except (ImportValidationError, ValueError) as e:
|
|
47
|
+
return {"messages": [e.__dict__]}
|
|
47
48
|
|
|
48
49
|
# get the translated model name to build
|
|
49
50
|
# a meaningful job description
|
|
@@ -104,7 +105,8 @@ class BaseImportImport(models.TransientModel):
|
|
|
104
105
|
)
|
|
105
106
|
return attachment
|
|
106
107
|
|
|
107
|
-
|
|
108
|
+
@staticmethod
|
|
109
|
+
def _read_csv_attachment(attachment, options):
|
|
108
110
|
decoded_datas = base64.decodebytes(attachment.datas)
|
|
109
111
|
encoding = options.get(OPT_ENCODING) or "utf-8"
|
|
110
112
|
f = TextIOWrapper(BytesIO(decoded_datas), encoding=encoding)
|
|
@@ -10,12 +10,10 @@ class QueueJob(models.Model):
|
|
|
10
10
|
_inherit = "queue.job"
|
|
11
11
|
|
|
12
12
|
def _related_action_attachment(self):
|
|
13
|
-
|
|
14
|
-
action = {
|
|
13
|
+
return {
|
|
15
14
|
"name": _("Attachment"),
|
|
16
15
|
"type": "ir.actions.act_window",
|
|
17
16
|
"res_model": "ir.attachment",
|
|
18
17
|
"view_mode": "form",
|
|
19
|
-
"res_id":
|
|
18
|
+
"res_id": self.kwargs.get("att_id"),
|
|
20
19
|
}
|
|
21
|
-
return action
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Sébastien Beau (Akretion) authored the initial prototype.
|
|
2
|
+
|
|
3
|
+
Stéphane Bidoul (ACSONE) extended it to version 1.0 to support
|
|
4
|
+
multi-line records, store data to import as attachments and let the user
|
|
5
|
+
control the asynchronous behaviour.
|
|
6
|
+
|
|
7
|
+
Other contributors include:
|
|
8
|
+
|
|
9
|
+
- Anthony Muschang (ACSONE)
|
|
10
|
+
|
|
11
|
+
- David Béal (Akretion)
|
|
12
|
+
|
|
13
|
+
- Jonathan Nemry (ACSONE)
|
|
14
|
+
|
|
15
|
+
- Laurent Mignon (ACSONE)
|
|
16
|
+
|
|
17
|
+
- Dennis Sluijk (Onestein)
|
|
18
|
+
|
|
19
|
+
- Guewen Baconnier (Camptocamp)
|
|
20
|
+
|
|
21
|
+
- [Trobz](https://trobz.com):
|
|
22
|
+
- Dzung Tran \<<dungtd@trobz.com>\>
|
|
23
|
+
- Do Anh Duy \<<duyda@trobz.com>\>
|
|
24
|
+
|
|
25
|
+
- Daniel Duque (FactorLibre)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
The user is presented with a new checkbox in the import screen. When
|
|
2
|
+
selected, the import is delayed in a background job.
|
|
3
|
+
|
|
4
|
+
This job in turn splits the CSV file in chunks of minimum 100 lines (or
|
|
5
|
+
more to align with record boundaries). Each chunk is then imported in a
|
|
6
|
+
separate background job.
|
|
7
|
+
|
|
8
|
+
When an import fails, the job is marked as such and the user can read
|
|
9
|
+
the error in the job status. The CSV chunk being imported is stored as
|
|
10
|
+
an attachment to the job, making it easy to download it, fix it and run
|
|
11
|
+
a new import, possibly in synchronous mode since the chunks are small.
|
|
12
|
+
|
|
13
|
+
Any file that can be imported by the standard import mechanism can also
|
|
14
|
+
be imported in the background.
|
|
15
|
+
|
|
16
|
+
This module's scope is limited to making standard imports asynchronous.
|
|
17
|
+
It does not attempt to transform the data nor automate ETL flows.
|
|
18
|
+
|
|
19
|
+
Other modules may benefit from this infrastructure in the following way
|
|
20
|
+
(as illustrated in the test suite):
|
|
21
|
+
|
|
22
|
+
1. create an instance of base_import.import and populate its fields
|
|
23
|
+
(res_model, file, file_name),
|
|
24
|
+
2. invoke the do method with appropriate options (header, encoding,
|
|
25
|
+
separator, quoting, use_queue, chunk_size).
|
|
@@ -367,12 +367,11 @@ ul.auto-toc {
|
|
|
367
367
|
!! This file is generated by oca-gen-addon-readme !!
|
|
368
368
|
!! changes will be overwritten. !!
|
|
369
369
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
370
|
-
!! source digest: sha256:
|
|
370
|
+
!! source digest: sha256:299c64b165f6348dfa062b8925a95235390daa6063964f21a5a1d0757cdf1614
|
|
371
371
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->
|
|
372
|
-
<p><a class="reference external image-reference" href="https://odoo-community.org/page/development-status"><img alt="Production/Stable" src="https://img.shields.io/badge/maturity-Production%2FStable-green.png" /></a> <a class="reference external image-reference" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/licence-AGPL--3-blue.png" /></a> <a class="reference external image-reference" href="https://github.com/OCA/queue/tree/
|
|
373
|
-
<p>This module extends the standard CSV import functionality
|
|
374
|
-
|
|
375
|
-
framework.</p>
|
|
372
|
+
<p><a class="reference external image-reference" href="https://odoo-community.org/page/development-status"><img alt="Production/Stable" src="https://img.shields.io/badge/maturity-Production%2FStable-green.png" /></a> <a class="reference external image-reference" href="http://www.gnu.org/licenses/agpl-3.0-standalone.html"><img alt="License: AGPL-3" src="https://img.shields.io/badge/licence-AGPL--3-blue.png" /></a> <a class="reference external image-reference" href="https://github.com/OCA/queue/tree/18.0/base_import_async"><img alt="OCA/queue" src="https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github" /></a> <a class="reference external image-reference" href="https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async"><img alt="Translate me on Weblate" src="https://img.shields.io/badge/weblate-Translate%20me-F47D42.png" /></a> <a class="reference external image-reference" href="https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0"><img alt="Try me on Runboat" src="https://img.shields.io/badge/runboat-Try%20me-875A7B.png" /></a></p>
|
|
373
|
+
<p>This module extends the standard CSV import functionality to import
|
|
374
|
+
files in the background using the OCA/queue framework.</p>
|
|
376
375
|
<p><strong>Table of contents</strong></p>
|
|
377
376
|
<div class="contents local topic" id="contents">
|
|
378
377
|
<ul class="simple">
|
|
@@ -394,38 +393,34 @@ framework.</p>
|
|
|
394
393
|
</div>
|
|
395
394
|
<div class="section" id="usage">
|
|
396
395
|
<h1><a class="toc-backref" href="#toc-entry-1">Usage</a></h1>
|
|
397
|
-
<p>The user is presented with a new checkbox in the import
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
<p>
|
|
409
|
-
|
|
410
|
-
<p>This module’s scope is limited to making standard imports
|
|
411
|
-
asynchronous. It does not attempt to transform the data nor
|
|
412
|
-
automate ETL flows.</p>
|
|
396
|
+
<p>The user is presented with a new checkbox in the import screen. When
|
|
397
|
+
selected, the import is delayed in a background job.</p>
|
|
398
|
+
<p>This job in turn splits the CSV file in chunks of minimum 100 lines (or
|
|
399
|
+
more to align with record boundaries). Each chunk is then imported in a
|
|
400
|
+
separate background job.</p>
|
|
401
|
+
<p>When an import fails, the job is marked as such and the user can read
|
|
402
|
+
the error in the job status. The CSV chunk being imported is stored as
|
|
403
|
+
an attachment to the job, making it easy to download it, fix it and run
|
|
404
|
+
a new import, possibly in synchronous mode since the chunks are small.</p>
|
|
405
|
+
<p>Any file that can be imported by the standard import mechanism can also
|
|
406
|
+
be imported in the background.</p>
|
|
407
|
+
<p>This module’s scope is limited to making standard imports asynchronous.
|
|
408
|
+
It does not attempt to transform the data nor automate ETL flows.</p>
|
|
413
409
|
<p>Other modules may benefit from this infrastructure in the following way
|
|
414
410
|
(as illustrated in the test suite):</p>
|
|
415
411
|
<ol class="arabic simple">
|
|
416
|
-
<li>create an instance of
|
|
417
|
-
(
|
|
418
|
-
<li>invoke the
|
|
419
|
-
|
|
420
|
-
<cite>use_queue</cite>, <cite>chunk_size</cite>).</li>
|
|
412
|
+
<li>create an instance of base_import.import and populate its fields
|
|
413
|
+
(res_model, file, file_name),</li>
|
|
414
|
+
<li>invoke the do method with appropriate options (header, encoding,
|
|
415
|
+
separator, quoting, use_queue, chunk_size).</li>
|
|
421
416
|
</ol>
|
|
422
417
|
</div>
|
|
423
418
|
<div class="section" id="known-issues-roadmap">
|
|
424
419
|
<h1><a class="toc-backref" href="#toc-entry-2">Known issues / Roadmap</a></h1>
|
|
425
420
|
<ul class="simple">
|
|
426
|
-
<li>There is currently no user interface to control the chunk size,
|
|
427
|
-
|
|
428
|
-
|
|
421
|
+
<li>There is currently no user interface to control the chunk size, which
|
|
422
|
+
is currently 100 by default. Should this proves to be an issue, it is
|
|
423
|
+
easy to add an option to extend the import screen.</li>
|
|
429
424
|
<li>Validation cannot be run in the background.</li>
|
|
430
425
|
</ul>
|
|
431
426
|
</div>
|
|
@@ -443,7 +438,7 @@ it is easy to add an option to extend the import screen.</li>
|
|
|
443
438
|
<p>Bugs are tracked on <a class="reference external" href="https://github.com/OCA/queue/issues">GitHub Issues</a>.
|
|
444
439
|
In case of trouble, please check there if your issue has already been reported.
|
|
445
440
|
If you spotted it first, help us to smash it by providing a detailed and welcomed
|
|
446
|
-
<a class="reference external" href="https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%
|
|
441
|
+
<a class="reference external" href="https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**">feedback</a>.</p>
|
|
447
442
|
<p>Do not contact contributors directly about support or help with technical issues.</p>
|
|
448
443
|
</div>
|
|
449
444
|
<div class="section" id="credits">
|
|
@@ -459,8 +454,8 @@ If you spotted it first, help us to smash it by providing a detailed and welcome
|
|
|
459
454
|
<h2><a class="toc-backref" href="#toc-entry-8">Contributors</a></h2>
|
|
460
455
|
<p>Sébastien Beau (Akretion) authored the initial prototype.</p>
|
|
461
456
|
<p>Stéphane Bidoul (ACSONE) extended it to version 1.0 to support
|
|
462
|
-
multi-line records, store data to import as attachments
|
|
463
|
-
|
|
457
|
+
multi-line records, store data to import as attachments and let the user
|
|
458
|
+
control the asynchronous behaviour.</p>
|
|
464
459
|
<p>Other contributors include:</p>
|
|
465
460
|
<ul class="simple">
|
|
466
461
|
<li>Anthony Muschang (ACSONE)</li>
|
|
@@ -469,20 +464,18 @@ and let the user control the asynchronous behaviour.</p>
|
|
|
469
464
|
<li>Laurent Mignon (ACSONE)</li>
|
|
470
465
|
<li>Dennis Sluijk (Onestein)</li>
|
|
471
466
|
<li>Guewen Baconnier (Camptocamp)</li>
|
|
472
|
-
<li><
|
|
473
|
-
<dt><a class="reference external" href="https://trobz.com">Trobz</a>:</dt>
|
|
474
|
-
<dd><ul class="first last">
|
|
467
|
+
<li><a class="reference external" href="https://trobz.com">Trobz</a>:<ul>
|
|
475
468
|
<li>Dzung Tran <<a class="reference external" href="mailto:dungtd@trobz.com">dungtd@trobz.com</a>></li>
|
|
469
|
+
<li>Do Anh Duy <<a class="reference external" href="mailto:duyda@trobz.com">duyda@trobz.com</a>></li>
|
|
476
470
|
</ul>
|
|
477
|
-
</dd>
|
|
478
|
-
</dl>
|
|
479
471
|
</li>
|
|
480
472
|
<li>Daniel Duque (FactorLibre)</li>
|
|
481
473
|
</ul>
|
|
482
474
|
</div>
|
|
483
475
|
<div class="section" id="other-credits">
|
|
484
476
|
<h2><a class="toc-backref" href="#toc-entry-9">Other credits</a></h2>
|
|
485
|
-
<p>The migration of this module from
|
|
477
|
+
<p>The migration of this module from 17.0 to 18.0 was financially supported
|
|
478
|
+
by Camptocamp</p>
|
|
486
479
|
</div>
|
|
487
480
|
<div class="section" id="maintainers">
|
|
488
481
|
<h2><a class="toc-backref" href="#toc-entry-10">Maintainers</a></h2>
|
|
@@ -493,7 +486,7 @@ and let the user control the asynchronous behaviour.</p>
|
|
|
493
486
|
<p>OCA, or the Odoo Community Association, is a nonprofit organization whose
|
|
494
487
|
mission is to support the collaborative development of Odoo features and
|
|
495
488
|
promote its widespread use.</p>
|
|
496
|
-
<p>This module is part of the <a class="reference external" href="https://github.com/OCA/queue/tree/
|
|
489
|
+
<p>This module is part of the <a class="reference external" href="https://github.com/OCA/queue/tree/18.0/base_import_async">OCA/queue</a> project on GitHub.</p>
|
|
497
490
|
<p>You are welcome to contribute. To learn how please visit <a class="reference external" href="https://odoo-community.org/page/Contribute">https://odoo-community.org/page/Contribute</a>.</p>
|
|
498
491
|
</div>
|
|
499
492
|
</div>
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import {BaseImportModel} from "@base_import/import_model";
|
|
2
|
+
import {_t} from "@web/core/l10n/translation";
|
|
3
|
+
import {patch} from "@web/core/utils/patch";
|
|
4
|
+
|
|
5
|
+
const {document} = globalThis;
|
|
6
|
+
|
|
7
|
+
patch(BaseImportModel.prototype, {
|
|
8
|
+
get importOptions() {
|
|
9
|
+
const options = super.importOptions;
|
|
10
|
+
const checkbox = document.querySelector("#oe_import_queue");
|
|
11
|
+
options.use_queue = checkbox ? checkbox.checked : false;
|
|
12
|
+
return options;
|
|
13
|
+
},
|
|
14
|
+
|
|
15
|
+
async executeImport(isTest, totalSteps, importProgress) {
|
|
16
|
+
const def = super.executeImport(isTest, totalSteps, importProgress);
|
|
17
|
+
const checkbox = document.querySelector("#oe_import_queue");
|
|
18
|
+
if (checkbox && checkbox.checked && !isTest) {
|
|
19
|
+
this._addMessage("warning", [
|
|
20
|
+
_t("Your request is being processed"),
|
|
21
|
+
_t("You can check the status of this job in menu 'Queue / Jobs'."),
|
|
22
|
+
]);
|
|
23
|
+
}
|
|
24
|
+
return def;
|
|
25
|
+
},
|
|
26
|
+
});
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
<?xml version="1.0" encoding="utf-8" ?>
|
|
2
|
-
<templates>
|
|
3
|
-
<t
|
|
4
|
-
|
|
2
|
+
<templates xml:space="preserve">
|
|
3
|
+
<t
|
|
4
|
+
t-name="base_import_async.ImportDataSidepanel"
|
|
5
|
+
t-inherit="ImportDataSidepanel"
|
|
6
|
+
t-inherit-mode="extension"
|
|
7
|
+
>
|
|
8
|
+
<xpath expr="//CheckBox" position="before">
|
|
5
9
|
<div
|
|
10
|
+
class="o-checkbox form-check"
|
|
6
11
|
title="When checked, the import will be executed as a background job,
|
|
7
12
|
after splitting your file in small chunks that will be processed independently.
|
|
8
13
|
Use this to import very large files."
|
|
@@ -14,6 +19,6 @@ Use this to import very large files."
|
|
|
14
19
|
/>
|
|
15
20
|
<label for="oe_import_queue">Import in the background</label>
|
|
16
21
|
</div>
|
|
17
|
-
</
|
|
22
|
+
</xpath>
|
|
18
23
|
</t>
|
|
19
24
|
</templates>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from . import test_base_import_import
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# Copyright 2024 Camptocamp
|
|
2
|
+
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
|
|
3
|
+
|
|
4
|
+
from odoo.tests.common import RecordCapturer, TransactionCase
|
|
5
|
+
|
|
6
|
+
from ..models.base_import_import import OPT_USE_QUEUE
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TestBaseImportImport(TransactionCase):
|
|
10
|
+
@classmethod
|
|
11
|
+
def setUpClass(cls):
|
|
12
|
+
super().setUpClass()
|
|
13
|
+
cls.res_partners = cls.env["res.partner"]
|
|
14
|
+
cls.import_wizard = cls.env["base_import.import"]
|
|
15
|
+
|
|
16
|
+
def test_normal_import_res_partners(self):
|
|
17
|
+
values = [
|
|
18
|
+
[
|
|
19
|
+
"name",
|
|
20
|
+
"email",
|
|
21
|
+
"is_company",
|
|
22
|
+
],
|
|
23
|
+
[
|
|
24
|
+
"partner 1",
|
|
25
|
+
"partner1@example.com",
|
|
26
|
+
"1",
|
|
27
|
+
],
|
|
28
|
+
[
|
|
29
|
+
"partner 2",
|
|
30
|
+
"partner2@example.com",
|
|
31
|
+
"0",
|
|
32
|
+
],
|
|
33
|
+
]
|
|
34
|
+
import_vals = {
|
|
35
|
+
"res_model": self.res_partners._name,
|
|
36
|
+
"file": "\n".join([";".join(values) for values in values]),
|
|
37
|
+
"file_type": "text/csv",
|
|
38
|
+
}
|
|
39
|
+
self.import_wizard |= self.import_wizard.create(import_vals)
|
|
40
|
+
opts = {"quoting": '"', "separator": ";", "has_headers": True}
|
|
41
|
+
preview = self.import_wizard.parse_preview(opts)
|
|
42
|
+
self.assertEqual(
|
|
43
|
+
preview["matches"],
|
|
44
|
+
{
|
|
45
|
+
0: ["name"],
|
|
46
|
+
1: ["email"],
|
|
47
|
+
2: ["is_company"],
|
|
48
|
+
},
|
|
49
|
+
)
|
|
50
|
+
with RecordCapturer(self.res_partners, []) as capture:
|
|
51
|
+
results = self.import_wizard.execute_import(
|
|
52
|
+
[fnames[0] for fnames in preview["matches"].values()],
|
|
53
|
+
[],
|
|
54
|
+
opts,
|
|
55
|
+
)
|
|
56
|
+
# if result is empty, no import error
|
|
57
|
+
self.assertItemsEqual(results["messages"], [])
|
|
58
|
+
records_created = capture.records
|
|
59
|
+
self.assertEqual(len(records_created), 2)
|
|
60
|
+
self.assertIn("partner1", records_created[0].email)
|
|
61
|
+
|
|
62
|
+
def test_wrong_import_res_partners(self):
|
|
63
|
+
values = [
|
|
64
|
+
[
|
|
65
|
+
"name",
|
|
66
|
+
"email",
|
|
67
|
+
"date", # Adding date field to trigger parsing error
|
|
68
|
+
],
|
|
69
|
+
[
|
|
70
|
+
"partner 1",
|
|
71
|
+
"partner1@example.com",
|
|
72
|
+
"21-13-2024",
|
|
73
|
+
],
|
|
74
|
+
[
|
|
75
|
+
"partner 2",
|
|
76
|
+
"partner2@example.com",
|
|
77
|
+
"2024-13-45",
|
|
78
|
+
],
|
|
79
|
+
]
|
|
80
|
+
opts = {
|
|
81
|
+
"quoting": '"',
|
|
82
|
+
"separator": ";",
|
|
83
|
+
"has_headers": True,
|
|
84
|
+
"date_format": "%Y-%m-%d", # Set specific date format
|
|
85
|
+
OPT_USE_QUEUE: True,
|
|
86
|
+
}
|
|
87
|
+
import_vals = {
|
|
88
|
+
"res_model": self.res_partners._name,
|
|
89
|
+
"file": "\n".join([";".join(row) for row in values]),
|
|
90
|
+
"file_type": "text/csv",
|
|
91
|
+
}
|
|
92
|
+
import_wizard = self.import_wizard.create(import_vals)
|
|
93
|
+
preview = import_wizard.parse_preview(opts)
|
|
94
|
+
results = import_wizard.execute_import(
|
|
95
|
+
[field[0] for field in preview["matches"].values()],
|
|
96
|
+
["name", "email", "date"], # Include date in fields to import
|
|
97
|
+
opts,
|
|
98
|
+
)
|
|
99
|
+
self.assertTrue(any(msg["type"] == "error" for msg in results["messages"]))
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: odoo-addon-base_import_async
|
|
3
|
-
Version:
|
|
3
|
+
Version: 18.0.1.0.0.4
|
|
4
|
+
Requires-Python: >=3.10
|
|
5
|
+
Requires-Dist: odoo-addon-queue_job==18.0.*
|
|
6
|
+
Requires-Dist: odoo==18.0.*
|
|
4
7
|
Summary: Import CSV files in the background
|
|
5
8
|
Home-page: https://github.com/OCA/queue
|
|
9
|
+
License: AGPL-3
|
|
6
10
|
Author: Akretion, ACSONE SA/NV, Odoo Community Association (OCA)
|
|
7
11
|
Author-email: support@odoo-community.org
|
|
8
|
-
License: AGPL-3
|
|
9
12
|
Classifier: Programming Language :: Python
|
|
10
13
|
Classifier: Framework :: Odoo
|
|
11
|
-
Classifier: Framework :: Odoo ::
|
|
14
|
+
Classifier: Framework :: Odoo :: 18.0
|
|
12
15
|
Classifier: License :: OSI Approved :: GNU Affero General Public License v3
|
|
13
16
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
-
Requires-Python: >=3.10
|
|
15
|
-
Requires-Dist: odoo-addon-queue-job<16.1dev,>=16.0dev
|
|
16
|
-
Requires-Dist: odoo<16.1dev,>=16.0a
|
|
17
17
|
|
|
18
18
|
===================
|
|
19
19
|
Asynchronous Import
|
|
@@ -24,7 +24,7 @@ Asynchronous Import
|
|
|
24
24
|
!! This file is generated by oca-gen-addon-readme !!
|
|
25
25
|
!! changes will be overwritten. !!
|
|
26
26
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
27
|
-
!! source digest: sha256:
|
|
27
|
+
!! source digest: sha256:299c64b165f6348dfa062b8925a95235390daa6063964f21a5a1d0757cdf1614
|
|
28
28
|
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
29
29
|
|
|
30
30
|
.. |badge1| image:: https://img.shields.io/badge/maturity-Production%2FStable-green.png
|
|
@@ -34,20 +34,19 @@ Asynchronous Import
|
|
|
34
34
|
:target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
|
|
35
35
|
:alt: License: AGPL-3
|
|
36
36
|
.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github
|
|
37
|
-
:target: https://github.com/OCA/queue/tree/
|
|
37
|
+
:target: https://github.com/OCA/queue/tree/18.0/base_import_async
|
|
38
38
|
:alt: OCA/queue
|
|
39
39
|
.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png
|
|
40
|
-
:target: https://translation.odoo-community.org/projects/queue-
|
|
40
|
+
:target: https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-base_import_async
|
|
41
41
|
:alt: Translate me on Weblate
|
|
42
42
|
.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png
|
|
43
|
-
:target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=
|
|
43
|
+
:target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0
|
|
44
44
|
:alt: Try me on Runboat
|
|
45
45
|
|
|
46
46
|
|badge1| |badge2| |badge3| |badge4| |badge5|
|
|
47
47
|
|
|
48
|
-
This module extends the standard CSV import functionality
|
|
49
|
-
|
|
50
|
-
framework.
|
|
48
|
+
This module extends the standard CSV import functionality to import
|
|
49
|
+
files in the background using the OCA/queue framework.
|
|
51
50
|
|
|
52
51
|
**Table of contents**
|
|
53
52
|
|
|
@@ -57,51 +56,47 @@ framework.
|
|
|
57
56
|
Usage
|
|
58
57
|
=====
|
|
59
58
|
|
|
60
|
-
The user is presented with a new checkbox in the import
|
|
61
|
-
|
|
62
|
-
job.
|
|
59
|
+
The user is presented with a new checkbox in the import screen. When
|
|
60
|
+
selected, the import is delayed in a background job.
|
|
63
61
|
|
|
64
|
-
This job in turn splits the CSV file in chunks of minimum
|
|
65
|
-
|
|
66
|
-
|
|
62
|
+
This job in turn splits the CSV file in chunks of minimum 100 lines (or
|
|
63
|
+
more to align with record boundaries). Each chunk is then imported in a
|
|
64
|
+
separate background job.
|
|
67
65
|
|
|
68
|
-
When an import fails, the job is marked as such and the
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
in synchronous mode since the chunks are small.
|
|
66
|
+
When an import fails, the job is marked as such and the user can read
|
|
67
|
+
the error in the job status. The CSV chunk being imported is stored as
|
|
68
|
+
an attachment to the job, making it easy to download it, fix it and run
|
|
69
|
+
a new import, possibly in synchronous mode since the chunks are small.
|
|
73
70
|
|
|
74
|
-
Any file that can be imported by the standard import mechanism
|
|
75
|
-
|
|
71
|
+
Any file that can be imported by the standard import mechanism can also
|
|
72
|
+
be imported in the background.
|
|
76
73
|
|
|
77
|
-
This module's scope is limited to making standard imports
|
|
78
|
-
|
|
79
|
-
automate ETL flows.
|
|
74
|
+
This module's scope is limited to making standard imports asynchronous.
|
|
75
|
+
It does not attempt to transform the data nor automate ETL flows.
|
|
80
76
|
|
|
81
77
|
Other modules may benefit from this infrastructure in the following way
|
|
82
78
|
(as illustrated in the test suite):
|
|
83
79
|
|
|
84
|
-
1. create an instance of
|
|
85
|
-
(
|
|
86
|
-
2. invoke the
|
|
87
|
-
|
|
88
|
-
`use_queue`, `chunk_size`).
|
|
80
|
+
1. create an instance of base_import.import and populate its fields
|
|
81
|
+
(res_model, file, file_name),
|
|
82
|
+
2. invoke the do method with appropriate options (header, encoding,
|
|
83
|
+
separator, quoting, use_queue, chunk_size).
|
|
89
84
|
|
|
90
85
|
Known issues / Roadmap
|
|
91
86
|
======================
|
|
92
87
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
88
|
+
- There is currently no user interface to control the chunk size, which
|
|
89
|
+
is currently 100 by default. Should this proves to be an issue, it is
|
|
90
|
+
easy to add an option to extend the import screen.
|
|
91
|
+
- Validation cannot be run in the background.
|
|
97
92
|
|
|
98
93
|
Changelog
|
|
99
94
|
=========
|
|
100
95
|
|
|
101
96
|
13.0.1.0.0 (2019-12-20)
|
|
102
|
-
|
|
97
|
+
-----------------------
|
|
103
98
|
|
|
104
|
-
|
|
99
|
+
- [MIGRATION] from 12.0 branched at rev. a7f8031
|
|
105
100
|
|
|
106
101
|
Bug Tracker
|
|
107
102
|
===========
|
|
@@ -109,7 +104,7 @@ Bug Tracker
|
|
|
109
104
|
Bugs are tracked on `GitHub Issues <https://github.com/OCA/queue/issues>`_.
|
|
110
105
|
In case of trouble, please check there if your issue has already been reported.
|
|
111
106
|
If you spotted it first, help us to smash it by providing a detailed and welcomed
|
|
112
|
-
`feedback <https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%
|
|
107
|
+
`feedback <https://github.com/OCA/queue/issues/new?body=module:%20base_import_async%0Aversion:%2018.0%0A%0A**Steps%20to%20reproduce**%0A-%20...%0A%0A**Current%20behavior**%0A%0A**Expected%20behavior**>`_.
|
|
113
108
|
|
|
114
109
|
Do not contact contributors directly about support or help with technical issues.
|
|
115
110
|
|
|
@@ -117,39 +112,49 @@ Credits
|
|
|
117
112
|
=======
|
|
118
113
|
|
|
119
114
|
Authors
|
|
120
|
-
|
|
115
|
+
-------
|
|
121
116
|
|
|
122
117
|
* Akretion
|
|
123
118
|
* ACSONE SA/NV
|
|
124
119
|
|
|
125
120
|
Contributors
|
|
126
|
-
|
|
121
|
+
------------
|
|
127
122
|
|
|
128
123
|
Sébastien Beau (Akretion) authored the initial prototype.
|
|
129
124
|
|
|
130
125
|
Stéphane Bidoul (ACSONE) extended it to version 1.0 to support
|
|
131
|
-
multi-line records, store data to import as attachments
|
|
132
|
-
|
|
126
|
+
multi-line records, store data to import as attachments and let the user
|
|
127
|
+
control the asynchronous behaviour.
|
|
133
128
|
|
|
134
129
|
Other contributors include:
|
|
135
130
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
131
|
+
- Anthony Muschang (ACSONE)
|
|
132
|
+
|
|
133
|
+
- David Béal (Akretion)
|
|
134
|
+
|
|
135
|
+
- Jonathan Nemry (ACSONE)
|
|
136
|
+
|
|
137
|
+
- Laurent Mignon (ACSONE)
|
|
138
|
+
|
|
139
|
+
- Dennis Sluijk (Onestein)
|
|
140
|
+
|
|
141
|
+
- Guewen Baconnier (Camptocamp)
|
|
142
|
+
|
|
143
|
+
- `Trobz <https://trobz.com>`__:
|
|
144
|
+
|
|
145
|
+
- Dzung Tran <dungtd@trobz.com>
|
|
146
|
+
- Do Anh Duy <duyda@trobz.com>
|
|
147
|
+
|
|
148
|
+
- Daniel Duque (FactorLibre)
|
|
145
149
|
|
|
146
150
|
Other credits
|
|
147
|
-
|
|
151
|
+
-------------
|
|
148
152
|
|
|
149
|
-
The migration of this module from
|
|
153
|
+
The migration of this module from 17.0 to 18.0 was financially supported
|
|
154
|
+
by Camptocamp
|
|
150
155
|
|
|
151
156
|
Maintainers
|
|
152
|
-
|
|
157
|
+
-----------
|
|
153
158
|
|
|
154
159
|
This module is maintained by the OCA.
|
|
155
160
|
|
|
@@ -161,6 +166,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose
|
|
|
161
166
|
mission is to support the collaborative development of Odoo features and
|
|
162
167
|
promote its widespread use.
|
|
163
168
|
|
|
164
|
-
This module is part of the `OCA/queue <https://github.com/OCA/queue/tree/
|
|
169
|
+
This module is part of the `OCA/queue <https://github.com/OCA/queue/tree/18.0/base_import_async>`_ project on GitHub.
|
|
165
170
|
|
|
166
171
|
You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
odoo/addons/base_import_async/README.rst,sha256=bC5FIxnKWgpy0DNUUVwUe-T_QnvNGNCaUsT1ocacjEo,5077
|
|
2
|
+
odoo/addons/base_import_async/__init__.py,sha256=eShEGRadAlxopoLqIQ4-N42FM7MWPSuBe0h_l1cXTPM,87
|
|
3
|
+
odoo/addons/base_import_async/__manifest__.py,sha256=SsVjSsfL53MwNZnVLy7BHbSh84eFi_jNkCgc_YD7SIA,868
|
|
4
|
+
odoo/addons/base_import_async/data/queue_job_function_data.xml,sha256=MhA59cNIFz0tybIEYYfas5vIgAL7YVje26-n0EKnwVE,763
|
|
5
|
+
odoo/addons/base_import_async/i18n/base_import_async.pot,sha256=aL1yvEPyrIqSFRMQMS-RgyQk8hrfe9dh2pvgvasX7zc,1981
|
|
6
|
+
odoo/addons/base_import_async/i18n/es.po,sha256=4pquZPN4cwoNJULxAPDDNgOO4bagakEJw7QRyU-QHHM,2774
|
|
7
|
+
odoo/addons/base_import_async/i18n/it.po,sha256=4WklbHLMT1sb8MfShTBaQQTP4Ruz1HpVi1PfTrbXuUc,2736
|
|
8
|
+
odoo/addons/base_import_async/i18n/tr.po,sha256=kKC4SZ-I034SLjmSIpCkLyUFpaiflEw_7_cCEDCBe9Y,2854
|
|
9
|
+
odoo/addons/base_import_async/models/__init__.py,sha256=O_Ai1JLN-RrWSSWZmCSu_fZ4kq-2nJiBEWOjerAN6Qo,123
|
|
10
|
+
odoo/addons/base_import_async/models/base_import_import.py,sha256=ZiNpKEGsFhPzYeFfw27joycWqul1PfVsKwrhzGuehQw,7331
|
|
11
|
+
odoo/addons/base_import_async/models/queue_job.py,sha256=8T8oe8hF8LK8Dq8T8PxLxHDtMLH4rTQuAcC6Qz0X6r4,491
|
|
12
|
+
odoo/addons/base_import_async/readme/CONTRIBUTORS.md,sha256=5qZjXXtS2bsEFx34uavcE9ZPJxD6WtgRYIACA8KlOek,568
|
|
13
|
+
odoo/addons/base_import_async/readme/CREDITS.md,sha256=JioPiNwd9zjoni_RFagda7cUI1Weu2O5mEvZ2ah8ROQ,87
|
|
14
|
+
odoo/addons/base_import_async/readme/DESCRIPTION.md,sha256=fm7FO_7wKtmir7kAqTJRTGvUTMDbe9zJkBP_z0bLpHM,123
|
|
15
|
+
odoo/addons/base_import_async/readme/HISTORY.md,sha256=24qB1TG1LDqfu9TT1j1gDhgiDDEbV6_5WNvQvYGlVoI,79
|
|
16
|
+
odoo/addons/base_import_async/readme/ROADMAP.md,sha256=cnTSBGOUAv-eGJLOGCMgL6lssfMNLagjKO_sBY2een4,243
|
|
17
|
+
odoo/addons/base_import_async/readme/USAGE.md,sha256=MRQCFtztLlD1vktg_aeTMw32MBRrkSek3KnDRCSgi0k,1150
|
|
18
|
+
odoo/addons/base_import_async/static/description/icon.png,sha256=6xBPJauaFOF0KDHfHgQopSc28kKvxMaeoQFQWZtfZDo,9455
|
|
19
|
+
odoo/addons/base_import_async/static/description/index.html,sha256=B7xlyuD6QZNu4wYSwxiKHcDiBnf7BOML2Un3rdM1kns,15652
|
|
20
|
+
odoo/addons/base_import_async/static/src/js/import_model.esm.js,sha256=7WqYaRt8Sz029SVO4Xy4l4_dht8NAqS_du2YqcutfpY,957
|
|
21
|
+
odoo/addons/base_import_async/static/src/xml/import_data_sidepanel.xml,sha256=mbCtHcDVShvbY5jN1RAmXhRkiZJP57N3Kig9AlPqjfY,865
|
|
22
|
+
odoo/addons/base_import_async/tests/__init__.py,sha256=zlJqLs-_34OrP2pEd6UIlGnkyqxVgRA6Xm7j30xu5F8,38
|
|
23
|
+
odoo/addons/base_import_async/tests/test_base_import_import.py,sha256=z9Sm1eI_a6pWS6zxc7TKvMUgjvCUn5yVe5ozT38jJ84,3245
|
|
24
|
+
odoo_addon_base_import_async-18.0.1.0.0.4.dist-info/METADATA,sha256=S_xIvFVb7OkIb2LcC0Xiyts16AohQ-ScGvahpvyhomQ,5702
|
|
25
|
+
odoo_addon_base_import_async-18.0.1.0.0.4.dist-info/WHEEL,sha256=9fEMia4zL7ZuZbnCOrcYogUhmn4XFIVaJ8G4YGI31xc,81
|
|
26
|
+
odoo_addon_base_import_async-18.0.1.0.0.4.dist-info/top_level.txt,sha256=QE6RBQ0QX5f4eFuUcGgU5Kbq1A_qJcDs-e_vpr6pmfU,4
|
|
27
|
+
odoo_addon_base_import_async-18.0.1.0.0.4.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
odoo
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
Sébastien Beau (Akretion) authored the initial prototype.
|
|
2
|
-
|
|
3
|
-
Stéphane Bidoul (ACSONE) extended it to version 1.0 to support
|
|
4
|
-
multi-line records, store data to import as attachments
|
|
5
|
-
and let the user control the asynchronous behaviour.
|
|
6
|
-
|
|
7
|
-
Other contributors include:
|
|
8
|
-
|
|
9
|
-
* Anthony Muschang (ACSONE)
|
|
10
|
-
* David Béal (Akretion)
|
|
11
|
-
* Jonathan Nemry (ACSONE)
|
|
12
|
-
* Laurent Mignon (ACSONE)
|
|
13
|
-
* Dennis Sluijk (Onestein)
|
|
14
|
-
* Guewen Baconnier (Camptocamp)
|
|
15
|
-
* `Trobz <https://trobz.com>`_:
|
|
16
|
-
* Dzung Tran <dungtd@trobz.com>
|
|
17
|
-
* Daniel Duque (FactorLibre)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
The user is presented with a new checkbox in the import
|
|
2
|
-
screen. When selected, the import is delayed in a background
|
|
3
|
-
job.
|
|
4
|
-
|
|
5
|
-
This job in turn splits the CSV file in chunks of minimum
|
|
6
|
-
100 lines (or more to align with record boundaries). Each
|
|
7
|
-
chunk is then imported in a separate background job.
|
|
8
|
-
|
|
9
|
-
When an import fails, the job is marked as such and the
|
|
10
|
-
user can read the error in the job status. The CSV chunk
|
|
11
|
-
being imported is stored as an attachment to the job, making
|
|
12
|
-
it easy to download it, fix it and run a new import, possibly
|
|
13
|
-
in synchronous mode since the chunks are small.
|
|
14
|
-
|
|
15
|
-
Any file that can be imported by the standard import mechanism
|
|
16
|
-
can also be imported in the background.
|
|
17
|
-
|
|
18
|
-
This module's scope is limited to making standard imports
|
|
19
|
-
asynchronous. It does not attempt to transform the data nor
|
|
20
|
-
automate ETL flows.
|
|
21
|
-
|
|
22
|
-
Other modules may benefit from this infrastructure in the following way
|
|
23
|
-
(as illustrated in the test suite):
|
|
24
|
-
|
|
25
|
-
1. create an instance of `base_import.import` and populate its fields
|
|
26
|
-
(`res_model`, `file`, `file_name`),
|
|
27
|
-
2. invoke the `do` method with appropriate options
|
|
28
|
-
(`header`, `encoding`, `separator`, `quoting`,
|
|
29
|
-
`use_queue`, `chunk_size`).
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
odoo.define("base_import_async.import", function (require) {
|
|
2
|
-
"use strict";
|
|
3
|
-
|
|
4
|
-
var core = require("web.core");
|
|
5
|
-
var _t = core._t;
|
|
6
|
-
var DataImport = require("base_import.import").DataImport;
|
|
7
|
-
|
|
8
|
-
DataImport.include({
|
|
9
|
-
import_options: function () {
|
|
10
|
-
var options = this._super.apply(this, arguments);
|
|
11
|
-
options.use_queue = this.$("input.oe_import_queue").prop("checked");
|
|
12
|
-
return options;
|
|
13
|
-
},
|
|
14
|
-
|
|
15
|
-
onimported: function () {
|
|
16
|
-
if (this.$("input.oe_import_queue").prop("checked")) {
|
|
17
|
-
this.displayNotification({
|
|
18
|
-
type: "warning",
|
|
19
|
-
title: _t("Your request is being processed"),
|
|
20
|
-
message: _t(
|
|
21
|
-
"You can check the status of this job in menu 'Queue / Jobs'."
|
|
22
|
-
),
|
|
23
|
-
});
|
|
24
|
-
this.exit();
|
|
25
|
-
} else {
|
|
26
|
-
this._super.apply(this, arguments);
|
|
27
|
-
}
|
|
28
|
-
},
|
|
29
|
-
});
|
|
30
|
-
});
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
odoo/addons/base_import_async/README.rst,sha256=flHh-8-1FRRUndQJLraC4HUfFWSWS0pN20MuzAM-heo,5062
|
|
2
|
-
odoo/addons/base_import_async/__init__.py,sha256=eShEGRadAlxopoLqIQ4-N42FM7MWPSuBe0h_l1cXTPM,87
|
|
3
|
-
odoo/addons/base_import_async/__manifest__.py,sha256=BOilPGveMUoe6lsSw9zHFZElsQF4zihS2x2QXWoVoW0,843
|
|
4
|
-
odoo/addons/base_import_async/data/queue_job_function_data.xml,sha256=khU0A8VUGwDbK7LqPRE2FUxDVhrkrlmdMw1tn63FsuQ,1077
|
|
5
|
-
odoo/addons/base_import_async/i18n/base_import_async.pot,sha256=nwOsvZ7eHAHGNxc1s0iuMmjFlr5Xi-4FqmmLns1pJLI,2050
|
|
6
|
-
odoo/addons/base_import_async/i18n/es.po,sha256=4pquZPN4cwoNJULxAPDDNgOO4bagakEJw7QRyU-QHHM,2774
|
|
7
|
-
odoo/addons/base_import_async/i18n/it.po,sha256=4WklbHLMT1sb8MfShTBaQQTP4Ruz1HpVi1PfTrbXuUc,2736
|
|
8
|
-
odoo/addons/base_import_async/i18n/tr.po,sha256=8S22-20MIS33QBY6BX2u_Cw3dpMVt1Atq_Q5t_qLqaU,3091
|
|
9
|
-
odoo/addons/base_import_async/models/__init__.py,sha256=O_Ai1JLN-RrWSSWZmCSu_fZ4kq-2nJiBEWOjerAN6Qo,123
|
|
10
|
-
odoo/addons/base_import_async/models/base_import_import.py,sha256=nfbJDz6UvHJVNcbJqJKmiPQpIIIzglSaU3nyBC0udyY,7260
|
|
11
|
-
odoo/addons/base_import_async/models/queue_job.py,sha256=EqqBGroTGnMoIpMBeO6a3C4hE9XIR6_ShHSylOk_kLI,539
|
|
12
|
-
odoo/addons/base_import_async/readme/CONTRIBUTORS.rst,sha256=u4L_ZT4Sx_MJVwndUIjR2JvTK9StCd30CdlRJTg0qC4,524
|
|
13
|
-
odoo/addons/base_import_async/readme/CREDITS.rst,sha256=XtPazYj6oqxvg62KjKqErJ8pssTcghFavMtYTDMOozc,87
|
|
14
|
-
odoo/addons/base_import_async/readme/DESCRIPTION.rst,sha256=xwjUiJw_gwj_R7Rg6_W2ifcX0Nmo_2DliOBAgEkWOZQ,123
|
|
15
|
-
odoo/addons/base_import_async/readme/HISTORY.rst,sha256=T6aa_cvCBaKeznUBeElAxpkZtjAwakycImWk4XjvQP8,98
|
|
16
|
-
odoo/addons/base_import_async/readme/ROADMAP.rst,sha256=H7QLRKOuskOesnFfIjHKIEyrZr10TSA4hbJlDnZwpS4,243
|
|
17
|
-
odoo/addons/base_import_async/readme/USAGE.rst,sha256=5Ftjm8p8wuAbkWftGQKFHgJNhJlo0AAC95D6FR79o14,1171
|
|
18
|
-
odoo/addons/base_import_async/static/description/icon.png,sha256=6xBPJauaFOF0KDHfHgQopSc28kKvxMaeoQFQWZtfZDo,9455
|
|
19
|
-
odoo/addons/base_import_async/static/description/index.html,sha256=9BKBVc2pamDGBb10SdEBg5CNBtRS4kFvfNfbS1XHdzw,15752
|
|
20
|
-
odoo/addons/base_import_async/static/src/js/import.js,sha256=LWkEcNmEPPcjS61Bs8vbwkhFvYPGG5hPeIErb6uXZmM,997
|
|
21
|
-
odoo/addons/base_import_async/static/src/xml/import.xml,sha256=67ASHYXFlGUnbJ1SxHLOlcLFIwuhisGVJw5buYP7JQc,706
|
|
22
|
-
odoo_addon_base_import_async-16.0.1.2.0.1.dist-info/METADATA,sha256=Swd0VtO9eQixEs_SV6WpEF2tmMa18QAwmW7GADVYvRM,5705
|
|
23
|
-
odoo_addon_base_import_async-16.0.1.2.0.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
24
|
-
odoo_addon_base_import_async-16.0.1.2.0.1.dist-info/top_level.txt,sha256=qBj40grFkGOfDZ2WDSw3y1RnDlgG0u8rP8pvGNdbz4w,5
|
|
25
|
-
odoo_addon_base_import_async-16.0.1.2.0.1.dist-info/RECORD,,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
odoo
|