g3w-admin-processing 1.2.1b1.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- g3w_admin_processing-1.2.1b1.dev1.dist-info/METADATA +258 -0
- g3w_admin_processing-1.2.1b1.dev1.dist-info/RECORD +80 -0
- g3w_admin_processing-1.2.1b1.dev1.dist-info/WHEEL +5 -0
- g3w_admin_processing-1.2.1b1.dev1.dist-info/licenses/LICENSE +373 -0
- g3w_admin_processing-1.2.1b1.dev1.dist-info/top_level.txt +1 -0
- qprocessing/__init__.py +7 -0
- qprocessing/admin.py +21 -0
- qprocessing/api/__init__.py +0 -0
- qprocessing/api/permissions.py +41 -0
- qprocessing/api/views/__init__.py +4 -0
- qprocessing/api/views/actions.py +84 -0
- qprocessing/api/views/admin.py +76 -0
- qprocessing/api/views/models.py +219 -0
- qprocessing/api/views/upload.py +177 -0
- qprocessing/apiurls.py +64 -0
- qprocessing/apps.py +38 -0
- qprocessing/configs.py +131 -0
- qprocessing/forms.py +133 -0
- qprocessing/migrations/0001_initial.py +24 -0
- qprocessing/migrations/0002_auto_20230615_1609.py +23 -0
- qprocessing/migrations/0003_qprocessinginputupload.py +32 -0
- qprocessing/migrations/0004_auto_20230809_0920.py +26 -0
- qprocessing/migrations/0005_alter_qprocessinginputupload_user.py +21 -0
- qprocessing/migrations/__init__.py +0 -0
- qprocessing/models.py +143 -0
- qprocessing/receivers.py +100 -0
- qprocessing/sitetrees.py +31 -0
- qprocessing/static/qprocessing/js/components/DrawInputVectorFeatures.js +180 -0
- qprocessing/static/qprocessing/js/components/InputFieldChooser.js +226 -0
- qprocessing/static/qprocessing/js/components/InputPrjRasterLayer.js +135 -0
- qprocessing/static/qprocessing/js/components/InputPrjVectorLayer.js +415 -0
- qprocessing/static/qprocessing/js/components/ModelPanel.js +496 -0
- qprocessing/static/qprocessing/js/components/ModelResults.js +73 -0
- qprocessing/static/qprocessing/js/components/OutputFile.js +63 -0
- qprocessing/static/qprocessing/js/components/OutputRasterLayer.js +63 -0
- qprocessing/static/qprocessing/js/components/OutputVectorLayer.js +115 -0
- qprocessing/static/qprocessing/js/components/UploadVectorFile.js +53 -0
- qprocessing/static/qprocessing/js/i18n/en.js +33 -0
- qprocessing/static/qprocessing/js/i18n/index.js +7 -0
- qprocessing/static/qprocessing/js/i18n/it.js +33 -0
- qprocessing/static/qprocessing/js/plugin.js +122 -0
- qprocessing/tasks.py +136 -0
- qprocessing/templates/qprocessing/project_form.html +70 -0
- qprocessing/templates/qprocessing/projects_list.html +72 -0
- qprocessing/tests/__init__.py +0 -0
- qprocessing/tests/base.py +104 -0
- qprocessing/tests/data/models/qbuffer.model3 +233 -0
- qprocessing/tests/data/models/qintersection.model3 +288 -0
- qprocessing/tests/data/models/qpointspolygons.model3 +222 -0
- qprocessing/tests/data/models/test.model3 +321 -0
- qprocessing/tests/data/models/test_buffer_all_inputs.model3 +269 -0
- qprocessing/tests/data/models/test_buffer_layer1.model3 +201 -0
- qprocessing/tests/data/project_data/buildings.geojson +15 -0
- qprocessing/tests/data/project_data/qprocessing.gpkg +0 -0
- qprocessing/tests/data/qprocessing_qprocessing.qgs +2813 -0
- qprocessing/tests/data/qprocessing_qprocessing_attachments.zip +0 -0
- qprocessing/tests/data/test_qgis_328.qgs +954 -0
- qprocessing/tests/data/test_qgis_328_attachments.zip +0 -0
- qprocessing/tests/data/upload/fail_shp_to_upload.zip +0 -0
- qprocessing/tests/data/upload/linestring_to_upload.geojson +9 -0
- qprocessing/tests/data/upload/point_to_upload.geojson +10 -0
- qprocessing/tests/data/upload/point_to_upload_espg4326.geojson +10 -0
- qprocessing/tests/data/upload/point_to_upload_espg4326.qmd +26 -0
- qprocessing/tests/data/upload/polygon_to_upload.geojson +9 -0
- qprocessing/tests/data/upload/shp_to_upload.cpg +1 -0
- qprocessing/tests/data/upload/shp_to_upload.dbf +0 -0
- qprocessing/tests/data/upload/shp_to_upload.prj +1 -0
- qprocessing/tests/data/upload/shp_to_upload.qix +0 -0
- qprocessing/tests/data/upload/shp_to_upload.shp +0 -0
- qprocessing/tests/data/upload/shp_to_upload.shx +0 -0
- qprocessing/tests/data/upload/shp_to_upload.zip +0 -0
- qprocessing/tests/test_algorithm.py +52 -0
- qprocessing/tests/test_api.py +173 -0
- qprocessing/tests/test_api_models.py +459 -0
- qprocessing/urls.py +33 -0
- qprocessing/utils/__init__.py +0 -0
- qprocessing/utils/data.py +283 -0
- qprocessing/utils/exceptions.py +20 -0
- qprocessing/utils/formtypes.py +521 -0
- qprocessing/views.py +96 -0
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
"""" Data qprocessing module
|
|
3
|
+
.. note:: This program is free software; you can redistribute it and/or modify
|
|
4
|
+
it under the terms of the Mozilla Public License 2.0.
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__author__ = 'lorenzetti@gis3w.it'
|
|
9
|
+
__date__ = '2023-05-09'
|
|
10
|
+
__copyright__ = 'Copyright 2015 - 2023, Gis3w'
|
|
11
|
+
__license__ = 'MPL 2.0'
|
|
12
|
+
|
|
13
|
+
from django.conf import settings
|
|
14
|
+
from django.urls import reverse
|
|
15
|
+
from django.utils.translation import gettext_lazy as _
|
|
16
|
+
from django.utils.text import slugify
|
|
17
|
+
from qgis.core import \
|
|
18
|
+
QgsProcessingModelAlgorithm, QgsProject, Qgis
|
|
19
|
+
from qdjango.models import Project as QdjangoProject
|
|
20
|
+
from core.utils.qgisapi import get_layer_fids_from_server_fids
|
|
21
|
+
from .formtypes import \
|
|
22
|
+
(MAPPING_PROCESSING_PARAMS_FORM_TYPE,
|
|
23
|
+
MAPPING_QPROCESSINGTYPE_FORMTYPE,
|
|
24
|
+
QgsProcessingOutputVectorLayer,
|
|
25
|
+
QgsProcessingOutputRasterLayer,
|
|
26
|
+
QgsProcessingOutputHtml,
|
|
27
|
+
QgsProcessingOutputFile)
|
|
28
|
+
from .exceptions import (
|
|
29
|
+
QProcessingInputException,
|
|
30
|
+
QProcessingOutputException)
|
|
31
|
+
|
|
32
|
+
import os
|
|
33
|
+
from cryptography.fernet import Fernet
|
|
34
|
+
import datetime
|
|
35
|
+
|
|
36
|
+
class QProcessingModel(object):
|
|
37
|
+
"""
|
|
38
|
+
A class to read a .model3 XMl file and retrieve information about inputs outputs end other.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, model_file):
|
|
42
|
+
self.model_file = model_file
|
|
43
|
+
|
|
44
|
+
# Instance QgsProcessingModelAlgorithm
|
|
45
|
+
self.model = QgsProcessingModelAlgorithm()
|
|
46
|
+
load = self.model.fromFile(self.model_file)
|
|
47
|
+
if not load:
|
|
48
|
+
raise Exception(f"The model file could not be loaded, check that the path is correct: {self.model_file}")
|
|
49
|
+
|
|
50
|
+
# Analyze .model3 file
|
|
51
|
+
# -------------------------------------------
|
|
52
|
+
# Get general metadata properties
|
|
53
|
+
self._get_generals()
|
|
54
|
+
|
|
55
|
+
#Get inputs and outputs properties
|
|
56
|
+
self._get_inputs()
|
|
57
|
+
|
|
58
|
+
# Get outputs
|
|
59
|
+
self._get_outputs()
|
|
60
|
+
|
|
61
|
+
def _get_generals(self):
|
|
62
|
+
"""
|
|
63
|
+
From a QgsProcessingModelAlgorithm instance get information general metadata information, i.e Model name etc.
|
|
64
|
+
|
|
65
|
+
:return: None
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
self.name = self.model.name()
|
|
69
|
+
self.display_name = self.model.displayName()
|
|
70
|
+
|
|
71
|
+
def _get_inputs(self):
|
|
72
|
+
"""
|
|
73
|
+
From a QgsProcessingModelAlgorithm instance get information about
|
|
74
|
+
inputs
|
|
75
|
+
|
|
76
|
+
:return: None
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
# Get inputs of first algorithm
|
|
80
|
+
self.inputs = {}
|
|
81
|
+
for op in self.model.orderedParameters():
|
|
82
|
+
dop = self.model.parameterDefinition(op.parameterName())
|
|
83
|
+
vmap = dop.toVariantMap()
|
|
84
|
+
qtype = vmap['parameter_type']
|
|
85
|
+
flags = self._read_flags(vmap['flags'])
|
|
86
|
+
|
|
87
|
+
# Checking the type of processing if it is managed
|
|
88
|
+
if qtype not in MAPPING_PROCESSING_PARAMS_FORM_TYPE:
|
|
89
|
+
raise QProcessingInputException(_(f'Processing input type`{qtype}` is not managed.'))
|
|
90
|
+
|
|
91
|
+
dt = {
|
|
92
|
+
'name': vmap['name'],
|
|
93
|
+
'label': vmap['description'],
|
|
94
|
+
'type': MAPPING_PROCESSING_PARAMS_FORM_TYPE[qtype],
|
|
95
|
+
'qprocessing_type': qtype,
|
|
96
|
+
'default': dop.defaultValue(),
|
|
97
|
+
'editable': True, # Editable by default: only for g3w-client complaint
|
|
98
|
+
'validate': {
|
|
99
|
+
"required": dop.FlagOptional not in flags
|
|
100
|
+
},
|
|
101
|
+
'input': {
|
|
102
|
+
'type': 'text',
|
|
103
|
+
'options': {}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
# Update `input` section by qtype
|
|
108
|
+
if qtype in MAPPING_QPROCESSINGTYPE_FORMTYPE:
|
|
109
|
+
dt.update(MAPPING_QPROCESSINGTYPE_FORMTYPE[qtype](**vmap).input_form)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
self.inputs.update({
|
|
113
|
+
vmap['name']: dt
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
def _get_outputs(self):
|
|
117
|
+
"""
|
|
118
|
+
From a QgsProcessingModelAlgorithm instance get information about
|
|
119
|
+
outputs
|
|
120
|
+
|
|
121
|
+
:return: None
|
|
122
|
+
"""
|
|
123
|
+
self.outputs = {}
|
|
124
|
+
for od in self.model.outputDefinitions():
|
|
125
|
+
qtype = od.type()
|
|
126
|
+
|
|
127
|
+
# Checking the type of processing if it is managed
|
|
128
|
+
if qtype not in MAPPING_PROCESSING_PARAMS_FORM_TYPE:
|
|
129
|
+
raise QProcessingOutputException(_(f'Processing output type`{qtype}` is not managed.'))
|
|
130
|
+
|
|
131
|
+
ot = {
|
|
132
|
+
'name': od.name(),
|
|
133
|
+
'label': od.description(),
|
|
134
|
+
'qprocessing_type': qtype,
|
|
135
|
+
'type': MAPPING_PROCESSING_PARAMS_FORM_TYPE[qtype],
|
|
136
|
+
'default': None,
|
|
137
|
+
'validate': {
|
|
138
|
+
"required": True
|
|
139
|
+
},
|
|
140
|
+
'input': {
|
|
141
|
+
'type': 'text',
|
|
142
|
+
'options': {}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
# Update `output` section by qtype
|
|
147
|
+
if qtype in MAPPING_QPROCESSINGTYPE_FORMTYPE:
|
|
148
|
+
ot.update(MAPPING_QPROCESSINGTYPE_FORMTYPE[qtype](**{}).input_form)
|
|
149
|
+
|
|
150
|
+
self.outputs.update({
|
|
151
|
+
od.name(): ot
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
##
|
|
155
|
+
def _read_flags(self, flags:int ):
|
|
156
|
+
|
|
157
|
+
def fit_bits(n:int):
|
|
158
|
+
"""
|
|
159
|
+
fn_bits: Calculate binary (base2) numbers from integer (base10)
|
|
160
|
+
"""
|
|
161
|
+
while n:
|
|
162
|
+
b = n & (~n + 1)
|
|
163
|
+
yield b
|
|
164
|
+
n ^= b
|
|
165
|
+
|
|
166
|
+
return [flag for flag in fit_bits(flags)]
|
|
167
|
+
|
|
168
|
+
def get_qgsprocessingparameter(self, name):
|
|
169
|
+
"""
|
|
170
|
+
Return the QgsProcessingModelParameter by name
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
return self.model.parameterComponent(name)
|
|
174
|
+
|
|
175
|
+
def render2dict(self):
|
|
176
|
+
"""
|
|
177
|
+
Render properties to json
|
|
178
|
+
|
|
179
|
+
:return: dict
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
return {
|
|
183
|
+
'name': self.name,
|
|
184
|
+
'display_name': self.display_name,
|
|
185
|
+
'inputs': list(self.inputs.values()),
|
|
186
|
+
'outputs': list(self.outputs.values())
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
def make_model_params(self, form_data:dict, qproject: QdjangoProject, **kwargs):
|
|
190
|
+
"""
|
|
191
|
+
Build and return a dict params for algorithm processing
|
|
192
|
+
:param form_data: dict data from g3w-client processing model form (usually request.data in a View).
|
|
193
|
+
:param qgs_project: An instace of QgsProject.
|
|
194
|
+
:return: tuple of QGSProject instance and dict params for algorithm processing
|
|
195
|
+
|
|
196
|
+
Change the values of form_data input by model inputs/outputs type
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
qgs_project = QgsProject()
|
|
201
|
+
flags = Qgis.ProjectReadFlags()
|
|
202
|
+
#flags |= Qgis.ProjectReadFlag.DontLoadLayouts
|
|
203
|
+
#flags |= Qgis.ProjectReadFlag.DontResolveLayers
|
|
204
|
+
qgs_project.read(str(qproject.qgis_file.path), flags)
|
|
205
|
+
|
|
206
|
+
#qgs_project = qproject.qgis_project
|
|
207
|
+
|
|
208
|
+
params = {}
|
|
209
|
+
params.update(form_data['inputs'])
|
|
210
|
+
params.update(form_data['outputs'])
|
|
211
|
+
|
|
212
|
+
# Input cases
|
|
213
|
+
# --------------------------------------
|
|
214
|
+
for k, v in form_data['inputs'].items():
|
|
215
|
+
|
|
216
|
+
formtype = MAPPING_QPROCESSINGTYPE_FORMTYPE[self.inputs[k]['qprocessing_type']]
|
|
217
|
+
params[k] = formtype.update_model_params(qgs_project, params[k])
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# Outputs cases
|
|
221
|
+
# --------------------------------------
|
|
222
|
+
|
|
223
|
+
# Make directory by user Id
|
|
224
|
+
if 'user' in kwargs:
|
|
225
|
+
save_path = f"{settings.QPROCESSING_OUTPUT_PATH}{kwargs['user'].pk}/"
|
|
226
|
+
else:
|
|
227
|
+
save_path = f"{settings.QPROCESSING_OUTPUT_PATH}nouser/"
|
|
228
|
+
|
|
229
|
+
if not os.path.exists(save_path):
|
|
230
|
+
os.mkdir(save_path)
|
|
231
|
+
|
|
232
|
+
for k, o in form_data['outputs'].items():
|
|
233
|
+
|
|
234
|
+
formtype = MAPPING_QPROCESSINGTYPE_FORMTYPE[self.outputs[k]['qprocessing_type']]
|
|
235
|
+
params[self.outputs[k]['name']] = formtype.update_model_params(qgs_project, o, self.outputs[k],
|
|
236
|
+
save_path=save_path)
|
|
237
|
+
|
|
238
|
+
return qgs_project, params
|
|
239
|
+
|
|
240
|
+
def make_outputs(self, pres, qprocessingproject_pk, project_pk):
|
|
241
|
+
"""
|
|
242
|
+
Refine processing running model output for g3w-client.
|
|
243
|
+
:param pres: Processing running model output dict.
|
|
244
|
+
:param qprocessingproject_pk: QProcessingProject model instance pk.
|
|
245
|
+
:param project_pk: Qdjango Project model instance pk.
|
|
246
|
+
:return: Return a dict with url to new geo data download.
|
|
247
|
+
"""
|
|
248
|
+
|
|
249
|
+
out = {}
|
|
250
|
+
for k, o in self.outputs.items():
|
|
251
|
+
if k in pres and o['qprocessing_type'] in [
|
|
252
|
+
QgsProcessingOutputVectorLayer('').type(),
|
|
253
|
+
QgsProcessingOutputRasterLayer('').type(),
|
|
254
|
+
QgsProcessingOutputHtml('').type(),
|
|
255
|
+
QgsProcessingOutputFile('').type(),
|
|
256
|
+
]:
|
|
257
|
+
f = Fernet(settings.QPROCESSING_CRYPTO_KEY)
|
|
258
|
+
out[k] = reverse('qprocessing-download-output', args=(qprocessingproject_pk, project_pk,
|
|
259
|
+
f.encrypt(pres[k].encode()).decode(),))
|
|
260
|
+
return out
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def validate(self):
|
|
266
|
+
"""
|
|
267
|
+
Proxy interface to QgsProcessingModelAlgorithm.validate method.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
return self.model.validate()
|
|
271
|
+
|
|
272
|
+
def process_algorithm(self, params, context, feedback):
|
|
273
|
+
"""
|
|
274
|
+
Proxy interface to QgsProcessingModelAlgorithm.processAlgorithm method.
|
|
275
|
+
"""
|
|
276
|
+
return self.model.processAlgorithm(params, context, feedback)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
"""" QProcessing custom exceptions
|
|
3
|
+
.. note:: This program is free software; you can redistribute it and/or modify
|
|
4
|
+
it under the terms of the Mozilla Public License 2.0.
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__author__ = 'lorenzetti@gis3w.it'
|
|
9
|
+
__date__ = '2023-06-15'
|
|
10
|
+
__copyright__ = 'Copyright 2015 - 2023, Gis3w'
|
|
11
|
+
__license__ = 'MPL 2.0'
|
|
12
|
+
|
|
13
|
+
class QProcessingException(Exception):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
class QProcessingInputException(QProcessingException):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
class QProcessingOutputException(QProcessingException):
|
|
20
|
+
pass
|