pyproc 0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyproc/__init__.py +10 -0
- pyproc/cli.py +807 -0
- pyproc/exceptions.py +18 -0
- pyproc/lpse.py +757 -0
- pyproc/text.py +43 -0
- pyproc/utils.py +90 -0
- pyproc-0.2.dist-info/METADATA +210 -0
- pyproc-0.2.dist-info/RECORD +11 -0
- pyproc-0.2.dist-info/WHEEL +4 -0
- pyproc-0.2.dist-info/entry_points.txt +2 -0
- pyproc-0.2.dist-info/licenses/LICENSE +21 -0
pyproc/lpse.py
ADDED
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import bs4
|
|
3
|
+
import requests
|
|
4
|
+
import re
|
|
5
|
+
import logging
|
|
6
|
+
import backoff
|
|
7
|
+
from . import utils
|
|
8
|
+
from bs4 import BeautifulSoup as Bs, NavigableString
|
|
9
|
+
from .exceptions import LpseVersionException, LpseServerExceptions, LpseHostExceptions
|
|
10
|
+
from enum import Enum
|
|
11
|
+
from abc import abstractmethod
|
|
12
|
+
from urllib.parse import urlparse
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class By(Enum):
|
|
16
|
+
KODE = 0
|
|
17
|
+
NAMA_PAKET = 1
|
|
18
|
+
INSTANSI = 2
|
|
19
|
+
HPS = 4
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class JenisPengadaan(Enum):
|
|
23
|
+
"""
|
|
24
|
+
Objek untuk menampung data kodifikasi jenis pengadaan
|
|
25
|
+
"""
|
|
26
|
+
PENGADAAN_BARANG = 0
|
|
27
|
+
JASA_KONSULTANSI_BADAN_USAHA_NON_KONSTRUKSI = 1
|
|
28
|
+
PEKERJAAN_KONSTRUKSI = 2
|
|
29
|
+
JASA_LAINNYA = 3
|
|
30
|
+
JASA_KONSULTANSI_PERORANGAN = 4
|
|
31
|
+
JASA_KONSULTANSI_BADAN_USAHA_KONSTRUKSI = 5
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Lpse(object):
|
|
35
|
+
|
|
36
|
+
def __init__(self, instansi, timeout=10):
|
|
37
|
+
self.session = requests.session()
|
|
38
|
+
self.session.verify = False
|
|
39
|
+
self.session.headers = {
|
|
40
|
+
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
|
|
41
|
+
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
|
42
|
+
'Chrome/102.0.5005.61 Safari/537.36'
|
|
43
|
+
}
|
|
44
|
+
self.timeout = timeout
|
|
45
|
+
self.auth_token = None
|
|
46
|
+
self.url = f"https://spse.inaproc.id/{instansi}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@staticmethod
|
|
50
|
+
def check_error(resp):
|
|
51
|
+
error_message = None
|
|
52
|
+
content = resp.text
|
|
53
|
+
|
|
54
|
+
if resp.status_code >= 400 or \
|
|
55
|
+
re.findall(r'Maaf, terjadi error pada aplikasi SPSE.', content) or \
|
|
56
|
+
re.findall(r'Terjadi Kesalahan', content):
|
|
57
|
+
error_message = "Terjadi error pada aplikasi SPSE."
|
|
58
|
+
error_code = re.findall(r'Kode Error: ([\da-zA-Z]+)', content)
|
|
59
|
+
|
|
60
|
+
if error_code:
|
|
61
|
+
error_message += ' Kode Error: ' + error_code[0]
|
|
62
|
+
elif re.findall('Halaman yang dituju tidak ditemukan', content):
|
|
63
|
+
error_message = "Paket tidak ditemukan"
|
|
64
|
+
|
|
65
|
+
if error_message is not None:
|
|
66
|
+
error_message = "{} - {}".format(
|
|
67
|
+
resp.url,
|
|
68
|
+
error_message
|
|
69
|
+
)
|
|
70
|
+
raise LpseServerExceptions(error_message)
|
|
71
|
+
|
|
72
|
+
def get_auth_token(self, from_cookies=True):
|
|
73
|
+
"""
|
|
74
|
+
Melakukan pengambilan auth token
|
|
75
|
+
:return: token (str)
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
r = self.session.get(self.url + '/lelang')
|
|
79
|
+
|
|
80
|
+
if from_cookies:
|
|
81
|
+
auth_token = re.findall(r'___AT=([A-Za-z0-9]+)&', self.session.cookies.get('SPSE_SESSION'))
|
|
82
|
+
|
|
83
|
+
if auth_token:
|
|
84
|
+
return auth_token[0]
|
|
85
|
+
|
|
86
|
+
return utils.parse_token(r.text)
|
|
87
|
+
|
|
88
|
+
@backoff.on_exception(backoff.fibo,
|
|
89
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
90
|
+
requests.exceptions.ConnectionError),
|
|
91
|
+
jitter=None, max_tries=3)
|
|
92
|
+
def get_paket(self, jenis_paket, start=0, length=0, data_only=False,
|
|
93
|
+
kategori=None, search_keyword=None, nama_penyedia=None,
|
|
94
|
+
order=By.KODE, tahun=None, ascending=False, instansi_id=None):
|
|
95
|
+
"""
|
|
96
|
+
Melakukan pencarian paket pengadaan
|
|
97
|
+
:param jenis_paket: Paket Pengadaan Lelang (lelang) atau Penunjukkan Langsung (pl)
|
|
98
|
+
:param start: index data awal
|
|
99
|
+
:param length: jumlah data yang ditampilkan
|
|
100
|
+
:param data_only: hanya menampilkan data tanpa menampilkan informasi lain
|
|
101
|
+
:param kategori: kategori pengadaan (lihat di lpse.JenisPengadaan)
|
|
102
|
+
:param search_keyword: keyword pencarian paket pengadaan
|
|
103
|
+
:param nama_penyedia: filter berdasarkan nama penyedia
|
|
104
|
+
:param order: Mengurutkan data berdasarkan kolom
|
|
105
|
+
:param tahun: Tahun Pengadaan
|
|
106
|
+
:param ascending: Ascending, descending jika diset False
|
|
107
|
+
:param instansi_id: Filter pencarian berdasarkan instansi atau satker tertentu
|
|
108
|
+
:return: dictionary dari hasil pencarian paket (atau list jika data_only=True)
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
# TODO: Header dari data berbeda untuk tiap SPSE masing-masing ILAP.
|
|
112
|
+
# Cek tiap LPSE tiap ilap untuk menentukan header dari data
|
|
113
|
+
|
|
114
|
+
if not self.auth_token:
|
|
115
|
+
self.auth_token = self.get_auth_token()
|
|
116
|
+
|
|
117
|
+
params = {
|
|
118
|
+
'draw': 1,
|
|
119
|
+
'start': start,
|
|
120
|
+
'length': length,
|
|
121
|
+
'tahun': tahun,
|
|
122
|
+
'search[value]': search_keyword if search_keyword else '',
|
|
123
|
+
'search[regex]': 'false',
|
|
124
|
+
'order[0][column]': order.value,
|
|
125
|
+
'order[0][dir]': 'asc' if ascending else 'desc',
|
|
126
|
+
'authenticityToken': self.auth_token,
|
|
127
|
+
'_': int(time.time()*1000)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
for i in range(0, 5):
|
|
131
|
+
params.update(
|
|
132
|
+
{
|
|
133
|
+
'columns[{}][data]'.format(i): i,
|
|
134
|
+
'columns[{}][name]'.format(i): '',
|
|
135
|
+
'columns[{}][searchable]'.format(i): 'true' if i != 3 else 'false',
|
|
136
|
+
'columns[{}][orderable]'.format(i): 'true' if i != 3 else 'false',
|
|
137
|
+
'columns[{}][search][value]'.format(i): '',
|
|
138
|
+
'columns[{}][search][regex]'.format(i): 'false'
|
|
139
|
+
}
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
if kategori:
|
|
143
|
+
params.update({'kategoriId': kategori.value})
|
|
144
|
+
|
|
145
|
+
if nama_penyedia:
|
|
146
|
+
params.update({'rekanan': nama_penyedia})
|
|
147
|
+
params.update({'rkn_nama': nama_penyedia})
|
|
148
|
+
|
|
149
|
+
if instansi_id:
|
|
150
|
+
params.update({'instansiId': instansi_id})
|
|
151
|
+
|
|
152
|
+
# prepare request GET dan POST untuk spse 4.5.20221227
|
|
153
|
+
headers = {
|
|
154
|
+
'X-Requested-With': 'XMLHttpRequest',
|
|
155
|
+
'Referer': self.url + '/lelang',
|
|
156
|
+
'Sec-Fetch-Mode': 'cors',
|
|
157
|
+
'Sec-Fetch-Site': 'same-origin',
|
|
158
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
|
159
|
+
'AppleWebKit/537.36 (KHTML, like Gecko) '
|
|
160
|
+
'Chrome/77.0.3865.90 Safari/537.36'
|
|
161
|
+
}
|
|
162
|
+
url = self.url + '/dt/' + jenis_paket
|
|
163
|
+
|
|
164
|
+
data = self.session.post(
|
|
165
|
+
url,
|
|
166
|
+
data=params,
|
|
167
|
+
verify=False,
|
|
168
|
+
timeout=self.timeout,
|
|
169
|
+
headers=headers
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
logging.debug(data.content)
|
|
173
|
+
self.check_error(data)
|
|
174
|
+
|
|
175
|
+
data.encoding = 'UTF-8'
|
|
176
|
+
|
|
177
|
+
if data_only:
|
|
178
|
+
return data.json()['data']
|
|
179
|
+
|
|
180
|
+
return data.json()
|
|
181
|
+
|
|
182
|
+
def get_paket_tender(self, start=0, length=0, data_only=False,
|
|
183
|
+
kategori=None, search_keyword=None, nama_penyedia=None,
|
|
184
|
+
order=By.KODE, tahun=None, ascending=False, instansi_id=None):
|
|
185
|
+
"""
|
|
186
|
+
Wrapper pencarian paket tender
|
|
187
|
+
:param start: index data awal
|
|
188
|
+
:param length: jumlah data yang ditampilkan
|
|
189
|
+
:param data_only: hanya menampilkan data tanpa menampilkan informasi lain
|
|
190
|
+
:param kategori: kategori pengadaan (lihat di pypro.kategori)
|
|
191
|
+
:param search_keyword: keyword pencarian paket pengadaan
|
|
192
|
+
:param nama_penyedia: filter berdasarkan nama penyedia
|
|
193
|
+
:param order: Mengurutkan data berdasarkan kolom
|
|
194
|
+
:param tahun: Tahun Pengadaan
|
|
195
|
+
:param ascending: Ascending, descending jika diset False
|
|
196
|
+
:param instansi_id: Filter pencarian berdasarkan instansi atau satker tertentu
|
|
197
|
+
:return: dictionary dari hasil pencarian paket (atau list jika data_only=True)
|
|
198
|
+
"""
|
|
199
|
+
return self.get_paket('lelang', start, length, data_only, kategori, search_keyword, nama_penyedia,
|
|
200
|
+
order, tahun, ascending, instansi_id)
|
|
201
|
+
|
|
202
|
+
def get_paket_non_tender(self, start=0, length=0, data_only=False, kategori=None, search_keyword=None,
|
|
203
|
+
order=By.KODE, tahun=None, ascending=False, instansi_id=None):
|
|
204
|
+
"""
|
|
205
|
+
Wrapper pencarian paket non tender
|
|
206
|
+
:param start: index data awal
|
|
207
|
+
:param length: jumlah data yang ditampilkan
|
|
208
|
+
:param data_only: hanya menampilkan data tanpa menampilkan informasi lain
|
|
209
|
+
:param kategori: kategori pengadaan (lihat di pypro.kategori)
|
|
210
|
+
:param search_keyword: keyword pencarian paket pengadaan
|
|
211
|
+
:param nama_penyedia: filter berdasarkan nama penyedia
|
|
212
|
+
:param order: Mengurutkan data berdasarkan kolom
|
|
213
|
+
:param tahun: Tahun pengadaan
|
|
214
|
+
:param ascending: Ascending, descending jika diset False
|
|
215
|
+
:param instansi_id: Filter pencarian berdasarkan instansi atau satker tertentu
|
|
216
|
+
:return: dictionary dari hasil pencarian paket (atau list jika data_only=True)
|
|
217
|
+
"""
|
|
218
|
+
return self.get_paket('pl', start, length, data_only, kategori, search_keyword, None, order, tahun,
|
|
219
|
+
ascending, instansi_id)
|
|
220
|
+
|
|
221
|
+
def detil_paket_tender(self, id_paket):
|
|
222
|
+
"""
|
|
223
|
+
Mengambil detil pengadaan
|
|
224
|
+
:param id_paket:
|
|
225
|
+
:return:
|
|
226
|
+
"""
|
|
227
|
+
return LpseDetil(self, id_paket)
|
|
228
|
+
|
|
229
|
+
def detil_paket_non_tender(self, id_paket):
|
|
230
|
+
"""
|
|
231
|
+
Mengambil detil pengadaan non tender (penunjukkan langsung)
|
|
232
|
+
:param id_paket: id_paket non tender
|
|
233
|
+
:return:
|
|
234
|
+
"""
|
|
235
|
+
return LpseDetilNonTender(self, id_paket)
|
|
236
|
+
|
|
237
|
+
def __del__(self):
|
|
238
|
+
self.session.close()
|
|
239
|
+
del self.session
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class BaseLpseDetil(object):
|
|
243
|
+
def __init__(self, lpse, id_paket):
|
|
244
|
+
self._lpse = lpse
|
|
245
|
+
self.id_paket = id_paket
|
|
246
|
+
self.pengumuman = None
|
|
247
|
+
self.peserta = None
|
|
248
|
+
self.hasil = None
|
|
249
|
+
self.pemenang = None
|
|
250
|
+
self.pemenang_berkontrak = None
|
|
251
|
+
self.jadwal = None
|
|
252
|
+
|
|
253
|
+
def get_all_detil(self):
|
|
254
|
+
info = {
|
|
255
|
+
'error': False,
|
|
256
|
+
'error_message': []
|
|
257
|
+
}
|
|
258
|
+
for name in ['get_pengumuman', 'get_peserta', 'get_hasil_evaluasi', 'get_pemenang', 'get_pemenang_berkontrak',
|
|
259
|
+
'get_jadwal']:
|
|
260
|
+
try:
|
|
261
|
+
getattr(self, name)()
|
|
262
|
+
except Exception as e:
|
|
263
|
+
info['error'] = True
|
|
264
|
+
info['error_message'].append(
|
|
265
|
+
'{} - {} - {}'.format(e, self.id_paket, name)
|
|
266
|
+
)
|
|
267
|
+
return info
|
|
268
|
+
|
|
269
|
+
def __str__(self):
|
|
270
|
+
return str(self.todict())
|
|
271
|
+
|
|
272
|
+
def todict(self):
|
|
273
|
+
data = self.__dict__.copy()
|
|
274
|
+
data.pop('_lpse')
|
|
275
|
+
return data
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
class LpseDetil(BaseLpseDetil):
|
|
279
|
+
|
|
280
|
+
@backoff.on_exception(backoff.fibo,
|
|
281
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
282
|
+
requests.exceptions.ConnectionError),
|
|
283
|
+
max_tries=3, jitter=None)
|
|
284
|
+
def get_pengumuman(self):
|
|
285
|
+
self.pengumuman = LpseDetilPengumumanParser(self._lpse, self.id_paket).get_detil()
|
|
286
|
+
|
|
287
|
+
return self.pengumuman
|
|
288
|
+
|
|
289
|
+
@backoff.on_exception(backoff.fibo,
|
|
290
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
291
|
+
requests.exceptions.ConnectionError),
|
|
292
|
+
max_tries=3, jitter=None)
|
|
293
|
+
def get_peserta(self):
|
|
294
|
+
self.peserta = LpseDetilPesertaParser(self._lpse, self.id_paket).get_detil()
|
|
295
|
+
|
|
296
|
+
return self.peserta
|
|
297
|
+
|
|
298
|
+
@backoff.on_exception(backoff.fibo,
|
|
299
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
300
|
+
requests.exceptions.ConnectionError),
|
|
301
|
+
max_tries=3, jitter=None)
|
|
302
|
+
def get_hasil_evaluasi(self):
|
|
303
|
+
self.hasil = LpseDetilHasilEvaluasiParser(self._lpse, self.id_paket).get_detil()
|
|
304
|
+
|
|
305
|
+
return self.hasil
|
|
306
|
+
|
|
307
|
+
@backoff.on_exception(backoff.fibo,
|
|
308
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
309
|
+
requests.exceptions.ConnectionError),
|
|
310
|
+
max_tries=3, jitter=None)
|
|
311
|
+
def get_pemenang(self, all=False, key='hasil_negosiasi'):
|
|
312
|
+
self.pemenang = LpseDetilPemenangParser(
|
|
313
|
+
self._lpse,
|
|
314
|
+
self.id_paket,
|
|
315
|
+
all=all,
|
|
316
|
+
key=key
|
|
317
|
+
).get_detil()
|
|
318
|
+
|
|
319
|
+
return self.pemenang
|
|
320
|
+
|
|
321
|
+
@backoff.on_exception(backoff.fibo,
|
|
322
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
323
|
+
requests.exceptions.ConnectionError),
|
|
324
|
+
max_tries=3, jitter=None)
|
|
325
|
+
def get_pemenang_berkontrak(self):
|
|
326
|
+
self.pemenang_berkontrak = LpseDetilPemenangBerkontrakParser(self._lpse, self.id_paket).get_detil()
|
|
327
|
+
|
|
328
|
+
return self.pemenang_berkontrak
|
|
329
|
+
|
|
330
|
+
@backoff.on_exception(backoff.fibo,
|
|
331
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
332
|
+
requests.exceptions.ConnectionError),
|
|
333
|
+
max_tries=3, jitter=None)
|
|
334
|
+
def get_jadwal(self):
|
|
335
|
+
self.jadwal = LpseDetilJadwalParser(self._lpse, self.id_paket).get_detil()
|
|
336
|
+
|
|
337
|
+
return self.jadwal
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
class LpseDetilNonTender(BaseLpseDetil):
|
|
341
|
+
|
|
342
|
+
@backoff.on_exception(backoff.fibo,
|
|
343
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
344
|
+
requests.exceptions.ConnectionError),
|
|
345
|
+
max_tries=3, jitter=None)
|
|
346
|
+
def get_pengumuman(self):
|
|
347
|
+
self.pengumuman = LpseDetilPengumumanNonTenderParser(self._lpse, self.id_paket).get_detil()
|
|
348
|
+
|
|
349
|
+
return self.pengumuman
|
|
350
|
+
|
|
351
|
+
@backoff.on_exception(backoff.fibo,
|
|
352
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
353
|
+
requests.exceptions.ConnectionError),
|
|
354
|
+
max_tries=3, jitter=None)
|
|
355
|
+
def get_peserta(self):
|
|
356
|
+
self.peserta = LpseDetilPesertaNonTenderParser(self._lpse, self.id_paket).get_detil()
|
|
357
|
+
|
|
358
|
+
return self.peserta
|
|
359
|
+
|
|
360
|
+
@backoff.on_exception(backoff.fibo,
|
|
361
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
362
|
+
requests.exceptions.ConnectionError),
|
|
363
|
+
max_tries=3, jitter=None)
|
|
364
|
+
def get_hasil_evaluasi(self):
|
|
365
|
+
self.hasil = LpseDetilHasilEvaluasiNonTenderParser(self._lpse, self.id_paket).get_detil()
|
|
366
|
+
|
|
367
|
+
return self.hasil
|
|
368
|
+
|
|
369
|
+
@backoff.on_exception(backoff.fibo,
|
|
370
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
371
|
+
requests.exceptions.ConnectionError),
|
|
372
|
+
max_tries=3, jitter=None)
|
|
373
|
+
def get_pemenang(self):
|
|
374
|
+
self.pemenang = LpseDetilPemenangNonTenderParser(self._lpse, self.id_paket).get_detil()
|
|
375
|
+
|
|
376
|
+
return self.pemenang
|
|
377
|
+
|
|
378
|
+
@backoff.on_exception(backoff.fibo,
|
|
379
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
380
|
+
requests.exceptions.ConnectionError),
|
|
381
|
+
max_tries=3, jitter=None)
|
|
382
|
+
def get_pemenang_berkontrak(self):
|
|
383
|
+
self.pemenang_berkontrak = LpseDetilPemenangBerkontrakNonTenderParser(self._lpse, self.id_paket).get_detil()
|
|
384
|
+
|
|
385
|
+
return self.pemenang_berkontrak
|
|
386
|
+
|
|
387
|
+
@backoff.on_exception(backoff.fibo,
|
|
388
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
389
|
+
requests.exceptions.ConnectionError),
|
|
390
|
+
max_tries=3, jitter=None)
|
|
391
|
+
def get_jadwal(self):
|
|
392
|
+
self.jadwal = LpseDetilJadwalNonTenderParser(self._lpse, self.id_paket).get_detil()
|
|
393
|
+
|
|
394
|
+
return self.jadwal
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
class BaseLpseDetilParser(object):
|
|
398
|
+
|
|
399
|
+
detil_path = None
|
|
400
|
+
|
|
401
|
+
def __init__(self, lpse, id_paket):
|
|
402
|
+
self.lpse = lpse
|
|
403
|
+
self.id_paket = id_paket
|
|
404
|
+
|
|
405
|
+
@backoff.on_exception(backoff.fibo,
|
|
406
|
+
(LpseServerExceptions, requests.exceptions.RequestException,
|
|
407
|
+
requests.exceptions.ConnectionError),
|
|
408
|
+
max_tries=3, jitter=None)
|
|
409
|
+
def get_detil(self):
|
|
410
|
+
url = self.lpse.url+self.detil_path.format(self.id_paket)
|
|
411
|
+
r = self.lpse.session.get(
|
|
412
|
+
url,
|
|
413
|
+
timeout=self.lpse.timeout,
|
|
414
|
+
headers={
|
|
415
|
+
"referer": self.lpse.url
|
|
416
|
+
}
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
self.lpse.check_error(r)
|
|
420
|
+
|
|
421
|
+
return self.parse_detil(r.content)
|
|
422
|
+
|
|
423
|
+
@abstractmethod
|
|
424
|
+
def parse_detil(self, content):
|
|
425
|
+
pass
|
|
426
|
+
|
|
427
|
+
@staticmethod
|
|
428
|
+
def parse_currency(nilai):
|
|
429
|
+
result = ''.join(re.findall(r'([\d+,])', nilai)).replace(',', '.')
|
|
430
|
+
try:
|
|
431
|
+
return float(result)
|
|
432
|
+
except ValueError:
|
|
433
|
+
return 0
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
class LpseDetilPengumumanParser(BaseLpseDetilParser):
|
|
437
|
+
|
|
438
|
+
detil_path = '/lelang/{}/pengumumanlelang'
|
|
439
|
+
|
|
440
|
+
def parse_detil(self, content):
|
|
441
|
+
soup = Bs(content, 'html5lib')
|
|
442
|
+
|
|
443
|
+
content = soup.find('div', {'class': 'content'})
|
|
444
|
+
table = content.find('table', {'class': 'table-bordered'}).find('tbody')
|
|
445
|
+
|
|
446
|
+
return self.parse_table(table)
|
|
447
|
+
|
|
448
|
+
def parse_table(self, table):
|
|
449
|
+
data = {}
|
|
450
|
+
|
|
451
|
+
for tr in table.find_all('tr', recursive=False):
|
|
452
|
+
ths = tr.find_all('th', recursive=False)
|
|
453
|
+
tds = tr.find_all('td', recursive=False)
|
|
454
|
+
|
|
455
|
+
for th, td in zip(ths, tds):
|
|
456
|
+
data_key = '_'.join(th.text.strip().split()).lower()
|
|
457
|
+
|
|
458
|
+
td_sub_table = td.find('table', recursive=False)
|
|
459
|
+
|
|
460
|
+
if td_sub_table and data_key == 'rencana_umum_pengadaan':
|
|
461
|
+
data_value = self.parse_rup(td_sub_table.find('tbody'))
|
|
462
|
+
elif data_key == 'syarat_kualifikasi':
|
|
463
|
+
# TODO: Buat parser syarat kualifikasi, tapi perlu tahu dulu kemungkinan format dan isinya
|
|
464
|
+
continue
|
|
465
|
+
elif data_key == 'lokasi_pekerjaan':
|
|
466
|
+
data_value = self.parse_lokasi_pekerjaan(td)
|
|
467
|
+
elif data_key in ('nilai_hps_paket', 'nilai_pagu_paket'):
|
|
468
|
+
data_value = self.parse_currency(' '.join(td.text.strip().split()))
|
|
469
|
+
elif data_key == 'peserta_tender':
|
|
470
|
+
try:
|
|
471
|
+
data_value = int(td.text.strip().split()[0])
|
|
472
|
+
except ValueError:
|
|
473
|
+
data_value = -1
|
|
474
|
+
elif data_key == 'nama_tender' or data_key == 'nama_paket':
|
|
475
|
+
data_value, label = self.parse_nama_tender(td)
|
|
476
|
+
data.update({
|
|
477
|
+
'label_paket': label
|
|
478
|
+
})
|
|
479
|
+
else:
|
|
480
|
+
data_value = ' '.join(td.text.strip().split())
|
|
481
|
+
|
|
482
|
+
data.update({
|
|
483
|
+
data_key: data_value
|
|
484
|
+
})
|
|
485
|
+
|
|
486
|
+
return data
|
|
487
|
+
|
|
488
|
+
def parse_rup(self, tbody_rup):
|
|
489
|
+
raw_data = []
|
|
490
|
+
for tr in tbody_rup.find_all('tr'):
|
|
491
|
+
raw_data.append([' '.join(i.text.strip().split()) for i in tr.children if not isinstance(i, NavigableString)])
|
|
492
|
+
|
|
493
|
+
header = ['_'.join(i.split()).lower() for i in raw_data[0]]
|
|
494
|
+
data = []
|
|
495
|
+
|
|
496
|
+
for row in raw_data[1:]:
|
|
497
|
+
item = {}
|
|
498
|
+
item.update(zip(header, row))
|
|
499
|
+
try:
|
|
500
|
+
item.pop('')
|
|
501
|
+
except KeyError:
|
|
502
|
+
pass
|
|
503
|
+
data.append(item)
|
|
504
|
+
|
|
505
|
+
return data
|
|
506
|
+
|
|
507
|
+
def parse_lokasi_pekerjaan(self, td_pekerjaan):
|
|
508
|
+
return [' '.join(li.text.strip().split()) for li in td_pekerjaan.find_all('li')]
|
|
509
|
+
|
|
510
|
+
def parse_nama_tender(self, element):
|
|
511
|
+
label = []
|
|
512
|
+
for i in element.find_all('span'):
|
|
513
|
+
label.append(i.text.strip())
|
|
514
|
+
i.decompose()
|
|
515
|
+
|
|
516
|
+
text = element.text.strip()
|
|
517
|
+
|
|
518
|
+
return text, label
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
class LpseDetilPesertaParser(BaseLpseDetilParser):
|
|
522
|
+
|
|
523
|
+
detil_path = '/lelang/{}/peserta'
|
|
524
|
+
|
|
525
|
+
def parse_detil(self, content):
|
|
526
|
+
soup = Bs(content, 'html5lib')
|
|
527
|
+
table = soup.find('div', {'class': 'content'})\
|
|
528
|
+
.find('table')
|
|
529
|
+
|
|
530
|
+
raw_data = [[i for i in tr.stripped_strings] for tr in table.find_all('tr')]
|
|
531
|
+
|
|
532
|
+
header = ['_'.join(i.strip().split()).lower() for i in raw_data[0]]
|
|
533
|
+
|
|
534
|
+
return [dict(zip(header, i)) for i in raw_data[1:]]
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
class LpseDetilHasilEvaluasiParser(BaseLpseDetilParser):
|
|
538
|
+
|
|
539
|
+
detil_path = '/evaluasi/{}/hasil'
|
|
540
|
+
header_ref = {
|
|
541
|
+
"a": "evaluasi_administrasi",
|
|
542
|
+
"t": "evaluasi_teknis",
|
|
543
|
+
"st": "skor_teknis",
|
|
544
|
+
"p_1": "penawaran",
|
|
545
|
+
"pt": "penawaran_terkoreksi",
|
|
546
|
+
"hn": "hasil_negosiasi",
|
|
547
|
+
"sh": "skor_harga",
|
|
548
|
+
"sa": "skor_akhir",
|
|
549
|
+
"b": "pembuktian_kualifikasi",
|
|
550
|
+
"k": "evaluasi_kualifikasi",
|
|
551
|
+
"sk": "skor_kualifikasi",
|
|
552
|
+
"sb": "skor_pembuktian",
|
|
553
|
+
"h": "evaluasi_harga_biaya",
|
|
554
|
+
"p_2": "pemenang",
|
|
555
|
+
"pk": "pemenang_berkontrak"
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
def parse_detil(self, content):
|
|
559
|
+
soup = Bs(content, 'html5lib')
|
|
560
|
+
table = soup.find('div', {'class': 'content'})\
|
|
561
|
+
.find('table')
|
|
562
|
+
|
|
563
|
+
if not table:
|
|
564
|
+
return
|
|
565
|
+
|
|
566
|
+
is_header = True
|
|
567
|
+
header = []
|
|
568
|
+
data = []
|
|
569
|
+
|
|
570
|
+
for tr in table.find_all('tr'):
|
|
571
|
+
|
|
572
|
+
if is_header:
|
|
573
|
+
header = ['_'.join(i.text.strip().split()).lower() for i in filter(lambda x: type(x) == bs4.element.Tag, tr.children)]
|
|
574
|
+
|
|
575
|
+
# fix duplicate header key for p
|
|
576
|
+
if header.count('p') > 1:
|
|
577
|
+
first_p_idx = header.index('p')
|
|
578
|
+
second_p_idx = header.index('p', first_p_idx + 1)
|
|
579
|
+
header[first_p_idx] = 'p_1'
|
|
580
|
+
header[second_p_idx] = 'p_2'
|
|
581
|
+
|
|
582
|
+
# map header key to reference
|
|
583
|
+
header = list(map(lambda x: self.header_ref.get(x, x), header))
|
|
584
|
+
|
|
585
|
+
is_header = False
|
|
586
|
+
else:
|
|
587
|
+
children = [self.parse_icon(i) for i in filter(lambda x: type(x) == bs4.element.Tag, tr.children)]
|
|
588
|
+
children_dict = self.parse_children(dict(zip(header, children)))
|
|
589
|
+
|
|
590
|
+
data.append(children_dict)
|
|
591
|
+
|
|
592
|
+
return data
|
|
593
|
+
|
|
594
|
+
def parse_children(self, children):
|
|
595
|
+
for key, value in children.items():
|
|
596
|
+
if key.startswith('s'):
|
|
597
|
+
try:
|
|
598
|
+
children[key] = float(value)
|
|
599
|
+
except ValueError:
|
|
600
|
+
children[key] = 0.0
|
|
601
|
+
elif key in ['penawaran', 'penawaran_terkoreksi', 'hasil_negosiasi']:
|
|
602
|
+
children[key] = self.parse_currency(value)
|
|
603
|
+
elif key in ['evaluasi_harga_biaya', 'pemenang', 'pemenang_berkontrak'] and children[key] != True:
|
|
604
|
+
children[key] = False
|
|
605
|
+
|
|
606
|
+
return children
|
|
607
|
+
|
|
608
|
+
def parse_nama_npwp(self, peserta):
|
|
609
|
+
return str(peserta).rsplit(' - ', maxsplit=1)
|
|
610
|
+
|
|
611
|
+
def parse_icon(self, child):
|
|
612
|
+
status = {
|
|
613
|
+
'fa-check': 1,
|
|
614
|
+
'fa-close': 0,
|
|
615
|
+
'fa-minus': None
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
icon = re.findall(r'fa (fa-.*)">', str(child))
|
|
619
|
+
if icon:
|
|
620
|
+
return status[icon[0]]
|
|
621
|
+
elif re.findall(r'star.gif', str(child)):
|
|
622
|
+
return True
|
|
623
|
+
return child.text.strip()
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
class LpseDetilPemenangParser(BaseLpseDetilParser):
|
|
627
|
+
|
|
628
|
+
detil_path = '/evaluasi/{}/pemenang'
|
|
629
|
+
|
|
630
|
+
def __init__(self, lpse, id_paket, all=False, key='hasil_negosiasi'):
|
|
631
|
+
super().__init__(lpse, id_paket)
|
|
632
|
+
self.key = key
|
|
633
|
+
self.all = all
|
|
634
|
+
|
|
635
|
+
def parse_detil(self, content):
|
|
636
|
+
soup = Bs(content, 'html5lib')
|
|
637
|
+
|
|
638
|
+
try:
|
|
639
|
+
table_pemenang = soup.find('div', {'class': 'content'})\
|
|
640
|
+
.table\
|
|
641
|
+
.tbody\
|
|
642
|
+
.find_all('tr', recursive=False)[-1]\
|
|
643
|
+
.find('table')
|
|
644
|
+
except AttributeError:
|
|
645
|
+
return
|
|
646
|
+
|
|
647
|
+
if table_pemenang:
|
|
648
|
+
header = ['_'.join(th.text.strip().split()).lower() for th in table_pemenang.find_all('th')]
|
|
649
|
+
all_pemenang = []
|
|
650
|
+
|
|
651
|
+
for tr in table_pemenang.find_all('tr'):
|
|
652
|
+
data = [' '.join(td.text.strip().split()) for td in tr.find_all('td')]
|
|
653
|
+
|
|
654
|
+
if data:
|
|
655
|
+
# set default dict untuk data pemenang karena nama header beda-beda
|
|
656
|
+
# ref: https://github.com/wakataw/pyproc/pull/53
|
|
657
|
+
pemenang = {
|
|
658
|
+
'nama_pemenang': None,
|
|
659
|
+
'alamat': None,
|
|
660
|
+
'npwp': None,
|
|
661
|
+
'harga_penawaran': 0,
|
|
662
|
+
'harga_terkoreksi': 0,
|
|
663
|
+
'hasil_negosiasi': 0,
|
|
664
|
+
'harga_negosiasi': 0
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
for i, v in zip(header, data):
|
|
668
|
+
if 'reverse_auction' in i:
|
|
669
|
+
i = 'hasil_negosiasi'
|
|
670
|
+
|
|
671
|
+
pemenang[i] = self.parse_currency(v) \
|
|
672
|
+
if (v.lower().startswith('rp') or i.startswith('harga') or i.startswith('hasil')) else v
|
|
673
|
+
|
|
674
|
+
all_pemenang.append(pemenang)
|
|
675
|
+
|
|
676
|
+
if not all_pemenang:
|
|
677
|
+
return []
|
|
678
|
+
elif self.all:
|
|
679
|
+
all_pemenang = self._check_col_harga_negosiasi(all_pemenang)
|
|
680
|
+
return all_pemenang
|
|
681
|
+
else:
|
|
682
|
+
try:
|
|
683
|
+
return [min(all_pemenang, key=lambda x: x[self.key])]
|
|
684
|
+
except KeyError:
|
|
685
|
+
# fallback ke kolom harga penawaran untuk sorting jika kolom hasil negosiasi tidak ditemukan
|
|
686
|
+
all_pemenang = self._check_col_harga_negosiasi(all_pemenang)
|
|
687
|
+
return [min(all_pemenang, key=lambda x: x['harga_penawaran'])]
|
|
688
|
+
return
|
|
689
|
+
|
|
690
|
+
@staticmethod
|
|
691
|
+
def _check_col_harga_negosiasi(all_pemenang):
|
|
692
|
+
if 'hasil_negosiasi' not in all_pemenang[0]:
|
|
693
|
+
all_pemenang[0]['hasil_negosiasi'] = ''
|
|
694
|
+
|
|
695
|
+
return all_pemenang
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
class LpseDetilPemenangBerkontrakParser(LpseDetilPemenangParser):
|
|
699
|
+
|
|
700
|
+
detil_path = '/evaluasi/{}/pemenangberkontrak'
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
class LpseDetilJadwalParser(BaseLpseDetilParser):
|
|
704
|
+
|
|
705
|
+
detil_path = '/lelang/{}/jadwal'
|
|
706
|
+
|
|
707
|
+
def parse_detil(self, content):
|
|
708
|
+
soup = Bs(content, 'html5lib')
|
|
709
|
+
table = soup.find('table')
|
|
710
|
+
|
|
711
|
+
if not table:
|
|
712
|
+
return
|
|
713
|
+
|
|
714
|
+
is_header = True
|
|
715
|
+
header = None
|
|
716
|
+
jadwal = []
|
|
717
|
+
|
|
718
|
+
for tr in table.find_all('tr'):
|
|
719
|
+
|
|
720
|
+
if is_header:
|
|
721
|
+
header = ['_'.join(th.text.strip().split()).lower() for th in tr.find_all('th')]
|
|
722
|
+
is_header = False
|
|
723
|
+
else:
|
|
724
|
+
data = [' '.join(td.text.strip().split()) for td in tr.find_all('td')]
|
|
725
|
+
jadwal.append(dict(zip(header, data)))
|
|
726
|
+
|
|
727
|
+
return jadwal
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
class LpseDetilPengumumanNonTenderParser(LpseDetilPengumumanParser):
|
|
731
|
+
|
|
732
|
+
detil_path = '/nontender/{}/pengumumanpl'
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
class LpseDetilPesertaNonTenderParser(LpseDetilPesertaParser):
|
|
736
|
+
|
|
737
|
+
detil_path = '/nontender/{}/peserta'
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
class LpseDetilHasilEvaluasiNonTenderParser(LpseDetilHasilEvaluasiParser):
|
|
741
|
+
|
|
742
|
+
detil_path = '/evaluasinontender/{}/hasil'
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
class LpseDetilPemenangNonTenderParser(LpseDetilPemenangParser):
|
|
746
|
+
|
|
747
|
+
detil_path = '/evaluasinontender/{}/pemenang'
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
class LpseDetilPemenangBerkontrakNonTenderParser(LpseDetilPemenangNonTenderParser):
|
|
751
|
+
|
|
752
|
+
detil_path = '/evaluasinontender/{}/pemenangberkontrak'
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
class LpseDetilJadwalNonTenderParser(LpseDetilJadwalParser):
|
|
756
|
+
|
|
757
|
+
detil_path = '/nontender/{}/jadwal'
|