tenderkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tenderkit/__init__.py +9 -0
- tenderkit/__main__.py +5 -0
- tenderkit/cli.py +248 -0
- tenderkit/documents.py +260 -0
- tenderkit/errors.py +52 -0
- tenderkit/http.py +94 -0
- tenderkit/interfaces.py +42 -0
- tenderkit/limits.py +7 -0
- tenderkit/mcp.py +221 -0
- tenderkit/model.py +78 -0
- tenderkit/pdf_ocr.py +78 -0
- tenderkit/platforms/__init__.py +33 -0
- tenderkit/platforms/backends.py +163 -0
- tenderkit/platforms/base.py +79 -0
- tenderkit/platforms/bk.py +78 -0
- tenderkit/platforms/ezamowienia.py +85 -0
- tenderkit/platforms/html.py +67 -0
- tenderkit/platforms/marketplanet_html.py +128 -0
- tenderkit/platforms/public_documents.py +206 -0
- tenderkit/platforms/pz_html.py +127 -0
- tenderkit/platforms/session.py +114 -0
- tenderkit/profile.py +216 -0
- tenderkit/profiles/__init__.py +13 -0
- tenderkit/profiles/leki.toml +5 -0
- tenderkit/profiles/oprogramowanie.toml +4 -0
- tenderkit/profiles/wyroby-medyczne.toml +4 -0
- tenderkit/queue.py +126 -0
- tenderkit/read_document.py +309 -0
- tenderkit/serialization.py +63 -0
- tenderkit/service.py +164 -0
- tenderkit/sources/__init__.py +1 -0
- tenderkit/sources/bk.py +145 -0
- tenderkit/sources/bzp.py +142 -0
- tenderkit/sources/mapping.py +264 -0
- tenderkit/sources/ted.py +116 -0
- tenderkit/sources/ted_xml.py +97 -0
- tenderkit/store/__init__.py +1 -0
- tenderkit/store/documents.py +168 -0
- tenderkit/store/state.py +202 -0
- tenderkit/ui.py +243 -0
- tenderkit/watch.py +247 -0
- tenderkit/web/index.html +59 -0
- tenderkit/web/style.css +221 -0
- tenderkit-0.1.0.dist-info/METADATA +219 -0
- tenderkit-0.1.0.dist-info/RECORD +49 -0
- tenderkit-0.1.0.dist-info/WHEEL +5 -0
- tenderkit-0.1.0.dist-info/entry_points.txt +2 -0
- tenderkit-0.1.0.dist-info/licenses/LICENSE +202 -0
- tenderkit-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Built-in profiles; no dependency on pharma or a user-specific filesystem path."""
|
|
2
|
+
from importlib.resources import files
|
|
3
|
+
|
|
4
|
+
from tenderkit.profile import load_profile
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def builtin_profiles():
|
|
8
|
+
return {path.name[:-5]: load_profile(path) for path in files(__name__).iterdir() if path.name.endswith('.toml')}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def resolve_profile(name_or_path):
|
|
12
|
+
profiles = builtin_profiles()
|
|
13
|
+
return profiles[name_or_path] if name_or_path in profiles else load_profile(name_or_path)
|
tenderkit/queue.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Durable bounded download jobs. An OS lock releases automatically if a worker dies."""
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from tenderkit.errors import classify_error
|
|
8
|
+
from tenderkit.limits import MAX_ATTEMPTS, MAX_DOCUMENTS, MAX_JOBS
|
|
9
|
+
from tenderkit.serialization import dumps
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@contextmanager
|
|
13
|
+
def worker_lock(path, *, suffix='.documents.lock'):
|
|
14
|
+
with open(str(path) + suffix, 'a+b') as stream:
|
|
15
|
+
if stream.tell() == 0:
|
|
16
|
+
stream.write(b'0')
|
|
17
|
+
stream.flush()
|
|
18
|
+
stream.seek(0)
|
|
19
|
+
try:
|
|
20
|
+
if os.name == 'nt':
|
|
21
|
+
import msvcrt
|
|
22
|
+
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
|
23
|
+
else:
|
|
24
|
+
import fcntl
|
|
25
|
+
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
26
|
+
except OSError:
|
|
27
|
+
yield False
|
|
28
|
+
return
|
|
29
|
+
try:
|
|
30
|
+
yield True
|
|
31
|
+
finally:
|
|
32
|
+
if os.name == 'nt':
|
|
33
|
+
stream.seek(0)
|
|
34
|
+
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
|
35
|
+
else:
|
|
36
|
+
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class SelectionConflict(ValueError):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Queue:
|
|
44
|
+
def __init__(self, state):
|
|
45
|
+
self.state = state
|
|
46
|
+
|
|
47
|
+
def enqueue(self, source, number, keys=None):
|
|
48
|
+
if self.state.get(source, number) is None:
|
|
49
|
+
raise ValueError('Notice not found in local state')
|
|
50
|
+
if keys is not None:
|
|
51
|
+
if (not isinstance(keys, (tuple, list)) or not 1 <= len(keys) <= MAX_DOCUMENTS
|
|
52
|
+
or any(not isinstance(k, str) or not k or len(k) > 8000 for k in keys)):
|
|
53
|
+
raise ValueError('Invalid document selection')
|
|
54
|
+
keys = list(dict.fromkeys(keys))
|
|
55
|
+
with self.state.connection() as conn:
|
|
56
|
+
conn.execute('begin immediate')
|
|
57
|
+
found = conn.execute("select * from fetch_jobs where source=? and number=? and status in ('QUEUED','RUNNING')", (source, number)).fetchone()
|
|
58
|
+
if found:
|
|
59
|
+
if json.loads(found['selection']) != keys:
|
|
60
|
+
raise SelectionConflict('A job with a different document selection is already active')
|
|
61
|
+
return found['id']
|
|
62
|
+
return conn.execute('insert into fetch_jobs(source,number,selection,status) values(?,?,?,?) returning id',
|
|
63
|
+
(source, number, dumps(keys), 'QUEUED')).fetchone()[0]
|
|
64
|
+
|
|
65
|
+
def job(self, job_id):
|
|
66
|
+
with self.state.connection() as conn:
|
|
67
|
+
row = conn.execute('select * from fetch_jobs where id=?', (job_id,)).fetchone()
|
|
68
|
+
if row is None:
|
|
69
|
+
raise ValueError('Download job not found')
|
|
70
|
+
result = dict(row)
|
|
71
|
+
result['selection'], result['summary'] = json.loads(result['selection']), json.loads(result['summary'])
|
|
72
|
+
return result
|
|
73
|
+
|
|
74
|
+
def recover(self):
|
|
75
|
+
# Called only while holding the OS worker lock, so no live owner can be running.
|
|
76
|
+
with self.state.connection() as conn:
|
|
77
|
+
rows = conn.execute("select id,attempts from fetch_jobs where status='RUNNING'").fetchall()
|
|
78
|
+
for row in rows:
|
|
79
|
+
conn.execute('update fetch_jobs set status=?,error=?,error_kind=?,next_attempt=0 where id=?',
|
|
80
|
+
('FAILED' if row['attempts'] >= MAX_ATTEMPTS else 'QUEUED', 'WORKER_LOST', 'TRANSIENT', row['id']))
|
|
81
|
+
if row['attempts'] >= MAX_ATTEMPTS:
|
|
82
|
+
conn.execute("update notices set coverage='FETCH_FAILED' where (source,number)=(select source,number from fetch_jobs where id=?)", (row['id'],))
|
|
83
|
+
return len(rows)
|
|
84
|
+
|
|
85
|
+
def beat(self, job_id, summary):
|
|
86
|
+
with self.state.connection() as conn:
|
|
87
|
+
conn.execute('update fetch_jobs set heartbeat=?,summary=? where id=?', (time.time(), dumps(summary), job_id))
|
|
88
|
+
|
|
89
|
+
def finish(self, job_id, error=None):
|
|
90
|
+
job = self.job(job_id)
|
|
91
|
+
kind = classify_error('FAILED' if error else 'DONE', error)
|
|
92
|
+
retry = bool(error) and job['attempts'] < (MAX_ATTEMPTS if kind == 'TRANSIENT' else 2 if kind == 'UNCERTAIN' else 1)
|
|
93
|
+
status = 'QUEUED' if retry else 'FAILED' if error else 'DONE'
|
|
94
|
+
with self.state.connection() as conn:
|
|
95
|
+
conn.execute('update fetch_jobs set status=?,error=?,error_kind=?,next_attempt=?,heartbeat=? where id=?',
|
|
96
|
+
(status, error, kind, time.time() + 60 * job['attempts'] if retry else 0, time.time(), job_id))
|
|
97
|
+
return self.job(job_id)
|
|
98
|
+
|
|
99
|
+
def run(self, process, *, max_jobs=1, job_id=None):
|
|
100
|
+
if type(max_jobs) is not int or not 1 <= max_jobs <= MAX_JOBS:
|
|
101
|
+
raise ValueError('max_jobs must be between 1 and 20')
|
|
102
|
+
results = []
|
|
103
|
+
with worker_lock(self.state.path) as locked:
|
|
104
|
+
if not locked:
|
|
105
|
+
return dict(locked=False, jobs=[])
|
|
106
|
+
recovered = self.recover()
|
|
107
|
+
for _ in range(max_jobs):
|
|
108
|
+
with self.state.connection() as conn:
|
|
109
|
+
conn.execute('begin immediate')
|
|
110
|
+
query = "select id from fetch_jobs where status='QUEUED' and next_attempt<=?"
|
|
111
|
+
args = [time.time()]
|
|
112
|
+
if job_id is not None:
|
|
113
|
+
query += ' and id=?'
|
|
114
|
+
args.append(job_id)
|
|
115
|
+
row = conn.execute(query + ' order by id limit 1', args).fetchone()
|
|
116
|
+
if row is None:
|
|
117
|
+
break
|
|
118
|
+
current = row[0]
|
|
119
|
+
conn.execute("update fetch_jobs set status='RUNNING',attempts=attempts+1,heartbeat=? where id=?", (time.time(), current))
|
|
120
|
+
error = None
|
|
121
|
+
try:
|
|
122
|
+
process(self.job(current))
|
|
123
|
+
except Exception as exc:
|
|
124
|
+
error = getattr(exc, 'code', type(exc).__name__)
|
|
125
|
+
results.append(self.finish(current, error))
|
|
126
|
+
return dict(locked=True, recovered=recovered, jobs=results)
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"""Bounded text extraction. OOXML uses stdlib, legacy XLS the optional xlrd; PDF and legacy DOC run an installed
|
|
2
|
+
CLI in a child process."""
|
|
3
|
+
from pathlib import Path, PurePosixPath
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
import time
|
|
10
|
+
from xml.etree import ElementTree as ET
|
|
11
|
+
from zipfile import ZipFile, BadZipFile
|
|
12
|
+
|
|
13
|
+
MAX_INPUT = 20 * 1024 * 1024
|
|
14
|
+
MAX_TEXT = 8 * 1024 * 1024
|
|
15
|
+
MAX_XML = 32 * 1024 * 1024
|
|
16
|
+
MAX_RESPONSE = 60000
|
|
17
|
+
CHUNK_CHARS = 12000
|
|
18
|
+
W = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
|
|
19
|
+
S = '{http://schemas.openxmlformats.org/spreadsheetml/2006/main}'
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TextReadError(ValueError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _chunks(text, label):
|
|
27
|
+
return [dict(text=text[i:i+CHUNK_CHARS], label=label) for i in range(0, len(text), CHUNK_CHARS)] or [dict(text='', label=label)]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _xml(archive, name):
|
|
31
|
+
data = archive.read(name)
|
|
32
|
+
if b'<!DOCTYPE' in data.upper() or b'<!ENTITY' in data.upper():
|
|
33
|
+
raise TextReadError('XML declarations are not supported')
|
|
34
|
+
return ET.fromstring(data)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _docx(archive):
|
|
38
|
+
root = _xml(archive, 'word/document.xml')
|
|
39
|
+
pages, text, size = [], [], 0
|
|
40
|
+
for paragraph in root.iter(W + 'p'):
|
|
41
|
+
for node in paragraph.iter():
|
|
42
|
+
if node.tag == W + 't':
|
|
43
|
+
size += len(node.text or '')
|
|
44
|
+
if size > MAX_TEXT:
|
|
45
|
+
raise TextReadError('Extracted text exceeds limit')
|
|
46
|
+
text.append(node.text or '')
|
|
47
|
+
elif node.tag == W + 'tab':
|
|
48
|
+
text.append('\t')
|
|
49
|
+
elif node.tag == W + 'lastRenderedPageBreak' or node.tag == W + 'br' and node.get(W + 'type') == 'page':
|
|
50
|
+
pages.extend(_chunks(''.join(text), 'document'))
|
|
51
|
+
text = []
|
|
52
|
+
elif node.tag == W + 'br':
|
|
53
|
+
text.append('\n')
|
|
54
|
+
text.append('\n')
|
|
55
|
+
pages.extend(_chunks(''.join(text), 'document'))
|
|
56
|
+
return pages
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _xlsx(archive):
|
|
60
|
+
shared = []
|
|
61
|
+
if 'xl/sharedStrings.xml' in archive.namelist():
|
|
62
|
+
shared = [''.join(node.itertext()) for node in _xml(archive, 'xl/sharedStrings.xml').findall(S + 'si')]
|
|
63
|
+
workbook = _xml(archive, 'xl/workbook.xml')
|
|
64
|
+
relationships = {node.get('Id'): node for node in _xml(archive, 'xl/_rels/workbook.xml.rels')}
|
|
65
|
+
relation_id = '{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id'
|
|
66
|
+
pages, size = [], 0
|
|
67
|
+
for sheet in workbook.findall(S + 'sheets/' + S + 'sheet'):
|
|
68
|
+
relation = relationships.get(sheet.get(relation_id))
|
|
69
|
+
if relation is None or relation.get('TargetMode') == 'External':
|
|
70
|
+
raise TextReadError('Unsupported worksheet relationship')
|
|
71
|
+
target = relation.get('Target', '')
|
|
72
|
+
path = PurePosixPath(target.lstrip('/')) if target.startswith('/') else PurePosixPath('xl') / target
|
|
73
|
+
if '..' in path.parts or not str(path).startswith('xl/'):
|
|
74
|
+
raise TextReadError('Invalid worksheet path')
|
|
75
|
+
root = _xml(archive, str(path))
|
|
76
|
+
lines = []
|
|
77
|
+
for row in root.iter(S + 'row'):
|
|
78
|
+
cells = []
|
|
79
|
+
for cell in row.findall(S + 'c'):
|
|
80
|
+
raw = cell.findtext(S + 'v') or ''
|
|
81
|
+
kind = cell.get('t')
|
|
82
|
+
if kind == 's':
|
|
83
|
+
index = int(raw)
|
|
84
|
+
if not 0 <= index < len(shared):
|
|
85
|
+
raise TextReadError('Invalid shared-string reference')
|
|
86
|
+
raw = shared[index]
|
|
87
|
+
elif kind == 'inlineStr':
|
|
88
|
+
raw = ''.join(n.text or '' for n in cell.iter(S + 't'))
|
|
89
|
+
elif cell.find(S + 'f') is not None:
|
|
90
|
+
# Keep formula text and cached value as data; never evaluate it.
|
|
91
|
+
raw = '=' + (cell.findtext(S + 'f') or '') + (' [cached: ' + raw + ']' if raw else '')
|
|
92
|
+
size += len(raw) + len(cell.get('r') or '?') + 3
|
|
93
|
+
if size > MAX_TEXT:
|
|
94
|
+
raise TextReadError('Extracted text exceeds limit')
|
|
95
|
+
cells.append((cell.get('r') or '?') + ': ' + raw)
|
|
96
|
+
lines.append('\t'.join(cells))
|
|
97
|
+
pages.extend(_chunks('\n'.join(lines), sheet.get('name') or 'sheet'))
|
|
98
|
+
return pages or [dict(text='', label='workbook')]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _bounded_text(command, target, *, deadline, limit, operation, capture=False):
|
|
102
|
+
"""Run a converter that writes text to target (or, with capture, to stdout redirected to target) within limits."""
|
|
103
|
+
if time.monotonic() >= deadline:
|
|
104
|
+
raise TextReadError(f'{operation} exceeded time or output limit')
|
|
105
|
+
output = open(target, 'wb') if capture else None
|
|
106
|
+
try:
|
|
107
|
+
process = subprocess.Popen(command, stdout=output or subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
108
|
+
stdin=subprocess.DEVNULL, creationflags=subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0)
|
|
109
|
+
except OSError as exc:
|
|
110
|
+
if output:
|
|
111
|
+
output.close()
|
|
112
|
+
raise TextReadError(f'{operation} failed: {exc}') from exc
|
|
113
|
+
try:
|
|
114
|
+
while process.poll() is None:
|
|
115
|
+
if time.monotonic() > deadline or target.exists() and target.stat().st_size > limit:
|
|
116
|
+
raise TextReadError(f'{operation} exceeded time or output limit')
|
|
117
|
+
time.sleep(0.05)
|
|
118
|
+
if process.returncode or not target.is_file():
|
|
119
|
+
raise TextReadError(f'{operation} failed')
|
|
120
|
+
if target.stat().st_size > limit:
|
|
121
|
+
raise TextReadError(f'{operation} exceeded time or output limit')
|
|
122
|
+
return target.read_text(encoding='utf-8', errors='replace')
|
|
123
|
+
finally:
|
|
124
|
+
if process.poll() is None:
|
|
125
|
+
process.kill()
|
|
126
|
+
process.wait()
|
|
127
|
+
if output:
|
|
128
|
+
output.close()
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _pdf(path):
|
|
132
|
+
with tempfile.TemporaryDirectory() as scratch:
|
|
133
|
+
target = Path(scratch) / 'text.txt'
|
|
134
|
+
executable = os.environ.get('TENDERKIT_PDFTOTEXT') or shutil.which('pdftotext')
|
|
135
|
+
if executable:
|
|
136
|
+
command = [executable, '-enc', 'UTF-8', '-layout', str(path), str(target)]
|
|
137
|
+
engine = 'pdftotext'
|
|
138
|
+
else:
|
|
139
|
+
executable = shutil.which('pymupdf') or str(Path(sys.executable).with_name('pymupdf.exe' if os.name == 'nt' else 'pymupdf'))
|
|
140
|
+
if not Path(executable).is_file():
|
|
141
|
+
raise TextReadError('PDF extraction needs pdftotext or the optional documents extra (PyMuPDF CLI)')
|
|
142
|
+
command = [executable, 'gettext', '-mode', 'simple', '-output', str(target), str(path)]
|
|
143
|
+
engine = 'pymupdf-cli'
|
|
144
|
+
text = _bounded_text(command, target, deadline=time.monotonic() + 30,
|
|
145
|
+
limit=MAX_TEXT, operation='PDF text extraction')
|
|
146
|
+
pages = text.split('\f')
|
|
147
|
+
if text.endswith('\f'):
|
|
148
|
+
pages.pop()
|
|
149
|
+
return [dict(text=p, label='PDF page') for p in pages or ['']], engine
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _xls_value(xlrd, cell, datemode):
|
|
153
|
+
if cell.ctype == xlrd.XL_CELL_NUMBER:
|
|
154
|
+
return str(int(cell.value)) if float(cell.value).is_integer() else repr(cell.value)
|
|
155
|
+
if cell.ctype == xlrd.XL_CELL_DATE:
|
|
156
|
+
try:
|
|
157
|
+
value = xlrd.xldate.xldate_as_datetime(cell.value, datemode)
|
|
158
|
+
except (ValueError, OverflowError, xlrd.xldate.XLDateError):
|
|
159
|
+
return repr(cell.value)
|
|
160
|
+
return value.date().isoformat() if value.time() == value.min.time() else value.isoformat(sep=' ')
|
|
161
|
+
if cell.ctype == xlrd.XL_CELL_BOOLEAN:
|
|
162
|
+
return 'TRUE' if cell.value else 'FALSE'
|
|
163
|
+
if cell.ctype == xlrd.XL_CELL_ERROR:
|
|
164
|
+
return xlrd.error_text_from_code.get(cell.value, '#ERROR')
|
|
165
|
+
return str(cell.value)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _xls(path):
|
|
169
|
+
"""Legacy Excel 97-2003 sheets in the same cell-reference layout as XLSX. xlrd exposes no formula text, so
|
|
170
|
+
formula cells give their cached value."""
|
|
171
|
+
try:
|
|
172
|
+
import xlrd
|
|
173
|
+
except ImportError as exc:
|
|
174
|
+
raise TextReadError('XLS extraction needs the optional documents extra (xlrd): '
|
|
175
|
+
'pip install "tenderkit[documents]"') from exc
|
|
176
|
+
try:
|
|
177
|
+
book = xlrd.open_workbook(str(path), on_demand=True)
|
|
178
|
+
except Exception as exc: # malformed input raises XLRDError, struct.error, AssertionError and others
|
|
179
|
+
raise TextReadError('Invalid or unsupported XLS workbook') from exc
|
|
180
|
+
pages, size = [], 0
|
|
181
|
+
try:
|
|
182
|
+
for index in range(book.nsheets):
|
|
183
|
+
sheet = book.sheet_by_index(index)
|
|
184
|
+
lines = []
|
|
185
|
+
for row in range(sheet.nrows):
|
|
186
|
+
cells = []
|
|
187
|
+
for column in range(sheet.ncols):
|
|
188
|
+
cell = sheet.cell(row, column)
|
|
189
|
+
if cell.ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
|
|
190
|
+
continue
|
|
191
|
+
raw, ref = _xls_value(xlrd, cell, book.datemode), xlrd.cellname(row, column)
|
|
192
|
+
size += len(raw) + len(ref) + 3
|
|
193
|
+
if size > MAX_TEXT:
|
|
194
|
+
raise TextReadError('Extracted text exceeds limit')
|
|
195
|
+
cells.append(ref + ': ' + raw)
|
|
196
|
+
lines.append('\t'.join(cells))
|
|
197
|
+
pages.extend(_chunks('\n'.join(lines), sheet.name or 'sheet'))
|
|
198
|
+
book.unload_sheet(index)
|
|
199
|
+
finally:
|
|
200
|
+
book.release_resources()
|
|
201
|
+
return pages or [dict(text='', label='workbook')]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _soffice():
|
|
205
|
+
found = os.environ.get('TENDERKIT_SOFFICE') or shutil.which('soffice') or shutil.which('libreoffice')
|
|
206
|
+
if found:
|
|
207
|
+
return found
|
|
208
|
+
for candidate in (Path(os.environ.get('ProgramFiles', r'C:\Program Files')) / 'LibreOffice' / 'program' / 'soffice.exe',
|
|
209
|
+
Path('/Applications/LibreOffice.app/Contents/MacOS/soffice')):
|
|
210
|
+
if candidate.is_file():
|
|
211
|
+
return str(candidate)
|
|
212
|
+
return None
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _doc_pages(text):
|
|
216
|
+
text = text.lstrip('\ufeff')
|
|
217
|
+
pages = [chunk for part in text.split('\f') for chunk in _chunks(part, 'document')]
|
|
218
|
+
if len(pages) > 1 and not pages[-1]['text'].strip():
|
|
219
|
+
pages.pop()
|
|
220
|
+
return pages
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _doc(path):
|
|
224
|
+
"""Legacy Word 97-2003 text through antiword or, failing that, LibreOffice; both run as bounded child processes."""
|
|
225
|
+
antiword = os.environ.get('TENDERKIT_ANTIWORD') or shutil.which('antiword')
|
|
226
|
+
soffice = _soffice()
|
|
227
|
+
if not antiword and not soffice:
|
|
228
|
+
raise TextReadError('DOC extraction needs antiword or LibreOffice (soffice) on PATH, '
|
|
229
|
+
'or TENDERKIT_ANTIWORD / TENDERKIT_SOFFICE naming one')
|
|
230
|
+
with tempfile.TemporaryDirectory() as scratch:
|
|
231
|
+
scratch = Path(scratch)
|
|
232
|
+
if antiword:
|
|
233
|
+
try:
|
|
234
|
+
text = _bounded_text([antiword, '-m', 'UTF-8.txt', '-w', '0', str(path)], scratch / 'antiword.txt',
|
|
235
|
+
deadline=time.monotonic() + 30, limit=MAX_TEXT, operation='DOC text extraction',
|
|
236
|
+
capture=True)
|
|
237
|
+
return _doc_pages(text), 'antiword'
|
|
238
|
+
except TextReadError:
|
|
239
|
+
# antiword reads only Word 97-2003 binaries; RTF or older Word saved as .doc may still convert below.
|
|
240
|
+
if not soffice:
|
|
241
|
+
raise
|
|
242
|
+
# A private copy with a short name, since LibreOffice names its output after the input, and a private profile,
|
|
243
|
+
# so the conversion never attaches to or waits for a LibreOffice the user has open.
|
|
244
|
+
source = scratch / 'document.doc'
|
|
245
|
+
shutil.copyfile(path, source)
|
|
246
|
+
text = _bounded_text([soffice, '--headless', '--norestore', '-env:UserInstallation=' + (scratch / 'profile').as_uri(),
|
|
247
|
+
'--convert-to', 'txt:Text (encoded):UTF8', '--outdir', str(scratch / 'out'), str(source)],
|
|
248
|
+
scratch / 'out' / 'document.txt', deadline=time.monotonic() + 90, limit=MAX_TEXT,
|
|
249
|
+
operation='DOC conversion with LibreOffice')
|
|
250
|
+
return _doc_pages(text), 'libreoffice'
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _sniff(path, kind):
|
|
254
|
+
"""The real format of a legacy-named file: portals often serve DOCX/XLSX under .doc/.xls."""
|
|
255
|
+
if kind in ('doc', 'xls'):
|
|
256
|
+
with open(path, 'rb') as stream:
|
|
257
|
+
if stream.read(4) == b'PK\x03\x04':
|
|
258
|
+
return kind + 'x'
|
|
259
|
+
return kind
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def read_document(path, *, page=1, pages=1, ocr=False, ocr_cache=None):
|
|
263
|
+
if type(page) is not int or page < 1 or type(pages) is not int or not 1 <= pages <= 5:
|
|
264
|
+
raise ValueError('Page must be positive; request 1 to 5 pages')
|
|
265
|
+
path = Path(path)
|
|
266
|
+
if path.stat().st_size > MAX_INPUT:
|
|
267
|
+
raise TextReadError('Text extraction input exceeds 20 MB limit')
|
|
268
|
+
kind = _sniff(path, path.suffix.lower().lstrip('.'))
|
|
269
|
+
if ocr and kind != 'pdf':
|
|
270
|
+
raise TextReadError('OCR is available only for PDFs')
|
|
271
|
+
engine, pagination = 'stdlib-ooxml', 'logical chunks (12000 characters, separated by explicit page breaks or worksheets)'
|
|
272
|
+
try:
|
|
273
|
+
if kind == 'pdf':
|
|
274
|
+
extracted, engine = _pdf(path)
|
|
275
|
+
pagination = 'physical PDF pages'
|
|
276
|
+
elif kind in ('docx', 'xlsx'):
|
|
277
|
+
with ZipFile(path) as archive:
|
|
278
|
+
infos = archive.infolist()
|
|
279
|
+
if len(infos) > 2000 or sum(i.file_size for i in infos) > MAX_XML:
|
|
280
|
+
raise TextReadError('Office archive exceeds extraction limits')
|
|
281
|
+
extracted = _docx(archive) if kind == 'docx' else _xlsx(archive)
|
|
282
|
+
if sum(len(p['text'].encode('utf-8')) for p in extracted) > MAX_TEXT:
|
|
283
|
+
raise TextReadError('Extracted text exceeds limit')
|
|
284
|
+
elif kind == 'xls':
|
|
285
|
+
extracted, engine = _xls(path), 'xlrd'
|
|
286
|
+
pagination = 'logical chunks (12000 characters, separated by worksheets)'
|
|
287
|
+
elif kind == 'doc':
|
|
288
|
+
extracted, engine = _doc(path)
|
|
289
|
+
pagination = 'logical chunks (12000 characters, separated by page breaks the converter reports)'
|
|
290
|
+
else:
|
|
291
|
+
raise TextReadError('Supported formats: PDF, DOCX, XLSX, DOC, XLS')
|
|
292
|
+
except (BadZipFile, ET.ParseError, KeyError, IndexError) as exc:
|
|
293
|
+
raise TextReadError('Invalid or unsupported document structure') from exc
|
|
294
|
+
if page > len(extracted):
|
|
295
|
+
raise ValueError('Page outside document')
|
|
296
|
+
selected = extracted[page-1:page-1+pages]
|
|
297
|
+
if kind == 'pdf':
|
|
298
|
+
for item in selected:
|
|
299
|
+
item['text_layer'] = bool(item['text'].strip())
|
|
300
|
+
item['text_source'] = 'embedded' if item['text_layer'] else None
|
|
301
|
+
if ocr and any(not item['text_layer'] for item in selected):
|
|
302
|
+
from tenderkit.pdf_ocr import recognize_pages
|
|
303
|
+
recognize_pages(path, selected, page, ocr_cache)
|
|
304
|
+
if sum(len(p['text']) for p in selected) > MAX_RESPONSE:
|
|
305
|
+
raise TextReadError('Selected pages exceed response text limit')
|
|
306
|
+
return dict(format=kind, engine=engine, pagination=pagination, total_pages=len(extracted),
|
|
307
|
+
pages=[dict(item, page=page+i) for i, item in enumerate(selected)],
|
|
308
|
+
next_page=page+len(selected) if page+len(selected) <= len(extracted) else None,
|
|
309
|
+
text_available=any(item['text'].strip() for item in selected))
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Lossless JSON notice records shared by SQLite, CLI and MCP."""
|
|
2
|
+
from dataclasses import asdict, replace
|
|
3
|
+
from datetime import date, datetime
|
|
4
|
+
from decimal import Decimal
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
from tenderkit.model import Award, Lot, Notice
|
|
8
|
+
from tenderkit.sources.mapping import bzp_scope, ted_scope
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def json_default(value):
|
|
12
|
+
if isinstance(value, (date, datetime)):
|
|
13
|
+
return value.isoformat()
|
|
14
|
+
if isinstance(value, Decimal):
|
|
15
|
+
return str(value)
|
|
16
|
+
raise TypeError(f'Cannot serialize {type(value).__name__}')
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def dumps(value):
|
|
20
|
+
return json.dumps(value, ensure_ascii=False, default=json_default, allow_nan=False)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def notice_record(notice, *, raw=True):
|
|
24
|
+
record = asdict(notice)
|
|
25
|
+
if not raw:
|
|
26
|
+
record.pop('raw')
|
|
27
|
+
return json.loads(dumps(record))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def notice_from_record(record):
|
|
31
|
+
data = dict(record)
|
|
32
|
+
data['published'] = date.fromisoformat(data['published']) if data.get('published') else None
|
|
33
|
+
data['deadline'] = datetime.fromisoformat(data['deadline']) if data.get('deadline') else None
|
|
34
|
+
data['value'] = Decimal(data['value']) if data.get('value') is not None else None
|
|
35
|
+
data['cpv'] = tuple(data.get('cpv') or [])
|
|
36
|
+
lots = []
|
|
37
|
+
for lot in data.get('lots') or []:
|
|
38
|
+
lot = dict(lot)
|
|
39
|
+
lot['cpv'] = tuple(lot.get('cpv') or [])
|
|
40
|
+
lot['value'] = Decimal(lot['value']) if lot.get('value') is not None else None
|
|
41
|
+
lots.append(Lot(**lot))
|
|
42
|
+
data['lots'] = tuple(lots)
|
|
43
|
+
data['lot_titles'] = tuple(data.get('lot_titles') or [])
|
|
44
|
+
if 'description' not in data:
|
|
45
|
+
raw = data.get('raw') or {}
|
|
46
|
+
if data['source'] == 'BZP':
|
|
47
|
+
description, titled_lots = bzp_scope(raw.get('htmlBody'))
|
|
48
|
+
merged = {lot.number: lot for lot in lots}
|
|
49
|
+
for lot in titled_lots:
|
|
50
|
+
existing = merged.get(lot.number)
|
|
51
|
+
merged[lot.number] = replace(existing, title=existing.title or lot.title) if existing else lot
|
|
52
|
+
data['description'] = description
|
|
53
|
+
data['lots'] = tuple(merged.values())
|
|
54
|
+
elif data['source'] == 'TED':
|
|
55
|
+
data['description'], data['lot_titles'] = ted_scope(raw)
|
|
56
|
+
awards = []
|
|
57
|
+
for award in data.get('awards') or []:
|
|
58
|
+
award = dict(award)
|
|
59
|
+
award['price'] = Decimal(award['price']) if award.get('price') is not None else None
|
|
60
|
+
award['date'] = date.fromisoformat(award['date']) if award.get('date') else None
|
|
61
|
+
awards.append(Award(**award))
|
|
62
|
+
data['awards'] = tuple(awards)
|
|
63
|
+
return Notice(**data)
|