tenderkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. tenderkit/__init__.py +9 -0
  2. tenderkit/__main__.py +5 -0
  3. tenderkit/cli.py +248 -0
  4. tenderkit/documents.py +260 -0
  5. tenderkit/errors.py +52 -0
  6. tenderkit/http.py +94 -0
  7. tenderkit/interfaces.py +42 -0
  8. tenderkit/limits.py +7 -0
  9. tenderkit/mcp.py +221 -0
  10. tenderkit/model.py +78 -0
  11. tenderkit/pdf_ocr.py +78 -0
  12. tenderkit/platforms/__init__.py +33 -0
  13. tenderkit/platforms/backends.py +163 -0
  14. tenderkit/platforms/base.py +79 -0
  15. tenderkit/platforms/bk.py +78 -0
  16. tenderkit/platforms/ezamowienia.py +85 -0
  17. tenderkit/platforms/html.py +67 -0
  18. tenderkit/platforms/marketplanet_html.py +128 -0
  19. tenderkit/platforms/public_documents.py +206 -0
  20. tenderkit/platforms/pz_html.py +127 -0
  21. tenderkit/platforms/session.py +114 -0
  22. tenderkit/profile.py +216 -0
  23. tenderkit/profiles/__init__.py +13 -0
  24. tenderkit/profiles/leki.toml +5 -0
  25. tenderkit/profiles/oprogramowanie.toml +4 -0
  26. tenderkit/profiles/wyroby-medyczne.toml +4 -0
  27. tenderkit/queue.py +126 -0
  28. tenderkit/read_document.py +309 -0
  29. tenderkit/serialization.py +63 -0
  30. tenderkit/service.py +164 -0
  31. tenderkit/sources/__init__.py +1 -0
  32. tenderkit/sources/bk.py +145 -0
  33. tenderkit/sources/bzp.py +142 -0
  34. tenderkit/sources/mapping.py +264 -0
  35. tenderkit/sources/ted.py +116 -0
  36. tenderkit/sources/ted_xml.py +97 -0
  37. tenderkit/store/__init__.py +1 -0
  38. tenderkit/store/documents.py +168 -0
  39. tenderkit/store/state.py +202 -0
  40. tenderkit/ui.py +243 -0
  41. tenderkit/watch.py +247 -0
  42. tenderkit/web/index.html +59 -0
  43. tenderkit/web/style.css +221 -0
  44. tenderkit-0.1.0.dist-info/METADATA +219 -0
  45. tenderkit-0.1.0.dist-info/RECORD +49 -0
  46. tenderkit-0.1.0.dist-info/WHEEL +5 -0
  47. tenderkit-0.1.0.dist-info/entry_points.txt +2 -0
  48. tenderkit-0.1.0.dist-info/licenses/LICENSE +202 -0
  49. tenderkit-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,13 @@
1
+ """Built-in profiles; no dependency on pharma or a user-specific filesystem path."""
2
+ from importlib.resources import files
3
+
4
+ from tenderkit.profile import load_profile
5
+
6
+
7
+ def builtin_profiles():
8
+ return {path.name[:-5]: load_profile(path) for path in files(__name__).iterdir() if path.name.endswith('.toml')}
9
+
10
+
11
+ def resolve_profile(name_or_path):
12
+ profiles = builtin_profiles()
13
+ return profiles[name_or_path] if name_or_path in profiles else load_profile(name_or_path)
@@ -0,0 +1,5 @@
1
+ name = "leki"
2
+ cpv = ["336*"]
3
+ # 336* also covers laboratory reagents (33696*) and disinfectants (336316*), which are not medicines.
4
+ exclude_cpv = ["33696*", "336316*"]
5
+ notice_types = ["ContractNotice", "SmallContractNotice", "CompetitionNotice", "cn-standard"]
@@ -0,0 +1,4 @@
1
+ name = "oprogramowanie"
2
+ cpv = ["72*", "48*"]
3
+ notice_types = ["ContractNotice", "cn-standard", "BKAnnouncement"]
4
+ # The legacy feed's TED query is narrower (72000000); parity reports must show this.
@@ -0,0 +1,4 @@
1
+ name = "wyroby-medyczne"
2
+ # Broad discovery scope from the specification; this is not an MDR classification.
3
+ cpv = ["331*", "332*", "334*"]
4
+ notice_types = ["ContractNotice", "SmallContractNotice", "CompetitionNotice", "cn-standard"]
tenderkit/queue.py ADDED
@@ -0,0 +1,126 @@
1
+ """Durable bounded download jobs. An OS lock releases automatically if a worker dies."""
2
+ from contextlib import contextmanager
3
+ import json
4
+ import os
5
+ import time
6
+
7
+ from tenderkit.errors import classify_error
8
+ from tenderkit.limits import MAX_ATTEMPTS, MAX_DOCUMENTS, MAX_JOBS
9
+ from tenderkit.serialization import dumps
10
+
11
+
12
+ @contextmanager
13
+ def worker_lock(path, *, suffix='.documents.lock'):
14
+ with open(str(path) + suffix, 'a+b') as stream:
15
+ if stream.tell() == 0:
16
+ stream.write(b'0')
17
+ stream.flush()
18
+ stream.seek(0)
19
+ try:
20
+ if os.name == 'nt':
21
+ import msvcrt
22
+ msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
23
+ else:
24
+ import fcntl
25
+ fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
26
+ except OSError:
27
+ yield False
28
+ return
29
+ try:
30
+ yield True
31
+ finally:
32
+ if os.name == 'nt':
33
+ stream.seek(0)
34
+ msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
35
+ else:
36
+ fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
37
+
38
+
39
+ class SelectionConflict(ValueError):
40
+ pass
41
+
42
+
43
+ class Queue:
44
+ def __init__(self, state):
45
+ self.state = state
46
+
47
+ def enqueue(self, source, number, keys=None):
48
+ if self.state.get(source, number) is None:
49
+ raise ValueError('Notice not found in local state')
50
+ if keys is not None:
51
+ if (not isinstance(keys, (tuple, list)) or not 1 <= len(keys) <= MAX_DOCUMENTS
52
+ or any(not isinstance(k, str) or not k or len(k) > 8000 for k in keys)):
53
+ raise ValueError('Invalid document selection')
54
+ keys = list(dict.fromkeys(keys))
55
+ with self.state.connection() as conn:
56
+ conn.execute('begin immediate')
57
+ found = conn.execute("select * from fetch_jobs where source=? and number=? and status in ('QUEUED','RUNNING')", (source, number)).fetchone()
58
+ if found:
59
+ if json.loads(found['selection']) != keys:
60
+ raise SelectionConflict('A job with a different document selection is already active')
61
+ return found['id']
62
+ return conn.execute('insert into fetch_jobs(source,number,selection,status) values(?,?,?,?) returning id',
63
+ (source, number, dumps(keys), 'QUEUED')).fetchone()[0]
64
+
65
+ def job(self, job_id):
66
+ with self.state.connection() as conn:
67
+ row = conn.execute('select * from fetch_jobs where id=?', (job_id,)).fetchone()
68
+ if row is None:
69
+ raise ValueError('Download job not found')
70
+ result = dict(row)
71
+ result['selection'], result['summary'] = json.loads(result['selection']), json.loads(result['summary'])
72
+ return result
73
+
74
+ def recover(self):
75
+ # Called only while holding the OS worker lock, so no live owner can be running.
76
+ with self.state.connection() as conn:
77
+ rows = conn.execute("select id,attempts from fetch_jobs where status='RUNNING'").fetchall()
78
+ for row in rows:
79
+ conn.execute('update fetch_jobs set status=?,error=?,error_kind=?,next_attempt=0 where id=?',
80
+ ('FAILED' if row['attempts'] >= MAX_ATTEMPTS else 'QUEUED', 'WORKER_LOST', 'TRANSIENT', row['id']))
81
+ if row['attempts'] >= MAX_ATTEMPTS:
82
+ conn.execute("update notices set coverage='FETCH_FAILED' where (source,number)=(select source,number from fetch_jobs where id=?)", (row['id'],))
83
+ return len(rows)
84
+
85
+ def beat(self, job_id, summary):
86
+ with self.state.connection() as conn:
87
+ conn.execute('update fetch_jobs set heartbeat=?,summary=? where id=?', (time.time(), dumps(summary), job_id))
88
+
89
+ def finish(self, job_id, error=None):
90
+ job = self.job(job_id)
91
+ kind = classify_error('FAILED' if error else 'DONE', error)
92
+ retry = bool(error) and job['attempts'] < (MAX_ATTEMPTS if kind == 'TRANSIENT' else 2 if kind == 'UNCERTAIN' else 1)
93
+ status = 'QUEUED' if retry else 'FAILED' if error else 'DONE'
94
+ with self.state.connection() as conn:
95
+ conn.execute('update fetch_jobs set status=?,error=?,error_kind=?,next_attempt=?,heartbeat=? where id=?',
96
+ (status, error, kind, time.time() + 60 * job['attempts'] if retry else 0, time.time(), job_id))
97
+ return self.job(job_id)
98
+
99
+ def run(self, process, *, max_jobs=1, job_id=None):
100
+ if type(max_jobs) is not int or not 1 <= max_jobs <= MAX_JOBS:
101
+ raise ValueError('max_jobs must be between 1 and 20')
102
+ results = []
103
+ with worker_lock(self.state.path) as locked:
104
+ if not locked:
105
+ return dict(locked=False, jobs=[])
106
+ recovered = self.recover()
107
+ for _ in range(max_jobs):
108
+ with self.state.connection() as conn:
109
+ conn.execute('begin immediate')
110
+ query = "select id from fetch_jobs where status='QUEUED' and next_attempt<=?"
111
+ args = [time.time()]
112
+ if job_id is not None:
113
+ query += ' and id=?'
114
+ args.append(job_id)
115
+ row = conn.execute(query + ' order by id limit 1', args).fetchone()
116
+ if row is None:
117
+ break
118
+ current = row[0]
119
+ conn.execute("update fetch_jobs set status='RUNNING',attempts=attempts+1,heartbeat=? where id=?", (time.time(), current))
120
+ error = None
121
+ try:
122
+ process(self.job(current))
123
+ except Exception as exc:
124
+ error = getattr(exc, 'code', type(exc).__name__)
125
+ results.append(self.finish(current, error))
126
+ return dict(locked=True, recovered=recovered, jobs=results)
@@ -0,0 +1,309 @@
1
+ """Bounded text extraction. OOXML uses stdlib, legacy XLS the optional xlrd; PDF and legacy DOC run an installed
2
+ CLI in a child process."""
3
+ from pathlib import Path, PurePosixPath
4
+ import os
5
+ import shutil
6
+ import subprocess
7
+ import sys
8
+ import tempfile
9
+ import time
10
+ from xml.etree import ElementTree as ET
11
+ from zipfile import ZipFile, BadZipFile
12
+
13
+ MAX_INPUT = 20 * 1024 * 1024
14
+ MAX_TEXT = 8 * 1024 * 1024
15
+ MAX_XML = 32 * 1024 * 1024
16
+ MAX_RESPONSE = 60000
17
+ CHUNK_CHARS = 12000
18
+ W = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
19
+ S = '{http://schemas.openxmlformats.org/spreadsheetml/2006/main}'
20
+
21
+
22
+ class TextReadError(ValueError):
23
+ pass
24
+
25
+
26
+ def _chunks(text, label):
27
+ return [dict(text=text[i:i+CHUNK_CHARS], label=label) for i in range(0, len(text), CHUNK_CHARS)] or [dict(text='', label=label)]
28
+
29
+
30
+ def _xml(archive, name):
31
+ data = archive.read(name)
32
+ if b'<!DOCTYPE' in data.upper() or b'<!ENTITY' in data.upper():
33
+ raise TextReadError('XML declarations are not supported')
34
+ return ET.fromstring(data)
35
+
36
+
37
+ def _docx(archive):
38
+ root = _xml(archive, 'word/document.xml')
39
+ pages, text, size = [], [], 0
40
+ for paragraph in root.iter(W + 'p'):
41
+ for node in paragraph.iter():
42
+ if node.tag == W + 't':
43
+ size += len(node.text or '')
44
+ if size > MAX_TEXT:
45
+ raise TextReadError('Extracted text exceeds limit')
46
+ text.append(node.text or '')
47
+ elif node.tag == W + 'tab':
48
+ text.append('\t')
49
+ elif node.tag == W + 'lastRenderedPageBreak' or node.tag == W + 'br' and node.get(W + 'type') == 'page':
50
+ pages.extend(_chunks(''.join(text), 'document'))
51
+ text = []
52
+ elif node.tag == W + 'br':
53
+ text.append('\n')
54
+ text.append('\n')
55
+ pages.extend(_chunks(''.join(text), 'document'))
56
+ return pages
57
+
58
+
59
+ def _xlsx(archive):
60
+ shared = []
61
+ if 'xl/sharedStrings.xml' in archive.namelist():
62
+ shared = [''.join(node.itertext()) for node in _xml(archive, 'xl/sharedStrings.xml').findall(S + 'si')]
63
+ workbook = _xml(archive, 'xl/workbook.xml')
64
+ relationships = {node.get('Id'): node for node in _xml(archive, 'xl/_rels/workbook.xml.rels')}
65
+ relation_id = '{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id'
66
+ pages, size = [], 0
67
+ for sheet in workbook.findall(S + 'sheets/' + S + 'sheet'):
68
+ relation = relationships.get(sheet.get(relation_id))
69
+ if relation is None or relation.get('TargetMode') == 'External':
70
+ raise TextReadError('Unsupported worksheet relationship')
71
+ target = relation.get('Target', '')
72
+ path = PurePosixPath(target.lstrip('/')) if target.startswith('/') else PurePosixPath('xl') / target
73
+ if '..' in path.parts or not str(path).startswith('xl/'):
74
+ raise TextReadError('Invalid worksheet path')
75
+ root = _xml(archive, str(path))
76
+ lines = []
77
+ for row in root.iter(S + 'row'):
78
+ cells = []
79
+ for cell in row.findall(S + 'c'):
80
+ raw = cell.findtext(S + 'v') or ''
81
+ kind = cell.get('t')
82
+ if kind == 's':
83
+ index = int(raw)
84
+ if not 0 <= index < len(shared):
85
+ raise TextReadError('Invalid shared-string reference')
86
+ raw = shared[index]
87
+ elif kind == 'inlineStr':
88
+ raw = ''.join(n.text or '' for n in cell.iter(S + 't'))
89
+ elif cell.find(S + 'f') is not None:
90
+ # Keep formula text and cached value as data; never evaluate it.
91
+ raw = '=' + (cell.findtext(S + 'f') or '') + (' [cached: ' + raw + ']' if raw else '')
92
+ size += len(raw) + len(cell.get('r') or '?') + 3
93
+ if size > MAX_TEXT:
94
+ raise TextReadError('Extracted text exceeds limit')
95
+ cells.append((cell.get('r') or '?') + ': ' + raw)
96
+ lines.append('\t'.join(cells))
97
+ pages.extend(_chunks('\n'.join(lines), sheet.get('name') or 'sheet'))
98
+ return pages or [dict(text='', label='workbook')]
99
+
100
+
101
+ def _bounded_text(command, target, *, deadline, limit, operation, capture=False):
102
+ """Run a converter that writes text to target (or, with capture, to stdout redirected to target) within limits."""
103
+ if time.monotonic() >= deadline:
104
+ raise TextReadError(f'{operation} exceeded time or output limit')
105
+ output = open(target, 'wb') if capture else None
106
+ try:
107
+ process = subprocess.Popen(command, stdout=output or subprocess.DEVNULL, stderr=subprocess.DEVNULL,
108
+ stdin=subprocess.DEVNULL, creationflags=subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0)
109
+ except OSError as exc:
110
+ if output:
111
+ output.close()
112
+ raise TextReadError(f'{operation} failed: {exc}') from exc
113
+ try:
114
+ while process.poll() is None:
115
+ if time.monotonic() > deadline or target.exists() and target.stat().st_size > limit:
116
+ raise TextReadError(f'{operation} exceeded time or output limit')
117
+ time.sleep(0.05)
118
+ if process.returncode or not target.is_file():
119
+ raise TextReadError(f'{operation} failed')
120
+ if target.stat().st_size > limit:
121
+ raise TextReadError(f'{operation} exceeded time or output limit')
122
+ return target.read_text(encoding='utf-8', errors='replace')
123
+ finally:
124
+ if process.poll() is None:
125
+ process.kill()
126
+ process.wait()
127
+ if output:
128
+ output.close()
129
+
130
+
131
+ def _pdf(path):
132
+ with tempfile.TemporaryDirectory() as scratch:
133
+ target = Path(scratch) / 'text.txt'
134
+ executable = os.environ.get('TENDERKIT_PDFTOTEXT') or shutil.which('pdftotext')
135
+ if executable:
136
+ command = [executable, '-enc', 'UTF-8', '-layout', str(path), str(target)]
137
+ engine = 'pdftotext'
138
+ else:
139
+ executable = shutil.which('pymupdf') or str(Path(sys.executable).with_name('pymupdf.exe' if os.name == 'nt' else 'pymupdf'))
140
+ if not Path(executable).is_file():
141
+ raise TextReadError('PDF extraction needs pdftotext or the optional documents extra (PyMuPDF CLI)')
142
+ command = [executable, 'gettext', '-mode', 'simple', '-output', str(target), str(path)]
143
+ engine = 'pymupdf-cli'
144
+ text = _bounded_text(command, target, deadline=time.monotonic() + 30,
145
+ limit=MAX_TEXT, operation='PDF text extraction')
146
+ pages = text.split('\f')
147
+ if text.endswith('\f'):
148
+ pages.pop()
149
+ return [dict(text=p, label='PDF page') for p in pages or ['']], engine
150
+
151
+
152
+ def _xls_value(xlrd, cell, datemode):
153
+ if cell.ctype == xlrd.XL_CELL_NUMBER:
154
+ return str(int(cell.value)) if float(cell.value).is_integer() else repr(cell.value)
155
+ if cell.ctype == xlrd.XL_CELL_DATE:
156
+ try:
157
+ value = xlrd.xldate.xldate_as_datetime(cell.value, datemode)
158
+ except (ValueError, OverflowError, xlrd.xldate.XLDateError):
159
+ return repr(cell.value)
160
+ return value.date().isoformat() if value.time() == value.min.time() else value.isoformat(sep=' ')
161
+ if cell.ctype == xlrd.XL_CELL_BOOLEAN:
162
+ return 'TRUE' if cell.value else 'FALSE'
163
+ if cell.ctype == xlrd.XL_CELL_ERROR:
164
+ return xlrd.error_text_from_code.get(cell.value, '#ERROR')
165
+ return str(cell.value)
166
+
167
+
168
+ def _xls(path):
169
+ """Legacy Excel 97-2003 sheets in the same cell-reference layout as XLSX. xlrd exposes no formula text, so
170
+ formula cells give their cached value."""
171
+ try:
172
+ import xlrd
173
+ except ImportError as exc:
174
+ raise TextReadError('XLS extraction needs the optional documents extra (xlrd): '
175
+ 'pip install "tenderkit[documents]"') from exc
176
+ try:
177
+ book = xlrd.open_workbook(str(path), on_demand=True)
178
+ except Exception as exc: # malformed input raises XLRDError, struct.error, AssertionError and others
179
+ raise TextReadError('Invalid or unsupported XLS workbook') from exc
180
+ pages, size = [], 0
181
+ try:
182
+ for index in range(book.nsheets):
183
+ sheet = book.sheet_by_index(index)
184
+ lines = []
185
+ for row in range(sheet.nrows):
186
+ cells = []
187
+ for column in range(sheet.ncols):
188
+ cell = sheet.cell(row, column)
189
+ if cell.ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
190
+ continue
191
+ raw, ref = _xls_value(xlrd, cell, book.datemode), xlrd.cellname(row, column)
192
+ size += len(raw) + len(ref) + 3
193
+ if size > MAX_TEXT:
194
+ raise TextReadError('Extracted text exceeds limit')
195
+ cells.append(ref + ': ' + raw)
196
+ lines.append('\t'.join(cells))
197
+ pages.extend(_chunks('\n'.join(lines), sheet.name or 'sheet'))
198
+ book.unload_sheet(index)
199
+ finally:
200
+ book.release_resources()
201
+ return pages or [dict(text='', label='workbook')]
202
+
203
+
204
+ def _soffice():
205
+ found = os.environ.get('TENDERKIT_SOFFICE') or shutil.which('soffice') or shutil.which('libreoffice')
206
+ if found:
207
+ return found
208
+ for candidate in (Path(os.environ.get('ProgramFiles', r'C:\Program Files')) / 'LibreOffice' / 'program' / 'soffice.exe',
209
+ Path('/Applications/LibreOffice.app/Contents/MacOS/soffice')):
210
+ if candidate.is_file():
211
+ return str(candidate)
212
+ return None
213
+
214
+
215
+ def _doc_pages(text):
216
+ text = text.lstrip('\ufeff')
217
+ pages = [chunk for part in text.split('\f') for chunk in _chunks(part, 'document')]
218
+ if len(pages) > 1 and not pages[-1]['text'].strip():
219
+ pages.pop()
220
+ return pages
221
+
222
+
223
+ def _doc(path):
224
+ """Legacy Word 97-2003 text through antiword or, failing that, LibreOffice; both run as bounded child processes."""
225
+ antiword = os.environ.get('TENDERKIT_ANTIWORD') or shutil.which('antiword')
226
+ soffice = _soffice()
227
+ if not antiword and not soffice:
228
+ raise TextReadError('DOC extraction needs antiword or LibreOffice (soffice) on PATH, '
229
+ 'or TENDERKIT_ANTIWORD / TENDERKIT_SOFFICE naming one')
230
+ with tempfile.TemporaryDirectory() as scratch:
231
+ scratch = Path(scratch)
232
+ if antiword:
233
+ try:
234
+ text = _bounded_text([antiword, '-m', 'UTF-8.txt', '-w', '0', str(path)], scratch / 'antiword.txt',
235
+ deadline=time.monotonic() + 30, limit=MAX_TEXT, operation='DOC text extraction',
236
+ capture=True)
237
+ return _doc_pages(text), 'antiword'
238
+ except TextReadError:
239
+ # antiword reads only Word 97-2003 binaries; RTF or older Word saved as .doc may still convert below.
240
+ if not soffice:
241
+ raise
242
+ # A private copy with a short name, since LibreOffice names its output after the input, and a private profile,
243
+ # so the conversion never attaches to or waits for a LibreOffice the user has open.
244
+ source = scratch / 'document.doc'
245
+ shutil.copyfile(path, source)
246
+ text = _bounded_text([soffice, '--headless', '--norestore', '-env:UserInstallation=' + (scratch / 'profile').as_uri(),
247
+ '--convert-to', 'txt:Text (encoded):UTF8', '--outdir', str(scratch / 'out'), str(source)],
248
+ scratch / 'out' / 'document.txt', deadline=time.monotonic() + 90, limit=MAX_TEXT,
249
+ operation='DOC conversion with LibreOffice')
250
+ return _doc_pages(text), 'libreoffice'
251
+
252
+
253
+ def _sniff(path, kind):
254
+ """The real format of a legacy-named file: portals often serve DOCX/XLSX under .doc/.xls."""
255
+ if kind in ('doc', 'xls'):
256
+ with open(path, 'rb') as stream:
257
+ if stream.read(4) == b'PK\x03\x04':
258
+ return kind + 'x'
259
+ return kind
260
+
261
+
262
+ def read_document(path, *, page=1, pages=1, ocr=False, ocr_cache=None):
263
+ if type(page) is not int or page < 1 or type(pages) is not int or not 1 <= pages <= 5:
264
+ raise ValueError('Page must be positive; request 1 to 5 pages')
265
+ path = Path(path)
266
+ if path.stat().st_size > MAX_INPUT:
267
+ raise TextReadError('Text extraction input exceeds 20 MB limit')
268
+ kind = _sniff(path, path.suffix.lower().lstrip('.'))
269
+ if ocr and kind != 'pdf':
270
+ raise TextReadError('OCR is available only for PDFs')
271
+ engine, pagination = 'stdlib-ooxml', 'logical chunks (12000 characters, separated by explicit page breaks or worksheets)'
272
+ try:
273
+ if kind == 'pdf':
274
+ extracted, engine = _pdf(path)
275
+ pagination = 'physical PDF pages'
276
+ elif kind in ('docx', 'xlsx'):
277
+ with ZipFile(path) as archive:
278
+ infos = archive.infolist()
279
+ if len(infos) > 2000 or sum(i.file_size for i in infos) > MAX_XML:
280
+ raise TextReadError('Office archive exceeds extraction limits')
281
+ extracted = _docx(archive) if kind == 'docx' else _xlsx(archive)
282
+ if sum(len(p['text'].encode('utf-8')) for p in extracted) > MAX_TEXT:
283
+ raise TextReadError('Extracted text exceeds limit')
284
+ elif kind == 'xls':
285
+ extracted, engine = _xls(path), 'xlrd'
286
+ pagination = 'logical chunks (12000 characters, separated by worksheets)'
287
+ elif kind == 'doc':
288
+ extracted, engine = _doc(path)
289
+ pagination = 'logical chunks (12000 characters, separated by page breaks the converter reports)'
290
+ else:
291
+ raise TextReadError('Supported formats: PDF, DOCX, XLSX, DOC, XLS')
292
+ except (BadZipFile, ET.ParseError, KeyError, IndexError) as exc:
293
+ raise TextReadError('Invalid or unsupported document structure') from exc
294
+ if page > len(extracted):
295
+ raise ValueError('Page outside document')
296
+ selected = extracted[page-1:page-1+pages]
297
+ if kind == 'pdf':
298
+ for item in selected:
299
+ item['text_layer'] = bool(item['text'].strip())
300
+ item['text_source'] = 'embedded' if item['text_layer'] else None
301
+ if ocr and any(not item['text_layer'] for item in selected):
302
+ from tenderkit.pdf_ocr import recognize_pages
303
+ recognize_pages(path, selected, page, ocr_cache)
304
+ if sum(len(p['text']) for p in selected) > MAX_RESPONSE:
305
+ raise TextReadError('Selected pages exceed response text limit')
306
+ return dict(format=kind, engine=engine, pagination=pagination, total_pages=len(extracted),
307
+ pages=[dict(item, page=page+i) for i, item in enumerate(selected)],
308
+ next_page=page+len(selected) if page+len(selected) <= len(extracted) else None,
309
+ text_available=any(item['text'].strip() for item in selected))
@@ -0,0 +1,63 @@
1
+ """Lossless JSON notice records shared by SQLite, CLI and MCP."""
2
+ from dataclasses import asdict, replace
3
+ from datetime import date, datetime
4
+ from decimal import Decimal
5
+ import json
6
+
7
+ from tenderkit.model import Award, Lot, Notice
8
+ from tenderkit.sources.mapping import bzp_scope, ted_scope
9
+
10
+
11
+ def json_default(value):
12
+ if isinstance(value, (date, datetime)):
13
+ return value.isoformat()
14
+ if isinstance(value, Decimal):
15
+ return str(value)
16
+ raise TypeError(f'Cannot serialize {type(value).__name__}')
17
+
18
+
19
+ def dumps(value):
20
+ return json.dumps(value, ensure_ascii=False, default=json_default, allow_nan=False)
21
+
22
+
23
+ def notice_record(notice, *, raw=True):
24
+ record = asdict(notice)
25
+ if not raw:
26
+ record.pop('raw')
27
+ return json.loads(dumps(record))
28
+
29
+
30
+ def notice_from_record(record):
31
+ data = dict(record)
32
+ data['published'] = date.fromisoformat(data['published']) if data.get('published') else None
33
+ data['deadline'] = datetime.fromisoformat(data['deadline']) if data.get('deadline') else None
34
+ data['value'] = Decimal(data['value']) if data.get('value') is not None else None
35
+ data['cpv'] = tuple(data.get('cpv') or [])
36
+ lots = []
37
+ for lot in data.get('lots') or []:
38
+ lot = dict(lot)
39
+ lot['cpv'] = tuple(lot.get('cpv') or [])
40
+ lot['value'] = Decimal(lot['value']) if lot.get('value') is not None else None
41
+ lots.append(Lot(**lot))
42
+ data['lots'] = tuple(lots)
43
+ data['lot_titles'] = tuple(data.get('lot_titles') or [])
44
+ if 'description' not in data:
45
+ raw = data.get('raw') or {}
46
+ if data['source'] == 'BZP':
47
+ description, titled_lots = bzp_scope(raw.get('htmlBody'))
48
+ merged = {lot.number: lot for lot in lots}
49
+ for lot in titled_lots:
50
+ existing = merged.get(lot.number)
51
+ merged[lot.number] = replace(existing, title=existing.title or lot.title) if existing else lot
52
+ data['description'] = description
53
+ data['lots'] = tuple(merged.values())
54
+ elif data['source'] == 'TED':
55
+ data['description'], data['lot_titles'] = ted_scope(raw)
56
+ awards = []
57
+ for award in data.get('awards') or []:
58
+ award = dict(award)
59
+ award['price'] = Decimal(award['price']) if award.get('price') is not None else None
60
+ award['date'] = date.fromisoformat(award['date']) if award.get('date') else None
61
+ awards.append(Award(**award))
62
+ data['awards'] = tuple(awards)
63
+ return Notice(**data)