tenderkit 0.1.0__py3-none-any.whl

tenderkit/service.py

@@ -0,0 +1,164 @@
+"""One application path for CLI and MCP: fetch to SQLite, then filter the same notices."""
+from dataclasses import asdict, replace
+from datetime import date, datetime, timedelta
+from importlib.metadata import entry_points
+import json
+from zoneinfo import ZoneInfo
+
+from tenderkit.interfaces import Source
+from tenderkit.profile import fold
+from tenderkit.serialization import dumps, notice_record
+from tenderkit.sources.bzp import Bzp
+from tenderkit.sources.bk import Bk
+from tenderkit.sources.ted import Ted
+
+MAX_WINDOW_DAYS = 120
+MAX_NOTICES = 20000
+MAX_RESULTS = 500
+
+
+def today():
+    return datetime.now(ZoneInfo('Europe/Warsaw')).date()
+
+
+def window(start=None, end=None):
+    end = date.fromisoformat(end) if isinstance(end, str) else end or today()
+    start = date.fromisoformat(start) if isinstance(start, str) else start or end - timedelta(days=29)
+    if not 0 <= (end - start).days < MAX_WINDOW_DAYS:
+        raise ValueError(f'Publication window must contain 1 to {MAX_WINDOW_DAYS} days')
+    return start, end
+
+
+def sources(*, include_bk=False):
+    registered = {'BZP': Bzp(), 'TED': Ted()}
+    if include_bk:
+        registered['BK'] = Bk()
+    for entry in entry_points(group='tenderkit.sources'):
+        source = entry.load()()
+        if not isinstance(source, Source) or source.name in registered:
+            raise ValueError(f'Invalid or duplicate source entry point: {entry.name}')
+        registered[source.name] = source
+    return registered
+
+
+def refresh(state, profile, start, end, adapters=None, include_results=False, include_amendments=False):
+    start, end = window(start, end)
+    adapters = sources() if adapters is None else adapters
+    summaries = []
+    query = profile.source_query()
+    if include_results or include_amendments:
+        query = replace(query, include_results=include_results, include_amendments=include_amendments)
+    for source, adapter in adapters.items():
+        try:
+            notices = []
+            for notice in adapter.fetch(start, end, query):
+                if len(notices) >= MAX_NOTICES:
+                    raise ValueError('Notice safety limit reached')
+                if notice.source != source or notice.published is None or not start <= notice.published <= end:
+                    raise ValueError('Source returned a notice outside its declared window')
+                notices.append(notice)
+            state.record_refresh(source, start, end, asdict(query), notices)
+            summaries.append(dict(source=source, status='COMPLETED', notices=len(notices)))
+        except Exception as exc:
+            state.record_refresh(source, start, end, asdict(query), error=type(exc).__name__)
+            summaries.append(dict(source=source, status='FAILED', error=type(exc).__name__))
+    return summaries
+
+
+def filtered(state, profile, *, start=None, end=None, source_names=(), on=None):
+    if start is not None or end is not None:
+        start, end = window(start, end)
+    on = on or today()
+    for notice in state.notices(start, end, source_names):
+        if profile.matches(notice, today=on):
+            yield notice
+
+
+def search(state, profile, *, start=None, end=None, limit=100, offset=0, source_names=()):
+    if type(limit) is not int or not 1 <= limit <= MAX_RESULTS or type(offset) is not int or not 0 <= offset <= MAX_NOTICES:
+        raise ValueError('Invalid result limit or offset')
+    found, more = [], False
+    for index, notice in enumerate(filtered(state, profile, start=start, end=end, source_names=source_names)):
+        if index < offset:
+            continue
+        if len(found) == limit:
+            more = True
+            break
+        found.append(matched_notice_record(notice, profile))
+    return dict(notices=found, offset=offset, next_offset=offset + len(found) if more else None)
+
+
+def matched_notice_record(notice, profile, *, raw=False):
+    record = notice_record(notice, raw=raw)
+    if profile.keywords_any:
+        record['keyword_matches'] = profile.keyword_matches(notice)
+    return record
+
+
+def _award_record(award, notice):
+    record = json_roundtrip(asdict(award))
+    return dict(record, source=notice.source, number=notice.number,
+                published=str(notice.published) if notice.published else None,
+                title=notice.title, buyer_name=notice.buyer_name)
+
+
+def json_roundtrip(value):
+    return json.loads(dumps(value))
+
+
+def latest_awards(notices):
+    """Latest formal result per source, procedure and lot; a later notice replaces an earlier one."""
+    latest = {}
+    for notice in sorted(notices, key=lambda n: (n.published or date.min, n.number)):
+        if not notice.awards:
+            continue
+        key = notice.procedure_id or notice.number
+        for award in notice.awards:
+            latest[(notice.source, key, award.lot)] = (award, notice)
+    return [latest[key] for key in sorted(latest, key=lambda k: (k[0], k[1], k[2]))]
+
+
+def procedure_results(state, source, number):
+    """One procedure's contract and result notices, linked by procedure_id, with the latest award per lot."""
+    notice = state.get(source, number)
+    if notice is None:
+        raise ValueError('Notice not found in local state')
+    related = [notice]
+    if notice.procedure_id:
+        related = [n for n in state.notices(sources=(source,)) if n.procedure_id == notice.procedure_id]
+        if not any(n.number == notice.number for n in related):
+            related.append(notice)
+    related.sort(key=lambda n: (n.published or date.min, n.number))
+    return dict(
+        notice=notice_record(notice, raw=False),
+        procedure_id=notice.procedure_id,
+        contracts=[notice_record(n, raw=False) for n in related if not n.awards],
+        results=[notice_record(n, raw=False) for n in related if n.awards],
+        awards=[_award_record(award, owner) for award, owner in latest_awards(related)])
+
+
+def search_results(state, *, buyer=None, cpv=(), start=None, end=None, limit=100, offset=0, source_names=()):
+    """Earlier awards by buyer text and CPV, latest per procedure and lot, paginated like notice search."""
+    if type(limit) is not int or not 1 <= limit <= MAX_RESULTS or type(offset) is not int or not 0 <= offset <= MAX_NOTICES:
+        raise ValueError('Invalid result limit or offset')
+    if start is not None or end is not None:
+        start, end = window(start, end)
+    patterns = tuple(str(code).strip() for code in cpv)
+    if any(not pattern or not (pattern.rstrip('*').isdigit()) for pattern in patterns):
+        raise ValueError('Result CPV must be digits, optionally ending in *')
+    folded_buyer = fold(buyer) if buyer else None
+    candidates = []
+    for notice in state.notices(start, end, source_names):
+        if not notice.awards:
+            continue
+        if folded_buyer and folded_buyer not in fold(notice.buyer_name):
+            continue
+        if patterns and not any(
+                str(code).startswith(pattern[:-1]) if pattern.endswith('*') else str(code) == pattern
+                for pattern in patterns for code in notice.cpv):
+            continue
+        candidates.append(notice)
+    awards = [_award_record(award, owner) for award, owner in latest_awards(candidates)]
+    page = awards[offset:offset + limit]
+    return dict(awards=page, offset=offset, total=len(awards),
+                next_offset=offset + len(page) if offset + len(page) < len(awards) else None)

tenderkit/sources/__init__.py

@@ -0,0 +1 @@
+"""Public notice sources. Network clients can be supplied for offline testing."""

tenderkit/sources/bk.py

@@ -0,0 +1,145 @@
+"""Bounded reads of public BK2021 notices from the portal's backend."""
+from datetime import date, datetime, timezone
+import re
+from zoneinfo import ZoneInfo
+
+from tenderkit.model import Lot, Notice
+from tenderkit.platforms.session import BK_HOST, PlatformSession
+from tenderkit.sources.mapping import CPV, amount, html_to_text, instant
+
+BASE = f'https://{BK_HOST}'
+WARSAW = ZoneInfo('Europe/Warsaw')
+NOTICE_TYPE = 'BKAnnouncement'
+STATUSES = ('PUBLISHED', 'CLOSED', 'RESOLVED', 'CANCELLED')
+MAX_TOTAL = 1000
+# Publication-date ordering can repeat records across page boundaries.
+# Request the bounded batch in one response when the portal permits it.
+PAGE_SIZE = MAX_TOTAL
+CPV_BATCH = 100
+
+
+class BkIncomplete(RuntimeError):
+    """The returned page or detail did not match the public search."""
+
+
+def _data(session, path, params=None):
+    response = session.get(BASE + path, params=params, timeout=30)
+    response.raise_for_status()
+    result = response.json()['data']
+    if not isinstance(result, dict):
+        raise BkIncomplete('Expected a BK data object')
+    return result
+
+
+def detail(session, identifier):
+    row = _data(session, f'/api/announcements/{identifier}')['advertisement']
+    if row['advertisement']['id'] != identifier or type(row['id']) is not int or row['id'] < 1:
+        raise BkIncomplete('BK detail does not match the search result')
+    return row
+
+
+def _codes(session, prefixes):
+    codes = set()
+    for prefix in dict.fromkeys(prefixes):
+        data = _data(session, '/api/dictionaries/cpv', {'search_key': prefix, 'request_id': 1})
+        rows = data['cpv_items']
+        if data['meta']['last_page'] != 1 or data['meta']['total'] != len(rows):
+            raise BkIncomplete('Incomplete BK CPV dictionary')
+        for row in rows:
+            code = row['code']
+            if not re.fullmatch(r'\d{8}-\d', code):
+                raise BkIncomplete('Invalid BK CPV code')
+            if code.startswith(prefix):
+                codes.add(code)
+    return sorted(codes)
+
+
+def _list(session, start, end, codes):
+    params = [('limit', PAGE_SIZE), ('sort', 'publicationDate'),
+              ('publicationDateRange[from]', str(start)), ('publicationDateRange[to]', str(end))]
+    params += [(f'status[{index}]', status) for index, status in enumerate(STATUSES)]
+    params += [('cpvItem[]', code) for code in codes]
+    found, expected, page = {}, None, 1
+    while True:
+        data = _data(session, '/api/announcements/search', [*params, ('page', page)])
+        total, rows = data['meta']['total'], data['advertisements']
+        if (type(total) is not int or not 0 <= total <= MAX_TOTAL
+                or expected is not None and total != expected
+                or not isinstance(rows, list) or len(rows) != min(PAGE_SIZE, total - len(found))):
+            raise BkIncomplete('BK count changed or exceeded the fetch limit; narrow the window')
+        expected = total
+        for row in rows:
+            identifier = row['id']
+            published = date.fromisoformat(row['publication_date'])
+            if type(identifier) is not int or identifier < 1 or identifier in found or not start <= published <= end:
+                raise BkIncomplete('Duplicate or invalid BK search result')
+            found[identifier] = row
+        if len(found) == total:
+            return found
+        page += 1
+
+
+def bk_notice(summary, row, acquired):
+    identifier = summary['id']
+    published = date.fromisoformat(row['publication_date'])
+    if published != date.fromisoformat(summary['publication_date']):
+        raise BkIncomplete('BK publication date changed during the fetch')
+    descriptions, lots = [], []
+    orders = row['orders']
+    for order in orders:
+        codes = []
+        for item in order['order_items']:
+            descriptions.append(html_to_text(item.get('description')).strip())
+            codes.extend(match for cpv in item['cpv_items'] for match in CPV.findall(cpv['code']))
+        estimate = amount(order.get('estimated_value'))
+        lots.append(Lot(str(order['id']), title=order.get('title'), cpv=tuple(dict.fromkeys(codes)),
+                        value=estimate, currency='PLN' if estimate is not None else None))
+    # A single order's estimate can be used directly. Variants and partial orders
+    # cannot be added safely without an explicit whole-notice total.
+    value = lots[0].value if len(orders) == 1 and not orders[0].get('is_variant') else None
+    deadline = instant(row.get('submission_deadline'))
+    if deadline is not None and deadline.tzinfo is None:
+        deadline = deadline.replace(tzinfo=WARSAW)
+    address = row.get('advertiser_address_details') or {}
+    url = f'{BASE}/ogloszenia/{identifier}'
+    return Notice(
+        'BK', str(identifier), procedure_id=str(identifier), title=row['title'], published=published,
+        deadline=deadline, buyer_name=summary.get('advertiser_name') or address.get('name'),
+        buyer_city=(address.get('address') or {}).get('locality'),
+        cpv=tuple(dict.fromkeys(code for lot in lots for code in lot.cpv)), lots=tuple(lots),
+        description='\n\n'.join(dict.fromkeys(text for text in descriptions if text)) or None,
+        value=value, currency='PLN' if value is not None else None, url=url, platform_url=url,
+        reference=row['advertisement'].get('number'), notice_type=NOTICE_TYPE, lot_count=len(lots),
+        raw={'version': row['id']}, source_attribution={
+            'publisher': 'Ministerstwo Funduszy i Polityki Regionalnej',
+            'acquired_at': acquired, 'processing': 'Selected and normalized from the public BK2021 notice',
+            'url': url,
+        })
+
+
+class Bk:
+    name = 'BK'
+
+    def __init__(self, session=None):
+        self.session = session or PlatformSession()
+
+    def fetch(self, start, end, query):
+        if query.notice_types and NOTICE_TYPE not in query.notice_types:
+            return
+        if any(not re.fullmatch(r'\d{1,8}', prefix) for prefix in query.cpv_prefixes):
+            raise ValueError('Invalid BK CPV prefix')
+        codes = _codes(self.session, query.cpv_prefixes)
+        batches = [codes[i:i + CPV_BATCH] for i in range(0, len(codes), CPV_BATCH)] if query.cpv_prefixes else [()]
+        found = {}
+        for batch in batches:
+            for identifier, row in _list(self.session, start, end, batch).items():
+                if identifier in found and any(
+                    found[identifier].get(key) != row.get(key)
+                    for key in ('publication_date', 'submission_deadline', 'title', 'advertiser_name')
+                ):
+                    raise BkIncomplete('BK search result changed between CPV batches')
+                found[identifier] = row
+            if len(found) > MAX_TOTAL:
+                raise BkIncomplete('BK detail request limit exceeded; narrow the window')
+        for identifier, row in found.items():
+            yield bk_notice(row, detail(self.session, identifier), datetime.now(timezone.utc).isoformat())

tenderkit/sources/bzp.py

@@ -0,0 +1,142 @@
+"""Bounded BZP pagination shared by ingestion and read-only discovery."""
+from datetime import datetime, timedelta, timezone
+import time
+from zoneinfo import ZoneInfo
+
+WARSAW = ZoneInfo('Europe/Warsaw')
+FRESH_NOTICE_TYPES = ('ContractNotice', 'SmallContractNotice', 'CompetitionNotice')
+RESULT_NOTICE_TYPES = ('TenderResultNotice',)
+AMENDMENT_TYPE = 'NoticeUpdateNotice'
+ENDPOINT = 'https://ezamowienia.gov.pl/mo-board/api/v1/notice'
+
+
+def timestamp(value):
+    result = datetime.fromisoformat(value.replace('Z', '+00:00'))
+    if result.tzinfo is None:
+        raise ValueError('Publication timestamp has no timezone')
+    return result.astimezone(timezone.utc)
+
+
+def bounds(date_from, date_to):
+    if date_from > date_to:
+        raise ValueError('Invalid date interval')
+    start = datetime.combine(date_from, datetime.min.time(), WARSAW)
+    end = datetime.combine(date_to + timedelta(days=1), datetime.min.time(), WARSAW)
+    return start.astimezone(timezone.utc), end.astimezone(timezone.utc)
+
+
+class FetchFailure(RuntimeError):
+    def __init__(self, message, audit, notices):
+        super().__init__(message)
+        self.audit, self.notices = audit, notices
+
+
+def fetch_window(session, date_from, date_to, kind, cpv_code='336', page_size=500,
+                 delay=0.3, observed_at=None, save_page=None, max_pages=100,
+                 retries=3):
+    """Fetch broad API date bounds, then enforce exact Warsaw bounds locally.
+
+    save_page is called before validation, preserving malformed/error evidence.
+    Failed windows carry partial notices but must never be marked complete.
+    """
+    if not 1 <= page_size <= 500 or max_pages < 1 or retries < 1:
+        raise ValueError('Invalid pagination configuration')
+    start, end = bounds(date_from, date_to)
+    observed_at = observed_at or datetime.now(timezone.utc)
+    if observed_at.tzinfo is None:
+        raise ValueError('Observation timestamp has no timezone')
+    cutoff = min(end, observed_at)
+    params = dict(NoticeType=kind, PageSize=page_size,
+                  PublicationDateFrom=start.strftime('%Y-%m-%dT00:00:00'),
+                  PublicationDateTo=end.strftime('%Y-%m-%dT23:59:59'))
+    if cpv_code:
+        params['CpvCode'] = cpv_code
+    audit = dict(notice_type=kind, from_date=str(date_from), to_date=str(date_to),
+                 start_inclusive=start.isoformat(), end_exclusive=end.isoformat(),
+                 observed_at=observed_at.isoformat(), pages=0, returned_unique=0,
+                 included=0, duplicates=0, outside_window=0, complete=False,
+                 provisional=date_to >= observed_at.astimezone(WARSAW).date())
+    seen, accepted = {}, {}
+    try:
+        for page_number in range(1, max_pages + 1):
+            for attempt in range(retries):
+                if delay:
+                    time.sleep(delay * (2 ** attempt))
+                try:
+                    response = session.get(ENDPOINT, params=params,
+                                           headers={'Accept': 'application/json'}, timeout=40)
+                    response.raise_for_status()
+                    page = response.json()
+                    break
+                except Exception:
+                    if attempt == retries - 1:
+                        raise
+            if save_page:
+                save_page(kind, page_number, dict(params), page)
+            audit['pages'] += 1
+            if not isinstance(page, list):
+                raise ValueError('Expected notice array, not an empty/error object')
+            before = len(seen)
+            for notice in page:
+                if not isinstance(notice, dict) or not notice.get('objectId'):
+                    raise ValueError('Notice is missing objectId')
+                nid = notice['objectId']
+                if nid in seen:
+                    if seen[nid] != notice:
+                        raise ValueError('Conflicting content for the same objectId')
+                    audit['duplicates'] += 1
+                    continue
+                seen[nid] = notice
+                audit['returned_unique'] = len(seen)
+                if notice.get('noticeType') != kind:
+                    raise ValueError('Unexpected notice type')
+                published = timestamp(notice['publicationDate'])
+                if not start <= published < cutoff:
+                    audit['outside_window'] += 1
+                    continue
+                accepted[nid] = notice
+                audit['included'] = len(accepted)
+            if len(page) < page_size:
+                audit['complete'] = not audit['provisional']
+                audit['fetch_succeeded'] = True
+                return list(accepted.values()), audit
+            cursor = page[-1]['objectId']
+            if len(seen) == before or cursor == params.get('SearchAfter'):
+                raise ValueError('Pagination stalled')
+            params['SearchAfter'] = cursor
+        raise ValueError('Pagination safety limit reached')
+    except Exception as exc:
+        audit.update(error_type=type(exc).__name__, fetch_succeeded=False,
+                     http_status=getattr(getattr(exc, 'response', None), 'status_code', None))
+        # Avoid printing connection strings or response bodies from exceptions.
+        raise FetchFailure(type(exc).__name__, audit, list(accepted.values())) from exc
+
+
+class Bzp:
+    name = 'BZP'
+
+    def __init__(self, session=None, delay=2.1):
+        from tenderkit.http import Session
+        self.session, self.delay = session or Session(), delay
+
+    def fetch(self, start, end, query):
+        from tenderkit.sources.mapping import bzp_amendment, bzp_notice
+        known = FRESH_NOTICE_TYPES + RESULT_NOTICE_TYPES
+        kinds = tuple(k for k in query.notice_types if k in known) if query.notice_types else FRESH_NOTICE_TYPES
+        if query.include_results:
+            kinds = kinds + tuple(k for k in RESULT_NOTICE_TYPES if k not in kinds)
+        found = {}
+        for kind in kinds:
+            for prefix in query.cpv_prefixes or ('',):
+                rows, _ = fetch_window(self.session, start, end, kind, cpv_code=prefix, delay=self.delay)
+                for raw in rows:
+                    notice = bzp_notice(raw)
+                    found[notice.number] = notice
+        if query.include_amendments and any(kind in FRESH_NOTICE_TYPES for kind in kinds):
+            # Amendment records do not carry CPV codes. Keep the reference and changed deadline.
+            rows, _ = fetch_window(self.session, start, end, AMENDMENT_TYPE, cpv_code='', delay=self.delay)
+            for raw in rows:
+                notice = bzp_amendment(raw)
+                if notice is not None:
+                    found[notice.number] = notice
+        return list(found.values())