tenderkit 0.1.0__py3-none-any.whl

tenderkit/store/state.py

@@ -0,0 +1,202 @@
+"""SQLite state: one transaction per successfully completed source window."""
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import sqlite3
+
+from tenderkit.serialization import dumps, notice_from_record, notice_record
+
+SCHEMA = '''
+create table if not exists notices (
+    source text not null, number text not null, published text,
+    payload text not null, coverage text not null default 'PENDING',
+    primary key(source, number)
+);
+create index if not exists notices_published on notices(published, source, number);
+create table if not exists refreshes (
+    id integer primary key, source text not null, start text not null, end text not null,
+    query text not null, status text not null, count integer not null,
+    error text, finished_at text not null
+);
+create table if not exists document_refs (
+    source text not null, number text not null, document_key text not null, payload text not null,
+    primary key(source,number,document_key)
+);
+create table if not exists procedure_links (
+    source text not null, number text not null, payload text not null,
+    primary key(source,number)
+);
+create table if not exists notice_documents (
+    source text not null, number text not null, document_key text not null, sha256 text not null,
+    payload text not null, fetched_at text not null,
+    primary key(source,number,document_key,sha256)
+);
+-- A document the platform refused with a permanent HTTP error (a withdrawn or replaced attachment). A whole-procedure
+-- download skips it; asking for its key again retries it, and a later successful download clears it.
+create table if not exists document_unavailable (
+    source text not null, number text not null, document_key text not null, error text not null, checked_at text not null,
+    primary key(source,number,document_key)
+);
+create table if not exists fetch_jobs (
+    id integer primary key, source text not null, number text not null, selection text,
+    status text not null, attempts integer not null default 0, error text, error_kind text,
+    next_attempt real not null default 0, heartbeat real, summary text not null default '{}'
+);
+create unique index if not exists active_document_job on fetch_jobs(source,number)
+    where status in ('QUEUED','RUNNING');
+create table if not exists watch_cursors (
+    profile text not null, source text not null, resume_from text not null,
+    primary key(profile,source)
+);
+create table if not exists watch_seen (
+    profile text not null, source text not null, identity text not null,
+    revision text not null, deadline text, primary key(profile,source,identity)
+);
+create table if not exists watch_ted_aliases (
+    profile text not null, alias text not null, identity text not null, primary key(profile,alias)
+);
+create table if not exists watch_runs (
+    id integer primary key, profile text not null, finished_at text not null, sources text not null
+);
+create index if not exists watch_runs_profile on watch_runs(profile,id);
+create table if not exists watch_events (
+    id integer primary key, run_id integer not null references watch_runs(id),
+    profile text not null, payload text not null, delivered integer not null default 0
+);
+create index if not exists watch_events_run on watch_events(run_id,id);
+create index if not exists watch_events_pending on watch_events(profile) where delivered=0;
+'''
+
+
+class State:
+    def __init__(self, path, *, read_only=False):
+        self.path = Path(path).expanduser()
+        self.read_only = read_only
+        if read_only:
+            return
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.connection() as conn:
+            conn.executescript(SCHEMA)
+
+    @contextmanager
+    def connection(self):
+        database = self.path.resolve().as_uri() + '?mode=ro' if self.read_only else self.path
+        conn = sqlite3.connect(database, timeout=30, uri=self.read_only)
+        conn.row_factory = sqlite3.Row
+        try:
+            with conn:
+                yield conn
+        finally:
+            conn.close()
+
+    def record_refresh(self, source, start, end, query, notices=None, error=None):
+        with self.connection() as conn:
+            for notice in notices or []:
+                conn.execute('''insert into notices(source,number,published,payload) values(?,?,?,?)
+                    on conflict(source,number) do update set published=excluded.published,payload=excluded.payload''',
+                    (notice.source, notice.number, str(notice.published) if notice.published else None,
+                     dumps(notice_record(notice))))
+            conn.execute('''insert into refreshes(source,start,end,query,status,count,error,finished_at)
+                values(?,?,?,?,?,?,?,?)''', (source, str(start), str(end), dumps(query),
+                    'FAILED' if error else 'COMPLETED', len(notices or []), error,
+                    datetime.now(timezone.utc).isoformat()))
+
+    def notices(self, start=None, end=None, sources=()):
+        conditions, params = [], []
+        if start:
+            conditions.append('published >= ?')
+            params.append(str(start))
+        if end:
+            conditions.append('published <= ?')
+            params.append(str(end))
+        if sources:
+            conditions.append('source in (' + ','.join('?' for _ in sources) + ')')
+            params.extend(sources)
+        where = ' where ' + ' and '.join(conditions) if conditions else ''
+        with self.connection() as conn:
+            for row in conn.execute('select payload from notices' + where + ' order by published desc,source,number', params):
+                yield notice_from_record(json.loads(row['payload']))
+
+    def get(self, source, number):
+        with self.connection() as conn:
+            row = conn.execute('select payload from notices where source=? and number=?', (source, number)).fetchone()
+            return notice_from_record(json.loads(row['payload'])) if row else None
+
+    def coverage(self):
+        with self.connection() as conn:
+            counts = [dict(r) for r in conn.execute('select source,coverage,count(*) as count from notices group by source,coverage')]
+            # Every source/query window is explicit; this is not a claim of complete catalogue coverage.
+            refreshes = [dict(r) for r in conn.execute('select * from refreshes order by id desc limit 50')]
+            for row in refreshes:
+                row['query'] = json.loads(row['query'])
+            jobs = [dict(r) for r in conn.execute('select id,source,number,status,attempts,error,error_kind,next_attempt,summary from fetch_jobs order by id desc limit 50')]
+            for job in jobs:
+                job['summary'] = json.loads(job['summary'])
+            return dict(counts=counts, refreshes=refreshes, jobs=jobs)
+
+    def set_coverage(self, source, number, status):
+        if status not in ('PENDING', 'FETCHED', 'NO_DOCUMENTS', 'UNSUPPORTED_PLATFORM', 'NO_REFERENCE', 'FETCH_FAILED'):
+            raise ValueError('Invalid coverage state')
+        with self.connection() as conn:
+            conn.execute('update notices set coverage=? where source=? and number=?', (status, source, number))
+
+    def save_listing(self, source, number, refs, link=None):
+        from dataclasses import asdict
+        with self.connection() as conn:
+            conn.execute('delete from document_refs where source=? and number=?', (source, number))
+            conn.executemany('insert into document_refs values(?,?,?,?)',
+                [(source, number, ref.key, dumps(asdict(ref))) for ref in refs])
+            if link is not None:
+                conn.execute('insert into procedure_links values(?,?,?) on conflict(source,number) do update set payload=excluded.payload',
+                             (source, number, dumps(asdict(link))))
+
+    def procedure_link(self, source, number):
+        with self.connection() as conn:
+            row = conn.execute('select payload from procedure_links where source=? and number=?', (source, number)).fetchone()
+            return json.loads(row[0]) if row else None
+
+    def listing(self, source, number):
+        with self.connection() as conn:
+            return [json.loads(row[0]) for row in conn.execute(
+                'select payload from document_refs where source=? and number=? order by rowid', (source, number))]
+
+    def record_document(self, source, number, record):
+        with self.connection() as conn:
+            conn.execute('''insert into notice_documents values(?,?,?,?,?,?)
+                on conflict(source,number,document_key,sha256) do update set payload=excluded.payload,fetched_at=excluded.fetched_at''',
+                (source, number, record['key'], record['sha256'], dumps(record), record['fetched_at']))
+
+    def mark_unavailable(self, source, number, key, error):
+        with self.connection() as conn:
+            conn.execute('''insert into document_unavailable values(?,?,?,?,?)
+                on conflict(source,number,document_key) do update set error=excluded.error,checked_at=excluded.checked_at''',
+                (source, number, key, error, datetime.now(timezone.utc).isoformat()))
+
+    def clear_unavailable(self, source, number, key):
+        with self.connection() as conn:
+            conn.execute('delete from document_unavailable where source=? and number=? and document_key=?', (source, number, key))
+
+    def unavailable(self, source, number):
+        """{document key: {error, checked_at}} of the documents the platform refused for good."""
+        with self.connection() as conn:
+            return {row['document_key']: dict(error=row['error'], checked_at=row['checked_at']) for row in conn.execute(
+                'select document_key,error,checked_at from document_unavailable where source=? and number=?', (source, number))}
+
+    def documents(self, source, number):
+        with self.connection() as conn:
+            # Return latest versions by key; earlier hashes stay registered and readable.
+            found = {}
+            for row in conn.execute('select document_key,payload from notice_documents where source=? and number=? order by fetched_at,rowid', (source, number)):
+                found[row['document_key']] = json.loads(row['payload'])
+            return found
+
+    def document(self, digest):
+        import re
+        if not re.fullmatch(r'[a-f0-9]{64}', digest or ''):
+            raise ValueError('Document id must be a SHA-256 digest')
+        with self.connection() as conn:
+            row = conn.execute('select source,number,payload from notice_documents where sha256=? order by fetched_at desc limit 1', (digest,)).fetchone()
+            if row is None:
+                raise ValueError('Document not found in local state')
+            return dict(source=row['source'], number=row['number'], **json.loads(row['payload']))

tenderkit/ui.py

@@ -0,0 +1,243 @@
+"""Loopback HTTP view over the same saved notices and profiles as the CLI."""
+from dataclasses import replace
+from datetime import date, datetime, timezone
+from html import escape
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from importlib.resources import files
+import re
+import sqlite3
+from string import Template
+from urllib.parse import parse_qsl, urlencode, urlsplit
+from zoneinfo import ZoneInfo
+
+from tenderkit.profile import fold
+from tenderkit.service import filtered
+from tenderkit.store.state import State
+
+PAGE_SIZE = 30
+PROFILE_LABELS = {
+    'oprogramowanie': 'Oprogramowanie',
+    'leki': 'Leki',
+    'wyroby-medyczne': 'Wyroby medyczne',
+}
+DEADLINE_LABELS = {
+    '': 'Dowolny',
+    'upcoming': 'Przed terminem',
+    'passed': 'Po terminie',
+    'today': 'Dziś, godzina nieznana',
+    'missing': 'Brak terminu',
+}
+SORT_LABELS = {
+    'newest': 'Najnowsze',
+    'oldest': 'Najstarsze',
+    'deadline': 'Termin rosnąco',
+    'title': 'Tytuł A–Z',
+    'buyer': 'Zamawiający A–Z',
+}
+
+
+def page_link(params):
+    return '/?' + escape(urlencode({key: value for key, value in params.items() if value != ''}))
+
+
+def options(labels, selected):
+    return ''.join(f'<option value="{key}"{" selected" if key == selected else ""}>{label}</option>'
+                   for key, label in labels.items())
+
+
+def local_deadline(notice, warsaw):
+    deadline = notice.deadline
+    if deadline is None:
+        return None
+    return deadline.astimezone(warsaw) if deadline.tzinfo else deadline.replace(tzinfo=warsaw)
+
+
+def deadline_date_only(notice):
+    if notice.source != 'TED':
+        return False
+    dates = notice.raw.get('deadline-receipt-tender-date-lot') or []
+    first = min((value for value in dates if value), default='')
+    return bool(first and len(first) <= 16 and 'T' not in first)
+
+
+def tab(label, href, active):
+    current = ' aria-current="page"' if active else ''
+    return f'<a href="{href}"{current}>{escape(label)}</a>'
+
+
+def notice_row(notice, deadline, date_only):
+    title = escape(notice.title or 'Ogłoszenie bez tytułu')
+    buyer = escape(notice.buyer_name or 'Nie podano zamawiającego')
+    published = escape(str(notice.published or ''))
+    due = ''
+    if deadline:
+        date_text = deadline.strftime('%d.%m.%Y' if date_only else '%d.%m.%Y, %H:%M')
+        due = f'<span>Termin: {date_text}</span>'
+    link = ''
+    for url in (notice.url, notice.platform_url):
+        if not url:
+            continue
+        try:
+            parts = urlsplit(url)
+        except ValueError:
+            continue
+        if parts.scheme in ('http', 'https') and parts.hostname:
+            link = (f'<a class="open" href="{escape(url)}" target="_blank" rel="noopener noreferrer"'
+                    f' aria-label="Otwórz ogłoszenie: {title}">↗</a>')
+            break
+    return f'''<li class="notice">
+      <div>
+        <div class="metadata"><span class="badge">{escape(notice.source)}</span><time>{published}</time>{due}</div>
+        <h2>{title}</h2>
+        <p>{buyer}</p>
+      </div>
+      {link}
+    </li>'''
+
+
+def render(template, state, profiles, params):
+    active, query, source = (params[key] for key in ('profile', 'q', 'source'))
+    cpv, deadline_filter, order = (params[key] for key in ('cpv', 'deadline', 'sort'))
+    start = date.fromisoformat(params['from']) if params['from'] else None
+    end = date.fromisoformat(params['to']) if params['to'] else None
+    page = int(params['page'])
+    profile = profiles[active]
+    terms = fold(query).split()
+    warsaw = ZoneInfo('Europe/Warsaw')
+    now = datetime.now(timezone.utc)
+    found_sources, matches = set(), []
+    offset = (page - 1) * PAGE_SIZE
+    for notice in filtered(state, profile):
+        found_sources.add(notice.source)
+        if source and notice.source != source:
+            continue
+        if start and (not notice.published or notice.published < start):
+            continue
+        if end and (not notice.published or notice.published > end):
+            continue
+        if cpv and not any(code.startswith(cpv) for code in notice.cpv):
+            continue
+        if terms:
+            haystack = fold(f'{notice.title or ""} {notice.buyer_name or ""}')
+            if not all(term in haystack for term in terms):
+                continue
+        deadline = local_deadline(notice, warsaw)
+        date_only = deadline_date_only(notice)
+        if deadline and date_only:
+            # Date-only TED fields were stored at midnight with the source offset.
+            deadline = notice.deadline.replace(tzinfo=warsaw)
+        deadline_utc = deadline.astimezone(timezone.utc) if deadline else None
+        if deadline is None:
+            status = 'missing'
+        elif date_only and deadline.date() == now.astimezone(warsaw).date():
+            status = 'today'
+        else:
+            status = 'upcoming' if deadline_utc >= now else 'passed'
+        if deadline_filter and status != deadline_filter:
+            continue
+        matches.append((replace(notice, raw={}, lots=()), deadline, deadline_utc, date_only))
+
+    if order == 'oldest':
+        matches.sort(key=lambda item: (item[0].published is None, item[0].published))
+    elif order == 'deadline':
+        matches.sort(key=lambda item: (item[2] is None, item[2]))
+    elif order in ('title', 'buyer'):
+        field = 'title' if order == 'title' else 'buyer_name'
+        matches.sort(key=lambda item: (not getattr(item[0], field), fold(getattr(item[0], field) or '')))
+    total = len(matches)
+    visible = matches[offset:offset + PAGE_SIZE]
+
+    def link(**changes):
+        return page_link({**params, 'page': 1, **changes})
+
+    profile_tabs = ''.join(tab(PROFILE_LABELS.get(name, name), link(profile=name, source=''), name == active)
+                           for name in profiles)
+    source_tabs = ''.join(tab(label, link(source=name), source == name)
+                          for name, label in [('', 'Wszystkie')] +
+                          [(name, name) for name in sorted(found_sources | ({source} if source else set()))])
+    count = f'{offset + 1}–{offset + len(visible)} z {total} ogłoszeń' if visible else f'{total} ogłoszeń'
+    pagination = []
+    if page > 1:
+        pagination.append(tab('← Poprzednia', link(page=page - 1), False))
+    if offset + PAGE_SIZE < total:
+        pagination.append(tab('Następna →', link(page=page + 1), False))
+    rows = ''.join(notice_row(notice, deadline, date_only) for notice, deadline, _, date_only in visible)
+    return template.substitute(
+        profile_tabs=profile_tabs, source_tabs=source_tabs, profile=escape(active),
+        source=escape(source), query=escape(query), count=count,
+        published_from=escape(params['from']), published_to=escape(params['to']), cpv=escape(cpv),
+        deadline_options=options(DEADLINE_LABELS, deadline_filter), sort_options=options(SORT_LABELS, order),
+        reset_link=page_link({'profile': active}),
+        criteria=''.join(f'<span>CPV {escape(code)}</span>' for code in profile.cpv)
+        + ''.join(f'<span>bez CPV {escape(code)}</span>' for code in profile.exclude_cpv),
+        rows=f'<ul class="notices">{rows}</ul>' if rows else
+             '<p class="empty">Brak wyników na tej stronie. Zmień filtry lub wybierz inny profil.</p>',
+        pagination=''.join(pagination),
+    )
+
+
+def serve(path, profiles, initial, port=8765):
+    if not 0 <= port <= 65535:
+        raise ValueError('Port must be between 0 and 65535')
+    state = State(path, read_only=True)
+    try:
+        with state.connection() as connection:
+            connection.execute('select 1 from notices limit 1').fetchone()
+    except sqlite3.Error as exc:
+        raise ValueError(f'Nie można otworzyć bazy ogłoszeń: {state.path}') from exc
+
+    assets = files('tenderkit').joinpath('web')
+    template = Template(assets.joinpath('index.html').read_text(encoding='utf-8'))
+    css = assets.joinpath('style.css').read_bytes()
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self):
+            port = self.server.server_port
+            if self.headers.get('Host') not in (f'127.0.0.1:{port}', f'localhost:{port}'):
+                return self.send_error(403)
+            path, _, params = self.path.partition('?')
+            if path == '/style.css':
+                return self.respond(css, 'text/css')
+            if path != '/':
+                return self.send_error(404)
+            try:
+                values = dict(parse_qsl(params, max_num_fields=12))
+                filters = dict(profile=initial, q='', source='', cpv='', deadline='', sort='newest', page='1')
+                filters.update({'from': '', 'to': ''})
+                filters = {key: values.get(key, default).strip() for key, default in filters.items()}
+                for key in ('from', 'to'):
+                    if filters[key]:
+                        filters[key] = date.fromisoformat(filters[key]).isoformat()
+                if (filters['profile'] not in profiles or len(filters['q']) > 200 or len(filters['source']) > 100
+                        or not re.fullmatch(r'[0-9]{0,8}', filters['cpv']) or int(filters['page']) < 1
+                        or filters['deadline'] not in DEADLINE_LABELS or filters['sort'] not in SORT_LABELS
+                        or filters['to'] and filters['from'] > filters['to']):
+                    raise ValueError
+            except ValueError:
+                return self.send_error(400, 'Invalid filter')
+            try:
+                body = render(template, state, profiles, filters)
+            except sqlite3.Error:
+                return self.send_error(503, 'Database unavailable')
+            self.respond(body.encode('utf-8'), 'text/html')
+
+        def respond(self, body, content_type):
+            self.send_response(200)
+            self.send_header('Content-Type', content_type + '; charset=utf-8')
+            self.send_header('Content-Length', str(len(body)))
+            self.send_header('Cache-Control', 'no-store')
+            self.send_header('Content-Security-Policy',
+                             "default-src 'none'; style-src 'self'; form-action 'self'; frame-ancestors 'none'")
+            self.send_header('Referrer-Policy', 'no-referrer')
+            self.end_headers()
+            self.wfile.write(body)
+
+        def log_message(self, format, *args):
+            pass
+
+    with ThreadingHTTPServer(('127.0.0.1', port), Handler) as server:
+        print(f'tenderkit UI: http://127.0.0.1:{server.server_port}/', flush=True)
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass