michraz 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- michraz/__init__.py +34 -0
- michraz/__main__.py +3 -0
- michraz/_closeable.py +25 -0
- michraz/_dates.py +68 -0
- michraz/_enums.py +14 -0
- michraz/_http.py +253 -0
- michraz/_version.py +10 -0
- michraz/cli.py +146 -0
- michraz/client.py +116 -0
- michraz/errors.py +34 -0
- michraz/export.py +126 -0
- michraz/filters.py +111 -0
- michraz/models.py +92 -0
- michraz/py.typed +0 -0
- michraz/sources/__init__.py +7 -0
- michraz/sources/base.py +112 -0
- michraz/sources/beersheva.py +204 -0
- michraz/sources/data/rmi_settlements.json +1 -0
- michraz/sources/rmi.py +220 -0
- michraz-0.1.0.dist-info/METADATA +126 -0
- michraz-0.1.0.dist-info/RECORD +25 -0
- michraz-0.1.0.dist-info/WHEEL +5 -0
- michraz-0.1.0.dist-info/entry_points.txt +2 -0
- michraz-0.1.0.dist-info/licenses/LICENSE +21 -0
- michraz-0.1.0.dist-info/top_level.txt +1 -0
michraz/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""michraz: one clean interface to Israeli public tenders.
|
|
2
|
+
|
|
3
|
+
from michraz import Michraz
|
|
4
|
+
|
|
5
|
+
with Michraz() as m:
|
|
6
|
+
for tender in m.search(keyword="גינון", open_only=True):
|
|
7
|
+
print(tender.deadline, tender.title)
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from michraz._version import __version__
|
|
11
|
+
from michraz.client import Michraz
|
|
12
|
+
from michraz.errors import MichrazError, SourceFormatError, SourceUnavailableError
|
|
13
|
+
from michraz.export import to_csv, to_dataframe, to_dicts
|
|
14
|
+
from michraz.models import Document, Tender
|
|
15
|
+
from michraz.sources import BeerShevaExtra, BeerShevaSource, RmiExtra, RmiSource, Source, SourceName
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"BeerShevaExtra",
|
|
19
|
+
"BeerShevaSource",
|
|
20
|
+
"Document",
|
|
21
|
+
"Michraz",
|
|
22
|
+
"MichrazError",
|
|
23
|
+
"RmiExtra",
|
|
24
|
+
"RmiSource",
|
|
25
|
+
"Source",
|
|
26
|
+
"SourceName",
|
|
27
|
+
"SourceFormatError",
|
|
28
|
+
"SourceUnavailableError",
|
|
29
|
+
"Tender",
|
|
30
|
+
"__version__",
|
|
31
|
+
"to_csv",
|
|
32
|
+
"to_dataframe",
|
|
33
|
+
"to_dicts",
|
|
34
|
+
]
|
michraz/__main__.py
ADDED
michraz/_closeable.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Context-manager behaviour shared by every object that owns a connection pool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from types import TracebackType
|
|
7
|
+
from typing import TypeVar
|
|
8
|
+
|
|
9
|
+
_T = TypeVar("_T", bound="Closeable")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Closeable(ABC):
|
|
13
|
+
"""An object with a :meth:`close` method that can be used in a ``with`` block."""
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def close(self) -> None:
|
|
17
|
+
"""Releases the resources this object owns."""
|
|
18
|
+
|
|
19
|
+
def __enter__(self: _T) -> _T:
|
|
20
|
+
return self
|
|
21
|
+
|
|
22
|
+
def __exit__(
|
|
23
|
+
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None
|
|
24
|
+
) -> None:
|
|
25
|
+
self.close()
|
michraz/_dates.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Date parsing helpers. Every datetime michraz produces is timezone-aware."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from zoneinfo import ZoneInfo
|
|
8
|
+
|
|
9
|
+
TIMEZONE_NAME = "Asia/Jerusalem"
|
|
10
|
+
JERUSALEM = ZoneInfo(TIMEZONE_NAME)
|
|
11
|
+
|
|
12
|
+
LOCAL_FORMAT = "%d/%m/%Y %H:%M:%S"
|
|
13
|
+
|
|
14
|
+
UTC_DESIGNATOR = "Z"
|
|
15
|
+
UTC_OFFSET = "+00:00"
|
|
16
|
+
|
|
17
|
+
_FRACTION = re.compile(r"\.(\d+)")
|
|
18
|
+
MICROSECOND_DIGITS = 6
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _pad_fraction(match: re.Match[str]) -> str:
|
|
22
|
+
# Python 3.10 only accepts fractions of exactly 3 or 6 digits; the RMI API sends 2 ("57.55").
|
|
23
|
+
return "." + match.group(1)[:MICROSECOND_DIGITS].ljust(MICROSECOND_DIGITS, "0")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_iso(value: str | None) -> datetime | None:
|
|
27
|
+
"""Parses an ISO 8601 timestamp and converts it to Asia/Jerusalem.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
value: Timestamp such as ``"2026-12-28T12:00:00+02:00"``. A value without an offset is taken as
|
|
31
|
+
Asia/Jerusalem local time.
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
An aware datetime, or None if ``value`` is None or blank.
|
|
35
|
+
|
|
36
|
+
Raises:
|
|
37
|
+
ValueError: If ``value`` is not a valid ISO 8601 timestamp.
|
|
38
|
+
"""
|
|
39
|
+
if value is None or not value.strip():
|
|
40
|
+
return None
|
|
41
|
+
text = _FRACTION.sub(_pad_fraction, value.strip().replace(UTC_DESIGNATOR, UTC_OFFSET))
|
|
42
|
+
parsed = datetime.fromisoformat(text)
|
|
43
|
+
if parsed.tzinfo is None:
|
|
44
|
+
return parsed.replace(tzinfo=JERUSALEM)
|
|
45
|
+
return parsed.astimezone(JERUSALEM)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def parse_local(value: str | None, fmt: str = LOCAL_FORMAT) -> datetime | None:
|
|
49
|
+
"""Parses a local Israeli timestamp such as ``"10/11/2026 12:00:00"``.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
value: Text to parse.
|
|
53
|
+
fmt: ``strptime`` format. Defaults to ``dd/mm/yyyy HH:MM:SS``.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
An aware datetime in Asia/Jerusalem, or None if ``value`` is None or blank.
|
|
57
|
+
|
|
58
|
+
Raises:
|
|
59
|
+
ValueError: If ``value`` does not match ``fmt``.
|
|
60
|
+
"""
|
|
61
|
+
if value is None or not value.strip():
|
|
62
|
+
return None
|
|
63
|
+
return datetime.strptime(value.strip(), fmt).replace(tzinfo=JERUSALEM)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def now() -> datetime:
|
|
67
|
+
"""Returns the current time in Asia/Jerusalem."""
|
|
68
|
+
return datetime.now(JERUSALEM)
|
michraz/_enums.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Enum base used across the package."""
|
|
2
|
+
|
|
3
|
+
from enum import Enum
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class StrEnum(str, Enum):
|
|
7
|
+
"""A string enum whose members equal, hash and print as their values.
|
|
8
|
+
|
|
9
|
+
``StrEnum.MEMBER == "value"`` is true and members work as dictionary keys next to plain strings, so code
|
|
10
|
+
that still passes the raw string keeps working. ``enum.StrEnum`` exists only from Python 3.11.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
def __str__(self) -> str:
|
|
14
|
+
return str(self.value)
|
michraz/_http.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Polite HTTP client shared by every source.
|
|
2
|
+
|
|
3
|
+
Government services are not built for heavy traffic, so this client sends an honest User-Agent, waits between
|
|
4
|
+
requests to the same host, caches successful responses on disk and backs off when a server asks it to.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import tempfile
|
|
14
|
+
import time
|
|
15
|
+
from collections.abc import Callable, Mapping
|
|
16
|
+
from datetime import datetime, timedelta, timezone
|
|
17
|
+
from email.utils import parsedate_to_datetime
|
|
18
|
+
from http import HTTPStatus
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
|
|
24
|
+
from michraz._closeable import Closeable
|
|
25
|
+
from michraz._enums import StrEnum
|
|
26
|
+
from michraz._version import __version__
|
|
27
|
+
from michraz.errors import SourceFormatError, SourceUnavailableError
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
PROJECT_URL = "https://github.com/Zikithezikit/michraz"
|
|
32
|
+
USER_AGENT = f"michraz/{__version__} (+{PROJECT_URL})"
|
|
33
|
+
HEADER_USER_AGENT = "User-Agent"
|
|
34
|
+
HEADER_ACCEPT = "Accept"
|
|
35
|
+
HEADER_RETRY_AFTER = "Retry-After"
|
|
36
|
+
JSON_MEDIA_TYPE = "application/json"
|
|
37
|
+
|
|
38
|
+
_CACHE_FETCHED_AT = "fetched_at"
|
|
39
|
+
_CACHE_DATA = "data"
|
|
40
|
+
CACHE_FILE_SUFFIX = ".json"
|
|
41
|
+
CACHE_TMP_SUFFIX = ".tmp"
|
|
42
|
+
DEFAULT_CACHE_DIR = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "michraz"
|
|
43
|
+
DEFAULT_CACHE_TTL = timedelta(hours=1)
|
|
44
|
+
DEFAULT_MIN_INTERVAL = 1.0
|
|
45
|
+
MIN_INTERVAL_FLOOR = 0.5
|
|
46
|
+
MAX_RETRIES = 2
|
|
47
|
+
BACKOFF_BASE = 1.0
|
|
48
|
+
MAX_RETRY_AFTER = 60.0
|
|
49
|
+
TIMEOUT = httpx.Timeout(30.0, connect=10.0)
|
|
50
|
+
SAMPLE_CHARS = 200
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class HttpMethod(StrEnum):
|
|
54
|
+
"""The HTTP methods michraz uses. Both only read data."""
|
|
55
|
+
|
|
56
|
+
GET = "GET"
|
|
57
|
+
POST = "POST"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# 429 and every 5xx are worth retrying; any other error status is final.
|
|
61
|
+
_REDIRECT_STATUS_START = HTTPStatus.MULTIPLE_CHOICES
|
|
62
|
+
_SERVER_ERROR_START = HTTPStatus.INTERNAL_SERVER_ERROR
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _truncate(text: str) -> str:
|
|
66
|
+
return text if len(text) <= SAMPLE_CHARS else text[:SAMPLE_CHARS] + "..."
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _retry_after(header: str | None, wall_clock: Callable[[], float]) -> float | None:
|
|
70
|
+
"""Parses a Retry-After header given in seconds or as an HTTP date."""
|
|
71
|
+
if not header:
|
|
72
|
+
return None
|
|
73
|
+
try:
|
|
74
|
+
return max(0.0, float(header))
|
|
75
|
+
except ValueError:
|
|
76
|
+
pass
|
|
77
|
+
try:
|
|
78
|
+
when = parsedate_to_datetime(header)
|
|
79
|
+
except (TypeError, ValueError):
|
|
80
|
+
return None
|
|
81
|
+
if when.tzinfo is None:
|
|
82
|
+
when = when.replace(tzinfo=timezone.utc)
|
|
83
|
+
return max(0.0, when.timestamp() - wall_clock())
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class _DiskCache:
|
|
87
|
+
"""JSON files on disk, one per request, valid for ``ttl``."""
|
|
88
|
+
|
|
89
|
+
def __init__(self, directory: Path, ttl: timedelta, wall_clock: Callable[[], float]) -> None:
|
|
90
|
+
self.directory = directory
|
|
91
|
+
self.ttl = ttl
|
|
92
|
+
self._wall_clock = wall_clock
|
|
93
|
+
|
|
94
|
+
def _path(self, key: str) -> Path:
|
|
95
|
+
return self.directory / f"{key}{CACHE_FILE_SUFFIX}"
|
|
96
|
+
|
|
97
|
+
def read(self, key: str) -> tuple[float, Any] | None:
|
|
98
|
+
"""Returns ``(fetched_at, data)`` regardless of age, or None if absent or unreadable."""
|
|
99
|
+
try:
|
|
100
|
+
entry = json.loads(self._path(key).read_text(encoding="utf-8"))
|
|
101
|
+
return float(entry[_CACHE_FETCHED_AT]), entry[_CACHE_DATA]
|
|
102
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
def fresh(self, key: str) -> tuple[float, Any] | None:
|
|
106
|
+
entry = self.read(key)
|
|
107
|
+
if entry is None or self._wall_clock() - entry[0] >= self.ttl.total_seconds():
|
|
108
|
+
return None
|
|
109
|
+
return entry
|
|
110
|
+
|
|
111
|
+
def write(self, key: str, data: Any) -> None:
|
|
112
|
+
payload = json.dumps({_CACHE_FETCHED_AT: self._wall_clock(), _CACHE_DATA: data}, ensure_ascii=False)
|
|
113
|
+
try:
|
|
114
|
+
self.directory.mkdir(parents=True, exist_ok=True)
|
|
115
|
+
with tempfile.NamedTemporaryFile(
|
|
116
|
+
"w", encoding="utf-8", dir=self.directory, suffix=CACHE_TMP_SUFFIX, delete=False
|
|
117
|
+
) as handle:
|
|
118
|
+
handle.write(payload)
|
|
119
|
+
os.replace(handle.name, self._path(key))
|
|
120
|
+
except OSError as exc: # a cache that cannot be written must never break a search
|
|
121
|
+
logger.debug("could not write cache entry: %s", exc)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _request_key(method: HttpMethod, url: str, payload: Any) -> str:
|
|
125
|
+
body = json.dumps(payload, sort_keys=True, ensure_ascii=False) if payload is not None else ""
|
|
126
|
+
return hashlib.sha256(f"{method}\n{url}\n{body}".encode()).hexdigest()
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class HttpClient(Closeable):
|
|
130
|
+
"""Rate-limited, caching JSON client with bounded retries.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
cache_dir: Directory for cached responses. None disables the cache.
|
|
134
|
+
cache_ttl: How long a cached response stays valid.
|
|
135
|
+
min_interval: Minimum seconds between requests to the same host. Must be at least 0.5.
|
|
136
|
+
client: An ``httpx.Client`` to use. By default one is created and owned by this object.
|
|
137
|
+
clock: Monotonic clock in seconds. Replace it in tests.
|
|
138
|
+
sleep: Sleep function. Replace it in tests so they do not wait.
|
|
139
|
+
wall_clock: Wall-clock time in seconds since the epoch, used for cache age.
|
|
140
|
+
|
|
141
|
+
Raises:
|
|
142
|
+
ValueError: If ``min_interval`` is below 0.5 seconds.
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
def __init__(
|
|
146
|
+
self,
|
|
147
|
+
*,
|
|
148
|
+
cache_dir: str | os.PathLike[str] | None = DEFAULT_CACHE_DIR,
|
|
149
|
+
cache_ttl: timedelta = DEFAULT_CACHE_TTL,
|
|
150
|
+
min_interval: float = DEFAULT_MIN_INTERVAL,
|
|
151
|
+
client: httpx.Client | None = None,
|
|
152
|
+
clock: Callable[[], float] = time.monotonic,
|
|
153
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
154
|
+
wall_clock: Callable[[], float] = time.time,
|
|
155
|
+
) -> None:
|
|
156
|
+
if min_interval < MIN_INTERVAL_FLOOR:
|
|
157
|
+
raise ValueError(f"min_interval must be at least {MIN_INTERVAL_FLOOR} seconds")
|
|
158
|
+
self.min_interval = min_interval
|
|
159
|
+
self._clock = clock
|
|
160
|
+
self._sleep = sleep
|
|
161
|
+
self._wall_clock = wall_clock
|
|
162
|
+
self._owns_client = client is None
|
|
163
|
+
self._client = client or httpx.Client(
|
|
164
|
+
headers={HEADER_USER_AGENT: USER_AGENT, HEADER_ACCEPT: JSON_MEDIA_TYPE}, timeout=TIMEOUT
|
|
165
|
+
)
|
|
166
|
+
self._cache = _DiskCache(Path(cache_dir).expanduser(), cache_ttl, wall_clock) if cache_dir else None
|
|
167
|
+
self._last_request: dict[str, float] = {}
|
|
168
|
+
|
|
169
|
+
def close(self) -> None:
|
|
170
|
+
"""Closes the underlying connection pool if this object created it."""
|
|
171
|
+
if self._owns_client:
|
|
172
|
+
self._client.close()
|
|
173
|
+
|
|
174
|
+
def get_json(self, source: str, url: str, params: Mapping[str, Any] | None = None) -> Any:
|
|
175
|
+
"""Fetches ``url`` with GET and returns the decoded JSON body.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
source: Name of the calling source, used in error messages.
|
|
179
|
+
url: Absolute URL.
|
|
180
|
+
params: Query parameters.
|
|
181
|
+
|
|
182
|
+
Raises:
|
|
183
|
+
SourceUnavailableError: If the host cannot be reached or answers with an error status.
|
|
184
|
+
SourceFormatError: If the body is not JSON.
|
|
185
|
+
"""
|
|
186
|
+
return self._json(source, HttpMethod.GET, self._full_url(url, params), None)
|
|
187
|
+
|
|
188
|
+
def post_json(self, source: str, url: str, payload: Any) -> Any:
|
|
189
|
+
"""Sends ``payload`` as JSON with POST and returns the decoded JSON body.
|
|
190
|
+
|
|
191
|
+
Raises:
|
|
192
|
+
SourceUnavailableError: If the host cannot be reached or answers with an error status.
|
|
193
|
+
SourceFormatError: If the body is not JSON.
|
|
194
|
+
"""
|
|
195
|
+
return self._json(source, HttpMethod.POST, url, payload)
|
|
196
|
+
|
|
197
|
+
def cached_at(
|
|
198
|
+
self, method: HttpMethod, url: str, params: Mapping[str, Any] | None = None, payload: Any = None
|
|
199
|
+
) -> datetime | None:
|
|
200
|
+
"""Returns when a request was last fetched into the cache, even if the entry has expired."""
|
|
201
|
+
if self._cache is None:
|
|
202
|
+
return None
|
|
203
|
+
entry = self._cache.read(_request_key(method, self._full_url(url, params), payload))
|
|
204
|
+
return datetime.fromtimestamp(entry[0], tz=timezone.utc) if entry else None
|
|
205
|
+
|
|
206
|
+
@staticmethod
|
|
207
|
+
def _full_url(url: str, params: Mapping[str, Any] | None) -> str:
|
|
208
|
+
return str(httpx.URL(url, params=params)) if params else url
|
|
209
|
+
|
|
210
|
+
def _json(self, source: str, method: HttpMethod, url: str, payload: Any) -> Any:
|
|
211
|
+
key = _request_key(method, url, payload)
|
|
212
|
+
if self._cache and (hit := self._cache.fresh(key)) is not None:
|
|
213
|
+
logger.debug("cache hit for %s", url)
|
|
214
|
+
return hit[1]
|
|
215
|
+
response = self._send(source, method, url, payload)
|
|
216
|
+
try:
|
|
217
|
+
data = response.json()
|
|
218
|
+
except ValueError as exc:
|
|
219
|
+
raise SourceFormatError(source, f"response is not JSON: {_truncate(response.text)!r}") from exc
|
|
220
|
+
if self._cache:
|
|
221
|
+
self._cache.write(key, data)
|
|
222
|
+
return data
|
|
223
|
+
|
|
224
|
+
def _throttle(self, host: str) -> None:
|
|
225
|
+
last = self._last_request.get(host)
|
|
226
|
+
if last is not None:
|
|
227
|
+
wait = last + self.min_interval - self._clock()
|
|
228
|
+
if wait > 0:
|
|
229
|
+
self._sleep(wait)
|
|
230
|
+
self._last_request[host] = self._clock()
|
|
231
|
+
|
|
232
|
+
def _send(self, source: str, method: HttpMethod, url: str, payload: Any) -> httpx.Response:
|
|
233
|
+
host = httpx.URL(url).host
|
|
234
|
+
for attempt in range(MAX_RETRIES + 1):
|
|
235
|
+
self._throttle(host)
|
|
236
|
+
try:
|
|
237
|
+
response = self._client.request(method.value, url, json=payload)
|
|
238
|
+
except httpx.HTTPError as exc:
|
|
239
|
+
raise SourceUnavailableError(source, f"request to {host} failed: {exc}") from exc
|
|
240
|
+
status = response.status_code
|
|
241
|
+
retryable = status == HTTPStatus.TOO_MANY_REQUESTS or status >= _SERVER_ERROR_START
|
|
242
|
+
if not retryable or attempt == MAX_RETRIES:
|
|
243
|
+
break
|
|
244
|
+
delay = _retry_after(response.headers.get(HEADER_RETRY_AFTER), self._wall_clock)
|
|
245
|
+
if delay is None:
|
|
246
|
+
delay = BACKOFF_BASE * 2**attempt
|
|
247
|
+
if delay > MAX_RETRY_AFTER:
|
|
248
|
+
raise SourceUnavailableError(source, f"{host} asked to wait {delay:.0f}s (HTTP {status}); giving up")
|
|
249
|
+
logger.warning("%s answered HTTP %d, retrying in %.1fs", host, status, delay)
|
|
250
|
+
self._sleep(delay)
|
|
251
|
+
if status >= _REDIRECT_STATUS_START:
|
|
252
|
+
raise SourceUnavailableError(source, f"HTTP {status} from {url}")
|
|
253
|
+
return response
|
michraz/_version.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Package version, read from the installed distribution metadata."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
DISTRIBUTION_NAME = "michraz"
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version(DISTRIBUTION_NAME)
|
|
9
|
+
except PackageNotFoundError: # running from a source tree that was never installed
|
|
10
|
+
__version__ = "0+unknown"
|
michraz/cli.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Command-line interface: ``michraz search`` and ``michraz sources``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import shutil
|
|
8
|
+
import sys
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from enum import IntEnum
|
|
12
|
+
|
|
13
|
+
from michraz import _dates
|
|
14
|
+
from michraz._enums import StrEnum
|
|
15
|
+
from michraz._http import DEFAULT_CACHE_DIR
|
|
16
|
+
from michraz._version import __version__
|
|
17
|
+
from michraz.client import Michraz
|
|
18
|
+
from michraz.errors import MichrazError
|
|
19
|
+
from michraz.export import to_csv, to_dicts
|
|
20
|
+
from michraz.models import Tender
|
|
21
|
+
from michraz.sources import BaseSource, SourceName
|
|
22
|
+
|
|
23
|
+
PROGRAM = "michraz"
|
|
24
|
+
DATE_FORMAT = "%Y-%m-%d %H:%M"
|
|
25
|
+
STATE_OPEN = "open"
|
|
26
|
+
STATE_CLOSED = "closed"
|
|
27
|
+
TABLE_HEADER = ("SOURCE", "NUMBER", "DEADLINE", "STATE")
|
|
28
|
+
TITLE_HEADER = "TITLE"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Command(StrEnum):
|
|
32
|
+
"""Subcommands of the ``michraz`` program."""
|
|
33
|
+
|
|
34
|
+
SEARCH = "search"
|
|
35
|
+
SOURCES = "sources"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ExitCode(IntEnum):
|
|
39
|
+
"""Process exit codes. argparse itself exits with 2 on a usage error."""
|
|
40
|
+
|
|
41
|
+
OK = 0
|
|
42
|
+
ERROR = 1
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
NO_DATE = "-"
|
|
46
|
+
MIN_TITLE_WIDTH = 20
|
|
47
|
+
COLUMN_GAP = 2
|
|
48
|
+
NUMBER_WIDTH = 14
|
|
49
|
+
ELLIPSIS = "\u2026"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
53
|
+
parser = argparse.ArgumentParser(prog=PROGRAM, description="Search Israeli public tenders.")
|
|
54
|
+
parser.add_argument("--version", action="version", version=f"michraz {__version__}")
|
|
55
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
56
|
+
common.add_argument("--no-cache", action="store_true", help="do not read or write the on-disk cache")
|
|
57
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
58
|
+
|
|
59
|
+
search = commands.add_parser(Command.SEARCH.value, parents=[common], help="search tenders")
|
|
60
|
+
search.add_argument("--keyword", "-k", help="text to look for in the title or number")
|
|
61
|
+
search.add_argument("--open", action="store_true", help="only tenders that can still receive submissions")
|
|
62
|
+
search.add_argument("--source", choices=[name.value for name in SourceName], help="search a single source")
|
|
63
|
+
search.add_argument("--csv", metavar="FILE", help="also write the results to a CSV file (UTF-8 with BOM)")
|
|
64
|
+
search.add_argument("--json", action="store_true", help="print JSON instead of a table")
|
|
65
|
+
|
|
66
|
+
commands.add_parser(
|
|
67
|
+
Command.SOURCES.value, parents=[common], help="list sources, their attribution and when each was last fetched"
|
|
68
|
+
)
|
|
69
|
+
return parser
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _when(value: datetime | None) -> str:
|
|
73
|
+
return value.astimezone(_dates.JERUSALEM).strftime(DATE_FORMAT) if value else NO_DATE
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _clip(text: str, width: int) -> str:
|
|
77
|
+
return text if len(text) <= width else text[: width - 1] + ELLIPSIS
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _print_table(tenders: Sequence[Tender]) -> None:
|
|
81
|
+
# Hebrew is printed in logical order and the terminal lays it out. The title goes last so the
|
|
82
|
+
# fixed-width columns stay aligned whatever the text direction.
|
|
83
|
+
header = TABLE_HEADER
|
|
84
|
+
rows = [
|
|
85
|
+
(
|
|
86
|
+
t.source,
|
|
87
|
+
_clip(t.number or NO_DATE, NUMBER_WIDTH),
|
|
88
|
+
_when(t.deadline),
|
|
89
|
+
STATE_OPEN if t.is_open() else STATE_CLOSED,
|
|
90
|
+
)
|
|
91
|
+
for t in tenders
|
|
92
|
+
]
|
|
93
|
+
widths = [max(len(cells[i]) for cells in [header, *rows]) for i in range(len(header))]
|
|
94
|
+
title_width = max(shutil.get_terminal_size().columns - sum(widths) - COLUMN_GAP * len(widths), MIN_TITLE_WIDTH)
|
|
95
|
+
|
|
96
|
+
def line(cells: Sequence[str], title: str) -> str:
|
|
97
|
+
fixed = (COLUMN_GAP * " ").join(cell.ljust(width) for cell, width in zip(cells, widths, strict=True))
|
|
98
|
+
return f"{fixed}{COLUMN_GAP * ' '}{title}"
|
|
99
|
+
|
|
100
|
+
print(line(header, TITLE_HEADER))
|
|
101
|
+
for cells, tender in zip(rows, tenders, strict=True):
|
|
102
|
+
print(line(cells, _clip(tender.title, title_width)))
|
|
103
|
+
print(f"\n{len(rows)} tender{'' if len(rows) == 1 else 's'}")
|
|
104
|
+
for credit in sorted({t.attribution for t in tenders}):
|
|
105
|
+
print(f"Data: {credit}")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _search(args: argparse.Namespace, m: Michraz) -> int:
|
|
109
|
+
tenders = m.search(keyword=args.keyword, open_only=args.open, source=args.source)
|
|
110
|
+
if args.csv:
|
|
111
|
+
count = to_csv(tenders, args.csv)
|
|
112
|
+
print(f"wrote {count} tenders to {args.csv}", file=sys.stderr)
|
|
113
|
+
if args.json:
|
|
114
|
+
print(json.dumps(to_dicts(tenders), ensure_ascii=False, indent=2))
|
|
115
|
+
else:
|
|
116
|
+
_print_table(tenders)
|
|
117
|
+
for message in m.last_errors:
|
|
118
|
+
print(f"warning: {message}", file=sys.stderr)
|
|
119
|
+
return ExitCode.OK
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _sources(m: Michraz) -> int:
|
|
123
|
+
for source in m.sources:
|
|
124
|
+
fetched = source.last_fetched() if isinstance(source, BaseSource) else None
|
|
125
|
+
print(f"{source.name}\n {source.description}\n data: {source.attribution}\n last fetch: {_when(fetched)}")
|
|
126
|
+
return ExitCode.OK
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
130
|
+
"""Runs the command-line interface.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
argv: Arguments without the program name. Defaults to ``sys.argv[1:]``.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
The process exit code: 0 on success, 1 if a source failed.
|
|
137
|
+
"""
|
|
138
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
139
|
+
sys.stdout.reconfigure(errors="replace") # a console that cannot show Hebrew must not crash the run
|
|
140
|
+
args = _build_parser().parse_args(argv)
|
|
141
|
+
try:
|
|
142
|
+
with Michraz(cache_dir=None if args.no_cache else DEFAULT_CACHE_DIR) as m:
|
|
143
|
+
return _search(args, m) if args.command == Command.SEARCH else _sources(m)
|
|
144
|
+
except MichrazError as exc:
|
|
145
|
+
print(f"michraz: error: {exc}", file=sys.stderr)
|
|
146
|
+
return ExitCode.ERROR
|
michraz/client.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""The :class:`Michraz` facade: one object to search every source."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from datetime import date, datetime, timedelta
|
|
9
|
+
|
|
10
|
+
from michraz._closeable import Closeable
|
|
11
|
+
from michraz._http import (
|
|
12
|
+
DEFAULT_CACHE_DIR,
|
|
13
|
+
DEFAULT_CACHE_TTL,
|
|
14
|
+
DEFAULT_MIN_INTERVAL,
|
|
15
|
+
HttpClient,
|
|
16
|
+
)
|
|
17
|
+
from michraz.filters import TenderFilter, sort_by_deadline
|
|
18
|
+
from michraz.models import Tender
|
|
19
|
+
from michraz.sources import BaseSource, BeerShevaSource, RmiSource, Source
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Michraz(Closeable):
|
|
25
|
+
"""Searches several tender sources through one polite HTTP client.
|
|
26
|
+
|
|
27
|
+
Use it as a context manager so the connection pool is closed:
|
|
28
|
+
|
|
29
|
+
with Michraz() as m:
|
|
30
|
+
tenders = m.search(keyword="גינון", open_only=True)
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
sources: Sources to search. Defaults to RMI and Beer Sheva.
|
|
34
|
+
cache_dir: Directory for cached responses. Defaults to ``~/.cache/michraz``. None disables the cache.
|
|
35
|
+
cache_ttl: How long a cached response stays valid.
|
|
36
|
+
min_interval: Minimum seconds between requests to the same host. Must be at least 0.5.
|
|
37
|
+
|
|
38
|
+
Attributes:
|
|
39
|
+
sources: The sources in use.
|
|
40
|
+
last_errors: Records skipped during the last :meth:`search`, one message each.
|
|
41
|
+
|
|
42
|
+
Raises:
|
|
43
|
+
ValueError: If ``min_interval`` is below 0.5 seconds or two sources share a name.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
sources: Sequence[Source] | None = None,
|
|
49
|
+
cache_dir: str | os.PathLike[str] | None = DEFAULT_CACHE_DIR,
|
|
50
|
+
cache_ttl: timedelta = DEFAULT_CACHE_TTL,
|
|
51
|
+
min_interval: float = DEFAULT_MIN_INTERVAL,
|
|
52
|
+
) -> None:
|
|
53
|
+
self._http = HttpClient(cache_dir=cache_dir, cache_ttl=cache_ttl, min_interval=min_interval)
|
|
54
|
+
self.sources: tuple[Source, ...] = tuple(sources) if sources is not None else (RmiSource(), BeerShevaSource())
|
|
55
|
+
names = [s.name for s in self.sources]
|
|
56
|
+
if len(set(names)) != len(names):
|
|
57
|
+
raise ValueError(f"source names must be unique, got {names}")
|
|
58
|
+
for source in self.sources:
|
|
59
|
+
if isinstance(source, BaseSource):
|
|
60
|
+
source.bind(self._http)
|
|
61
|
+
self.last_errors: list[str] = []
|
|
62
|
+
|
|
63
|
+
def search(
|
|
64
|
+
self,
|
|
65
|
+
keyword: str | None = None,
|
|
66
|
+
open_only: bool = False,
|
|
67
|
+
source: str | None = None,
|
|
68
|
+
closing_before: datetime | date | None = None,
|
|
69
|
+
closing_after: datetime | date | None = None,
|
|
70
|
+
) -> list[Tender]:
|
|
71
|
+
"""Fetches tenders from the sources and filters them.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
keyword: Keep tenders whose title or number contains this text (case-insensitive, NFKC).
|
|
75
|
+
open_only: Keep only tenders that can still receive submissions.
|
|
76
|
+
source: Search only the source with this name.
|
|
77
|
+
closing_before: Keep tenders whose deadline is before this moment. A date means the start of
|
|
78
|
+
that day in Asia/Jerusalem.
|
|
79
|
+
closing_after: Keep tenders whose deadline is at or after this moment.
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
Matching tenders sorted by deadline, earliest first. Tenders without a deadline come last.
|
|
83
|
+
|
|
84
|
+
Raises:
|
|
85
|
+
ValueError: If ``source`` names no known source or a datetime bound is naive.
|
|
86
|
+
SourceUnavailableError: If a source cannot be reached.
|
|
87
|
+
SourceFormatError: If a source answers in an unexpected shape.
|
|
88
|
+
"""
|
|
89
|
+
criteria = TenderFilter(
|
|
90
|
+
keyword=keyword, open_only=open_only, closing_before=closing_before, closing_after=closing_after
|
|
91
|
+
)
|
|
92
|
+
self.last_errors = []
|
|
93
|
+
tenders: list[Tender] = []
|
|
94
|
+
for src in self._select(source):
|
|
95
|
+
tenders.extend(src.fetch())
|
|
96
|
+
self.last_errors.extend(src.errors)
|
|
97
|
+
return sort_by_deadline(criteria.apply(tenders))
|
|
98
|
+
|
|
99
|
+
def _select(self, name: str | None) -> tuple[Source, ...]:
|
|
100
|
+
if name is None:
|
|
101
|
+
return self.sources
|
|
102
|
+
chosen = tuple(s for s in self.sources if s.name == name)
|
|
103
|
+
if not chosen:
|
|
104
|
+
known = ", ".join(s.name for s in self.sources)
|
|
105
|
+
raise ValueError(f"unknown source {name!r}; known sources: {known}")
|
|
106
|
+
return chosen
|
|
107
|
+
|
|
108
|
+
def __repr__(self) -> str:
|
|
109
|
+
return f"{type(self).__name__}(sources={[s.name for s in self.sources]})"
|
|
110
|
+
|
|
111
|
+
def close(self) -> None:
|
|
112
|
+
"""Closes the shared connection pool."""
|
|
113
|
+
self._http.close()
|
|
114
|
+
for source in self.sources:
|
|
115
|
+
if isinstance(source, BaseSource):
|
|
116
|
+
source.close()
|