thousandproblems 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,463 @@
1
+ """Fetch problems from a TPTP library: an unpacked folder, a tarball, or the TPTP website.
2
+
3
+ >>> import thousandproblems as tptp
4
+ >>> problem = tptp.fetch_problem("SET001-1", "/data/TPTP-v9.0.0") # doctest: +SKIP
5
+ >>> problem = tptp.fetch_problem("SET001-1", "/data/TPTP-v9.0.0.tgz") # doctest: +SKIP
6
+ >>> problem = tptp.fetch_problem("SET001-1") # doctest: +SKIP
7
+
8
+ All three understand the standard layout, in which problem ``SET001-1`` lives at
9
+ ``Problems/SET/SET001-1.p`` and includes such as ``include('Axioms/SET001-0.ax')`` are
10
+ relative to the library root::
11
+
12
+ TPTP-v9.0.0/
13
+ Problems/SET/SET001-1.p
14
+ Axioms/SET001-0.ax
15
+
16
+ Use :func:`open_library` to get a :class:`Library` when you want to read several
17
+ problems, and :func:`fetch_problem` for a single one. With no source given, the folder
18
+ named by the ``TPTP`` environment variable is used if there is one, and the website
19
+ otherwise.
20
+
21
+ Online access
22
+ -------------
23
+ The website allows only a few requests per minute (at the time of writing, 6), so
24
+ the online library fetches a problem *together with the axiom files it includes* in
25
+ one request, and keeps everything it downloads in a cache directory
26
+ (``$THOUSANDPROBLEMS_CACHE``, or the platform's usual cache location). Reading a problem
27
+ twice, or an axiom file shared by several problems, costs one request. If the server
28
+ refuses a request for going too fast, :class:`RateLimited` is raised, with the time
29
+ to wait; nothing is retried automatically. Only the current release is available
30
+ online; for a particular version, download its tarball and open that.
31
+
32
+ Only the standard library is used, and nothing in this module is imported until you
33
+ ask for it.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import io
39
+ import os
40
+ import posixpath
41
+ import re
42
+ import sys
43
+ import tarfile
44
+ import urllib.error
45
+ import urllib.parse
46
+ import urllib.request
47
+ from pathlib import Path
48
+ from typing import Optional, Union
49
+
50
+ from .nodes import AnnotatedFormula, ParseError, TPTPFile, _error_at
51
+
52
+ __all__ = [
53
+ "DirectoryLibrary", "Library", "LibraryError", "OnlineLibrary", "ProblemNotFound",
54
+ "RateLimited", "TarballLibrary", "fetch_problem", "open_library",
55
+ ]
56
+
57
+ DEFAULT_URL = "https://seetptp.tptp.org"
58
+
59
+ #: ``SET001-1``, ``SEU140+3``, ``SYN000_1``, ``SYO000^1``, ``SYN000-1.002``: three
60
+ #: capitals for the domain, three digits, then whatever the naming scheme allows.
61
+ _NAME_RE = re.compile(r"[A-Z]{3}[0-9]{3}[A-Za-z0-9_+\-^=#.]*")
62
+
63
+
64
+ class LibraryError(Exception):
65
+ """A problem with reaching or reading a TPTP library."""
66
+
67
+
68
+ class ProblemNotFound(LibraryError, FileNotFoundError):
69
+ """The library has no such problem or file."""
70
+
71
+
72
+ class RateLimited(LibraryError):
73
+ """The TPTP server refused a request for coming too fast.
74
+
75
+ ``retry_after`` is the number of seconds the server asked us to wait, if it said.
76
+ """
77
+
78
+ def __init__(self, message: str, retry_after: Optional[float] = None):
79
+ super().__init__(message)
80
+ self.retry_after = retry_after
81
+
82
+
83
+ # ---------------------------------------------------------------------------------
84
+ # Names and paths
85
+ # ---------------------------------------------------------------------------------
86
+
87
+
88
+ def problem_path(name: str) -> str:
89
+ """The library path of a problem: ``'SET001-1'`` -> ``'Problems/SET/SET001-1.p'``."""
90
+ stem = name[:-2] if name.endswith(".p") else name
91
+ if not _NAME_RE.fullmatch(stem) or ".." in stem:
92
+ raise ValueError(
93
+ f"{name!r} is not a TPTP problem name (expected something like 'SET001-1' or 'SEU140+3')")
94
+ return f"Problems/{stem[:3]}/{stem}.p"
95
+
96
+
97
+ def _clean(path: str) -> Optional[str]:
98
+ """A library-relative path in normal form, or None if it points outside the library."""
99
+ path = path.replace("\\", "/")
100
+ if "\x00" in path or path.startswith("/") or re.match(r"[A-Za-z]:", path):
101
+ return None
102
+ norm = posixpath.normpath(path)
103
+ if norm == "." or norm == ".." or norm.startswith("../"):
104
+ return None
105
+ return norm
106
+
107
+
108
+ def _decode(data: bytes) -> str:
109
+ return data.decode("utf-8", errors="replace")
110
+
111
+
112
+ # ---------------------------------------------------------------------------------
113
+ # Libraries
114
+ # ---------------------------------------------------------------------------------
115
+
116
+
117
+ class Library:
118
+ """A TPTP library. Paths are relative to its root, with forward slashes."""
119
+
120
+ #: Shown in error messages and set as the ``filename`` of what is parsed.
121
+ label = "TPTP"
122
+
123
+ def read_optional(self, path: str) -> Optional[str]:
124
+ """The text of the file at ``path``, or ``None`` if the library has no such file."""
125
+ raise NotImplementedError
126
+
127
+ def read(self, path: str) -> str:
128
+ """The text of the file at ``path``; raises :class:`ProblemNotFound` if absent."""
129
+ clean = _clean(path)
130
+ text = None if clean is None else self.read_optional(clean)
131
+ if text is None:
132
+ raise ProblemNotFound(f"{path!r} is not in {self.label}")
133
+ return text
134
+
135
+ def problem_text(self, name: str) -> str:
136
+ """The source of problem ``name``, e.g. ``'SET001-1'``."""
137
+ try:
138
+ return self.read(problem_path(name))
139
+ except ProblemNotFound:
140
+ raise ProblemNotFound(f"problem {name!r} is not in {self.label}") from None
141
+
142
+ def problem(self, name: str, *, expand_includes: bool = False) -> TPTPFile:
143
+ """Parse problem ``name``.
144
+
145
+ With ``expand_includes``, each ``include`` is replaced by the formulae it selects,
146
+ as in :func:`thousandproblems.parse_file`. Include paths are read relative to the
147
+ library root, then relative to the including file.
148
+ """
149
+ from . import parse
150
+
151
+ path = problem_path(name)
152
+ try:
153
+ text = self.read(path)
154
+ except ProblemNotFound:
155
+ raise ProblemNotFound(f"problem {name!r} is not in {self.label}") from None
156
+ parsed = parse(text, path)
157
+ return self._expand(path, text, parsed, ()) if expand_includes else parsed
158
+
159
+ def problem_names(self, domain: Optional[str] = None) -> list[str]:
160
+ """The names of the problems in the library, sorted; optionally only one domain."""
161
+ raise NotImplementedError(f"{type(self).__name__} cannot list its problems")
162
+
163
+ def close(self) -> None:
164
+ """Release any open files."""
165
+
166
+ def __enter__(self) -> "Library":
167
+ return self
168
+
169
+ def __exit__(self, *exc) -> None:
170
+ self.close()
171
+
172
+ def __repr__(self) -> str:
173
+ return f"<{type(self).__name__} {self.label}>"
174
+
175
+ def _expand(self, path: str, text: str, parsed: TPTPFile, stack: tuple[str, ...]) -> TPTPFile:
176
+ from . import parse
177
+
178
+ inputs = []
179
+ for item in parsed.inputs:
180
+ if isinstance(item, AnnotatedFormula):
181
+ inputs.append(item)
182
+ continue
183
+
184
+ def fail(message: str, item=item) -> ParseError:
185
+ return _error_at(message, text, item.line, 1, path)
186
+
187
+ candidates = []
188
+ for candidate in (item.path, posixpath.join(posixpath.dirname(path), item.path)):
189
+ clean = _clean(candidate)
190
+ if clean is not None and clean not in candidates:
191
+ candidates.append(clean)
192
+ target = body = None
193
+ for candidate in candidates:
194
+ body = self.read_optional(candidate)
195
+ if body is not None:
196
+ target = candidate
197
+ break
198
+ if target is None:
199
+ tried = ", ".join(candidates) or item.path
200
+ raise fail(f"included file {item.path!r} not found (tried {tried})")
201
+ if target == path or target in stack:
202
+ raise fail(f"circular include of {item.path!r}")
203
+ child = parse(body, target)
204
+ formulae = self._expand(target, body, child, (*stack, path)).formulae
205
+ if isinstance(item.selection, tuple):
206
+ selected = [f for f in formulae if f.name in item.selection]
207
+ missing = set(item.selection) - {f.name for f in selected}
208
+ if missing:
209
+ raise fail(f"{item.path!r} has no formulae named {sorted(missing)}")
210
+ formulae = tuple(selected)
211
+ inputs.extend(formulae)
212
+ return TPTPFile(tuple(inputs), parsed.comments, parsed.filename)
213
+
214
+
215
+ class DirectoryLibrary(Library):
216
+ """An unpacked TPTP distribution, e.g. ``TPTP-v9.0.0/`` (the folder that holds ``Problems``)."""
217
+
218
+ def __init__(self, root: Union[str, os.PathLike]):
219
+ self.root = Path(root)
220
+ if not (self.root / "Problems").is_dir():
221
+ raise LibraryError(
222
+ f"{str(self.root)!r} does not look like a TPTP library: it has no 'Problems' folder. "
223
+ "Point at the folder that contains Problems/ and Axioms/.")
224
+ self.label = str(self.root)
225
+
226
+ def read_optional(self, path: str) -> Optional[str]:
227
+ clean = _clean(path)
228
+ if clean is None:
229
+ return None
230
+ file = self.root / clean
231
+ try:
232
+ return file.read_text(encoding="utf-8", errors="replace") if file.is_file() else None
233
+ except OSError:
234
+ return None
235
+
236
+ def problem(self, name: str, *, expand_includes: bool = False) -> TPTPFile:
237
+ from . import parse_file
238
+
239
+ file = self.root / problem_path(name)
240
+ if not file.is_file():
241
+ raise ProblemNotFound(f"problem {name!r} is not in {self.label}")
242
+ return parse_file(file, expand_includes=expand_includes, tptp_root=self.root)
243
+
244
+ def problem_names(self, domain: Optional[str] = None) -> list[str]:
245
+ pattern = f"{domain}/*.p" if domain else "*/*.p"
246
+ return sorted(p.stem for p in (self.root / "Problems").glob(pattern))
247
+
248
+
249
+ class TarballLibrary(Library):
250
+ """A TPTP distribution tarball (``.tar``, ``.tgz``, ``.tar.gz``, ``.tar.bz2``, ``.tar.xz``).
251
+
252
+ The archive may have a top-level folder such as ``TPTP-v9.0.0/``, or none.
253
+
254
+ The first lookup reads the whole archive to index it, which takes a while for the
255
+ full distribution when it is compressed; after that each lookup seeks to its file,
256
+ which in a compressed archive means decompressing up to that point again. For
257
+ heavy use, unpack the tarball and use :class:`DirectoryLibrary`.
258
+ """
259
+
260
+ def __init__(self, path: Union[str, os.PathLike]):
261
+ self.path = Path(path)
262
+ if not self.path.is_file():
263
+ raise FileNotFoundError(f"no such tarball: {str(self.path)!r}")
264
+ self.label = str(self.path)
265
+ self._tar: Optional[tarfile.TarFile] = None
266
+ self._index: Optional[dict[str, tarfile.TarInfo]] = None
267
+
268
+ def _members(self) -> dict[str, tarfile.TarInfo]:
269
+ if self._index is not None:
270
+ return self._index
271
+ try:
272
+ self._tar = tarfile.open(self.path, "r:*")
273
+ files = {}
274
+ for member in self._tar:
275
+ if member.isfile():
276
+ files[member.name[2:] if member.name.startswith("./") else member.name] = member
277
+ except (tarfile.TarError, OSError, EOFError) as e:
278
+ raise LibraryError(f"cannot read {self.label}: {e}") from e
279
+ prefix = None
280
+ for name in files:
281
+ parts = name.split("/")
282
+ if "Problems" in parts[:-1]:
283
+ prefix = "/".join(parts[:parts.index("Problems")])
284
+ break
285
+ if prefix is None:
286
+ raise LibraryError(f"{self.label} has no 'Problems' folder; is it a TPTP distribution?")
287
+ cut = len(prefix) + 1 if prefix else 0
288
+ self._index = {name[cut:]: m for name, m in files.items() if name.startswith(prefix + "/" if prefix else "")}
289
+ return self._index
290
+
291
+ def read_optional(self, path: str) -> Optional[str]:
292
+ member = self._members().get(path)
293
+ if member is None:
294
+ return None
295
+ handle = self._tar.extractfile(member)
296
+ if handle is None:
297
+ return None
298
+ with handle:
299
+ return _decode(handle.read())
300
+
301
+ def problem_names(self, domain: Optional[str] = None) -> list[str]:
302
+ names = []
303
+ for path in self._members():
304
+ parts = path.split("/")
305
+ if len(parts) == 3 and parts[0] == "Problems" and parts[2].endswith(".p"):
306
+ if domain is None or parts[1] == domain:
307
+ names.append(parts[2][:-2])
308
+ return sorted(names)
309
+
310
+ def close(self) -> None:
311
+ if self._tar is not None:
312
+ self._tar.close()
313
+ self._tar = self._index = None
314
+
315
+
316
+ def default_cache_dir() -> Path:
317
+ """Where downloads are kept unless told otherwise."""
318
+ if os.environ.get("THOUSANDPROBLEMS_CACHE"):
319
+ return Path(os.environ["THOUSANDPROBLEMS_CACHE"])
320
+ if sys.platform == "darwin":
321
+ return Path.home() / "Library" / "Caches" / "thousandproblems"
322
+ if sys.platform == "win32":
323
+ return Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) / "thousandproblems" / "Cache"
324
+ return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "thousandproblems"
325
+
326
+
327
+ class OnlineLibrary(Library):
328
+ """The current TPTP release, fetched over HTTP from the TPTP website.
329
+
330
+ :param base_url: the server; the default is the TPTP project's own.
331
+ :param cache: ``True`` (default) to keep downloads in :func:`default_cache_dir`,
332
+ ``False`` to keep them in memory only, or a directory to use as the cache.
333
+ :param timeout: seconds to wait for the server.
334
+ """
335
+
336
+ def __init__(self, base_url: str = DEFAULT_URL, *, cache: Union[bool, str, os.PathLike] = True,
337
+ timeout: float = 30.0):
338
+ self.base_url = base_url.rstrip("/")
339
+ self.timeout = timeout
340
+ self.label = self.base_url
341
+ if cache is True:
342
+ cache = default_cache_dir()
343
+ host = re.sub(r"[^A-Za-z0-9._-]+", "_", urllib.parse.urlsplit(self.base_url).netloc or "server")
344
+ self.cache_dir: Optional[Path] = Path(cache) / host if cache else None
345
+ self._memory: dict[str, str] = {}
346
+
347
+ # The server hands out a .tgz holding the requested file and, for a problem, every
348
+ # axiom file it includes, laid out from the library root.
349
+ def _download_url(self, path: str) -> str:
350
+ return f"{self.base_url}/download/TPTPWorld/TPTP/{urllib.parse.quote(path, safe='/+')}"
351
+
352
+ def _fetch(self, path: str) -> Optional[dict[str, str]]:
353
+ from . import __version__
354
+
355
+ request = urllib.request.Request(
356
+ self._download_url(path), headers={"User-Agent": f"thousandproblems/{__version__}"})
357
+ try:
358
+ with urllib.request.urlopen(request, timeout=self.timeout) as response:
359
+ data = response.read()
360
+ except urllib.error.HTTPError as e:
361
+ e.close()
362
+ if e.code == 429:
363
+ retry = e.headers.get("Retry-After")
364
+ wait = float(retry) if retry and retry.replace(".", "", 1).isdigit() else None
365
+ hint = f"; wait {wait:g} s" if wait else "; the TPTP server allows only a few requests a minute"
366
+ raise RateLimited(f"the TPTP server is rate limiting requests{hint}", wait) from None
367
+ if e.code == 404:
368
+ return None
369
+ raise LibraryError(f"{self._download_url(path)}: HTTP {e.code} {e.reason}") from e
370
+ except (urllib.error.URLError, OSError) as e:
371
+ raise LibraryError(f"cannot reach {self.base_url}: {getattr(e, 'reason', e)}") from e
372
+ if data[:2] != b"\x1f\x8b": # a missing file comes back as an HTML page, status 200
373
+ return None
374
+ files = {}
375
+ try:
376
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
377
+ for member in tar:
378
+ if not member.isfile():
379
+ continue
380
+ name = member.name[2:] if member.name.startswith("./") else member.name
381
+ # the requested file is stored under its bare name
382
+ key = path if "/" not in name and name == posixpath.basename(path) else name
383
+ clean = _clean(key)
384
+ if clean is not None:
385
+ files[clean] = _decode(tar.extractfile(member).read())
386
+ except (tarfile.TarError, EOFError) as e:
387
+ raise LibraryError(f"unreadable download for {path!r}: {e}") from e
388
+ return files if path in files else None
389
+
390
+ def _cache_file(self, path: str) -> Optional[Path]:
391
+ return self.cache_dir / path if self.cache_dir else None
392
+
393
+ def read_optional(self, path: str) -> Optional[str]:
394
+ if path in self._memory:
395
+ return self._memory[path]
396
+ cached = self._cache_file(path)
397
+ if cached is not None:
398
+ try:
399
+ text = _decode(cached.read_bytes())
400
+ self._memory[path] = text
401
+ return text
402
+ except OSError:
403
+ pass
404
+ files = self._fetch(path)
405
+ if files is None:
406
+ return None
407
+ for name, text in files.items():
408
+ self._memory[name] = text
409
+ target = self._cache_file(name)
410
+ if target is not None:
411
+ try:
412
+ target.parent.mkdir(parents=True, exist_ok=True)
413
+ temporary = target.with_name(f".{target.name}.{os.getpid()}.tmp")
414
+ temporary.write_bytes(text.encode("utf-8")) # bytes: no newline translation on Windows
415
+ os.replace(temporary, target)
416
+ except OSError:
417
+ pass # a cache that can't be written is not an error
418
+ return files[path]
419
+
420
+ def problem_names(self, domain: Optional[str] = None) -> list[str]:
421
+ raise NotImplementedError("the TPTP website cannot be listed; use a local copy of the library")
422
+
423
+
424
+ # ---------------------------------------------------------------------------------
425
+ # Entry points
426
+ # ---------------------------------------------------------------------------------
427
+
428
+
429
+ def open_library(source: Union[None, str, os.PathLike, Library] = None) -> Library:
430
+ """A :class:`Library` for ``source``.
431
+
432
+ ``source`` can be a folder, a tarball, an ``http(s)://`` URL of a TPTP server, an
433
+ existing :class:`Library` (returned as is), or ``None`` for the folder named by the
434
+ ``TPTP`` environment variable if that is set, else the TPTP website.
435
+ """
436
+ if isinstance(source, Library):
437
+ return source
438
+ if source is None:
439
+ root = os.environ.get("TPTP")
440
+ return DirectoryLibrary(root) if root and Path(root).is_dir() else OnlineLibrary()
441
+ text = os.fspath(source)
442
+ if re.match(r"https?://", text):
443
+ return OnlineLibrary(text)
444
+ path = Path(text)
445
+ if path.is_dir():
446
+ return DirectoryLibrary(path)
447
+ if path.is_file():
448
+ return TarballLibrary(path)
449
+ raise FileNotFoundError(f"no such TPTP library (folder or tarball): {text!r}")
450
+
451
+
452
+ def fetch_problem(name: str, source: Union[None, str, os.PathLike, Library] = None, *,
453
+ expand_includes: bool = False) -> TPTPFile:
454
+ """Parse problem ``name`` (e.g. ``'SET001-1'``) from a TPTP library; see :func:`open_library`.
455
+
456
+ A library opened here is closed again before returning.
457
+ """
458
+ library = open_library(source)
459
+ try:
460
+ return library.problem(name, expand_includes=expand_includes)
461
+ finally:
462
+ if library is not source:
463
+ library.close()