nibs 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nibs/__init__.py +6 -0
- nibs/__main__.py +5 -0
- nibs/cache.py +48 -0
- nibs/cli.py +125 -0
- nibs/page.py +223 -0
- nibs/paths.py +14 -0
- nibs/usage.py +48 -0
- nibs-0.1.0.dist-info/METADATA +93 -0
- nibs-0.1.0.dist-info/RECORD +12 -0
- nibs-0.1.0.dist-info/WHEEL +4 -0
- nibs-0.1.0.dist-info/entry_points.txt +2 -0
- nibs-0.1.0.dist-info/licenses/LICENSE +21 -0
nibs/__init__.py
ADDED
nibs/__main__.py
ADDED
nibs/cache.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Pages already read, so that toc, section and find on one page fetch it once."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
from .paths import cache_dir
|
|
9
|
+
|
|
10
|
+
# How long a fetched page is reused; the same default as Claude Code's WebFetch.
|
|
11
|
+
TTL_SECONDS = 15 * 60
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _path(url):
|
|
15
|
+
return cache_dir() / (hashlib.sha256(url.encode()).hexdigest() + ".json")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get(url):
|
|
19
|
+
try:
|
|
20
|
+
with open(_path(url), encoding="utf-8") as f:
|
|
21
|
+
entry = json.load(f)
|
|
22
|
+
except (OSError, ValueError):
|
|
23
|
+
return None
|
|
24
|
+
if entry.get("url") != url or time.time() - entry.get("time", 0) > TTL_SECONDS:
|
|
25
|
+
return None
|
|
26
|
+
return entry.get("text")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def put(url, text):
|
|
30
|
+
try:
|
|
31
|
+
cache_dir().mkdir(parents=True, exist_ok=True)
|
|
32
|
+
_prune()
|
|
33
|
+
tmp = _path(url).with_suffix(".tmp")
|
|
34
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
35
|
+
json.dump({"url": url, "time": time.time(), "text": text}, f)
|
|
36
|
+
os.replace(tmp, _path(url))
|
|
37
|
+
except OSError:
|
|
38
|
+
pass # A page that cannot be cached is fetched again next time.
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _prune():
|
|
42
|
+
now = time.time()
|
|
43
|
+
for p in cache_dir().glob("*.json"):
|
|
44
|
+
try:
|
|
45
|
+
if now - p.stat().st_mtime > TTL_SECONDS:
|
|
46
|
+
p.unlink()
|
|
47
|
+
except OSError:
|
|
48
|
+
pass
|
nibs/cli.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""nibs: read only the part of a web page you need.
|
|
2
|
+
|
|
3
|
+
nibs toc <url> headings with their numbers and sizes
|
|
4
|
+
nibs section <url> <n|text> one section (number from toc, or heading text)
|
|
5
|
+
nibs find <url> <regex> matching lines, each with the section it is in
|
|
6
|
+
nibs stats how much of the pages nibs has printed so far
|
|
7
|
+
|
|
8
|
+
Pages are fetched as Markdown when the site serves it (the URL itself, or the
|
|
9
|
+
URL with .md added); otherwise the HTML is turned into headings and text.
|
|
10
|
+
A page is fetched once and reused for 15 minutes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import contextlib
|
|
15
|
+
import io
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
from . import __version__, page, usage
|
|
20
|
+
|
|
21
|
+
NO_HEADINGS = (
|
|
22
|
+
"[nibs: no headings found. If the page shows more in a browser, it is likely "
|
|
23
|
+
"drawn by JavaScript, which nibs does not run]"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def cmd_toc(text, _args):
|
|
28
|
+
lines, secs = page.sections(text)
|
|
29
|
+
print(f"{len(text):,} chars, {len(secs)} headings")
|
|
30
|
+
for n, (level, title, start, end) in enumerate(secs, 1):
|
|
31
|
+
size = sum(len(line) + 1 for line in lines[start:end])
|
|
32
|
+
print(f'{n:3}. {" " * (level - 1)}{title} ({size:,})')
|
|
33
|
+
if not secs:
|
|
34
|
+
print(NO_HEADINGS)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def cmd_section(text, args):
|
|
38
|
+
lines, secs = page.sections(text)
|
|
39
|
+
if not secs:
|
|
40
|
+
print(NO_HEADINGS, file=sys.stderr)
|
|
41
|
+
return 1
|
|
42
|
+
# A number picks the section with that number in toc; anything else, or a
|
|
43
|
+
# number toc does not have (a heading such as "2024"), is heading text.
|
|
44
|
+
q = args.query
|
|
45
|
+
try:
|
|
46
|
+
n = int(q) if q.isascii() and q.isdigit() else 0
|
|
47
|
+
except ValueError: # more digits than int() converts
|
|
48
|
+
n = 0
|
|
49
|
+
if 0 < n <= len(secs):
|
|
50
|
+
picked = [secs[n - 1]]
|
|
51
|
+
else:
|
|
52
|
+
q = q.lower()
|
|
53
|
+
picked = [s for s in secs if s[1].lower() == q] or [s for s in secs if q in s[1].lower()]
|
|
54
|
+
if not picked:
|
|
55
|
+
print(f"nibs: no heading matches {args.query!r}; see nibs toc", file=sys.stderr)
|
|
56
|
+
return 1
|
|
57
|
+
if len(picked) > 1:
|
|
58
|
+
print(f"{len(picked)} headings match; showing the first. Others: " + "; ".join(s[1] for s in picked[1:6]))
|
|
59
|
+
_, _, start, end = picked[0]
|
|
60
|
+
body = "\n".join(lines[start:end]).strip()
|
|
61
|
+
if len(body) > args.max:
|
|
62
|
+
print(body[: args.max])
|
|
63
|
+
print(f"\n[nibs: cut at {args.max:,} of {len(body):,} chars; pick a subsection from nibs toc, or pass --max]")
|
|
64
|
+
else:
|
|
65
|
+
print(body)
|
|
66
|
+
return 0
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cmd_find(text, args):
|
|
70
|
+
lines, secs = page.sections(text)
|
|
71
|
+
pattern = re.compile(args.regex, re.IGNORECASE)
|
|
72
|
+
shown, last = 0, None
|
|
73
|
+
for i, line in enumerate(lines):
|
|
74
|
+
if not pattern.search(line):
|
|
75
|
+
continue
|
|
76
|
+
sec = next((s for s in reversed(secs) if s[2] <= i), None)
|
|
77
|
+
if sec is not last:
|
|
78
|
+
print(f'## {sec[1] if sec else "(before the first heading)"}')
|
|
79
|
+
last = sec
|
|
80
|
+
print(f" {line.strip()[:300]}")
|
|
81
|
+
shown += 1
|
|
82
|
+
if shown >= args.limit:
|
|
83
|
+
print(f"[nibs: stopped at {args.limit} matches; pass --limit]")
|
|
84
|
+
break
|
|
85
|
+
if not shown:
|
|
86
|
+
print("no match")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def parser():
|
|
90
|
+
ap = argparse.ArgumentParser(prog="nibs", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
91
|
+
ap.add_argument("--version", action="version", version=f"nibs {__version__}")
|
|
92
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
93
|
+
p = sub.add_parser("toc", help="list the headings")
|
|
94
|
+
p.add_argument("url")
|
|
95
|
+
p = sub.add_parser("section", help="print one section")
|
|
96
|
+
p.add_argument("url")
|
|
97
|
+
p.add_argument("query", help="a number from toc, or heading text")
|
|
98
|
+
p.add_argument("--max", type=int, default=8000, help="most characters to print (default 8000)")
|
|
99
|
+
p = sub.add_parser("find", help="print matching lines with their sections")
|
|
100
|
+
p.add_argument("url")
|
|
101
|
+
p.add_argument("regex")
|
|
102
|
+
p.add_argument("--limit", type=int, default=30, help="most lines to print (default 30)")
|
|
103
|
+
sub.add_parser("stats", help="how much nibs has saved so far")
|
|
104
|
+
return ap
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main(argv=None):
|
|
108
|
+
args = parser().parse_args(argv)
|
|
109
|
+
if args.cmd == "stats":
|
|
110
|
+
return usage.print_stats()
|
|
111
|
+
if args.cmd == "find":
|
|
112
|
+
try:
|
|
113
|
+
re.compile(args.regex)
|
|
114
|
+
except re.error as err:
|
|
115
|
+
sys.exit(f"nibs: bad regex {args.regex!r}: {err}")
|
|
116
|
+
try:
|
|
117
|
+
text = page.load(args.url)
|
|
118
|
+
except page.FETCH_ERRORS as err:
|
|
119
|
+
sys.exit(f"nibs: cannot fetch {args.url}: {err}")
|
|
120
|
+
out = io.StringIO()
|
|
121
|
+
with contextlib.redirect_stdout(out):
|
|
122
|
+
status = {"toc": cmd_toc, "section": cmd_section, "find": cmd_find}[args.cmd](text, args)
|
|
123
|
+
sys.stdout.write(out.getvalue())
|
|
124
|
+
usage.record(args.cmd, args.url, len(text), len(out.getvalue()))
|
|
125
|
+
return status or 0
|
nibs/page.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Fetching a page as Markdown-like text and splitting it by its headings."""
|
|
2
|
+
|
|
3
|
+
import http.client
|
|
4
|
+
import re
|
|
5
|
+
import urllib.error
|
|
6
|
+
import urllib.request
|
|
7
|
+
from html.parser import HTMLParser
|
|
8
|
+
|
|
9
|
+
from . import __version__, cache
|
|
10
|
+
|
|
11
|
+
MAX_BYTES = 5_000_000
|
|
12
|
+
USER_AGENT = f"nibs/{__version__} (+https://github.com/nhinata/nibs)"
|
|
13
|
+
CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]")
|
|
14
|
+
# What fetching can raise: URLError and the socket and SSL errors that reading
|
|
15
|
+
# the body raises are all OSError; http.client raises HTTPException subclasses
|
|
16
|
+
# such as IncompleteRead; a malformed URL raises ValueError.
|
|
17
|
+
FETCH_ERRORS = (OSError, http.client.HTTPException, ValueError)
|
|
18
|
+
|
|
19
|
+
# Web pages only: an opener with no handlers for file:, ftp: or data: URLs, so
|
|
20
|
+
# that neither a URL given to nibs nor a redirect can read anything else.
|
|
21
|
+
# ProxyHandler adds a handler for every scheme it has a proxy for (ftp_proxy
|
|
22
|
+
# would make ftp: URLs go through the proxy), so it gets the web ones only.
|
|
23
|
+
_WEB = ("http", "https")
|
|
24
|
+
_OPENER = urllib.request.OpenerDirector()
|
|
25
|
+
for _handler in (
|
|
26
|
+
urllib.request.ProxyHandler({k: v for k, v in urllib.request.getproxies().items() if k.lower() in _WEB}),
|
|
27
|
+
urllib.request.UnknownHandler(),
|
|
28
|
+
urllib.request.HTTPHandler(),
|
|
29
|
+
urllib.request.HTTPSHandler(),
|
|
30
|
+
urllib.request.HTTPDefaultErrorHandler(),
|
|
31
|
+
urllib.request.HTTPRedirectHandler(),
|
|
32
|
+
urllib.request.HTTPErrorProcessor(),
|
|
33
|
+
):
|
|
34
|
+
_OPENER.add_handler(_handler)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def fetch(url):
|
|
38
|
+
"""(content type, text) of the response."""
|
|
39
|
+
req = urllib.request.Request(
|
|
40
|
+
url,
|
|
41
|
+
headers={"User-Agent": USER_AGENT, "Accept": "text/markdown, text/html;q=0.9, */*;q=0.5"},
|
|
42
|
+
)
|
|
43
|
+
with _OPENER.open(req, timeout=20) as res:
|
|
44
|
+
body = res.read(MAX_BYTES + 1)
|
|
45
|
+
ctype = res.headers.get("Content-Type", "")
|
|
46
|
+
return ctype.split(";")[0].strip(), decode(ctype, body[:MAX_BYTES])
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def decode(ctype, body):
|
|
50
|
+
try:
|
|
51
|
+
return body.decode(encoding(ctype, body), "replace")
|
|
52
|
+
except (LookupError, UnicodeError): # a charset Python does not know, or one that cannot decode text
|
|
53
|
+
return body.decode("utf-8", "replace")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
CHARSET = re.compile(rb"""charset\s*=\s*["']?([\w.:-]+)""", re.IGNORECASE)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def encoding(ctype, body):
|
|
60
|
+
"""The charset in the Content-Type header, else, for an HTML page, one a <meta> tag
|
|
61
|
+
declares in the first 1,024 bytes (<meta charset> or <meta http-equiv>, as browsers
|
|
62
|
+
look), else UTF-8."""
|
|
63
|
+
m = CHARSET.search(ctype.encode("latin-1", "replace"))
|
|
64
|
+
if m is None and looks_like_html(ctype, body[:1024].decode("latin-1")):
|
|
65
|
+
m = re.search(rb"<meta[^>]*?" + CHARSET.pattern, body[:1024], re.IGNORECASE)
|
|
66
|
+
return m.group(1).decode("ascii") if m else "utf-8"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def looks_like_html(ctype, text):
|
|
70
|
+
return "html" in ctype or text.lstrip()[:15].lower().startswith(("<!doctype", "<html"))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class TextOfHtml(HTMLParser):
|
|
74
|
+
"""Headings become Markdown headings, <pre> becomes a code block, other block text becomes lines.
|
|
75
|
+
|
|
76
|
+
Only heading tags make headings and only <pre> makes code blocks: a text line
|
|
77
|
+
that sections() would read as a heading or a fence is escaped with a backslash.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
SKIP = {"script", "style", "nav", "aside", "noscript", "svg"}
|
|
81
|
+
# The page's own header and footer (banner, site links) are left out; inside
|
|
82
|
+
# <article>, <main> or <section> they belong to that content and are kept.
|
|
83
|
+
PAGE_ONLY = {"header", "footer"}
|
|
84
|
+
SECTIONING = {"article", "main", "section"}
|
|
85
|
+
BLOCK = {"p", "li", "td", "th", "blockquote", "dt", "dd", "div", "section", "article", "main", "tr", "br", "form"}
|
|
86
|
+
|
|
87
|
+
def __init__(self):
|
|
88
|
+
super().__init__(convert_charrefs=True)
|
|
89
|
+
self.lines, self.buf, self.skip, self.heading = [], [], 0, 0
|
|
90
|
+
self.sectioning, self.page_only, self.pre = 0, [], None
|
|
91
|
+
|
|
92
|
+
def flush(self):
|
|
93
|
+
text = " ".join("".join(self.buf).split())
|
|
94
|
+
if text:
|
|
95
|
+
if self.heading:
|
|
96
|
+
text = "#" * self.heading + " " + text
|
|
97
|
+
elif HEADING.match(text) or FENCE.match(text):
|
|
98
|
+
text = "\\" + text # text, not a heading or a fence
|
|
99
|
+
self.lines.append(text)
|
|
100
|
+
self.buf = []
|
|
101
|
+
|
|
102
|
+
def handle_starttag(self, tag, attrs):
|
|
103
|
+
if self.pre is not None:
|
|
104
|
+
if tag == "br":
|
|
105
|
+
self.pre.append("\n")
|
|
106
|
+
return
|
|
107
|
+
if tag in self.PAGE_ONLY:
|
|
108
|
+
self.flush()
|
|
109
|
+
skipped = self.sectioning == 0
|
|
110
|
+
self.page_only.append(skipped)
|
|
111
|
+
self.skip += skipped
|
|
112
|
+
elif tag in self.SKIP:
|
|
113
|
+
self.skip += 1
|
|
114
|
+
elif re.fullmatch(r"h[1-6]", tag):
|
|
115
|
+
self.flush()
|
|
116
|
+
self.heading = int(tag[1])
|
|
117
|
+
elif tag == "pre":
|
|
118
|
+
self.flush()
|
|
119
|
+
self.pre = []
|
|
120
|
+
elif tag in self.BLOCK:
|
|
121
|
+
self.flush()
|
|
122
|
+
if tag in self.SECTIONING:
|
|
123
|
+
self.sectioning += 1
|
|
124
|
+
|
|
125
|
+
def handle_endtag(self, tag):
|
|
126
|
+
if self.pre is not None:
|
|
127
|
+
if tag == "pre":
|
|
128
|
+
self.end_pre()
|
|
129
|
+
return
|
|
130
|
+
if tag in self.PAGE_ONLY:
|
|
131
|
+
self.flush()
|
|
132
|
+
if self.page_only and self.page_only.pop():
|
|
133
|
+
self.skip = max(0, self.skip - 1)
|
|
134
|
+
elif tag in self.SKIP:
|
|
135
|
+
self.skip = max(0, self.skip - 1)
|
|
136
|
+
elif re.fullmatch(r"h[1-6]", tag):
|
|
137
|
+
self.flush()
|
|
138
|
+
self.heading = 0
|
|
139
|
+
elif tag in self.BLOCK:
|
|
140
|
+
self.flush()
|
|
141
|
+
if tag in self.SECTIONING:
|
|
142
|
+
self.sectioning = max(0, self.sectioning - 1)
|
|
143
|
+
|
|
144
|
+
def end_pre(self):
|
|
145
|
+
code = "".join(self.pre).strip("\n")
|
|
146
|
+
self.pre = None
|
|
147
|
+
if self.skip or not code.strip():
|
|
148
|
+
return
|
|
149
|
+
# Longer than any run of backticks in the code, so that nothing in it closes the fence.
|
|
150
|
+
fence = "`" * max(3, max((len(r) for r in re.findall(r"`+", code)), default=0) + 1)
|
|
151
|
+
self.lines.append(f"{fence}\n{code}\n{fence}")
|
|
152
|
+
|
|
153
|
+
def handle_data(self, data):
|
|
154
|
+
if self.pre is not None:
|
|
155
|
+
self.pre.append(data)
|
|
156
|
+
elif not self.skip:
|
|
157
|
+
self.buf.append(data)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def html_to_text(html):
|
|
161
|
+
parser = TextOfHtml()
|
|
162
|
+
parser.feed(html)
|
|
163
|
+
parser.close()
|
|
164
|
+
if parser.pre is not None: # a <pre> the page never closed
|
|
165
|
+
parser.end_pre()
|
|
166
|
+
parser.flush()
|
|
167
|
+
return "\n\n".join(parser.lines)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def load(url):
|
|
171
|
+
"""The page as Markdown-like text, from the cache when it was read recently."""
|
|
172
|
+
text = cache.get(url)
|
|
173
|
+
if text is None:
|
|
174
|
+
text = _download(url)
|
|
175
|
+
cache.put(url, text)
|
|
176
|
+
return text
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _download(url):
|
|
180
|
+
ctype, text = fetch(url)
|
|
181
|
+
if looks_like_html(ctype, text) and not url.endswith(".md"):
|
|
182
|
+
# Many documentation sites serve the page's Markdown at the same URL plus .md.
|
|
183
|
+
try:
|
|
184
|
+
mctype, mtext = fetch(url.split("#")[0].rstrip("/") + ".md")
|
|
185
|
+
if not looks_like_html(mctype, mtext) and mtext.strip():
|
|
186
|
+
return CONTROL.sub("", mtext)
|
|
187
|
+
except FETCH_ERRORS:
|
|
188
|
+
pass
|
|
189
|
+
if looks_like_html(ctype, text):
|
|
190
|
+
text = html_to_text(text)
|
|
191
|
+
return CONTROL.sub("", text)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# The Markdown structure sections() reads (README: Headings and code blocks).
|
|
195
|
+
HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
|
|
196
|
+
FENCE = re.compile(r"^(`{3,}|~{3,})(.*)$")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def sections(text):
|
|
200
|
+
"""(lines, [(level, title, start, end)]) over the lines of `text`.
|
|
201
|
+
|
|
202
|
+
The Markdown nibs reads is described in README (Headings and code blocks): only
|
|
203
|
+
`#` headings and code fences, by nibs' own small rule, not all of CommonMark.
|
|
204
|
+
"""
|
|
205
|
+
lines = text.split("\n")
|
|
206
|
+
heads, fence = [], None
|
|
207
|
+
for i, line in enumerate(lines):
|
|
208
|
+
m = FENCE.match(line.strip())
|
|
209
|
+
if fence is not None:
|
|
210
|
+
if m and m.group(1)[0] == fence[0] and len(m.group(1)) >= len(fence) and not m.group(2).strip():
|
|
211
|
+
fence = None
|
|
212
|
+
continue
|
|
213
|
+
if m:
|
|
214
|
+
fence = m.group(1)
|
|
215
|
+
continue
|
|
216
|
+
h = HEADING.match(line)
|
|
217
|
+
if h:
|
|
218
|
+
heads.append((len(h.group(1)), h.group(2), i))
|
|
219
|
+
out = []
|
|
220
|
+
for n, (level, title, start) in enumerate(heads):
|
|
221
|
+
end = next((s for lv, _, s in heads[n + 1:] if lv <= level), len(lines))
|
|
222
|
+
out.append((level, title, start, end))
|
|
223
|
+
return lines, out
|
nibs/paths.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def cache_dir():
|
|
6
|
+
base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
|
|
7
|
+
return Path(base) / "nibs"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def usage_log():
|
|
11
|
+
if os.environ.get("NIBS_LOG"):
|
|
12
|
+
return Path(os.environ["NIBS_LOG"])
|
|
13
|
+
base = os.environ.get("XDG_STATE_HOME") or Path.home() / ".local" / "state"
|
|
14
|
+
return Path(base) / "nibs" / "usage.jsonl"
|
nibs/usage.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""A local record of what nibs printed, to compare with reading whole pages.
|
|
2
|
+
|
|
3
|
+
One JSON line per run: the command, the URL, the page's size and how much was
|
|
4
|
+
printed. Nothing is sent anywhere.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
from .paths import usage_log
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def record(cmd, url, page_chars, printed_chars):
|
|
14
|
+
path = usage_log()
|
|
15
|
+
try:
|
|
16
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
17
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
18
|
+
f.write(json.dumps({"time": time.strftime("%Y-%m-%dT%H:%M:%S"), "cmd": cmd, "url": url,
|
|
19
|
+
"page": page_chars, "printed": printed_chars}) + "\n")
|
|
20
|
+
except OSError:
|
|
21
|
+
pass # Measuring must never stop the lookup.
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def print_stats():
|
|
25
|
+
rows, unreadable = [], 0
|
|
26
|
+
try:
|
|
27
|
+
with open(usage_log(), "rb") as f:
|
|
28
|
+
for line in f:
|
|
29
|
+
if not line.strip():
|
|
30
|
+
continue
|
|
31
|
+
try:
|
|
32
|
+
row = json.loads(line.decode("utf-8"))
|
|
33
|
+
rows.append({"time": str(row["time"]), "url": str(row["url"]),
|
|
34
|
+
"page": int(row["page"]), "printed": int(row["printed"])})
|
|
35
|
+
except (ValueError, TypeError, KeyError, OverflowError):
|
|
36
|
+
unreadable += 1 # a line cut short by a full disk or a killed run
|
|
37
|
+
except FileNotFoundError:
|
|
38
|
+
pass
|
|
39
|
+
# Without nibs, each page would have been read whole, about once a day.
|
|
40
|
+
pages = {(r["time"][:10], r["url"]): r["page"] for r in rows}
|
|
41
|
+
whole = sum(pages.values())
|
|
42
|
+
printed = sum(r["printed"] for r in rows)
|
|
43
|
+
print(f"{len(rows)} runs over {len(pages)} page-days")
|
|
44
|
+
if unreadable:
|
|
45
|
+
print(f"skipped {unreadable} unreadable lines in {usage_log()}")
|
|
46
|
+
if whole:
|
|
47
|
+
print(f"printed {printed:,} chars; reading each page whole once a day: {whole:,} ({printed / whole:.0%} of that)")
|
|
48
|
+
return 0
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: nibs
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Read a web page by its headings: list them, then print only the section you need
|
|
5
|
+
Project-URL: Repository, https://github.com/nhinata/nibs
|
|
6
|
+
Author: nhinata
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: agent,cli,llm,markdown,tokens,web
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
14
|
+
Classifier: Topic :: Utilities
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# nibs
|
|
19
|
+
|
|
20
|
+
Read a web page by its headings: list them, then print only the section you need.
|
|
21
|
+
|
|
22
|
+
An AI agent that reads a whole documentation page to answer one question puts tens of thousands of characters into its context, most of which it never uses. nibs lets it look at the page's headings first and then take just one section, in the page's own words.
|
|
23
|
+
|
|
24
|
+
```console
|
|
25
|
+
$ nibs toc https://docs.python.org/3/library/argparse.html
|
|
26
|
+
82,019 chars, 57 headings
|
|
27
|
+
1. Navigation (254)
|
|
28
|
+
2. argparse — Parser for command-line options, arguments and subcommands¶ (81,665)
|
|
29
|
+
3. ArgumentParser objects¶ (19,217)
|
|
30
|
+
4. prog¶ (1,483)
|
|
31
|
+
5. usage¶ (893)
|
|
32
|
+
...
|
|
33
|
+
$ nibs find https://docs.python.org/3/library/argparse.html allow_abbrev --limit 2
|
|
34
|
+
## ArgumentParser objects¶
|
|
35
|
+
class argparse.ArgumentParser(prog=None, usage=None, description=None, epilog=None, ...
|
|
36
|
+
allow_abbrev - Allows long options to be abbreviated if the abbreviation is unambiguous (default: True)
|
|
37
|
+
[nibs: stopped at 2 matches; pass --limit]
|
|
38
|
+
$ nibs section https://docs.python.org/3/library/argparse.html 4
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
pipx install nibs # or: uv tool install nibs
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Python 3.11 or later. No other dependencies.
|
|
48
|
+
|
|
49
|
+
## Commands
|
|
50
|
+
|
|
51
|
+
```sh
|
|
52
|
+
nibs toc <url> # headings, numbered, with each section's size in characters
|
|
53
|
+
nibs section <url> <n|heading> # one section, by its number in toc or by heading text
|
|
54
|
+
nibs find <url> <regex> # matching lines, each under the heading it belongs to
|
|
55
|
+
nibs stats # characters printed so far, against reading each page whole
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
- `section` prints at most 8,000 characters; `--max` changes that. A heading given as text matches exactly first, then as a case-insensitive substring.
|
|
59
|
+
- `find` prints at most 30 lines; `--limit` changes that.
|
|
60
|
+
|
|
61
|
+
## How a page is read
|
|
62
|
+
|
|
63
|
+
- Only `http` and `https` URLs are read, also when a page redirects.
|
|
64
|
+
- If the site serves Markdown, either at the URL itself or at the URL with `.md` added, nibs uses it. Many documentation sites do.
|
|
65
|
+
- Otherwise nibs turns the HTML into headings, text and code blocks (from `<pre>`). It leaves out scripts, styles, navigation, sidebars, and the page's own header and footer; a header or footer inside an article is kept. Only heading tags become headings, and only `<pre>` becomes a code block: a line of text that would read as either (see below) gets a backslash in front.
|
|
66
|
+
- Control characters in the page are removed before printing.
|
|
67
|
+
- nibs does not run JavaScript. A page that builds its content in the browser comes back with little or no text; when no headings are found, nibs says so.
|
|
68
|
+
- A fetched page is kept for 15 minutes in `~/.cache/nibs` (or `$XDG_CACHE_HOME/nibs`), so `toc`, `section` and `find` on the same page fetch it once.
|
|
69
|
+
|
|
70
|
+
### Headings and code blocks
|
|
71
|
+
|
|
72
|
+
nibs splits a page by its Markdown headings. It reads only what it needs for that, by this rule rather than all of CommonMark:
|
|
73
|
+
|
|
74
|
+
- A heading is a line that starts with 1 to 6 `#` and a space.
|
|
75
|
+
- A line that starts with three or more `` ` `` or `~` opens a code block, after any spaces. The block ends at a line of the same character, at least as long, with nothing after it. Headings inside a code block are not headings.
|
|
76
|
+
- Nothing else is read as structure: not headings underlined with `===` or `---`, not code indented by four spaces.
|
|
77
|
+
|
|
78
|
+
## Using it from an agent
|
|
79
|
+
|
|
80
|
+
Tell the agent in its instructions, for example in `CLAUDE.md` or `AGENTS.md`:
|
|
81
|
+
|
|
82
|
+
```markdown
|
|
83
|
+
To read documentation on the web, run `nibs toc <url>` to see the headings,
|
|
84
|
+
then `nibs section <url> <n>` for the part you need, or `nibs find <url> <regex>`.
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## The usage record
|
|
88
|
+
|
|
89
|
+
Each run adds one line to `~/.local/state/nibs/usage.jsonl` (or `$XDG_STATE_HOME/nibs/usage.jsonl`): the time, the command, the URL, the page's size and how many characters were printed. `nibs stats` reads it. Set `NIBS_LOG` to another path to keep the record there. Nothing is sent anywhere.
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
MIT
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
nibs/__init__.py,sha256=5xx2ZI7buRzpKsEkI7XHjE_2kipeKlncrBmI-GlZ4H4,210
|
|
2
|
+
nibs/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
+
nibs/cache.py,sha256=VvxOFbVTUZmxAeaDsEU5FgBnLnuQA26pDrZkm6Dv1mg,1279
|
|
4
|
+
nibs/cli.py,sha256=BDgKCdfJNbGFaLj9DiRA_ZYUcom-EvG4UFlHnlpqiZY,4706
|
|
5
|
+
nibs/page.py,sha256=mf0C3REncEa-S2r8lql5nKvd9VR0upkx4e9yWZEj4hM,8191
|
|
6
|
+
nibs/paths.py,sha256=bcaFuNWR51p6WVXadlswUbD7Th1a5zLjxB3t_hzH9oI,380
|
|
7
|
+
nibs/usage.py,sha256=EHI7UEZocsTjB7r4c_IScLHtlkZZy5aXE3Igc3Qae4M,1876
|
|
8
|
+
nibs-0.1.0.dist-info/METADATA,sha256=ikaaEw0N7NFCxCgFZcVMWt86CW7OtULKY7vYGbmT1Y0,4513
|
|
9
|
+
nibs-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
10
|
+
nibs-0.1.0.dist-info/entry_points.txt,sha256=3b9vCkBq416TK_EkfT-mHu-kQr4H98TRueh5buUNJRo,39
|
|
11
|
+
nibs-0.1.0.dist-info/licenses/LICENSE,sha256=RTp9_jaasWtI0jk_CouMJRPrHb8c2zdd19HbUZrksDM,1064
|
|
12
|
+
nibs-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 nhinata
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|