recala-cli 0.6.0__py3-none-any.whl

recala_cli/commands/index.py

@@ -0,0 +1,293 @@
+"""`recala index`: is Google indexing the company's pages, and what can be done about it, honestly.
+
+What Google allows, and what this command does:
+
+* **Read**: Search Console's URL Inspection, per published article (indexed or not, coverage, last crawl, Google's
+  canonical against the declared one, mobile) and its sitemaps (health, errors, last read). Recala's server inspects,
+  caches for 24 hours and respects the per-property quota; the CLI reads its answers.
+* **Notify**: an IndexNow ping (Bing, Yandex, and ChatGPT search through Bing) for an article, and resubmitting the
+  sitemap when the Search Console connection holds the write scope (else Google re-reads it on its own).
+* **Not possible**: Google's "Request indexing" button has no public API for normal pages (the Indexing API is only for
+  job postings and livestreams, and using it otherwise breaks Google's rules). Every answer prints the Search Console
+  link where the person can press it.
+
+The server side is the indexing tracker (its own PR); until it is deployed every route answers 404, and the CLI says
+"Indexing tracker not enabled yet".
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+import click
+
+from recala_cli.client import RecalaClient
+from recala_cli.endpoints import (
+    ARTICLE_INDEXING,
+    ARTICLE_INDEXING_SUBMIT,
+    ARTICLES_LIST,
+    INDEXING_SITEMAPS,
+    INDEXING_SITEMAPS_SUBMIT,
+)
+from recala_cli.errors import CliError, ExitCode
+from recala_cli.human import describe_fields
+from recala_cli.ids import id_arguments, resolve_article_id
+from recala_cli.labels import resource_label
+from recala_cli.polling import indicator, open_in_browser
+from recala_cli.runtime import command_runner, common_options, confirm, open_client, optional_read, accepted_yes_option, dry_run_option, accepted_yes_option, dry_run_option
+from recala_cli.views import company_timezone, local_time, timezone_label
+
+NOT_ENABLED = "Indexing tracker not enabled yet on this Recala server."
+REQUEST_INDEXING = ("For a priority crawl in Google, use URL Inspection -> Request indexing in Search Console "
+                    "(Google has no public API for it):")
+_PUBLISHED = {"published", "updated"}
+_INDEXED = {"PASS", "indexed", "INDEXED"}
+#: Google's URL Inspection verdicts in words. VERDICT_UNSPECIFIED and anything unknown say nothing, so they are hidden.
+_INDEXING_WORDS = {"PASS": "indexed", "PARTIAL": "partly indexed", "FAIL": "not indexed (Google found a problem)",
+                   "NEUTRAL": "not indexed"}
+_MOBILE_WORDS = {"PASS": "works on phones", "PARTIAL": "works on phones with some problems",
+                 "FAIL": "does not work well on phones"}
+_PARALLEL_READS = 6
+
+
+def tracker_off() -> CliError:
+    return CliError(ExitCode.CONFLICT, "indexing_not_enabled", NOT_ENABLED, 404)
+
+
+def route_missing(error: CliError) -> bool:
+    """FastAPI's own 404 for a route that is not there (an unknown article answers with its own message)."""
+    return error.status == 404 and error.message == "Not Found"
+
+
+def read_indexing(client: RecalaClient, article_id: str) -> dict[str, Any]:
+    try:
+        state: dict[str, Any] = client.call(ARTICLE_INDEXING, path={"article_id": article_id})
+    except CliError as error:
+        if route_missing(error):
+            raise tracker_off() from error
+        raise
+    return state
+
+
+def published_article_ids(client: RecalaClient, limit: int = 50) -> list[str]:
+    listed = client.call(ARTICLES_LIST, params={"limit": limit, "offset": 0}) or {}
+    return [str(a["id"]) for a in listed.get("articles") or [] if a.get("cms_publish_status") in _PUBLISHED]
+
+
+def is_indexed(state: dict[str, Any]) -> bool:
+    return str(state.get("verdict") or "") in _INDEXED or str(state.get("indexing_state") or "").lower() == "indexed"
+
+
+def read_indexing_all(client: RecalaClient, ids: list[str],
+                      on_read: Callable[[int], None] | None = None) -> list[dict[str, Any]]:
+    """Each article's indexing state, read in parallel and returned in the order asked."""
+    done = 0
+
+    def read(article_id: str) -> dict[str, Any]:
+        nonlocal done
+        state = read_indexing(client, article_id)
+        done += 1
+        if on_read is not None:
+            on_read(done)
+        return state
+
+    with ThreadPoolExecutor(max_workers=_PARALLEL_READS) as pool:
+        return list(pool.map(read, ids))
+
+
+def indexing_summary(client: RecalaClient, limit: int = 20) -> dict[str, Any] | None:
+    """For `recala status`: how many published articles Google has indexed (None while the tracker is off). One line
+    of a dashboard never takes the dashboard down: any other refusal reads as "unavailable"."""
+    try:
+        ids = published_article_ids(client)[:limit]
+        states = read_indexing_all(client, ids)
+    except CliError as error:
+        return None if error.code == "indexing_not_enabled" else {"unavailable": error.message}
+    return {"published": len(ids), "indexed": sum(1 for s in states if is_indexed(s))}
+
+
+def _when(value: str | None, zone: str | None, *, empty: str = "-") -> str:
+    return local_time(value, zone) if value else empty
+
+
+def article_lines(state: dict[str, Any], zone: str | None = None) -> list[str]:
+    verdict = _INDEXING_WORDS.get(str(state.get("verdict") or ""))
+    indexed = "indexed" if is_indexed(state) else verdict or "not indexed"
+    lines = [f"{state.get('url')}", f"  Google:        {indexed}"]
+    if state.get("coverage_state"):
+        lines.append(f"  Coverage:      {state['coverage_state']}")
+    lines.append(f"  Last crawl:    {_when(state.get('last_crawl'), zone, empty='never')}")
+    google, declared = state.get("google_canonical"), state.get("user_canonical")
+    same = google == declared if google and declared else None
+    lines.append(f"  Canonical:     {'Google agrees' if same else ('Google chose ' + str(google)) if google else '-'}"
+                 + (f" (declared {declared})" if declared and not same else ""))
+    mobile = _MOBILE_WORDS.get(str(state.get("mobile") or ""))
+    if mobile:
+        lines.append(f"  Mobile:        {mobile}")
+    if state.get("indexnow_submitted_at"):
+        lines.append(f"  IndexNow:      sent {_when(state['indexnow_submitted_at'], zone)}")
+    lines.append(f"  Checked:       {_when(state.get('inspected_at'), zone)}"
+                 f"  (next {_when(state.get('next_check_at'), zone)})")
+    if not is_indexed(state) and state.get("request_indexing_url"):
+        lines.append(f"  {REQUEST_INDEXING}\n  {state['request_indexing_url']}")
+    return lines
+
+
+def describe_statuses(ctx: click.Context, result: dict[str, Any]) -> str:
+    rows = result.get("articles") or []
+    if not rows:
+        return "No published articles to check yet."
+    zone = company_timezone(ctx, result)
+    lines = [timezone_label(zone)]
+    for state in rows:
+        lines.extend(article_lines(state, zone))
+    lines.append(f"{sum(1 for s in rows if is_indexed(s))} of {len(rows)} indexed in Google.")
+    return "\n".join(lines)
+
+
+def describe_sitemaps(result: dict[str, Any], zone: str | None = None) -> str:
+    sitemaps = result.get("sitemaps") or []
+    if not sitemaps:
+        return "Search Console holds no sitemap for this site. Submit one: recala index sitemaps submit"
+    lines = []
+    for sitemap in sitemaps:
+        errors, warnings = sitemap.get("errors") or 0, sitemap.get("warnings") or 0
+        health = "healthy" if not int(errors) else f"{errors} error(s)"
+        read = _when(sitemap.get("last_downloaded") or sitemap.get("last_read"), zone, empty="not yet")
+        lines.append(f"{sitemap.get('path')}  {health}, {warnings} warning(s)")
+        lines.append(f"  Last read by Google: {read}   Submitted: {_when(sitemap.get('last_submitted'), zone)}"
+                     + ("   (pending)" if sitemap.get("is_pending") else ""))
+    return "\n".join(lines)
+
+
+def describe_sitemaps_in_zone(ctx: click.Context, result: dict[str, Any]) -> str:
+    if not result.get("sitemaps"):
+        return describe_sitemaps(result)
+    zone = company_timezone(ctx, result)
+    return f"{timezone_label(zone)}\n{describe_sitemaps(result, zone)}"
+
+
+@click.group()
+def index() -> None:
+    """Indexing in Google, IndexNow and the sitemap.
+
+    What Google has indexed, IndexNow pings, and the sitemap in Search Console."""
+
+
+@index.group("sitemaps", invoke_without_command=True)
+@common_options
+@click.pass_context
+def sitemaps(ctx: click.Context, as_json: bool, org_flag: str | None) -> None:
+    """The site's sitemaps in Search Console.
+
+    Health, errors, last read (`submit` resubmits)."""
+    if ctx.invoked_subcommand is None:
+        ctx.invoke(show_sitemaps, as_json=as_json, org_flag=org_flag)
+
+
+@sitemaps.command("show")
+@common_options
+@command_runner(contextual_human=describe_sitemaps_in_zone)
+def show_sitemaps(ctx: click.Context, org_flag: str | None) -> dict[str, Any]:
+    """The site's sitemaps in Search Console.
+
+    Health, errors and last read."""
+    try:
+        answer: dict[str, Any] = open_client(ctx, org_flag).call(INDEXING_SITEMAPS)
+    except CliError as error:
+        if route_missing(error):
+            raise tracker_off() from error
+        raise
+    return answer
+
+
+@sitemaps.command("submit")
+@common_options
+@command_runner(human=lambda r: r.get("message") or f"Submitted {r.get('feedpath')} to Search Console.")
+@click.option("--feedpath", default=None, help="The sitemap URL (default: the first one Search Console holds).")
+@accepted_yes_option
+@dry_run_option
+def submit_sitemap(ctx: click.Context, org_flag: str | None, feedpath: str | None) -> dict[str, Any]:
+    """Resubmit the sitemap to Search Console.
+
+    Needs the connection's write scope; it offers the upgrade."""
+    client = open_client(ctx, org_flag)
+    if not feedpath:
+        feeds = (optional_read(client, INDEXING_SITEMAPS) or {}).get("sitemaps") or []
+        feedpath = feeds[0].get("path") if feeds else None
+    if not feedpath:
+        raise CliError(ExitCode.USAGE, "missing_input", "Name the sitemap: --feedpath https://…/sitemap.xml")
+    confirm(ctx, client, f"Submit {feedpath} to Search Console?")
+    try:
+        submitted: dict[str, Any] = client.call(INDEXING_SITEMAPS_SUBMIT, json={"feedpath": feedpath})
+    except CliError as refusal:
+        if route_missing(refusal):
+            raise tracker_off() from refusal
+        upgrade = (refusal.details or {}).get("upgrade_url")
+        if refusal.code != "scope_upgrade_required" or not upgrade:
+            raise
+        open_in_browser(ctx, upgrade, why="Recala can read Search Console but not submit sitemaps. Allow it here:")
+        return {"feedpath": feedpath, "submitted": False, "upgrade_url": upgrade,
+                "message": "Google re-reads your sitemap on its own. To submit from Recala, allow the write access "
+                           "in the page just opened, then run this again."}
+    return {"feedpath": feedpath, **submitted}
+
+
+@index.command("status")
+@common_options
+@command_runner(contextual_human=describe_statuses)
+@click.option("--all-published", is_flag=True, help="Every published article (at most 50).")
+@click.argument("article_ids", nargs=-1)
+@id_arguments(article_ids="article")
+def status(ctx: click.Context, org_flag: str | None, all_published: bool, article_ids: tuple[str, ...]) -> dict[str, Any]:
+    """Google's view of each article (cached 24 h).
+
+    Indexed or not, coverage, last crawl, canonical, mobile."""
+    if bool(article_ids) == all_published:
+        raise click.UsageError("Name one or more articles, or pass --all-published.")
+    client = open_client(ctx, org_flag)
+    ids = [resolve_article_id(client, selector) for selector in article_ids] if article_ids else published_article_ids(client)
+    with indicator(ctx, f"Reading Google's index for {len(ids)} article(s)", tty_only=True) as spinner:
+        progress = (lambda done: spinner.update(f"Reading Google's index: {done}/{len(ids)}")) if spinner else None
+        states = read_indexing_all(client, ids, progress)
+        if spinner is not None:
+            spinner.done(f"Read {len(ids)} article(s)")
+    return {"articles": [{"article_id": article_id, **state} for article_id, state in zip(ids, states)]}
+
+
+def describe_submission(ctx: click.Context, result: dict[str, Any]) -> str:
+    from recala_cli.views import localise_instants
+
+    zone = company_timezone(ctx, result)
+    lines = article_lines(result["indexing"], zone) if result.get("indexing") else []
+    indexnow = localise_instants(result.get("indexnow") or "sent", zone)[0]
+    return "\n".join([*lines, f"IndexNow: {describe_fields(indexnow)}", REQUEST_INDEXING,
+                      result.get("request_indexing_url") or ""])
+
+
+@index.command("submit")
+@common_options
+@command_runner(contextual_human=describe_submission)
+@click.argument("article_id")
+@id_arguments(article_id="article")
+@accepted_yes_option
+@dry_run_option
+def submit(ctx: click.Context, org_flag: str | None, article_id: str) -> dict[str, Any]:
+    """Ping IndexNow for an article and re-check it.
+
+    IndexNow reaches Bing, Yandex and ChatGPT search; prints Google's Request-indexing link."""
+    client = open_client(ctx, org_flag)
+    article_id = resolve_article_id(client, article_id)
+    label = resource_label(ctx, client, "article", article_id)
+    confirm(ctx, client, f"Tell IndexNow about {label} and re-check it in Google?")
+    try:
+        answer: dict[str, Any] = client.call(ARTICLE_INDEXING_SUBMIT, path={"article_id": article_id})
+    except CliError as refusal:
+        if route_missing(refusal):
+            raise tracker_off() from refusal
+        raise
+    state = answer.get("indexing") if isinstance(answer.get("indexing"), dict) else answer
+    return {"article_id": article_id, "indexnow": answer.get("indexnow") or answer.get("indexnow_submitted_at"),
+            "indexing": state, "request_indexing_url": (state or {}).get("request_indexing_url")}

recala_cli/commands/keys.py

@@ -0,0 +1,98 @@
+"""`recala keys`: organization API keys for agents and CI. A person's session only; a key never manages keys."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import click
+
+from recala_cli.endpoints import KEY_EVENTS, KEY_REVOKE, KEYS_CREATE, KEYS_LIST
+from recala_cli.labels import resource_label
+from recala_cli.runtime import command_runner, common_options, confirm, ignored_yes_option, open_client, yes_option, dry_run_option
+
+SCOPES = ("read", "content:write", "content:edit", "publish", "billing:read", "settings:write")
+
+
+def _describe_keys(result: dict[str, Any]) -> str:
+    rows = [f"{k['id']}  {k['prefix']}  {k['status']:<8} {','.join(k['scopes'])}  {k['name']}" for k in result["keys"]]
+    return "\n".join(rows) if rows else "No API keys."
+
+
+def _describe_created(result: dict[str, Any]) -> str:
+    return (f"{result['key']}\n\nThis is the only time the key is shown. Store it in a secret manager.\n"
+            f"Scopes: {', '.join(result['scopes'])}. Expires {result['expires_at']}.")
+
+
+@click.group()
+def keys() -> None:
+    """Organization API keys for agents and CI."""
+
+
+@keys.command("list")
+@common_options
+@command_runner(human=_describe_keys)
+def list_keys(ctx: click.Context, org_flag: str | None) -> dict[str, Any]:
+    """Owners see every key of the company; others see their own."""
+    listed: dict[str, Any] = open_client(ctx, org_flag).call(KEYS_LIST)
+    return listed
+
+
+@keys.command("create")
+@common_options
+@command_runner(human=_describe_created)
+@yes_option
+@click.option("--name", required=True, help="What the key is for, e.g. \"Claude agent\".")
+@click.option("--scope", "scopes", type=click.Choice(SCOPES), multiple=True, required=True, help="Repeat for each scope.")
+@click.option("--expires-days", type=click.IntRange(1, 365), default=90, show_default=True)
+@click.option("--rate", "rate_limit", type=click.IntRange(1, 600), default=60, show_default=True, help="Requests per minute.")
+@click.option("--credit-cap", type=click.IntRange(0, 100), default=5, show_default=True, help="Articles per month.")
+@click.option("--publish-cap", type=click.IntRange(0, 50), default=3, show_default=True,
+              help="Articles approved or published per day.")
+@click.option("--sign-as", "signer", default=None, help="Your own email: needed for a key that can approve and publish.")
+@click.option("--person", "people", multiple=True,
+              help="With publish: a person (id from recala authors) whose articles this key may approve and publish. Repeat for each.")
+@dry_run_option
+def create_key(ctx: click.Context, org_flag: str | None, yes: bool, name: str, scopes: tuple[str, ...], expires_days: int,
+               rate_limit: int, credit_cap: int, publish_cap: int, signer: str | None, people: tuple[str, ...]) -> dict[str, Any]:
+    """Create a key. It is printed once; Recala keeps only its hash."""
+    body: dict[str, Any] = {
+        "name": name, "scopes": list(dict.fromkeys(scopes)), "expires_in_days": expires_days,
+        "rate_limit_per_minute": rate_limit, "credit_cap_per_month": credit_cap, "publish_cap_per_day": publish_cap,
+    }
+    client = open_client(ctx, org_flag)
+    if "publish" in scopes:
+        confirm(ctx, client, f"Create key \"{name}\"? It will approve and publish the articles of the people you name.",
+                publishes=True, yes=yes)
+        body["publish_signer"] = signer
+        body["publish_person_ids"] = list(dict.fromkeys(people))
+    else:
+        confirm(ctx, client, f"Create key \"{name}\" with {', '.join(body['scopes'])}?")
+    created: dict[str, Any] = client.call(KEYS_CREATE, json=body)
+    return created
+
+
+@keys.command("revoke")
+@common_options
+@command_runner(human=lambda key: f"Revoked {key['prefix']}.")
+@ignored_yes_option
+@click.argument("key_id")
+@dry_run_option
+def revoke_key(ctx: click.Context, org_flag: str | None, yes: bool, key_id: str) -> dict[str, Any]:
+    """Revoke a key: its next request is refused."""
+    client = open_client(ctx, org_flag)
+    label = resource_label(ctx, client, "key", key_id)
+    confirm(ctx, client, f"Revoke {label}? Every agent that uses it stops.", yes=yes)
+    revoked: dict[str, Any] = client.call(KEY_REVOKE, path={"key_id": key_id})
+    return revoked
+
+
+@keys.command("events")
+@common_options
+@command_runner(human=lambda r: "\n".join(f"{e.get('created_at', '')}  {e['event']:<8} {e.get('route') or ''}"
+                                          for e in r["events"]))
+@click.option("--limit", type=click.IntRange(1, 200), default=50, show_default=True)
+@click.argument("key_id")
+def key_events(ctx: click.Context, org_flag: str | None, limit: int, key_id: str) -> dict[str, Any]:
+    """The key's audit trail, newest first."""
+    events: dict[str, Any] = open_client(ctx, org_flag).call(KEY_EVENTS, path={"key_id": key_id}, params={"limit": limit})
+    return events

recala_cli/commands/knowledge.py

@@ -0,0 +1,245 @@
+"""`recala knowledge`: the company's sources (documents and pages) the articles are written from.
+
+An owner's upload is the company's own material (founder, 5 Oct: owner uploads are
+admitted automatically). The server admits it itself when its flag allows; when it
+answers "not admitted", the CLI admits it with the same route the Library's
+"Company's own material" choice uses. Both happen under the one confirmation.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import click
+
+from recala_cli.client import RecalaClient
+from recala_cli.endpoints import (
+    KNOWLEDGE_ADMIT,
+    KNOWLEDGE_LIST,
+    KNOWLEDGE_PAGE,
+    KNOWLEDGE_REMOVE,
+    KNOWLEDGE_UPLOAD,
+    KNOWLEDGE_URL,
+    KNOWLEDGE_URLS,
+)
+from recala_cli.errors import CliError, ExitCode
+from recala_cli.ids import resolve_source_id
+from recala_cli.labels import action_sentence, resource_label
+from recala_cli.runtime import command_runner, common_options, confirm, ignored_yes_option, open_client, accepted_yes_option, dry_run_option
+
+#: What the API reads (routers/knowledge.py upload): the type travels as Content-Type.
+CONTENT_TYPES = {
+    ".pdf": "application/pdf",
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".md": "text/markdown",
+    ".txt": "text/plain",
+    ".csv": "text/csv",
+}
+#: The Library's "Company's own material": reviewed before it reaches a published page.
+OWN_MATERIAL = {"provenance_class": "first_party", "publication_permission": "review_only"}
+
+
+#: How `knowledge admit --as` names the API's provenance classes.
+ADMIT_KINDS = {"own": "first_party", "proposed": "proposed", "researched": "researched"}
+ADMIT_WORDS = {"own": "the company's own material", "proposed": "proposed material", "researched": "researched material"}
+
+
+def content_type_of(file: Path) -> str:
+    content_type = CONTENT_TYPES.get(file.suffix.lower())
+    if content_type is None:
+        raise click.BadParameter(f"{file.name}: Recala reads {', '.join(sorted(CONTENT_TYPES))} files.", param_hint="FILE")
+    return content_type
+
+
+def admission_state(client: RecalaClient, document_id: str) -> str:
+    listed = client.call(KNOWLEDGE_LIST) or {}
+    for document in listed.get("documents") or []:
+        if str(document.get("id")) == document_id:
+            return str(document.get("admission_state") or "unknown")
+    return "unknown"
+
+
+def upload_own_material(client: RecalaClient, file: Path) -> dict[str, Any]:
+    """Upload one file and make sure it is admitted as the company's own material. The caller confirmed both."""
+    uploaded: dict[str, Any] = client.call(
+        KNOWLEDGE_UPLOAD, content=file.read_bytes(),
+        extra_headers={"Content-Type": content_type_of(file), "X-Filename": file.name},
+    )
+    document_id = str(uploaded.get("document_id") or "")
+    if not document_id or uploaded.get("status") != "ready":
+        return {**uploaded, "admission_state": "not_admitted"}
+    state = admission_state(client, document_id)
+    if state == "not_admitted":
+        try:
+            client.call(KNOWLEDGE_ADMIT, path={"doc_id": document_id}, json=OWN_MATERIAL)
+            state = admission_state(client, document_id)
+        except CliError as refusal:  # the upload stands; the person can admit it in the Library
+            return {**uploaded, "admission_state": state, "admission_error": refusal.message}
+    return {**uploaded, "admission_state": state}
+
+
+def _describe_upload(result: dict[str, Any]) -> str:
+    state = result.get("admission_state")
+    admitted = "admitted as the company's own material" if state == "admitted" else f"admission: {state}"
+    line = f"{result.get('filename')}: {result.get('status')}, {result.get('chunk_count', 0)} passages, {admitted}"
+    return line + (f" ({result['admission_error']})" if result.get("admission_error") else "")
+
+
+@click.group()
+def knowledge() -> None:
+    """Sources: the documents and pages Recala writes from."""
+
+
+def _describe_sources(listed: dict[str, Any]) -> str:
+    """Each source by name, its state and admission in words, its short id last, and the next step when it is not admitted."""
+    documents = listed.get("documents") or []
+    if not documents:
+        return "No sources yet."
+    admitted = sum(1 for document in documents if document.get("admission_state") == "admitted")
+    lines = [f"{len(documents)} source{'' if len(documents) == 1 else 's'}, {admitted} admitted:"]
+    for document in documents:
+        states = " · ".join(str(document[key]).replace("_", " ") for key in ("status", "admission_state")
+                            if document.get(key))
+        short_id = str(document["id"])[:8]
+        lines.append(f"  {document.get('filename') or 'Untitled source'}  {states}  [{short_id}]")
+        if document.get("status") == "ready" and document.get("admission_state") == "not_admitted":
+            lines.append(f"    Next: recala knowledge admit {short_id}")
+    return "\n".join(lines)
+
+
+@knowledge.command("list")
+@common_options
+@command_runner(human=_describe_sources)
+def list_documents(ctx: click.Context, org_flag: str | None) -> dict[str, Any]:
+    """Every source, its state and its admission.
+
+    Each document's processing state and whether it is admitted."""
+    listed: dict[str, Any] = open_client(ctx, org_flag).call(KNOWLEDGE_LIST)
+    return listed
+
+
+@knowledge.command("add")
+@common_options
+@command_runner(human=_describe_upload)
+@click.argument("file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@accepted_yes_option
+@dry_run_option
+def add_document(ctx: click.Context, org_flag: str | None, file: Path) -> dict[str, Any]:
+    """Upload a file as the company's own material.
+
+    PDF, DOCX, Markdown, text or CSV, 10 MB at most."""
+    content_type_of(file)
+    client = open_client(ctx, org_flag)
+    confirm(ctx, client, f"Upload {file.name} and admit it as the company's own material?")
+    result = upload_own_material(client, file)
+    if not result.get("success", True) and result.get("status") == "error":
+        raise CliError(ExitCode.UNAVAILABLE, "upload_failed", result.get("error_message") or "The upload failed.")
+    return result
+
+
+def with_admission(client: RecalaClient, page: dict[str, Any]) -> dict[str, Any]:
+    """The added page plus its admission (state and the server's reason). The add stands if that read fails."""
+    if not page.get("document_id"):
+        return page
+    try:
+        document = client.call(KNOWLEDGE_PAGE, path={"doc_id": str(page["document_id"])}) or {}
+    except CliError:
+        return page
+    return {**page, "admission": document["admission"]} if document.get("admission") else page
+
+
+def _describe_added_page(page: dict[str, Any], url: str | None = None) -> str:
+    name = page.get("title") or page.get("url") or url or "Page"
+    if page.get("success") is False:
+        return f"{name}: not added ({page.get('error') or 'the page could not be read'})"
+    passages = int(page.get("chunk_count") or 0)
+    line = f"{name}: added, {passages} passage{'' if passages == 1 else 's'}"
+    admission = page.get("admission") or {}
+    if admission.get("state") == "admitted":
+        return line + "; admitted"
+    return line + (f"; {str(admission['state']).replace('_', ' ')}: {admission.get('reason')}" if admission else "")
+
+
+def _describe_added_pages(ctx: click.Context, result: dict[str, Any]) -> str:
+    if "results" in result:
+        return "\n".join(_describe_added_page(page) for page in result["results"]) or "No pages added."
+    if "document_id" not in result and "success" not in result:
+        return result.get("message") or "Added source page(s)."
+    return _describe_added_page(result, ctx.params["urls"][0])
+
+
+@knowledge.command("add-url")
+@common_options
+@command_runner(contextual_human=_describe_added_pages)
+@click.argument("urls", nargs=-1, required=True)
+@accepted_yes_option
+@dry_run_option
+def add_urls(ctx: click.Context, org_flag: str | None, urls: tuple[str, ...]) -> dict[str, Any]:
+    """Add one or more pages as sources.
+
+    Says whether each was admitted, and why not."""
+    client = open_client(ctx, org_flag)
+    confirm(ctx, client, f"Add {len(urls)} page(s) as sources?")
+    if len(urls) == 1:
+        return with_admission(client, client.call(KNOWLEDGE_URL, json={"url": urls[0]}) or {})
+    many: dict[str, Any] = client.call(KNOWLEDGE_URLS, json={"urls": list(urls)}) or {}
+    return {**many, "results": [with_admission(client, page) for page in many.get("results") or []]}
+
+
+@knowledge.command("remove")
+@common_options
+@command_runner(contextual_human=action_sentence("Removed"))
+@ignored_yes_option
+@click.argument("document_id", metavar="SOURCE_ID")
+@dry_run_option
+def remove_document(ctx: click.Context, org_flag: str | None, yes: bool, document_id: str) -> dict[str, Any]:
+    """Remove a source and everything derived from it."""
+    client = open_client(ctx, org_flag)
+    document_id = resolve_source_id(client, document_id)
+    label = resource_label(ctx, client, "source", document_id)
+    confirm(ctx, client, f"Remove {label}? This also erases what was learned from it.", yes=yes)
+    removed: dict[str, Any] = client.call(KNOWLEDGE_REMOVE, path={"doc_id": document_id})
+    return removed
+
+
+def describe_admission(ctx: click.Context, result: dict[str, Any]) -> str:
+    kind = ADMIT_WORDS[ctx.params["kind"]]
+    if result.get("dry_run"):
+        return f"Dry run — would admit {ctx.meta['result_subject']} as {kind}."
+    return f"Admitted: {ctx.meta['result_subject']} as {kind}."
+
+
+@knowledge.command("admit")
+@common_options
+@command_runner(contextual_human=describe_admission)
+@ignored_yes_option
+@click.argument("source_id", metavar="SOURCE_ID")
+@click.option("--as", "kind", type=click.Choice(tuple(ADMIT_KINDS)), default="own", show_default=True,
+              help="What kind of material it is.")
+@click.option("--dry-run", is_flag=True, help="Check the source can be admitted without admitting it.")
+def admit_source(ctx: click.Context, org_flag: str | None, yes: bool, source_id: str, kind: str, dry_run: bool) -> dict[str, Any]:
+    """Admit a ready source for use in articles.
+
+    Needed for a source that is ready but not admitted. SOURCE_ID is the 8-character id from `recala knowledge list`."""
+    client = open_client(ctx, org_flag)
+    document_id = resolve_source_id(client, source_id)
+    label = resource_label(ctx, client, "source", document_id)
+    listed = (client.call(KNOWLEDGE_LIST) or {}).get("documents") or []
+    document = next((row for row in listed if str(row.get("id")) == document_id), None)
+    if document is None:
+        raise CliError(ExitCode.NOT_FOUND, "source_not_found",
+                       "No source with that id in this company. Find it with: recala knowledge list")
+    if document.get("admission_state") == "admitted":
+        raise CliError(ExitCode.CONFLICT, "already_admitted", f"{label} is already admitted.")
+    if document.get("status") != "ready":
+        raise CliError(ExitCode.CONFLICT, "source_not_ready",
+                       f"{label} is still {str(document.get('status') or 'being read').replace('_', ' ')}; "
+                       "admit it once it is ready (recala knowledge list).")
+    if dry_run:
+        return {"dry_run": True, "document_id": document_id, "provenance_class": ADMIT_KINDS[kind]}
+    confirm(ctx, client, f"Admit {label} as {ADMIT_WORDS[kind]}?")
+    admitted: dict[str, Any] = client.call(
+        KNOWLEDGE_ADMIT, path={"doc_id": document_id},
+        json={"provenance_class": ADMIT_KINDS[kind], "publication_permission": "review_only"})
+    return admitted