arcsecond 4.2.1__tar.gz → 4.3.1__tar.gz

{arcsecond-4.2.1 → arcsecond-4.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: arcsecond
-Version: 4.2.1
+Version: 4.3.1
 Summary: CLI for arcsecond.io
 Project-URL: Homepage, https://github.com/arcsecond-io/cli
 Project-URL: Issues, https://github.com/arcsecond-io/cli/issues

{arcsecond-4.2.1 → arcsecond-4.3.1}/arcsecond/hosting/backups.py

@@ -21,6 +21,9 @@ from .utils import _read_env_value as _read_env_value_from
 DB_CONTAINER = "arcsecond-db"
 API_CONTAINER = "arcsecond-api"
 WORKER_CONTAINER = "arcsecond-worker"
+# Since docker-compose.yml 7.3; absent on older installs, which the stop loop
+# and the docker-start fallback both tolerate, as for alerts below.
+DATA_WORKER_CONTAINER = "arcsecond-dataworker"
 BEAT_CONTAINER = "arcsecond-beat"
 WEB_CONTAINER = "arcsecond-web"
 # Optional service: absent on most installs. The stop loop skips it when it
@@ -31,6 +34,7 @@ ALERTS_CONTAINER = "arcsecond-alerts"
 SERVICES_TO_STOP = [
     API_CONTAINER,
     WORKER_CONTAINER,
+    DATA_WORKER_CONTAINER,
     ALERTS_CONTAINER,
     BEAT_CONTAINER,
     WEB_CONTAINER,

{arcsecond-4.2.1 → arcsecond-4.3.1}/arcsecond/hosting/composedoc.py

@@ -251,6 +251,37 @@ ENV_KEYS: Dict[str, tuple] = {
         "How long a new member may use the installation before verifying their email, "
         "on an installation with no mail server.",
     ),
+    "ARCSECOND_DOWNLOADS_PER_CLIENT": (
+        OPERATOR,
+        "How many downloads one computer may have open at once; past it, the next one is refused with a 429 "
+        "until one finishes. Default 6. Then `arcsecond restart web`.",
+    ),
+    "ARCSECOND_DOWNLOADS_TOTAL": (
+        OPERATOR,
+        "How many downloads the installation serves at once, all computers together. Default 24.",
+    ),
+    "ARCSECOND_DOWNLOAD_RATE": (
+        OPERATOR,
+        "The speed of each download, in bytes per second, `k` and `m` allowed; `0` for no limit. Default `20m`. "
+        "Lower it when the observers work remotely over a link the downloads would otherwise fill.",
+    ),
+    "ARCSECOND_BACKEND_MEMORY": (
+        OPERATOR,
+        "The most memory the backend container may use, e.g. `6g` (the default). Past it, Docker restarts the "
+        "busiest of its processes rather than letting the machine take memory from the database.",
+    ),
+    "ARCSECOND_WORKER_MEMORY": (
+        OPERATOR,
+        "The same ceiling for the worker, which runs the exposures and procedures. Default `6g`.",
+    ),
+    "ARCSECOND_DATA_WORKER_MEMORY": (
+        OPERATOR,
+        "The same ceiling for the data worker: zip archives, previews, storage pushes, backups. Default `3g`.",
+    ),
+    "ARCSECOND_DATA_WORKER_CONCURRENCY": (
+        OPERATOR,
+        "How many jobs the data worker runs at once. Default 2; each can hold a few hundred MB.",
+    ),
 }
 
 

{arcsecond-4.2.1 → arcsecond-4.3.1}/arcsecond/hosting/database.py

@@ -44,16 +44,20 @@ SERVICES_TO_RECREATE = ["backend", "worker", "beat"]
 
 
 def _services_to_recreate():
-    """The optional alerts consumer joins the recreate list only when the
-    operator's compose file actually carries it — naming an unknown service
-    makes `docker compose up` fail outright on installs without the block."""
+    """The optional alerts consumer, and the data worker of docker-compose.yml
+    7.3 and later, join the recreate list only when the operator's compose
+    file actually carries them — naming an unknown service makes
+    `docker compose up` fail outright on installs without them."""
     services = list(SERVICES_TO_RECREATE)
     compose_path = Path.cwd() / "docker-compose.yml"
     try:
-        if "# >>> arcsecond:alerts" in compose_path.read_text(encoding="utf-8"):
-            services.append("alerts")
+        text = compose_path.read_text(encoding="utf-8")
     except OSError:
-        pass
+        return services
+    if "container_name: arcsecond-dataworker" in text:
+        services.append("dataworker")
+    if "# >>> arcsecond:alerts" in text:
+        services.append("alerts")
     return services
 
 

{arcsecond-4.2.1 → arcsecond-4.3.1}/arcsecond/hosting/docker/docker-compose.yml

@@ -1,6 +1,6 @@
 # === Arcsecond.local config. ===
 # *** DO NOT change any name or port number. ***
-# Version 6.4
+# Version 7.3
 
 services:
   # Database (PostgresQL)
@@ -54,6 +54,10 @@ services:
         bind:
           create_host_path: true
     stop_grace_period: 60s
+    # A ceiling, so no burst of work here can take the memory Postgres needs.
+    # Downloads no longer pass through these processes (the web container
+    # sends them), but rendering a large frame for review still does.
+    mem_limit: ${ARCSECOND_BACKEND_MEMORY:-6g}
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:8800/healthcheck/" ]
       interval: 10s
@@ -61,7 +65,8 @@ services:
       retries: 12
       start_period: 120s
 
-  # Arcsecond worker, for offloading background work.
+  # Arcsecond worker, for offloading background work: the Control Room's
+  # exposures and procedures, and everything else someone is waiting on.
   worker:
     image: ghcr.io/arcsecond-io/arcsecond-api:latest
     container_name: arcsecond-worker
@@ -72,6 +77,9 @@ services:
         condition: service_healthy
     env_file:
       - .env
+    environment:
+      # The default queue only: the data worker below takes the rest.
+      ARCSECOND_WORKER_QUEUES: celery
     volumes:
       # Leave /data as is, it's a path inside the container, not in the host machine.
       # SHARED_DATA_PATH must be a path of your host machine, specified in the .env file.
@@ -81,6 +89,36 @@ services:
         bind:
           create_host_path: true
     stop_grace_period: 120s
+    mem_limit: ${ARCSECOND_WORKER_MEMORY:-6g}
+
+  # Arcsecond data worker, for the background work nobody at the telescope is
+  # waiting on: zip archives, previews of uploaded files, pushes to your
+  # storages, database backups, keograms, catalogue refreshes. On its own
+  # queue, so a long zip can never hold up a frame the observer is waiting for.
+  dataworker:
+    image: ghcr.io/arcsecond-io/arcsecond-api:latest
+    container_name: arcsecond-dataworker
+    restart: unless-stopped
+    entrypoint: /home/app/entrypoint-worker.sh
+    depends_on:
+      backend:
+        condition: service_healthy
+    env_file:
+      - .env
+    environment:
+      ARCSECOND_WORKER_QUEUES: data
+      # Few at a time: this work is heavy and none of it is urgent.
+      ARCSECOND_WORKER_CONCURRENCY: ${ARCSECOND_DATA_WORKER_CONCURRENCY:-2}
+    volumes:
+      # Leave /data as is, it's a path inside the container, not in the host machine.
+      # SHARED_DATA_PATH must be a path of your host machine, specified in the .env file.
+      - type: bind
+        source: ${SHARED_DATA_PATH}
+        target: /data
+        bind:
+          create_host_path: true
+    stop_grace_period: 120s
+    mem_limit: ${ARCSECOND_DATA_WORKER_MEMORY:-3g}
 
   # Arcsecond beat, to launch specific background tasks at specific times.
   beat:
@@ -114,6 +152,23 @@ services:
         condition: service_started
     ports:
       - "5555:5555"
+    environment:
+      # Download limits: how many downloads one computer may have open at
+      # once, how many in total, and the speed of each (bytes per second,
+      # k and m allowed, 0 for none). They keep downloads from filling the
+      # link a remote observer drives the telescope through.
+      ARCSECOND_DOWNLOADS_PER_CLIENT: ${ARCSECOND_DOWNLOADS_PER_CLIENT:-6}
+      ARCSECOND_DOWNLOADS_TOTAL: ${ARCSECOND_DOWNLOADS_TOTAL:-24}
+      ARCSECOND_DOWNLOAD_RATE: ${ARCSECOND_DOWNLOAD_RATE:-20m}
+    volumes:
+      # The same folder, read-only: this container sends the stored files
+      # itself, once the backend has said who may read them.
+      - type: bind
+        source: ${SHARED_DATA_PATH}
+        target: /data
+        read_only: true
+        bind:
+          create_host_path: true
 
   # >>> arcsecond:alerts
   # Arcsecond transient-alerts consumer (optional).

{arcsecond-4.2.1 → arcsecond-4.3.1}/arcsecond/hosting/lifecycle.py

@@ -272,7 +272,7 @@ def status(directory):
 @basic_options
 def logs(directory, service, follow, tail):
     """Show recent logs. Services are named as in `arcsecond status`:
-    backend, worker, beat, web, db, broker, platesolver, alerts."""
+    backend, worker, dataworker, beat, web, db, broker, platesolver, alerts."""
     install = stack.resolve_install_dir(directory)
     stack.ensure_docker()
     args = ["logs", "--tail", str(tail)]

{arcsecond-4.2.1 → arcsecond-4.3.1}/arcsecond/hosting/token.py

@@ -4,7 +4,8 @@ From the operator's side there is one fact: Arcsecond sent a token, and it
 has to be entered once so the machine can download Arcsecond.local. What it
 is underneath — a login to the container registry the images are pulled
 from, ghcr.io, as the organisation user — is this module's business, not
-theirs, and the words "registry" and "docker" stay out of what they read.
+theirs, and the words "registry" and "docker" stay out of what they read —
+except when Docker's own password store is what failed, which only they can fix.
 
 The token is asked for without echo, or read from standard input for a
 script, and handed to Docker over a pipe. There is deliberately no
@@ -96,7 +97,10 @@ def store_token(token: str) -> None:
         token=token + "\n",
     )
     if result.returncode != 0:
-        detail = (result.stderr or result.stdout or "").strip().splitlines()
+        output = (result.stderr or result.stdout or "").strip()
+        if _is_credential_store_failure(output):
+            raise ArcsecondError(_credential_store_advice(output))
+        detail = output.splitlines()
         reason = detail[-1] if detail else "Docker gave no reason"
         raise ArcsecondError(
             f"The token was refused: {reason}\n"
@@ -105,6 +109,43 @@ def store_token(token: str) -> None:
         )
 
 
+# Docker only saves the credential once the registry has accepted it, so these
+# words mean the token was good and the machine's own keychain failed. Docker
+# Desktop on Linux keeps credentials in `pass`, which is empty until someone
+# runs `pass init` — it says so in its install notes, and nobody reads them.
+_CREDENTIAL_STORE_FAILURES = ("error saving credentials", "error storing credentials")
+
+
+def _is_credential_store_failure(output: str) -> bool:
+    lowered = output.lower()
+    return any(words in lowered for words in _CREDENTIAL_STORE_FAILURES)
+
+
+def _credential_store_advice(output: str) -> str:
+    lowered = output.lower()
+    head = (
+        "The token is good — Arcsecond accepted it — but this machine could not "
+        "save it: Docker keeps it in a password store that refused to write.\n"
+    )
+    if "pass init" in lowered or "pass not initialized" in lowered:
+        return head + (
+            "Docker Desktop on Linux uses `pass`, and it has not been set up yet. "
+            "Set it up once, then run `arcsecond token set` again:\n"
+            "    gpg --generate-key          (any name and e-mail; note the key id it prints)\n"
+            "    pass init <that key id>\n"
+            "Or, if this machine runs Docker Engine without Docker Desktop, remove the "
+            f'"credsStore" line from {docker_config_path()} and Docker will keep the '
+            "token in that file instead."
+        )
+    detail = output.splitlines()[-1] if output else ""
+    return head + (
+        f"It said: {detail}\n"
+        'Unlock or repair that store, or remove the "credsStore" line from '
+        f"{docker_config_path()} so Docker keeps the token in that file, then run "
+        "`arcsecond token set` again."
+    )
+
+
 def prompt_for_token() -> str:
     return click.prompt(
         "Access token from Arcsecond (nothing is shown while you type)", hide_input=True

{arcsecond-4.2.1 → arcsecond-4.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "arcsecond"
-version = "4.2.1"
+version = "4.3.1"
 authors = [
     { name = "Cedric Foellmi", email = "cedric@arcsecond.io" },
 ]

{arcsecond-4.2.1 → arcsecond-4.3.1}/tests/test_hosting_composedoc.py

@@ -19,6 +19,7 @@ def test_every_service_of_the_template_is_read():
         "broker",
         "backend",
         "worker",
+        "dataworker",
         "beat",
         "platesolver",
         "web",

{arcsecond-4.2.1 → arcsecond-4.3.1}/tests/test_hosting_database.py

@@ -328,3 +328,20 @@ def test_failed_recreate_reports_but_keeps_the_new_password(tmp_path, monkeypatc
     # The rotation itself succeeded, so the file must keep the new value.
     assert database._read_env_value("POSTGRES_PASSWORD", env_path) == cluster.password
     assert "password was changed successfully" in result.output
+
+
+def test_the_data_worker_is_recreated_only_where_the_compose_file_has_it(tmp_path, monkeypatch):
+    """docker-compose.yml 7.3 added it; naming it on an older install would
+    make `docker compose up` fail outright."""
+    monkeypatch.chdir(tmp_path)
+    compose = Path(tmp_path) / "docker-compose.yml"
+
+    compose.write_text("services:\n  worker:\n    container_name: arcsecond-worker\n", encoding="utf-8")
+    assert database._services_to_recreate() == ["backend", "worker", "beat"]
+
+    compose.write_text(
+        "services:\n  dataworker:\n    container_name: arcsecond-dataworker\n"
+        "  # >>> arcsecond:alerts\n  alerts:\n  # <<< arcsecond:alerts\n",
+        encoding="utf-8",
+    )
+    assert database._services_to_recreate() == ["backend", "worker", "beat", "dataworker", "alerts"]

{arcsecond-4.2.1 → arcsecond-4.3.1}/tests/test_hosting_token.py

@@ -65,6 +65,42 @@ def test_a_refused_token_is_explained_without_docker_words(docker):
     assert "registry" not in result.output.lower()
 
 
+def _login_fails_with(monkeypatch, stderr):
+    def fake_run(cmd, input=None, **kwargs):
+        return subprocess.CompletedProcess(cmd, 1, stdout="", stderr=stderr)
+
+    monkeypatch.setattr(token.subprocess, "run", fake_run)
+
+
+def test_an_uninitialised_pass_store_is_not_a_refused_token(docker, monkeypatch):
+    _login_fails_with(
+        monkeypatch,
+        "Error saving credentials: error storing credentials - err: exit status 1, "
+        'out: `pass not initialized: exit status 1: Error: password store is empty. Try "pass init".`',
+    )
+    result = _invoke("token", "set", input="ghp_good\n")
+    assert result.exit_code == 1
+    assert (
+        "token is good" in result.output
+        and "refused" not in result.output.split("\n")[0]
+    )
+    assert "pass init" in result.output and "gpg --generate-key" in result.output
+    assert "team@arcsecond.io" not in result.output
+
+
+def test_any_other_credential_store_failure_names_the_config(
+    docker, monkeypatch, tmp_path
+):
+    _login_fails_with(
+        monkeypatch,
+        "Error saving credentials: error storing credentials - err: exit status 1, out: `keychain locked`",
+    )
+    result = _invoke("token", "set", input="ghp_good\n")
+    assert result.exit_code == 1
+    assert "token is good" in result.output and "keychain locked" in result.output
+    assert "credsStore" in result.output and "gpg" not in result.output
+
+
 def test_chevrons_and_empty_input_never_reach_docker(docker):
     assert "placeholder" in _invoke("token", "set", input="<ghp_good>\n").output
     assert _invoke("token", "set", "--stdin", input="\n").exit_code == 1

{arcsecond-4.2.1 → arcsecond-4.3.1}/uv.lock

@@ -155,7 +155,7 @@ wheels = [
 
 [[package]]
 name = "arcsecond"
-version = "4.2.1"
+version = "4.3.1"
 source = { editable = "." }
 dependencies = [
     { name = "alpyca" },