arcsecond 4.2.1__tar.gz → 4.3.0__tar.gz

{arcsecond-4.2.1 → arcsecond-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: arcsecond
-Version: 4.2.1
+Version: 4.3.0
 Summary: CLI for arcsecond.io
 Project-URL: Homepage, https://github.com/arcsecond-io/cli
 Project-URL: Issues, https://github.com/arcsecond-io/cli/issues

{arcsecond-4.2.1 → arcsecond-4.3.0}/arcsecond/hosting/backups.py

@@ -21,6 +21,9 @@ from .utils import _read_env_value as _read_env_value_from
 DB_CONTAINER = "arcsecond-db"
 API_CONTAINER = "arcsecond-api"
 WORKER_CONTAINER = "arcsecond-worker"
+# Since docker-compose.yml 7.3; absent on older installs, which the stop loop
+# and the docker-start fallback both tolerate, as for alerts below.
+DATA_WORKER_CONTAINER = "arcsecond-dataworker"
 BEAT_CONTAINER = "arcsecond-beat"
 WEB_CONTAINER = "arcsecond-web"
 # Optional service: absent on most installs. The stop loop skips it when it
@@ -31,6 +34,7 @@ ALERTS_CONTAINER = "arcsecond-alerts"
 SERVICES_TO_STOP = [
     API_CONTAINER,
     WORKER_CONTAINER,
+    DATA_WORKER_CONTAINER,
     ALERTS_CONTAINER,
     BEAT_CONTAINER,
     WEB_CONTAINER,

{arcsecond-4.2.1 → arcsecond-4.3.0}/arcsecond/hosting/composedoc.py

@@ -251,6 +251,37 @@ ENV_KEYS: Dict[str, tuple] = {
         "How long a new member may use the installation before verifying their email, "
         "on an installation with no mail server.",
     ),
+    "ARCSECOND_DOWNLOADS_PER_CLIENT": (
+        OPERATOR,
+        "How many downloads one computer may have open at once; past it, the next one is refused with a 429 "
+        "until one finishes. Default 6. Then `arcsecond restart web`.",
+    ),
+    "ARCSECOND_DOWNLOADS_TOTAL": (
+        OPERATOR,
+        "How many downloads the installation serves at once, all computers together. Default 24.",
+    ),
+    "ARCSECOND_DOWNLOAD_RATE": (
+        OPERATOR,
+        "The speed of each download, in bytes per second, `k` and `m` allowed; `0` for no limit. Default `20m`. "
+        "Lower it when the observers work remotely over a link the downloads would otherwise fill.",
+    ),
+    "ARCSECOND_BACKEND_MEMORY": (
+        OPERATOR,
+        "The most memory the backend container may use, e.g. `6g` (the default). Past it, Docker restarts the "
+        "busiest of its processes rather than letting the machine take memory from the database.",
+    ),
+    "ARCSECOND_WORKER_MEMORY": (
+        OPERATOR,
+        "The same ceiling for the worker, which runs the exposures and procedures. Default `6g`.",
+    ),
+    "ARCSECOND_DATA_WORKER_MEMORY": (
+        OPERATOR,
+        "The same ceiling for the data worker: zip archives, previews, storage pushes, backups. Default `3g`.",
+    ),
+    "ARCSECOND_DATA_WORKER_CONCURRENCY": (
+        OPERATOR,
+        "How many jobs the data worker runs at once. Default 2; each can hold a few hundred MB.",
+    ),
 }
 
 

{arcsecond-4.2.1 → arcsecond-4.3.0}/arcsecond/hosting/database.py

@@ -44,16 +44,20 @@ SERVICES_TO_RECREATE = ["backend", "worker", "beat"]
 
 
 def _services_to_recreate():
-    """The optional alerts consumer joins the recreate list only when the
-    operator's compose file actually carries it — naming an unknown service
-    makes `docker compose up` fail outright on installs without the block."""
+    """The optional alerts consumer, and the data worker of docker-compose.yml
+    7.3 and later, join the recreate list only when the operator's compose
+    file actually carries them — naming an unknown service makes
+    `docker compose up` fail outright on installs without them."""
     services = list(SERVICES_TO_RECREATE)
     compose_path = Path.cwd() / "docker-compose.yml"
     try:
-        if "# >>> arcsecond:alerts" in compose_path.read_text(encoding="utf-8"):
-            services.append("alerts")
+        text = compose_path.read_text(encoding="utf-8")
     except OSError:
-        pass
+        return services
+    if "container_name: arcsecond-dataworker" in text:
+        services.append("dataworker")
+    if "# >>> arcsecond:alerts" in text:
+        services.append("alerts")
     return services
 
 

{arcsecond-4.2.1 → arcsecond-4.3.0}/arcsecond/hosting/docker/docker-compose.yml

@@ -1,6 +1,6 @@
 # === Arcsecond.local config. ===
 # *** DO NOT change any name or port number. ***
-# Version 6.4
+# Version 7.3
 
 services:
   # Database (PostgresQL)
@@ -54,6 +54,10 @@ services:
         bind:
           create_host_path: true
     stop_grace_period: 60s
+    # A ceiling, so no burst of work here can take the memory Postgres needs.
+    # Downloads no longer pass through these processes (the web container
+    # sends them), but rendering a large frame for review still does.
+    mem_limit: ${ARCSECOND_BACKEND_MEMORY:-6g}
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:8800/healthcheck/" ]
       interval: 10s
@@ -61,7 +65,8 @@ services:
       retries: 12
       start_period: 120s
 
-  # Arcsecond worker, for offloading background work.
+  # Arcsecond worker, for offloading background work: the Control Room's
+  # exposures and procedures, and everything else someone is waiting on.
   worker:
     image: ghcr.io/arcsecond-io/arcsecond-api:latest
     container_name: arcsecond-worker
@@ -72,6 +77,9 @@ services:
         condition: service_healthy
     env_file:
       - .env
+    environment:
+      # The default queue only: the data worker below takes the rest.
+      ARCSECOND_WORKER_QUEUES: celery
     volumes:
       # Leave /data as is, it's a path inside the container, not in the host machine.
       # SHARED_DATA_PATH must be a path of your host machine, specified in the .env file.
@@ -81,6 +89,36 @@ services:
         bind:
           create_host_path: true
     stop_grace_period: 120s
+    mem_limit: ${ARCSECOND_WORKER_MEMORY:-6g}
+
+  # Arcsecond data worker, for the background work nobody at the telescope is
+  # waiting on: zip archives, previews of uploaded files, pushes to your
+  # storages, database backups, keograms, catalogue refreshes. On its own
+  # queue, so a long zip can never hold up a frame the observer is waiting for.
+  dataworker:
+    image: ghcr.io/arcsecond-io/arcsecond-api:latest
+    container_name: arcsecond-dataworker
+    restart: unless-stopped
+    entrypoint: /home/app/entrypoint-worker.sh
+    depends_on:
+      backend:
+        condition: service_healthy
+    env_file:
+      - .env
+    environment:
+      ARCSECOND_WORKER_QUEUES: data
+      # Few at a time: this work is heavy and none of it is urgent.
+      ARCSECOND_WORKER_CONCURRENCY: ${ARCSECOND_DATA_WORKER_CONCURRENCY:-2}
+    volumes:
+      # Leave /data as is, it's a path inside the container, not in the host machine.
+      # SHARED_DATA_PATH must be a path of your host machine, specified in the .env file.
+      - type: bind
+        source: ${SHARED_DATA_PATH}
+        target: /data
+        bind:
+          create_host_path: true
+    stop_grace_period: 120s
+    mem_limit: ${ARCSECOND_DATA_WORKER_MEMORY:-3g}
 
   # Arcsecond beat, to launch specific background tasks at specific times.
   beat:
@@ -114,6 +152,23 @@ services:
         condition: service_started
     ports:
       - "5555:5555"
+    environment:
+      # Download limits: how many downloads one computer may have open at
+      # once, how many in total, and the speed of each (bytes per second,
+      # k and m allowed, 0 for none). They keep downloads from filling the
+      # link a remote observer drives the telescope through.
+      ARCSECOND_DOWNLOADS_PER_CLIENT: ${ARCSECOND_DOWNLOADS_PER_CLIENT:-6}
+      ARCSECOND_DOWNLOADS_TOTAL: ${ARCSECOND_DOWNLOADS_TOTAL:-24}
+      ARCSECOND_DOWNLOAD_RATE: ${ARCSECOND_DOWNLOAD_RATE:-20m}
+    volumes:
+      # The same folder, read-only: this container sends the stored files
+      # itself, once the backend has said who may read them.
+      - type: bind
+        source: ${SHARED_DATA_PATH}
+        target: /data
+        read_only: true
+        bind:
+          create_host_path: true
 
   # >>> arcsecond:alerts
   # Arcsecond transient-alerts consumer (optional).

{arcsecond-4.2.1 → arcsecond-4.3.0}/arcsecond/hosting/lifecycle.py

@@ -272,7 +272,7 @@ def status(directory):
 @basic_options
 def logs(directory, service, follow, tail):
     """Show recent logs. Services are named as in `arcsecond status`:
-    backend, worker, beat, web, db, broker, platesolver, alerts."""
+    backend, worker, dataworker, beat, web, db, broker, platesolver, alerts."""
     install = stack.resolve_install_dir(directory)
     stack.ensure_docker()
     args = ["logs", "--tail", str(tail)]

{arcsecond-4.2.1 → arcsecond-4.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "arcsecond"
-version = "4.2.1"
+version = "4.3.0"
 authors = [
     { name = "Cedric Foellmi", email = "cedric@arcsecond.io" },
 ]

{arcsecond-4.2.1 → arcsecond-4.3.0}/tests/test_hosting_composedoc.py

@@ -19,6 +19,7 @@ def test_every_service_of_the_template_is_read():
         "broker",
         "backend",
         "worker",
+        "dataworker",
         "beat",
         "platesolver",
         "web",

{arcsecond-4.2.1 → arcsecond-4.3.0}/tests/test_hosting_database.py

@@ -328,3 +328,20 @@ def test_failed_recreate_reports_but_keeps_the_new_password(tmp_path, monkeypatc
     # The rotation itself succeeded, so the file must keep the new value.
     assert database._read_env_value("POSTGRES_PASSWORD", env_path) == cluster.password
     assert "password was changed successfully" in result.output
+
+
+def test_the_data_worker_is_recreated_only_where_the_compose_file_has_it(tmp_path, monkeypatch):
+    """docker-compose.yml 7.3 added it; naming it on an older install would
+    make `docker compose up` fail outright."""
+    monkeypatch.chdir(tmp_path)
+    compose = Path(tmp_path) / "docker-compose.yml"
+
+    compose.write_text("services:\n  worker:\n    container_name: arcsecond-worker\n", encoding="utf-8")
+    assert database._services_to_recreate() == ["backend", "worker", "beat"]
+
+    compose.write_text(
+        "services:\n  dataworker:\n    container_name: arcsecond-dataworker\n"
+        "  # >>> arcsecond:alerts\n  alerts:\n  # <<< arcsecond:alerts\n",
+        encoding="utf-8",
+    )
+    assert database._services_to_recreate() == ["backend", "worker", "beat", "dataworker", "alerts"]

{arcsecond-4.2.1 → arcsecond-4.3.0}/uv.lock

@@ -155,7 +155,7 @@ wheels = [
 
 [[package]]
 name = "arcsecond"
-version = "4.2.1"
+version = "4.3.0"
 source = { editable = "." }
 dependencies = [
     { name = "alpyca" },