girder 5.0.22.dev2__tar.gz → 5.0.23.dev7__tar.gz

girder-5.0.23.dev7/.circleci/check_npm_package.js

@@ -0,0 +1,153 @@
+#!/usr/bin/env node
+// Check that a packed npm tarball is usable: every entry point that
+// package.json declares is in the tarball, every "files" entry matched
+// something, and the relative references inside packed CSS (url(...)) and
+// JavaScript (sourceMappingURL) resolve to packed files, and no runtime
+// dependency uses a local specifier (file:, link:, workspace:) that cannot
+// resolve once installed from the registry.  Any further
+// arguments are paths that must also be in the package.
+//
+// Usage: check_npm_package.js <package.tgz> [required-path ...]
+'use strict';
+
+const childProcess = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const METADATA_FILES = new Set(['package.json', 'readme', 'readme.md', 'readme.rst', 'readme.txt', 'license', 'license.txt', 'license.md']);
+
+function listFiles(root) {
+  const result = [];
+  for (const entry of fs.readdirSync(root, {withFileTypes: true, recursive: true})) {
+    if (entry.isFile()) {
+      result.push(path.relative(root, path.join(entry.parentPath, entry.name)).split(path.sep).join('/'));
+    }
+  }
+  return result.sort();
+}
+
+function normalize(ref) {
+  return path.posix.normalize(ref.replace(/^\.\//, ''));
+}
+
+// Collect every path that package.json says a consumer can load.
+function entryPoints(pkg) {
+  const result = [];
+  for (const key of ['main', 'module', 'types', 'typings', 'style']) {
+    if (typeof pkg[key] === 'string') {
+      result.push([key, pkg[key]]);
+    }
+  }
+  if (typeof pkg.browser === 'string') {
+    result.push(['browser', pkg.browser]);
+  }
+  if (typeof pkg.bin === 'string') {
+    result.push(['bin', pkg.bin]);
+  } else if (pkg.bin) {
+    for (const [name, value] of Object.entries(pkg.bin)) {
+      result.push([`bin.${name}`, value]);
+    }
+  }
+  const walk = (value, where) => {
+    if (typeof value === 'string') {
+      if (!value.includes('*')) {
+        result.push([where, value]);
+      }
+    } else if (value && typeof value === 'object') {
+      for (const [key, child] of Object.entries(value)) {
+        walk(child, `${where}[${JSON.stringify(key)}]`);
+      }
+    }
+  };
+  walk(pkg.exports, 'exports');
+  return result;
+}
+
+function checkPackage(root, required) {
+  const errors = [];
+  const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
+  const files = listFiles(root);
+  const fileSet = new Set(files);
+
+  if (!files.some((file) => !METADATA_FILES.has(file.toLowerCase()))) {
+    errors.push('the package contains no files besides package.json, README, and LICENSE');
+  }
+
+  for (const [where, ref] of entryPoints(pkg)) {
+    if (!fileSet.has(normalize(ref))) {
+      errors.push(`${where} refers to ${ref}, which is not in the package`);
+    }
+  }
+
+  for (const field of ['dependencies', 'peerDependencies', 'optionalDependencies']) {
+    for (const [name, spec] of Object.entries(pkg[field] || {})) {
+      if (/^(file|link|workspace):/.test(spec)) {
+        errors.push(`${field} has ${name}@${spec}, which cannot be installed from the registry`);
+      }
+    }
+  }
+
+  for (const ref of required) {
+    if (!fileSet.has(normalize(ref))) {
+      errors.push(`${ref} is required but is not in the package`);
+    }
+  }
+
+  for (const entry of pkg.files || []) {
+    if (/[*?[\]{}!]/.test(entry)) {
+      continue;
+    }
+    const prefix = normalize(entry).replace(/\/$/, '');
+    if (!files.some((file) => file === prefix || file.startsWith(`${prefix}/`))) {
+      errors.push(`"files" lists ${entry}, but nothing in the package matches it`);
+    }
+  }
+
+  for (const file of files) {
+    const ext = path.extname(file);
+    let refs = [];
+    if (ext === '.css') {
+      const text = fs.readFileSync(path.join(root, file), 'utf8');
+      refs = [...text.matchAll(/url\(\s*(['"]?)([^'")]+)\1\s*\)/g)].map((match) => match[2]);
+    } else if (['.js', '.cjs', '.mjs'].includes(ext)) {
+      const text = fs.readFileSync(path.join(root, file), 'utf8');
+      refs = [...text.matchAll(/^\/\/# sourceMappingURL=(\S+)\s*$/gm)].map((match) => match[1]);
+    }
+    for (const ref of refs) {
+      if (/^(data:|[a-z][a-z0-9+.-]*:|\/|#)/i.test(ref)) {
+        continue;
+      }
+      const target = path.posix.normalize(path.posix.join(path.posix.dirname(file), ref.replace(/[?#].*$/, '')));
+      if (!fileSet.has(target)) {
+        errors.push(`${file} refers to ${ref}, but ${target} is not in the package`);
+      }
+    }
+  }
+  return {pkg, files, errors};
+}
+
+function main() {
+  const [tarball, ...required] = process.argv.slice(2);
+  if (!tarball) {
+    console.error('Usage: check_npm_package.js <package.tgz> [required-path ...]');
+    process.exit(2);
+  }
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'npm-package-check-'));
+  try {
+    childProcess.execFileSync('tar', ['-xzf', path.resolve(tarball), '-C', tmpDir]);
+    const {pkg, files, errors} = checkPackage(path.join(tmpDir, 'package'), required);
+    if (errors.length) {
+      console.error(`${pkg.name}@${pkg.version} is not usable:`);
+      for (const error of errors) {
+        console.error(`  ${error}`);
+      }
+      process.exit(1);
+    }
+    console.log(`${pkg.name}@${pkg.version}: ${files.length} files, entry points and references are valid`);
+  } finally {
+    fs.rmSync(tmpDir, {recursive: true, force: true});
+  }
+}
+
+main();

{girder-5.0.22.dev2 → girder-5.0.23.dev7}/.circleci/config.yml

@@ -23,6 +23,12 @@ executors:
     working_directory: /home/circleci/project
 
 commands:
+  install-ttfautohint:
+    description: "Install ttfautohint so the Fontello build produces a hinted font"
+    steps:
+      - run:
+          name: Install ttfautohint
+          command: sudo apt-get update && sudo apt-get install -y ttfautohint
   prerequisites:
     description: "Install prerequisites and run required background services"
     steps:
@@ -163,7 +169,10 @@ jobs:
       - build-plugins
       - run:
           name: Run pytest tests
-          command: tox -e pytest_circleci -- -x
+          command: |
+            # tox refuses to stage packages in group/world-writable directories
+            umask 0022
+            tox -e pytest_circleci -- -x
           working_directory: girder
       - store_test_results:
           path: girder/build/test/results
@@ -472,11 +481,23 @@ jobs:
           command: tox -e publish
           working_directory: girder
 
+  check-publish-npm:
+    executor: py3
+    steps:
+      - checkout:
+          path: girder
+      - install-ttfautohint
+      - run:
+          name: Check that npm packages can be built and packed
+          command: ./.circleci/publish_npm.sh --dry-run
+          working_directory: girder
+
   publish-release:
     executor: py3
     steps:
       - checkout:
           path: girder
+      - install-ttfautohint
       - run:
           name: Publish npm packages
           command: npm run publish-all
@@ -554,6 +575,10 @@ workflows:
           filters:
             tags:
               only: /^v[0-9]+.*/
+      - check-publish-npm:
+          filters:
+            tags:
+              only: /^v[0-9]+.*/
       - publish:
           requires:
             - check-publish
@@ -570,6 +595,7 @@ workflows:
       - publish-release:
           requires:
             - check-publish
+            - check-publish-npm
             - server-lint-test
             - web-lint-test
             - coverage

girder-5.0.23.dev7/.circleci/publish_npm.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+set -e
+# Publish npm packages for selected locations.
+# Every package is installed, versioned, packed (its prepack script builds
+# it), and checked before any is published, and the checked tarballs are what
+# get published.  Publishing authenticates through CircleCI OIDC.  With
+# --dry-run, everything runs except the upload.
+
+dry_run=()
+if [[ "$1" == "--dry-run" ]]; then
+  dry_run=(--dry-run --tag dry-run)
+fi
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+export npm_config_prefix="$HOME/.npm-global"
+npm install -g 'npm@^11.5.1'
+export PATH="$npm_config_prefix/bin:$PATH"
+hash -r
+npm --version
+
+GIT_VERSION=$(git describe --tags)
+readonly GIT_VERSION
+readonly PUBLISHED_NPM_PACKAGES=(
+  # Published Fontello is used by all builds, and is critical to publish
+  girder/web/fontello
+  # The raw JS source is used by some downstream 'external builds'
+  girder/web
+  # These plugins were published to support downstream external builds, and should be kept updated
+  plugins/jobs/girder_jobs/web_client
+  plugins/oauth/girder_oauth/web_client
+  plugins/gravatar/girder_gravatar/web_client
+)
+
+# Files a package must contain beyond what its package.json declares
+required_files() {
+  case "$1" in
+    girder/web)
+      # The full source tree for downstream external builds
+      echo dist-lib/src/index.ts dist-lib/src/views/App.js
+      ;;
+  esac
+}
+
+pack_root=$(mktemp -d)
+trap 'rm -rf "$pack_root"' EXIT
+tarballs=()
+for directory in "${PUBLISHED_NPM_PACKAGES[@]}"; do
+  pushd "$directory"
+  npm ci
+  npm version --allow-same-version --no-git-tag-version "$GIT_VERSION"
+  pack_dir="$pack_root/${directory//\//_}"
+  mkdir "$pack_dir"
+  npm pack --pack-destination "$pack_dir"
+  tarball=("$pack_dir"/*.tgz)
+  # shellcheck disable=SC2046
+  node "$script_dir/check_npm_package.js" "${tarball[0]}" $(required_files "$directory")
+  tarballs+=("${tarball[0]}")
+  popd
+done
+
+for tarball in "${tarballs[@]}"; do
+  if [[ ${#dry_run[@]} -eq 0 ]]; then
+    NPM_ID_TOKEN=$(circleci run oidc get --claims '{"aud": "npm:registry.npmjs.org"}')
+    export NPM_ID_TOKEN
+  fi
+  npm publish "$tarball" --loglevel warn --access public "${dry_run[@]}"
+done

{girder-5.0.22.dev2/girder.egg-info → girder-5.0.23.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: girder
-Version: 5.0.22.dev2
+Version: 5.0.23.dev7
 Summary: Web-based data management platform
 Home-page: https://girder.readthedocs.org
 Author: Kitware, Inc.

{girder-5.0.22.dev2 → girder-5.0.23.dev7}/docs/plugins.rst

@@ -402,6 +402,51 @@ instead. The potential error values are:
   current authentication policy.
 
 
+Local Checkout
+--------------
+`PyPI package <https://pypi.org/project/girder-local-checkout/>`__: ``girder-local-checkout``
+
+Data on an S3 assetstore is slower to access than data on a filesystem assetstore.  This plugin
+lets users with write access to a folder temporarily "checkout" the folder: every file in the
+folder (recursively) whose data lives on an S3 assetstore is copied to a filesystem assetstore for
+a chosen number of days, and Girder serves the local copy while the checkout lasts.  Files added to
+the folder after a checkout are not copied until the checkout is renewed.
+
+The folder actions menu gains a **Make files local** item.  Choosing it shows how many files can
+be made local, how many already are, and, for a checked-out folder, the days remaining.  The
+duration can be lengthened or shortened, or the checkout released; copying and releasing happen in
+background tasks on the ``local`` worker queue, so a worker must be available.
+
+Each checkout is tracked per folder.  A file keeps its local copy until the latest expiration of
+every checkout that covers it, so checking out a parent folder for a short time never shortens a
+longer checkout of a subfolder.  When a file's local copy is released, the file once again refers
+to its S3 data; deleting or replacing a checked-out file removes both the local copy and the S3
+data.  An S3 assetstore cannot be deleted while a checked-out file's original data is on it.
+
+Expired checkouts are cleaned up daily, before each new checkout, and on demand via
+``POST /local_checkout/cleanup``.  Free space on the filesystem assetstore is computed with the
+``GIRDER_FILESYSTEMASSETSTORE_RESERVED_MBYTES`` reservation applied.  If a checkout needs more space
+than is free, local copies that expire sooner than the new checkout are released first; if there
+is still not enough room, the checkout is left partial and says so.  Likewise, an upload to the
+filesystem assetstore that would not otherwise fit releases local copies, soonest-expiring first,
+rather than failing.
+
+Administrators can list all checkouts, the files and space they use, and each assetstore's free
+space with ``GET /local_checkout``.  The plugin's configuration page shows the same summary and
+has these settings:
+
+* **Assetstore for local copies**: the filesystem assetstore that receives the copies.  By default
+  the current default assetstore is used.  If that is not a filesystem assetstore, the user picks
+  one of the filesystem assetstores when checking out a folder (via the ``assetstoreId``
+  parameter of ``PUT /folder/{id}/local_checkout``, which may be omitted when there is only one),
+  and the feature is unavailable only if there are no filesystem assetstores.
+* **Maximum checkout duration**: the most days a checkout may last (default 90).
+* **Only administrators may checkout folders**: restrict the feature to site administrators;
+  other users do not see the folder action.
+
+This plugin requires the Jobs plugin and currently only copies files from S3 assetstores.
+
+
 README
 ------
 `PyPI package <https://pypi.org/project/girder-readme/>`__: ``girder-readme``