bids-validator-deno 2.3.1.dev0__tar.gz → 2.3.2__tar.gz

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bids-validator-deno
-Version: 2.3.1.dev0
+Version: 2.3.2
 Summary: Typescript implementation of the BIDS validator
 Keywords: BIDS,BIDS validator
 Author: bids-standard developers

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@bids/validator",
-  "version": "2.3.1-dev",
+  "version": "2.3.2",
   "exports": {
     ".": "./src/bids-validator.ts",
     "./main": "./src/main.ts",
@@ -37,6 +37,7 @@
     "@libs/xml": "jsr:@libs/xml@^7.0.3",
     "@mango/nifti": "npm:@bids/nifti-reader-js@^0.6.9",
     "@std/assert": "jsr:@std/assert@^1.0.16",
+    "@std/async": "jsr:@std/async@^1.1.0",
     "@std/fmt": "jsr:@std/fmt@^1.0.8",
     "@std/fs": "jsr:@std/fs@^1.0.21",
     "@std/io": "jsr:@std/io@^0.225.2",

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/pyproject.toml

@@ -18,7 +18,7 @@ keywords = [
     "BIDS validator",
 ]
 dynamic = []
-version = "2.3.1-dev"
+version = "2.3.2"
 
 [project.license]
 text = "MIT"

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/src/files/access.ts

@@ -1,8 +1,9 @@
 import { type BIDSFile } from '../types/filetree.ts'
 import { type Issue } from '../types/issues.ts'
+import { filememoize } from '../utils/memoize.ts'
+import { logger } from '../utils/logger.ts'
 
 function IOErrorToIssue(err: { code: string; name: string }): Issue {
-  const subcode = err.name
   let issueMessage: string | undefined = undefined
   if (err.code === 'ENOENT' || err.code === 'ELOOP') {
     issueMessage = 'Possible dangling symbolic link'
@@ -14,22 +15,29 @@ export async function openStream(
   file: BIDSFile,
 ): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
   return file.stream().catch((err: any) => {
+    logger.debug(`Error opening stream from file ${file.path}: ${err}`)
     throw { location: file.path, ...IOErrorToIssue(err) }
   })
 }
 
-export async function readBytes(
+async function _readBytes(
   file: BIDSFile,
   size: number,
   offset = 0,
 ): Promise<Uint8Array<ArrayBuffer>> {
   return file.readBytes(size, offset).catch((err: any) => {
+    logger.debug(`Error reading bytes from file ${file.path}: ${err}`)
     throw { location: file.path, ...IOErrorToIssue(err) }
   })
 }
 
-export async function readText(file: BIDSFile): Promise<string> {
+export const readBytes = filememoize(_readBytes)
+
+async function _readText(file: BIDSFile): Promise<string> {
   return file.text().catch((err: any) => {
+    logger.debug(`Error reading text from file ${file.path}: ${err}`)
     throw { location: file.path, ...IOErrorToIssue(err) }
   })
 }
+
+export const readText = filememoize(_readText)

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/src/files/gzip.ts

@@ -18,7 +18,7 @@ import { readBytes } from './access.ts'
  */
 export async function parseGzip(
   file: BIDSFile,
-  maxBytes: number = 512,
+  maxBytes: number = 1024,
 ): Promise<Gzip | undefined> {
   const buf = await readBytes(file, maxBytes)
   const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength)

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/src/files/openers.ts

@@ -3,9 +3,11 @@
  *
  * These classes implement stream, text and random bytes access to BIDS resources.
  */
+import { retry } from '@std/async'
 import { join } from '@std/path'
 import { type FileOpener } from '../types/filetree.ts'
 import { createUTF8Stream } from './streams.ts'
+import { logger } from '../utils/logger.ts'
 
 export class FsFileOpener implements FileOpener {
   path: string
@@ -97,6 +99,15 @@ export class BrowserFileOpener implements FileOpener {
   }
 }
 
+class HttpError extends Error {
+  status: number
+
+  constructor(status: number, message: string) {
+    super(`HTTP Error ${status}: ${message}`)
+    this.status = status
+  }
+}
+
 export class HTTPOpener implements FileOpener {
   url: string
   size: number
@@ -106,28 +117,53 @@ export class HTTPOpener implements FileOpener {
     this.size = size
   }
 
-  async _fetch(headers = {}): Promise<Response> {
-    return fetch(this.url, { headers }).then((response) => {
+  async _fetch(options: RequestInit = {}): Promise<Response> {
+    // Fetch with retries, for transient errors
+    return retry(async () => {
+      const response = await fetch(this.url, options)
       if (!response.ok || !response.body) {
-        throw new Error(`Failed to fetch ${this.url}: ${response.status} ${response.statusText}`)
+        throw new HttpError(response.status, response.statusText)
       }
       return response
+    }, {
+      isRetriable: (error) => {
+        logger.info(`Failed to fetch ${this.url}: ${error}, retrying`)
+        return (
+          error instanceof TypeError ||
+          error instanceof HttpError && (error.status == 429 || error.status >= 500)
+        )
+      },
     })
   }
 
   async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    // Streams should not timeout
     return this._fetch().then((response) => response.body!)
   }
 
   async text(): Promise<string> {
-    return this._fetch().then((response) => response.text())
+    // Timeout after 5 seconds and retry; many connections can result in timeouts
+    return await retry(
+      () => this._fetch({ signal: AbortSignal.timeout(5000) }).then((response) => response.text()),
+      {
+        isRetriable: (error) => error instanceof DOMException && error.name === 'TimeoutError',
+      },
+    )
   }
 
   async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
     const headers = new Headers()
     headers.append('Range', `bytes=${offset}-${offset + size - 1}`)
-    const response = await this._fetch({ headers })
-    return new Uint8Array(await response.arrayBuffer())
+    // Timeout after 5 seconds and retry; many connections can result in timeouts
+    return await retry(
+      () =>
+        this._fetch({ headers, signal: AbortSignal.timeout(5000) }).then((response) =>
+          response.bytes()
+        ),
+      {
+        isRetriable: (error) => error instanceof DOMException && error.name === 'TimeoutError',
+      },
+    )
   }
 }
 

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/src/schema/context.ts

@@ -317,7 +317,7 @@ export class BIDSContext implements Context {
     if (!this.extension.endsWith('.gz')) {
       return
     }
-    this.gzip = await parseGzip(this.file, 512).catch((error) => {
+    this.gzip = await parseGzip(this.file, 1024).catch((error) => {
       logger.debug('Error parsing gzip header', error)
       return undefined
     })

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/src/schema/walk.ts

@@ -4,6 +4,7 @@ import type { DatasetIssues } from '../issues/datasetIssues.ts'
 import { NullFileOpener } from '../files/openers.ts'
 import { loadTSV } from '../files/tsv.ts'
 import { loadJSON } from '../files/json.ts'
+import { readBytes, readText } from '../files/access.ts'
 import { queuedAsyncIterator } from '../utils/queue.ts'
 
 function* quickWalk(dir: FileTree): Generator<BIDSFile> {
@@ -54,6 +55,8 @@ async function* _walkFileTree(
   yield () => {
     loadTSV.cache.delete(fileTree.path)
     loadJSON.cache.delete(fileTree.path)
+    readBytes.cache.delete(fileTree.path)
+    readText.cache.delete(fileTree.path)
   }
 }
 

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/src/utils/memoize.ts

@@ -17,11 +17,11 @@ export const memoize = <A extends any[], R>(
   return cached
 }
 
-export function filememoize<F extends FileLike, T>(
-  fn: (file: F, ...args: any[]) => T,
-): WithCache<(file: F, ...args: any[]) => T> {
-  const cache = new Map<string, Map<string, T>>()
-  const cached = function (this: any, file: F, ...args: any[]): T {
+export function filememoize<F extends FileLike, A extends any[], R>(
+  fn: (file: F, ...args: A) => R,
+): WithCache<(file: F, ...args: A) => R> {
+  const cache = new Map<string, Map<string, R>>()
+  const cached = function (this: any, file: F, ...args: A): R {
     let subcache = cache.get(file.parent.path)
     if (!subcache) {
       subcache = new Map()

{bids_validator_deno-2.3.1.dev0 → bids_validator_deno-2.3.2}/src/validators/bids.ts

@@ -107,6 +107,7 @@ export async function validate(
     return false
   })
 
+  logger.info('Performing file-level validation...')
   for await (const context of walkFileTree(dsContext, 20)) {
     if (
       dsContext.dataset_description.DatasetType == 'raw' &&
@@ -121,6 +122,7 @@ export async function validate(
     await summary.update(context)
   }
 
+  logger.info('Performing dataset-level validation...')
   for (const check of perDSChecks) {
     await check(schema as unknown as GenericSchema, dsContext)
   }