bids-validator-deno 2.1.1__tar.gz → 2.2.1__tar.gz

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bids-validator-deno
-Version: 2.1.1
+Version: 2.2.1
 Summary: Typescript implementation of the BIDS validator
 Keywords: BIDS,BIDS validator
 Author: bids-standard developers

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@bids/validator",
-  "version": "2.1.1",
+  "version": "2.2.1",
   "exports": {
     ".": "./src/bids-validator.ts",
     "./main": "./src/main.ts",
@@ -31,7 +31,7 @@
     "@bids/schema": "jsr:@bids/schema@~1.1.0",
     "@cliffy/command": "jsr:@effigies/cliffy-command@1.0.0-dev.8",
     "@cliffy/table": "jsr:@effigies/cliffy-table@1.0.0-dev.5",
-    "@hed/validator": "npm:hed-validator@~4.0.1",
+    "@hed/validator": "npm:hed-validator@~4.1.4",
     "@ignore": "npm:ignore@^7.0.5",
     "@libs/xml": "jsr:@libs/xml@^6.0.8",
     "@mango/nifti": "npm:@bids/nifti-reader-js@^0.6.9",
@@ -42,7 +42,9 @@
     "@std/log": "jsr:@std/log@^0.224.14",
     "@std/path": "jsr:@std/path@^1.1.2",
     "@std/streams": "jsr:@std/streams@^1.0.12",
-    "@std/yaml": "jsr:@std/yaml@^1.0.9"
+    "@std/yaml": "jsr:@std/yaml@^1.0.9",
+    "isomorphic-git": "npm:isomorphic-git@^1.27.1",
+    "hash-wasm": "npm:hash-wasm@^4.12.0"
   },
   "tasks": {
     "test": "deno test -A src/"

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/pyproject.toml

@@ -18,7 +18,7 @@ keywords = [
     "BIDS validator",
 ]
 dynamic = []
-version = "2.1.1"
+version = "2.2.1"
 
 [project.license]
 text = "MIT"

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/src/files/access.ts

@@ -10,12 +10,12 @@ function IOErrorToIssue(err: { code: string; name: string }): Issue {
   return { code: 'FILE_READ', subCode: err.name, issueMessage }
 }
 
-export function openStream(file: BIDSFile): ReadableStream<Uint8Array<ArrayBuffer>> {
-  try {
-    return file.stream
-  } catch (err: any) {
+export async function openStream(
+  file: BIDSFile,
+): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+  return file.stream().catch((err: any) => {
     throw { location: file.path, ...IOErrorToIssue(err) }
-  }
+  })
 }
 
 export async function readBytes(

bids_validator_deno-2.2.1/src/files/browser.ts

@@ -0,0 +1,34 @@
+import { BIDSFile, FileTree } from '../types/filetree.ts'
+import { filesToTree } from './filetree.ts'
+import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
+import { BrowserFileOpener } from './openers.ts'
+
+/**
+ * Browser implement of BIDSFile wrapping native File/FileList types
+ */
+export class BIDSFileBrowser extends BIDSFile {
+  constructor(file: File, ignore: FileIgnoreRules, parent?: FileTree) {
+    const relativePath = file.webkitRelativePath
+    const prefixLength = relativePath.indexOf('/')
+    const opener = new BrowserFileOpener(file)
+    super(relativePath.substring(prefixLength), opener, ignore, parent)
+  }
+}
+
+/**
+ * Convert from FileList (created with webkitDirectory: true) to FileTree for validator use
+ */
+export async function fileListToTree(files: File[]): Promise<FileTree> {
+  const ignore = new FileIgnoreRules([])
+  const root = new FileTree('/', '/', undefined)
+  const tree = filesToTree(files.map((f) => new BIDSFileBrowser(f, ignore, root)), ignore)
+  const bidsignore = tree.get('.bidsignore')
+  if (bidsignore) {
+    try {
+      ignore.add(await readBidsIgnore(bidsignore as BIDSFile))
+    } catch (err) {
+      console.log(`Failed to read '.bidsignore' file with the following error:\n${err}`)
+    }
+  }
+  return tree
+}

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/src/files/deno.test.ts

@@ -31,7 +31,7 @@ Deno.test('Deno implementation of BIDSFile', async (t) => {
   })
   await t.step('can be read as ReadableStream', async () => {
     const file = new BIDSFileDeno(testDir, testFilename, ignore)
-    const stream = file.stream
+    const stream = await file.stream()
     const streamReader = stream.getReader()
     const denoReader = readerFromStreamReader(streamReader)
     const fileBuffer = await readAll(denoReader)

bids_validator_deno-2.2.1/src/files/deno.ts

@@ -0,0 +1,99 @@
+/**
+ * Deno specific implementation for reading files
+ */
+import { basename, join } from '@std/path'
+import * as posix from '@std/path/posix'
+import { BIDSFile, type FileOpener, FileTree } from '../types/filetree.ts'
+import { requestReadPermission } from '../setup/requestPermissions.ts'
+import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
+import { FsFileOpener, HTTPOpener } from './openers.ts'
+import { resolveAnnexedFile } from './repo.ts'
+import fs from 'node:fs'
+
+export class BIDSFileDeno extends BIDSFile {
+  constructor(datasetPath: string, path: string, ignore?: FileIgnoreRules, parent?: FileTree) {
+    super(path, new FsFileOpener(datasetPath, path), ignore, parent)
+  }
+}
+
+type ReadFileTreeOptions = {
+  rootPath: string
+  relativePath: string
+  ignore: FileIgnoreRules
+  prune: FileIgnoreRules
+  parent?: FileTree
+  preferredRemote?: string
+}
+
+async function _readFileTree({
+  rootPath,
+  relativePath,
+  ignore,
+  prune,
+  parent,
+  preferredRemote,
+}: ReadFileTreeOptions): Promise<FileTree> {
+  await requestReadPermission()
+  const name = basename(relativePath)
+  const tree = new FileTree(relativePath, name, parent, ignore)
+
+  // Opaque cache for passing to git operations
+  const cache = {}
+
+  for await (const dirEntry of Deno.readDir(join(rootPath, relativePath))) {
+    const thisPath = posix.join(relativePath, dirEntry.name)
+    if (prune.test(thisPath)) {
+      continue
+    }
+    if (dirEntry.isFile || dirEntry.isSymlink) {
+      let opener: FileOpener
+      const fullPath = join(rootPath, thisPath)
+      try {
+        const fileInfo = await Deno.stat(fullPath)
+        opener = new FsFileOpener(rootPath, thisPath, fileInfo)
+      } catch (error) {
+        try {
+          const { url, size } = await resolveAnnexedFile(fullPath, preferredRemote, { cache, fs })
+          opener = new HTTPOpener(url, size)
+        } catch (_) {
+          throw error
+        }
+      }
+      tree.files.push(new BIDSFile(thisPath, opener, ignore, tree))
+    }
+    if (dirEntry.isDirectory) {
+      const dirTree = await _readFileTree({
+        rootPath,
+        relativePath: thisPath,
+        ignore,
+        prune,
+        parent: tree,
+        preferredRemote,
+      })
+      tree.directories.push(dirTree)
+    }
+  }
+  return tree
+}
+
+/**
+ * Read in the target directory structure and return a FileTree
+ */
+export async function readFileTree(
+  rootPath: string,
+  prune?: FileIgnoreRules,
+  preferredRemote?: string,
+): Promise<FileTree> {
+  prune ??= new FileIgnoreRules([], false)
+  const ignore = new FileIgnoreRules([])
+  const tree = await _readFileTree({ rootPath, relativePath: '/', ignore, prune, preferredRemote })
+  const bidsignore = tree.get('.bidsignore')
+  if (bidsignore) {
+    try {
+      ignore.add(await readBidsIgnore(bidsignore as BIDSFile))
+    } catch (err) {
+      console.log(`Failed to read '.bidsignore' file with the following error:\n${err}`)
+    }
+  }
+  return tree
+}

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/src/files/filetree.test.ts

@@ -1,7 +1,18 @@
 import { assert, assertEquals } from '@std/assert'
-import type { FileIgnoreRules } from './ignore.ts'
-import type { BIDSFile, FileTree } from '../types/filetree.ts'
-import { filesToTree, pathsToTree } from './filetree.ts'
+import { FileIgnoreRules } from './ignore.ts'
+import { BIDSFile, type FileOpener, type FileTree } from '../types/filetree.ts'
+import { filesToTree } from './filetree.ts'
+import { StringOpener } from './openers.test.ts'
+
+export function pathToFile(path: string, ignored: boolean = false): BIDSFile {
+  const name = path.split('/').pop() as string
+  return new BIDSFile(path, new StringOpener(''), ignored)
+}
+
+export function pathsToTree(paths: string[], ignore?: string[]): FileTree {
+  const ignoreRules = new FileIgnoreRules(ignore ?? [])
+  return filesToTree(paths.map((path) => pathToFile(path, ignoreRules.test(path))))
+}
 
 Deno.test('FileTree generation', async (t) => {
   await t.step('converts a basic list', async () => {

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/src/files/filetree.ts

@@ -1,31 +1,8 @@
 import { parse, SEPARATOR_PATTERN } from '@std/path'
 import * as posix from '@std/path/posix'
-import { type BIDSFile, FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { FileIgnoreRules } from './ignore.ts'
 
-const nullFile = {
-  size: 0,
-  stream: new ReadableStream({
-    start(controller) {
-      controller.close()
-    },
-  }),
-  text: () => Promise.resolve(''),
-  readBytes: async (size: number, offset?: number) => new Uint8Array(),
-  parent: new FileTree('', '/'),
-  viewed: false,
-}
-
-export function pathToFile(path: string, ignored: boolean = false): BIDSFile {
-  const name = path.split('/').pop() as string
-  return { name, path, ignored, ...nullFile }
-}
-
-export function pathsToTree(paths: string[], ignore?: string[]): FileTree {
-  const ignoreRules = new FileIgnoreRules(ignore ?? [])
-  return filesToTree(paths.map((path) => pathToFile(path, ignoreRules.test(path))))
-}
-
 export function filesToTree(fileList: BIDSFile[], ignore?: FileIgnoreRules): FileTree {
   ignore = ignore ?? new FileIgnoreRules([])
   const tree: FileTree = new FileTree('/', '/')

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/src/files/inheritance.test.ts

@@ -1,6 +1,6 @@
 import { assert, assertEquals, assertThrows } from '@std/assert'
 import type { BIDSFile } from '../types/filetree.ts'
-import { pathsToTree } from './filetree.ts'
+import { pathsToTree } from './filetree.test.ts'
 import { walkBack } from './inheritance.ts'
 
 Deno.test('walkback inheritance tests', async (t) => {

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/src/files/inheritance.ts

@@ -1,5 +1,6 @@
 import type { BIDSFile, FileTree } from '../types/filetree.ts'
 import { readEntities } from '../schema/entities.ts'
+import { loadJSON } from './json.ts'
 
 type Ret<T> = T extends [string, ...string[]] ? (BIDSFile | BIDSFile[]) : BIDSFile
 
@@ -77,3 +78,20 @@ export function* walkBack<T extends string[]>(
     fileTree = fileTree.parent
   }
 }
+
+export async function readSidecars(
+  source: BIDSFile,
+): Promise<Map<string, Record<string, unknown>>> {
+  const ret: Map<string, Record<string, unknown>> = new Map()
+  for (const file of walkBack(source)) {
+    try {
+      ret.set(file.path, await loadJSON(file))
+    } catch (e: any) {
+      // Expect JSON parsing errors to be handled when the file is loaded directly
+      if (!e?.code) {
+        throw e
+      }
+    }
+  }
+  return ret
+}

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.1}/src/files/json.test.ts

@@ -3,7 +3,7 @@ import type { BIDSFile } from '../types/filetree.ts'
 import type { FileIgnoreRules } from './ignore.ts'
 import { testAsyncFileAccess } from './access.test.ts'
 
-import { pathsToTree } from '../files/filetree.ts'
+import { pathsToTree } from '../files/filetree.test.ts'
 import { loadJSON } from './json.ts'
 
 function encodeUTF16(text: string) {

bids_validator_deno-2.2.1/src/files/openers.test.ts

@@ -0,0 +1,154 @@
+/**
+ * File openers for use in testing
+ */
+import { assertEquals } from '@std/assert'
+import { type FileOpener } from '../types/filetree.ts'
+import { streamFromString } from '../tests/utils.ts'
+import { createUTF8Stream } from './streams.ts'
+
+const textEncoder = new TextEncoder()
+const textDecoder = new TextDecoder()
+
+export class BytesOpener implements FileOpener {
+  contents: Uint8Array<ArrayBuffer>
+  size: number
+
+  constructor(contents: Uint8Array<ArrayBuffer>) {
+    this.contents = contents
+    this.size = contents.length
+  }
+
+  async text(): Promise<string> {
+    return textDecoder.decode(this.contents)
+  }
+
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    return this.contents.slice(offset, offset + size)
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    const contents = this.contents
+    return new ReadableStream({
+      start(controller) {
+        controller.enqueue(contents)
+        controller.close()
+      },
+    })
+  }
+}
+
+export class StringOpener extends BytesOpener {
+  constructor(contents: string) {
+    super(textEncoder.encode(contents))
+  }
+}
+
+export class StreamOpener implements FileOpener {
+  #stream: ReadableStream<Uint8Array<ArrayBuffer>>
+  size: number
+
+  constructor(stream: ReadableStream<Uint8Array<ArrayBuffer>>, size: number) {
+    this.#stream = stream
+    this.size = size
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    const tee = this.#stream.tee()
+    this.#stream = tee[1]
+    return tee[0]
+  }
+
+  async text(): Promise<string> {
+    const stream = await this.stream()
+    const reader = stream.pipeThrough(createUTF8Stream()).getReader()
+    const chunks: string[] = []
+    try {
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) break
+        chunks.push(value)
+      }
+      return chunks.join('')
+    } finally {
+      reader.releaseLock()
+    }
+  }
+
+  // Not a thoroughly tested implementation
+  // Do not move this out of test files without adding more substantial tests
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    const stream = await this.stream()
+    const reader = stream.getReader()
+    const result = new Uint8Array(size)
+    let pos = 0
+    try {
+      while (pos < offset + size) {
+        const { done, value } = await reader.read()
+        if (done) break
+        if (pos + value.length > offset) {
+          result.set(
+            value.subarray(Math.max(0, offset - pos), Math.min(value.length, offset + size - pos)),
+            Math.max(0, pos - offset),
+          )
+        }
+        pos += value.length
+      }
+    } finally {
+      reader.releaseLock()
+    }
+    return result.subarray(0, Math.min(size, pos - offset))
+  }
+}
+
+export class CompressedStringOpener extends StreamOpener {
+  constructor(contents: string) {
+    // Use uncompressed length as size, for simplicity
+    super(streamFromString(contents).pipeThrough(new CompressionStream('gzip')), contents.length)
+  }
+}
+
+async function testOpener(t: Deno.TestContext, opener: FileOpener) {
+  await t.step('size', async () => {
+    assertEquals(opener.size, 13)
+  })
+  await t.step('text()', async () => {
+    assertEquals(await opener.text(), 'Hello, world!')
+  })
+  await t.step('readBytes()', async () => {
+    assertEquals(textDecoder.decode(await opener.readBytes(5)), 'Hello')
+    assertEquals(textDecoder.decode(await opener.readBytes(7, 5)), ', world')
+  })
+  await t.step('stream()', async () => {
+    const stream = await opener.stream()
+    const chunks: string[] = []
+    for await (const chunk of stream) {
+      chunks.push(textDecoder.decode(chunk))
+    }
+    assertEquals(chunks.join(''), 'Hello, world!')
+  })
+}
+
+Deno.test('Validate BytesOpener', async (t) => {
+  await testOpener(t, new BytesOpener(textEncoder.encode('Hello, world!')))
+})
+
+Deno.test('Validate StringOpener', async (t) => {
+  await testOpener(t, new StringOpener('Hello, world!'))
+})
+
+Deno.test('Validate StreamOpener', async (t) => {
+  await testOpener(t, new StreamOpener(streamFromString('Hello, world!'), 13))
+})
+
+Deno.test('Validate CompressedStringOpener', async (t) => {
+  const opener = new CompressedStringOpener('Hello, world!')
+  await t.step('Decompress', async () => {
+    const stream = await opener.stream()
+    const decompressedStream = stream.pipeThrough(new DecompressionStream('gzip'))
+    const chunks: string[] = []
+    for await (const chunk of decompressedStream) {
+      chunks.push(textDecoder.decode(chunk))
+    }
+    assertEquals(chunks.join(''), 'Hello, world!')
+  })
+})

bids_validator_deno-2.2.1/src/files/openers.ts

@@ -0,0 +1,128 @@
+/**
+ * BIDS file openers
+ *
+ * These classes implement stream, text and random bytes access to BIDS resources.
+ */
+import { join } from '@std/path'
+import { type FileOpener } from '../types/filetree.ts'
+import { createUTF8Stream } from './streams.ts'
+
+export class FsFileOpener implements FileOpener {
+  path: string
+  fileInfo!: Deno.FileInfo
+
+  constructor(datasetPath: string, path: string, fileInfo?: Deno.FileInfo) {
+    this.path = join(datasetPath, path)
+    if (fileInfo) {
+      this.fileInfo = fileInfo
+    } else {
+      try {
+        this.fileInfo = Deno.statSync(this.path)
+      } catch (error) {
+        if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
+          this.fileInfo = Deno.lstatSync(this.path)
+        }
+      }
+    }
+  }
+
+  get size(): number {
+    return this.fileInfo.size
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    const handle = await this.open()
+    return handle.readable
+  }
+
+  /**
+   * Read the entire file and decode as utf-8 text
+   */
+  async text(): Promise<string> {
+    const stream = await this.stream()
+    const reader = stream.pipeThrough(createUTF8Stream()).getReader()
+    const chunks: string[] = []
+    try {
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) break
+        chunks.push(value)
+      }
+      return chunks.join('')
+    } finally {
+      reader.releaseLock()
+    }
+  }
+
+  /**
+   * Read bytes in a range efficiently from a given file
+   *
+   * Reads up to size bytes, starting at offset.
+   * If EOF is encountered, the resulting array may be smaller.
+   */
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    const handle = await this.open()
+    const buf = new Uint8Array(size)
+    await handle.seek(offset, Deno.SeekMode.Start)
+    const nbytes = await handle.read(buf) ?? 0
+    await handle.close()
+    return buf.subarray(0, nbytes)
+  }
+
+  async open(): Promise<Deno.FsFile> {
+    return Deno.open(this.path, { read: true, write: false })
+  }
+}
+
+export class BrowserFileOpener implements FileOpener {
+  file: File
+  constructor(file: File) {
+    this.file = file
+  }
+
+  get size(): number {
+    return this.file.size
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    return Promise.resolve(this.file.stream() as ReadableStream<Uint8Array<ArrayBuffer>>)
+  }
+
+  async text(): Promise<string> {
+    return this.file.text()
+  }
+
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    return new Uint8Array(await this.file.slice(offset, size).arrayBuffer())
+  }
+}
+
+export class HTTPOpener implements FileOpener {
+  url: string
+  size: number
+
+  constructor(url: string, size: number = -1) {
+    this.url = url
+    this.size = size
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    const response = await fetch(this.url)
+    if (!response.ok || !response.body) {
+      throw new Error(`Failed to fetch ${this.url}: ${response.status} ${response.statusText}`)
+    }
+    return response.body
+  }
+
+  async text(): Promise<string> {
+    const response = await fetch(this.url)
+    return response.text()
+  }
+
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    const headers = new Headers()
+    headers.append('Range', `bytes=${offset}-${offset + size - 1}`)
+    const response = await fetch(this.url, { headers })
+    return new Uint8Array(await response.arrayBuffer())
+  }
+}