bids-validator-deno 2.1.1__tar.gz → 2.2.0__tar.gz

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bids-validator-deno
-Version: 2.1.1
+Version: 2.2.0
 Summary: Typescript implementation of the BIDS validator
 Keywords: BIDS,BIDS validator
 Author: bids-standard developers

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@bids/validator",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "exports": {
     ".": "./src/bids-validator.ts",
     "./main": "./src/main.ts",
@@ -31,7 +31,7 @@
     "@bids/schema": "jsr:@bids/schema@~1.1.0",
     "@cliffy/command": "jsr:@effigies/cliffy-command@1.0.0-dev.8",
     "@cliffy/table": "jsr:@effigies/cliffy-table@1.0.0-dev.5",
-    "@hed/validator": "npm:hed-validator@~4.0.1",
+    "@hed/validator": "npm:hed-validator@~4.1.4",
     "@ignore": "npm:ignore@^7.0.5",
     "@libs/xml": "jsr:@libs/xml@^6.0.8",
     "@mango/nifti": "npm:@bids/nifti-reader-js@^0.6.9",
@@ -42,7 +42,9 @@
     "@std/log": "jsr:@std/log@^0.224.14",
     "@std/path": "jsr:@std/path@^1.1.2",
     "@std/streams": "jsr:@std/streams@^1.0.12",
-    "@std/yaml": "jsr:@std/yaml@^1.0.9"
+    "@std/yaml": "jsr:@std/yaml@^1.0.9",
+    "isomorphic-git": "npm:isomorphic-git@^1.27.1",
+    "hash-wasm": "npm:hash-wasm@^4.12.0"
   },
   "tasks": {
     "test": "deno test -A src/"

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/pyproject.toml

@@ -18,7 +18,7 @@ keywords = [
     "BIDS validator",
 ]
 dynamic = []
-version = "2.1.1"
+version = "2.2.0"
 
 [project.license]
 text = "MIT"

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/src/files/access.ts

@@ -10,12 +10,12 @@ function IOErrorToIssue(err: { code: string; name: string }): Issue {
   return { code: 'FILE_READ', subCode: err.name, issueMessage }
 }
 
-export function openStream(file: BIDSFile): ReadableStream<Uint8Array<ArrayBuffer>> {
-  try {
-    return file.stream
-  } catch (err: any) {
+export async function openStream(
+  file: BIDSFile,
+): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+  return file.stream().catch((err: any) => {
     throw { location: file.path, ...IOErrorToIssue(err) }
-  }
+  })
 }
 
 export async function readBytes(

bids_validator_deno-2.2.0/src/files/browser.ts

@@ -0,0 +1,34 @@
+import { BIDSFile, FileTree } from '../types/filetree.ts'
+import { filesToTree } from './filetree.ts'
+import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
+import { BrowserFileOpener } from './openers.ts'
+
+/**
+ * Browser implement of BIDSFile wrapping native File/FileList types
+ */
+export class BIDSFileBrowser extends BIDSFile {
+  constructor(file: File, ignore: FileIgnoreRules, parent?: FileTree) {
+    const relativePath = file.webkitRelativePath
+    const prefixLength = relativePath.indexOf('/')
+    const opener = new BrowserFileOpener(file)
+    super(relativePath.substring(prefixLength), opener, ignore, parent)
+  }
+}
+
+/**
+ * Convert from FileList (created with webkitDirectory: true) to FileTree for validator use
+ */
+export async function fileListToTree(files: File[]): Promise<FileTree> {
+  const ignore = new FileIgnoreRules([])
+  const root = new FileTree('/', '/', undefined)
+  const tree = filesToTree(files.map((f) => new BIDSFileBrowser(f, ignore, root)), ignore)
+  const bidsignore = tree.get('.bidsignore')
+  if (bidsignore) {
+    try {
+      ignore.add(await readBidsIgnore(bidsignore as BIDSFile))
+    } catch (err) {
+      console.log(`Failed to read '.bidsignore' file with the following error:\n${err}`)
+    }
+  }
+  return tree
+}

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/src/files/deno.test.ts

@@ -31,7 +31,7 @@ Deno.test('Deno implementation of BIDSFile', async (t) => {
   })
   await t.step('can be read as ReadableStream', async () => {
     const file = new BIDSFileDeno(testDir, testFilename, ignore)
-    const stream = file.stream
+    const stream = await file.stream()
     const streamReader = stream.getReader()
     const denoReader = readerFromStreamReader(streamReader)
     const fileBuffer = await readAll(denoReader)

bids_validator_deno-2.2.0/src/files/deno.ts

@@ -0,0 +1,99 @@
+/**
+ * Deno specific implementation for reading files
+ */
+import { basename, join } from '@std/path'
+import * as posix from '@std/path/posix'
+import { BIDSFile, type FileOpener, FileTree } from '../types/filetree.ts'
+import { requestReadPermission } from '../setup/requestPermissions.ts'
+import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
+import { FsFileOpener, HTTPOpener } from './openers.ts'
+import { resolveAnnexedFile } from './repo.ts'
+import fs from 'node:fs'
+
+export class BIDSFileDeno extends BIDSFile {
+  constructor(datasetPath: string, path: string, ignore?: FileIgnoreRules, parent?: FileTree) {
+    super(path, new FsFileOpener(datasetPath, path), ignore, parent)
+  }
+}
+
+type ReadFileTreeOptions = {
+  rootPath: string
+  relativePath: string
+  ignore: FileIgnoreRules
+  prune: FileIgnoreRules
+  parent?: FileTree
+  preferredRemote?: string
+}
+
+async function _readFileTree({
+  rootPath,
+  relativePath,
+  ignore,
+  prune,
+  parent,
+  preferredRemote,
+}: ReadFileTreeOptions): Promise<FileTree> {
+  await requestReadPermission()
+  const name = basename(relativePath)
+  const tree = new FileTree(relativePath, name, parent, ignore)
+
+  // Opaque cache for passing to git operations
+  const cache = {}
+
+  for await (const dirEntry of Deno.readDir(join(rootPath, relativePath))) {
+    const thisPath = posix.join(relativePath, dirEntry.name)
+    if (prune.test(thisPath)) {
+      continue
+    }
+    if (dirEntry.isFile || dirEntry.isSymlink) {
+      let opener: FileOpener
+      const fullPath = join(rootPath, thisPath)
+      try {
+        const fileInfo = await Deno.stat(fullPath)
+        opener = new FsFileOpener(rootPath, thisPath, fileInfo)
+      } catch (error) {
+        try {
+          const { url, size } = await resolveAnnexedFile(fullPath, preferredRemote, { cache, fs })
+          opener = new HTTPOpener(url, size)
+        } catch (_) {
+          throw error
+        }
+      }
+      tree.files.push(new BIDSFile(thisPath, opener, ignore, tree))
+    }
+    if (dirEntry.isDirectory) {
+      const dirTree = await _readFileTree({
+        rootPath,
+        relativePath: thisPath,
+        ignore,
+        prune,
+        parent: tree,
+        preferredRemote,
+      })
+      tree.directories.push(dirTree)
+    }
+  }
+  return tree
+}
+
+/**
+ * Read in the target directory structure and return a FileTree
+ */
+export async function readFileTree(
+  rootPath: string,
+  prune?: FileIgnoreRules,
+  preferredRemote?: string,
+): Promise<FileTree> {
+  prune ??= new FileIgnoreRules([], false)
+  const ignore = new FileIgnoreRules([])
+  const tree = await _readFileTree({rootPath, relativePath: '/', ignore, prune, preferredRemote})
+  const bidsignore = tree.get('.bidsignore')
+  if (bidsignore) {
+    try {
+      ignore.add(await readBidsIgnore(bidsignore as BIDSFile))
+    } catch (err) {
+      console.log(`Failed to read '.bidsignore' file with the following error:\n${err}`)
+    }
+  }
+  return tree
+}

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/src/files/filetree.test.ts

@@ -1,7 +1,25 @@
 import { assert, assertEquals } from '@std/assert'
-import type { FileIgnoreRules } from './ignore.ts'
-import type { BIDSFile, FileTree } from '../types/filetree.ts'
-import { filesToTree, pathsToTree } from './filetree.ts'
+import { FileIgnoreRules } from './ignore.ts'
+import { BIDSFile, type FileOpener, type FileTree } from '../types/filetree.ts'
+import { filesToTree } from './filetree.ts'
+import { asyncStreamFromString } from '../tests/utils.ts'
+
+class NullFileOpener implements FileOpener {
+  size = 0
+  stream = () => asyncStreamFromString('')
+  text = () => Promise.resolve('')
+  readBytes = async (size: number, offset?: number) => new Uint8Array()
+}
+
+export function pathToFile(path: string, ignored: boolean = false): BIDSFile {
+  const name = path.split('/').pop() as string
+  return new BIDSFile(path, new NullFileOpener(), ignored)
+}
+
+export function pathsToTree(paths: string[], ignore?: string[]): FileTree {
+  const ignoreRules = new FileIgnoreRules(ignore ?? [])
+  return filesToTree(paths.map((path) => pathToFile(path, ignoreRules.test(path))))
+}
 
 Deno.test('FileTree generation', async (t) => {
   await t.step('converts a basic list', async () => {

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/src/files/filetree.ts

@@ -1,31 +1,8 @@
 import { parse, SEPARATOR_PATTERN } from '@std/path'
 import * as posix from '@std/path/posix'
-import { type BIDSFile, FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { FileIgnoreRules } from './ignore.ts'
 
-const nullFile = {
-  size: 0,
-  stream: new ReadableStream({
-    start(controller) {
-      controller.close()
-    },
-  }),
-  text: () => Promise.resolve(''),
-  readBytes: async (size: number, offset?: number) => new Uint8Array(),
-  parent: new FileTree('', '/'),
-  viewed: false,
-}
-
-export function pathToFile(path: string, ignored: boolean = false): BIDSFile {
-  const name = path.split('/').pop() as string
-  return { name, path, ignored, ...nullFile }
-}
-
-export function pathsToTree(paths: string[], ignore?: string[]): FileTree {
-  const ignoreRules = new FileIgnoreRules(ignore ?? [])
-  return filesToTree(paths.map((path) => pathToFile(path, ignoreRules.test(path))))
-}
-
 export function filesToTree(fileList: BIDSFile[], ignore?: FileIgnoreRules): FileTree {
   ignore = ignore ?? new FileIgnoreRules([])
   const tree: FileTree = new FileTree('/', '/')

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/src/files/inheritance.test.ts

@@ -1,6 +1,6 @@
 import { assert, assertEquals, assertThrows } from '@std/assert'
 import type { BIDSFile } from '../types/filetree.ts'
-import { pathsToTree } from './filetree.ts'
+import { pathsToTree } from './filetree.test.ts'
 import { walkBack } from './inheritance.ts'
 
 Deno.test('walkback inheritance tests', async (t) => {

{bids_validator_deno-2.1.1 → bids_validator_deno-2.2.0}/src/files/json.test.ts

@@ -3,7 +3,7 @@ import type { BIDSFile } from '../types/filetree.ts'
 import type { FileIgnoreRules } from './ignore.ts'
 import { testAsyncFileAccess } from './access.test.ts'
 
-import { pathsToTree } from '../files/filetree.ts'
+import { pathsToTree } from '../files/filetree.test.ts'
 import { loadJSON } from './json.ts'
 
 function encodeUTF16(text: string) {

bids_validator_deno-2.2.0/src/files/openers.ts

@@ -0,0 +1,128 @@
+/**
+ * BIDS file openers
+ *
+ * These classes implement stream, text and random bytes access to BIDS resources.
+ */
+import { join } from '@std/path'
+import { type FileOpener } from '../types/filetree.ts'
+import { createUTF8Stream } from './streams.ts'
+
+export class FsFileOpener implements FileOpener {
+  path: string
+  fileInfo!: Deno.FileInfo
+
+  constructor(datasetPath: string, path: string, fileInfo?: Deno.FileInfo) {
+    this.path = join(datasetPath, path)
+    if (fileInfo) {
+      this.fileInfo = fileInfo
+    } else {
+      try {
+        this.fileInfo = Deno.statSync(this.path)
+      } catch (error) {
+        if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
+          this.fileInfo = Deno.lstatSync(this.path)
+        }
+      }
+    }
+  }
+
+  get size(): number {
+    return this.fileInfo.size
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    const handle = await this.open()
+    return handle.readable
+  }
+
+  /**
+   * Read the entire file and decode as utf-8 text
+   */
+  async text(): Promise<string> {
+    const stream = await this.stream()
+    const reader = stream.pipeThrough(createUTF8Stream()).getReader()
+    const chunks: string[] = []
+    try {
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) break
+        chunks.push(value)
+      }
+      return chunks.join('')
+    } finally {
+      reader.releaseLock()
+    }
+  }
+
+  /**
+   * Read bytes in a range efficiently from a given file
+   *
+   * Reads up to size bytes, starting at offset.
+   * If EOF is encountered, the resulting array may be smaller.
+   */
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    const handle = await this.open()
+    const buf = new Uint8Array(size)
+    await handle.seek(offset, Deno.SeekMode.Start)
+    const nbytes = await handle.read(buf) ?? 0
+    await handle.close()
+    return buf.subarray(0, nbytes)
+  }
+
+  async open(): Promise<Deno.FsFile> {
+    return Deno.open(this.path, { read: true, write: false })
+  }
+}
+
+export class BrowserFileOpener implements FileOpener {
+  file: File
+  constructor(file: File) {
+    this.file = file
+  }
+
+  get size(): number {
+    return this.file.size
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    return Promise.resolve(this.file.stream() as ReadableStream<Uint8Array<ArrayBuffer>>)
+  }
+
+  async text(): Promise<string> {
+    return this.file.text()
+  }
+
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    return new Uint8Array(await this.file.slice(offset, size).arrayBuffer())
+  }
+}
+
+export class HTTPOpener implements FileOpener {
+  url: string
+  size: number
+
+  constructor(url: string, size: number = -1) {
+    this.url = url
+    this.size = size
+  }
+
+  async stream(): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+    const response = await fetch(this.url)
+    if (!response.ok || !response.body) {
+      throw new Error(`Failed to fetch ${this.url}: ${response.status} ${response.statusText}`)
+    }
+    return response.body
+  }
+
+  async text(): Promise<string> {
+    const response = await fetch(this.url)
+    return response.text()
+  }
+
+  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
+    const headers = new Headers()
+    headers.append('Range', `bytes=${offset}-${offset + size - 1}`)
+    const response = await fetch(this.url, { headers })
+    return new Uint8Array(await response.arrayBuffer())
+  }
+}

bids_validator_deno-2.2.0/src/files/repo.ts

@@ -0,0 +1,141 @@
+/**
+ * Utilities for reading git-annex metadata
+ */
+import { dirname, join, parse, SEPARATOR_PATTERN } from '@std/path'
+import { default as git } from 'isomorphic-git'
+import { createMD5 } from 'hash-wasm'
+
+const textDecoder = new TextDecoder('utf-8')
+export const annexKeyRegex =
+  /^(?<hashname>[A-Z0-9]+)-s(?<size>\d+)--(?<digest>[0-9a-fA-F]+)(?<ext>\.[\w\-. ]*)?/
+export const rmetLineRegex =
+  /^(?<timestamp>\d+(\.\d+)?)s (?<uuid>[0-9a-fA-F-]+):V \+(?<version>[^#]+)#(?<path>.+)/
+
+type Rmet = {
+  timestamp: number
+  uuid: string
+  version: string
+  path: string
+}
+
+export async function readAnnexPath(
+  filepath: string,
+  options: any,
+): Promise<string> {
+  const oid = await git.resolveRef({ ref: 'git-annex', ...options })
+  const { blob } = await git.readBlob({ oid, filepath, ...options })
+  return textDecoder.decode(blob)
+}
+
+/**
+ * git-annex hashDirLower implementation based on https://git-annex.branchable.com/internals/hashing/
+ * Compute the directory path from a git-annex filename
+ */
+export async function hashDirLower(annexKey: string): Promise<[string, string]> {
+  const computeMD5 = await createMD5()
+  computeMD5.init()
+  computeMD5.update(annexKey)
+  const digest = computeMD5.digest('hex')
+  return [digest.slice(0, 3), digest.slice(3, 6)]
+}
+
+/**
+ * Read remote metadata entries for a given annex key
+ *
+ * *.log.rmet
+ * Path: {md5(key)[0:3]}/{md5(key)[3:6]}/{key}.log.rmet
+ * Contents:
+ *   <timestamp> <uuid>:V +<version>#<path>
+ *
+ * The general form is <uuid>:<key> [+-]<value> and is an append-only log
+ * We may at some point care about doing this correctly.
+ */
+async function readRmet(key: string, options: any): Promise<Record<string, Rmet>> {
+  const hashDirs = await hashDirLower(key)
+  const rmet = await readAnnexPath(join(...hashDirs, `${key}.log.rmet`), options)
+  const ret: Record<string, Rmet> = {}
+  for (const line of rmet.split('\n')) {
+    const match = line.match(rmetLineRegex)
+    if (match) {
+      ret[match!.groups!.uuid] = match!.groups as unknown as Rmet
+    }
+  }
+  return ret
+}
+
+/**
+ * Read special remote configuration from remote.log
+ *
+ * remote.log
+ *  <uuid> [<key>=<value>]...
+ *  keys of interest:
+ *    name
+ *    type (S3)
+ *    publicurl
+ *    timestamp
+ */
+export async function readRemotes(options: any): Promise<Record<string, Record<string, string>>> {
+  const remotesText = await readAnnexPath('remote.log', options)
+  const byUUID: Record<string, Record<string, string>> = {}
+  for (const line of remotesText.split('\n')) {
+    const [uuid, ...keyvals] = line.split(' ')
+    byUUID[uuid] = Object.fromEntries(keyvals.map((kv) => kv.split('=')))
+  }
+  return byUUID
+}
+
+/**
+ * Resolve an annexed file location to an HTTP URL, if a public S3 remote is available
+ */
+export async function resolveAnnexedFile(
+  path: string,
+  remote?: string,
+  options?: any,
+): Promise<{ url: string; size: number }> {
+  // path is known to be a symlink
+  const target = await Deno.readLink(path)
+  const { dir, base } = parse(target)
+
+  if (!options?.gitdir) {
+    const dirs = dir.split(SEPARATOR_PATTERN)
+    const gitdir = join(dirname(path), ...dirs.slice(0, dirs.indexOf('.git') + 1))
+    options = { ...options, gitdir }
+  }
+
+  const size = +base.match(annexKeyRegex)?.groups?.size!
+
+  const rmet = await readRmet(base, options)
+  const remotes = await readRemotes(options)
+  let uuid: string
+  if (remote) {
+    let matching: string | undefined
+    for (const [u, r] of Object.entries(remotes)) {
+      // Only consider public S3 remotes.
+      // This will need to be expanded for other types of remotes in future
+      if (!r?.publicurl) {
+        continue
+      }
+      if (r.name === remote) {
+        matching = u
+        break
+      }
+    }
+    if (!matching) {
+      throw new Error(`No remote named ${remote}`)
+    }
+    uuid = matching
+  } else {
+    // Take the newest remote (reverse sort timestamps, take first)
+    uuid = Object.entries(rmet).toSorted((a, b) => +b[1].timestamp - +a[1].timestamp)[0][0]
+  }
+  const { publicurl } = remotes[uuid]
+
+  if (!publicurl) {
+    throw new Error(`No publicurl found for remote ${uuid}`)
+  }
+
+  const metadata = rmet[uuid]
+  const url = `${publicurl}/${metadata.path}?versionId=${metadata.version}`
+
+  return { url, size }
+}