bids-validator-deno 2.0.11__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of bids-validator-deno might be problematic. Click here for more details.

Files changed (106) hide show
  1. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/PKG-INFO +1 -1
  2. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/deno.json +2 -2
  3. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/pyproject.toml +1 -1
  4. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/bids-validator.ts +1 -0
  5. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/nifti.test.ts +25 -2
  6. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/nifti.ts +83 -0
  7. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/tsv.test.ts +111 -2
  8. bids_validator_deno-2.1.0/src/files/tsv.ts +101 -0
  9. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/issues/list.ts +11 -8
  10. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/applyRules.ts +1 -1
  11. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/context.test.ts +7 -1
  12. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/context.ts +61 -40
  13. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/tables.test.ts +21 -2
  14. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/tables.ts +19 -5
  15. bids_validator_deno-2.1.0/src/tests/local/nifti_rules.test.ts +111 -0
  16. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/bids.ts +1 -0
  17. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/filenameIdentify.test.ts +2 -2
  18. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/internal/unusedFile.ts +9 -5
  19. bids_validator_deno-2.0.11/src/files/tsv.ts +0 -68
  20. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/LICENSE +0 -0
  21. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/README.md +0 -0
  22. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/pdm_build.py +0 -0
  23. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/.git-meta.json +0 -0
  24. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/browser.test.ts +0 -0
  25. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/browser.ts +0 -0
  26. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/deno.test.ts +0 -0
  27. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/deno.ts +0 -0
  28. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/dwi.test.ts +0 -0
  29. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/dwi.ts +0 -0
  30. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/filetree.test.ts +0 -0
  31. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/filetree.ts +0 -0
  32. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/gzip.test.ts +0 -0
  33. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/gzip.ts +0 -0
  34. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/ignore.test.ts +0 -0
  35. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/ignore.ts +0 -0
  36. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/inheritance.test.ts +0 -0
  37. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/inheritance.ts +0 -0
  38. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/json.test.ts +0 -0
  39. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/json.ts +0 -0
  40. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/streams.test.ts +0 -0
  41. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/streams.ts +0 -0
  42. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/tiff.test.ts +0 -0
  43. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/files/tiff.ts +0 -0
  44. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/issues/datasetIssues.test.ts +0 -0
  45. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/issues/datasetIssues.ts +0 -0
  46. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/issues/list.test.ts +0 -0
  47. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/main.ts +0 -0
  48. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/applyRules.test.ts +0 -0
  49. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/associations.ts +0 -0
  50. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/entities.test.ts +0 -0
  51. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/entities.ts +0 -0
  52. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/expressionLanguage.test.ts +0 -0
  53. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/expressionLanguage.ts +0 -0
  54. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/fixtures.test.ts +0 -0
  55. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/modalities.ts +0 -0
  56. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/walk.test.ts +0 -0
  57. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/schema/walk.ts +0 -0
  58. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/setup/loadSchema.test.ts +0 -0
  59. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/setup/loadSchema.ts +0 -0
  60. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/setup/options.test.ts +0 -0
  61. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/setup/options.ts +0 -0
  62. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/setup/requestPermissions.ts +0 -0
  63. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/summary/collectSubjectMetadata.ts +0 -0
  64. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/summary/summary.test.ts +0 -0
  65. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/summary/summary.ts +0 -0
  66. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/README.md +0 -0
  67. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/bom-utf16.tsv +0 -0
  68. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/bom-utf8.json +0 -0
  69. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/generate-filenames.ts +0 -0
  70. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/bids_examples.test.ts +0 -0
  71. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/common.ts +0 -0
  72. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/derivatives.test.ts +0 -0
  73. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/empty_files.test.ts +0 -0
  74. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/hed-integration.test.ts +0 -0
  75. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/valid_dataset.test.ts +0 -0
  76. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/valid_filenames.test.ts +0 -0
  77. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/local/valid_headers.test.ts +0 -0
  78. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/nullReadBytes.ts +0 -0
  79. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/regression.test.ts +0 -0
  80. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/schema-expression-language.test.ts +0 -0
  81. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/simple-dataset.ts +0 -0
  82. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/tests/utils.ts +0 -0
  83. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/types/check.ts +0 -0
  84. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/types/columns.test.ts +0 -0
  85. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/types/columns.ts +0 -0
  86. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/types/filetree.ts +0 -0
  87. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/types/issues.ts +0 -0
  88. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/types/schema.ts +0 -0
  89. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/types/validation-result.ts +0 -0
  90. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/utils/errors.ts +0 -0
  91. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/utils/logger.test.ts +0 -0
  92. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/utils/logger.ts +0 -0
  93. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/utils/memoize.ts +0 -0
  94. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/utils/objectPathHandler.ts +0 -0
  95. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/utils/output.ts +0 -0
  96. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/bids.test.ts +0 -0
  97. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/citation.test.ts +0 -0
  98. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/citation.ts +0 -0
  99. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/filenameIdentify.ts +0 -0
  100. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/filenameValidate.test.ts +0 -0
  101. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/filenameValidate.ts +0 -0
  102. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/hed.ts +0 -0
  103. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/internal/emptyFile.ts +0 -0
  104. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/json.ts +0 -0
  105. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/validators/validateFiles.test.ts +0 -0
  106. {bids_validator_deno-2.0.11 → bids_validator_deno-2.1.0}/src/version.ts +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: bids-validator-deno
3
- Version: 2.0.11
3
+ Version: 2.1.0
4
4
  Summary: Typescript implementation of the BIDS validator
5
5
  Keywords: BIDS,BIDS validator
6
6
  Author: bids-standard developers
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bids/validator",
3
- "version": "2.0.11",
3
+ "version": "2.1.0",
4
4
  "exports": {
5
5
  ".": "./src/bids-validator.ts",
6
6
  "./main": "./src/main.ts",
@@ -28,7 +28,7 @@
28
28
  },
29
29
  "imports": {
30
30
  "@ajv": "npm:ajv@8.17.1",
31
- "@bids/schema": "jsr:@bids/schema@~1.0.14",
31
+ "@bids/schema": "jsr:@bids/schema@~1.1.0",
32
32
  "@cliffy/command": "jsr:@effigies/cliffy-command@1.0.0-dev.8",
33
33
  "@cliffy/table": "jsr:@effigies/cliffy-table@1.0.0-dev.5",
34
34
  "@hed/validator": "npm:hed-validator@4.0.1",
@@ -18,7 +18,7 @@ keywords = [
18
18
  "BIDS validator",
19
19
  ]
20
20
  dynamic = []
21
- version = "2.0.11"
21
+ version = "2.1.0"
22
22
 
23
23
  [project.license]
24
24
  text = "MIT"
@@ -6,3 +6,4 @@ const errors = result.issues.get({ severity: 'error' })
6
6
  if (errors.length) {
7
7
  Deno.exit(16)
8
8
  }
9
+ Deno.exit(0)
@@ -1,8 +1,8 @@
1
- import { assert, assertObjectMatch } from '@std/assert'
1
+ import { assert, assertEquals, assertObjectMatch } from '@std/assert'
2
2
  import { FileIgnoreRules } from './ignore.ts'
3
3
  import { BIDSFileDeno } from './deno.ts'
4
4
 
5
- import { loadHeader } from './nifti.ts'
5
+ import { loadHeader, axisCodes } from './nifti.ts'
6
6
 
7
7
  Deno.test('Test loading nifti header', async (t) => {
8
8
  const ignore = new FileIgnoreRules([])
@@ -73,3 +73,26 @@ Deno.test('Test loading nifti header', async (t) => {
73
73
  })
74
74
  })
75
75
  })
76
+
77
+ Deno.test('Test extracting axis codes', async (t) => {
78
+ await t.step('Identify RAS', async () => {
79
+ const affine = [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]
80
+ assertEquals(axisCodes(affine), ['R', 'A', 'S'])
81
+ })
82
+ await t.step('Identify LPS (flips)', async () => {
83
+ const affine = [[-1, 0, 0, 0], [0, -1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]
84
+ assertEquals(axisCodes(affine), ['L', 'P', 'S'])
85
+ })
86
+ await t.step('Identify SPL (flips + swap)', async () => {
87
+ const affine = [[0, 0, -1, 0], [0, -1, 0, 0], [1, 0, 0, 0], [0, 0, 0, 1]]
88
+ assertEquals(axisCodes(affine), ['S', 'P', 'L'])
89
+ })
90
+ await t.step('Identify SLP (flips + rotate)', async () => {
91
+ const affine = [[0, -1, 0, 0], [0, 0, -1, 0], [1, 0, 0, 0], [0, 0, 0, 1]]
92
+ assertEquals(axisCodes(affine), ['S', 'L', 'P'])
93
+ })
94
+ await t.step('Identify ASR (rotate)', async () => {
95
+ const affine = [[0, 0, 1, 0], [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1]]
96
+ assertEquals(axisCodes(affine), ['A', 'S', 'R'])
97
+ })
98
+ })
@@ -65,8 +65,91 @@ export async function loadHeader(file: BIDSFile): Promise<NiftiHeader> {
65
65
  },
66
66
  qform_code: header.qform_code,
67
67
  sform_code: header.sform_code,
68
+ axis_codes: axisCodes(header.affine),
68
69
  } as NiftiHeader
69
70
  } catch (err) {
70
71
  throw { code: 'NIFTI_HEADER_UNREADABLE' }
71
72
  }
72
73
  }
74
+
75
+ /** Vector addition */
76
+ function add(a: number[], b: number[]): number[] {
77
+ return a.map((x, i) => x + b[i])
78
+ }
79
+
80
+ /** Vector subtraction */
81
+ function sub(a: number[], b: number[]): number[] {
82
+ return a.map((x, i) => x - b[i])
83
+ }
84
+
85
+ /** Scalar multiplication */
86
+ function scale(vec: number[], scalar: number): number[] {
87
+ return vec.map((x) => x * scalar)
88
+ }
89
+
90
+ /** Dot product */
91
+ function dot(a: number[], b: number[]): number {
92
+ return a.map((x, i) => x * b[i]).reduce((acc, x) => acc + x, 0)
93
+ }
94
+
95
+ function argMax(arr: number[]): number {
96
+ return arr.reduce((acc, x, i) => (x > arr[acc] ? i : acc), 0)
97
+ }
98
+
99
+ /**
100
+ * Identify the nearest principle axes of an image affine.
101
+ *
102
+ * Affines transform indices in a data array into mm right, anterior and superior of
103
+ * an origin in "world coordinates". If moving along an axis in the positive direction
104
+ * predominantly moves right, that axis is labeled "R".
105
+ *
106
+ * @example The identity matrix is in "RAS" orientation:
107
+ *
108
+ * # Usage
109
+ *
110
+ * ```ts
111
+ * const affine = [[1, 0, 0, 0],
112
+ * [0, 1, 0, 0],
113
+ * [0, 0, 1, 0],
114
+ * [0, 0, 0, 1]]
115
+ *
116
+ * axisCodes(affine)
117
+ * ```
118
+ *
119
+ * # Result
120
+ * ```ts
121
+ * ['R', 'A', 'S']
122
+ * ```
123
+ *
124
+ * @returns character codes describing the orientation of an image affine.
125
+ */
126
+ export function axisCodes(affine: number[][]): string[] {
127
+ // This function is an extract of the Python function transforms3d.affines.decompose44
128
+ // (https://github.com/matthew-brett/transforms3d/blob/6a43a98/transforms3d/affines.py#L10-L153)
129
+ //
130
+ // As an optimization, this only orthogonalizes the basis,
131
+ // and does not normalize to unit vectors.
132
+
133
+ // Operate on columns, which are the cosines that project input coordinates onto output axes
134
+ const [cosX, cosY, cosZ] = [0, 1, 2].map((j) => [0, 1, 2].map((i) => affine[i][j]))
135
+
136
+ // Orthogonalize cosY with respect to cosX
137
+ const orthY = sub(cosY, scale(cosX, dot(cosX, cosY)))
138
+
139
+ // Orthogonalize cosZ with respect to cosX and orthY
140
+ const orthZ = sub(
141
+ cosZ, add(scale(cosX, dot(cosX, cosZ)), scale(orthY, dot(orthY, cosZ)))
142
+ )
143
+
144
+ const basis = [cosX, orthY, orthZ]
145
+ const maxIndices = basis.map((row) => argMax(row.map(Math.abs)))
146
+
147
+ // Check that indices are 0, 1 and 2 in some order
148
+ if (maxIndices.toSorted().some((idx, i) => idx !== i)) {
149
+ throw { key: 'AMBIGUOUS_AFFINE' }
150
+ }
151
+
152
+ // Positive/negative codes for each world axis
153
+ const codes = ['RL', 'AP', 'SI']
154
+ return maxIndices.map((idx, i) => codes[idx][basis[i][idx] > 0 ? 0 : 1])
155
+ }
@@ -6,7 +6,7 @@ import {
6
6
  assertStrictEquals,
7
7
  } from '@std/assert'
8
8
  import { pathToFile } from './filetree.ts'
9
- import { loadTSV } from './tsv.ts'
9
+ import { loadTSV, loadTSVGZ } from './tsv.ts'
10
10
  import { streamFromString } from '../tests/utils.ts'
11
11
  import { ColumnsMap } from '../types/columns.ts'
12
12
 
@@ -64,7 +64,7 @@ Deno.test('TSV loading', async (t) => {
64
64
  try {
65
65
  await loadTSV(file)
66
66
  } catch (e: any) {
67
- assertObjectMatch(e, { code: 'TSV_EQUAL_ROWS', location: '/mismatched_row.tsv', line: 3 })
67
+ assertObjectMatch(e, { code: 'TSV_EQUAL_ROWS', line: 3 })
68
68
  }
69
69
  })
70
70
 
@@ -178,3 +178,112 @@ Deno.test('TSV loading', async (t) => {
178
178
  // Tests will have populated the memoization cache
179
179
  loadTSV.cache.clear()
180
180
  })
181
+
182
+ Deno.test('TSVGZ loading', async (t) => {
183
+ await t.step('No header and empty file produces empty map', async () => {
184
+ const file = pathToFile('/empty.tsv.gz')
185
+ file.stream = streamFromString('').pipeThrough(new CompressionStream('gzip'))
186
+
187
+ const map = await loadTSVGZ(file, [])
188
+ // map.size looks for a column called map, so work around it
189
+ assertEquals(Object.keys(map).length, 0)
190
+ })
191
+
192
+ await t.step('Empty file produces header-only map', async () => {
193
+ const file = pathToFile('/empty.tsv.gz')
194
+ file.stream = streamFromString('').pipeThrough(new CompressionStream('gzip'))
195
+
196
+ const map = await loadTSVGZ(file, ['a', 'b', 'c'])
197
+ assertEquals(map.a, [])
198
+ assertEquals(map.b, [])
199
+ assertEquals(map.c, [])
200
+ })
201
+
202
+ await t.step('Single column file produces single column maps', async () => {
203
+ const file = pathToFile('/single_column.tsv')
204
+ file.stream = streamFromString('1\n2\n3\n').pipeThrough(new CompressionStream('gzip'))
205
+
206
+ const map = await loadTSVGZ(file, ['a'])
207
+ assertEquals(map.a, ['1', '2', '3'])
208
+ })
209
+
210
+ await t.step('Mismatched header length throws issue', async () => {
211
+ const file = pathToFile('/single_column.tsv.gz')
212
+ file.stream = streamFromString('1\n2\n3\n').pipeThrough(new CompressionStream('gzip'))
213
+
214
+ try {
215
+ await loadTSVGZ(file, ['a', 'b'])
216
+ } catch (e: any) {
217
+ assertObjectMatch(e, { code: 'TSV_EQUAL_ROWS', line: 1 })
218
+ }
219
+ })
220
+
221
+ await t.step('Missing final newline is ignored', async () => {
222
+ const file = pathToFile('/missing_newline.tsv.gz')
223
+ file.stream = streamFromString('1\n2\n3').pipeThrough(new CompressionStream('gzip'))
224
+
225
+ const map = await loadTSVGZ(file, ['a'])
226
+ assertEquals(map.a, ['1', '2', '3'])
227
+ })
228
+
229
+ await t.step('Empty row throws issue', async () => {
230
+ const file = pathToFile('/empty_row.tsv.gz')
231
+ file.stream = streamFromString('1\t2\t3\n\n4\t5\t6\n').pipeThrough(new CompressionStream('gzip'))
232
+
233
+ try {
234
+ await loadTSVGZ(file, ['a', 'b', 'c'])
235
+ } catch (e: any) {
236
+ assertObjectMatch(e, { code: 'TSV_EMPTY_LINE', line: 2 })
237
+ }
238
+ })
239
+
240
+ await t.step('Mislabeled TSV throws issue', async () => {
241
+ const file = pathToFile('/mismatched_row.tsv.gz')
242
+ file.stream = streamFromString('a\tb\tc\n1\t2\t3\n4\t5\n')
243
+
244
+ try {
245
+ await loadTSVGZ(file, ['a', 'b', 'c'])
246
+ } catch (e: any) {
247
+ assertObjectMatch(e, { code: 'INVALID_GZIP' })
248
+ }
249
+ })
250
+
251
+ await t.step('maxRows limits the number of rows read', async () => {
252
+ const file = pathToFile('/long.tsv.gz')
253
+ // Use 1500 to avoid overlap with default initial capacity
254
+ const headers = ['a', 'b', 'c']
255
+ const text = '1\t2\t3\n'.repeat(1500)
256
+ file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
257
+
258
+ let map = await loadTSVGZ(file, headers, 0)
259
+ assertEquals(map.a, [])
260
+ assertEquals(map.b, [])
261
+ assertEquals(map.c, [])
262
+
263
+ file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
264
+ map = await loadTSVGZ(file, headers, 1)
265
+ assertEquals(map.a, ['1'])
266
+ assertEquals(map.b, ['2'])
267
+ assertEquals(map.c, ['3'])
268
+
269
+ file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
270
+ map = await loadTSVGZ(file, headers, 2)
271
+ assertEquals(map.a, ['1', '1'])
272
+ assertEquals(map.b, ['2', '2'])
273
+ assertEquals(map.c, ['3', '3'])
274
+
275
+ file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
276
+ map = await loadTSVGZ(file, headers, -1)
277
+ assertEquals(map.a, Array(1500).fill('1'))
278
+ assertEquals(map.b, Array(1500).fill('2'))
279
+ assertEquals(map.c, Array(1500).fill('3'))
280
+
281
+ // Check that maxRows does not truncate shorter files
282
+ file.stream = streamFromString('1\t2\t3\n4\t5\t6\n7\t8\t9\n').pipeThrough(new CompressionStream('gzip'))
283
+ map = await loadTSVGZ(file, headers, 4)
284
+ assertEquals(map.a, ['1', '4', '7'])
285
+ assertEquals(map.b, ['2', '5', '8'])
286
+ assertEquals(map.c, ['3', '6', '9'])
287
+ })
288
+
289
+ })
@@ -0,0 +1,101 @@
1
+ /*
2
+ * TSV
3
+ * Module for parsing TSV
4
+ */
5
+ import { TextLineStream } from '@std/streams'
6
+ import { ColumnsMap } from '../types/columns.ts'
7
+ import type { BIDSFile } from '../types/filetree.ts'
8
+ import { filememoizeAsync } from '../utils/memoize.ts'
9
+ import { createUTF8Stream } from './streams.ts'
10
+
11
+ async function loadColumns(
12
+ reader: ReadableStreamDefaultReader<string>,
13
+ headers: string[],
14
+ maxRows: number,
15
+ startRow: number = 0,
16
+ ): Promise<ColumnsMap> {
17
+ // Initialize columns in array for construction efficiency
18
+ const initialCapacity = maxRows >= 0 ? maxRows : 1000
19
+ const columns: string[][] = headers.map(() => new Array<string>(initialCapacity))
20
+
21
+ maxRows = maxRows >= 0 ? maxRows : Infinity
22
+ let rowIndex = 0 // Keep in scope after loop
23
+ for (; rowIndex < maxRows; rowIndex++) {
24
+ const { done, value } = await reader.read()
25
+ if (done) break
26
+
27
+ // Expect a newline at the end of the file, but otherwise error on empty lines
28
+ if (!value) {
29
+ const nextRow = await reader.read()
30
+ if (nextRow.done) break
31
+ throw { code: 'TSV_EMPTY_LINE', line: rowIndex + startRow + 1 }
32
+ }
33
+
34
+ const values = value.split('\t')
35
+ if (values.length !== headers.length) {
36
+ throw { code: 'TSV_EQUAL_ROWS', line: rowIndex + startRow + 1 }
37
+ }
38
+ columns.forEach((column, columnIndex) => {
39
+ // Double array size if we exceed the current capacity
40
+ if (rowIndex >= column.length) {
41
+ column.length = column.length * 2
42
+ }
43
+ column[rowIndex] = values[columnIndex]
44
+ })
45
+ }
46
+
47
+ // Construct map, truncating columns to number of rows read
48
+ return new ColumnsMap(
49
+ headers.map((header, index) => [header, columns[index].slice(0, rowIndex)]),
50
+ )
51
+ }
52
+
53
+ export async function loadTSVGZ(
54
+ file: BIDSFile,
55
+ headers: string[],
56
+ maxRows: number = -1,
57
+ ): Promise<ColumnsMap> {
58
+ const reader = file.stream
59
+ .pipeThrough(new DecompressionStream('gzip'))
60
+ .pipeThrough(createUTF8Stream())
61
+ .pipeThrough(new TextLineStream())
62
+ .getReader()
63
+
64
+ try {
65
+ return await loadColumns(reader, headers, maxRows)
66
+ } catch (e: any) {
67
+ // Cancel the reader if we interrupted the read
68
+ // Cancelling for I/O errors will just re-trigger the error
69
+ if (e.code) {
70
+ await reader.cancel()
71
+ throw e
72
+ }
73
+ throw { code: 'INVALID_GZIP', location: file.path }
74
+ }
75
+ }
76
+
77
+ async function _loadTSV(file: BIDSFile, maxRows: number = -1): Promise<ColumnsMap> {
78
+ const reader = file.stream
79
+ .pipeThrough(createUTF8Stream())
80
+ .pipeThrough(new TextLineStream())
81
+ .getReader()
82
+
83
+ try {
84
+ const headerRow = await reader.read()
85
+ const headers = (headerRow.done || !headerRow.value) ? [] : headerRow.value.split('\t')
86
+
87
+ if (new Set(headers).size !== headers.length) {
88
+ throw {
89
+ code: 'TSV_COLUMN_HEADER_DUPLICATE',
90
+ location: file.path,
91
+ issueMessage: headers.join(', '),
92
+ }
93
+ }
94
+
95
+ return await loadColumns(reader, headers, maxRows, 1)
96
+ } finally {
97
+ await reader.cancel()
98
+ }
99
+ }
100
+
101
+ export const loadTSV = filememoizeAsync(_loadTSV)
@@ -123,16 +123,19 @@ export const bidsIssues: IssueDefinitionRecord = {
123
123
  reason:
124
124
  'A value in a column did not match the acceptable type for that column headers specified format.',
125
125
  },
126
- TSV_VALUE_INCORRECT_TYPE_NONREQUIRED: {
127
- severity: 'warning',
128
- reason:
129
- 'A value in a column did not match the acceptable type for that column headers specified format.',
130
- },
131
126
  TSV_COLUMN_TYPE_REDEFINED: {
132
127
  severity: 'warning',
133
128
  reason:
134
129
  'A column required in a TSV file has been redefined in a sidecar file. This redefinition is being ignored.',
135
130
  },
131
+ TSV_PSEUDO_AGE_DEPRECATED: {
132
+ severity: 'warning',
133
+ reason: 'Use of the value "89+" in column "age" is deprecated. Use 89 for all ages 89 and over.',
134
+ },
135
+ INVALID_GZIP: {
136
+ severity: 'error',
137
+ reason: 'The gzip file could not be decompressed. It may be corrupt or misnamed.',
138
+ },
136
139
  MULTIPLE_INHERITABLE_FILES: {
137
140
  severity: 'error',
138
141
  reason: 'Multiple files in a directory were found to be valid candidates for inheritance.',
@@ -170,9 +173,9 @@ export const bidsIssues: IssueDefinitionRecord = {
170
173
  severity: 'error',
171
174
  reason: 'A json sidecar file was found without a corresponding data file',
172
175
  },
173
- UNSUPPORTED_DATASET_TYPE: {
174
- severity: 'error',
175
- reason: 'This DatasetType is not supported by the application.',
176
+ SIDECAR_FIELD_OVERRIDE: {
177
+ severity: 'warning',
178
+ reason: 'Sidecar files should not override values assigned at a higher level.',
176
179
  },
177
180
  BLACKLISTED_MODALITY: {
178
181
  severity: 'error',
@@ -146,7 +146,7 @@ function mapEvalCheck(statements: string[], context: BIDSContext): boolean {
146
146
  * Classic rules interpreted like selectors. Examples in specification:
147
147
  * schema/rules/checks/*
148
148
  */
149
- function evalRuleChecks(
149
+ export function evalRuleChecks(
150
150
  rule: GenericRule,
151
151
  context: BIDSContext,
152
152
  schema: GenericSchema,
@@ -1,4 +1,4 @@
1
- import { assert, assertObjectMatch } from '@std/assert'
1
+ import { assert, assertEquals, assertObjectMatch } from '@std/assert'
2
2
  import type { DatasetIssues } from '../issues/datasetIssues.ts'
3
3
  import { BIDSContext } from './context.ts'
4
4
  import { dataFile, rootFileTree } from './fixtures.test.ts'
@@ -21,6 +21,12 @@ Deno.test('test context LoadSidecar', async (t) => {
21
21
  anatValue: 'anat',
22
22
  })
23
23
  })
24
+ await t.step('Warnings are emitted for overriding sidecar fields', () => {
25
+ assertEquals(
26
+ context.dataset.issues.get({ code: 'SIDECAR_FIELD_OVERRIDE' }).length,
27
+ 2,
28
+ )
29
+ })
24
30
  })
25
31
 
26
32
  Deno.test('test context loadSubjects', async (t) => {
@@ -18,7 +18,7 @@ import { readEntities } from './entities.ts'
18
18
  import { DatasetIssues } from '../issues/datasetIssues.ts'
19
19
  import { walkBack } from '../files/inheritance.ts'
20
20
  import { parseGzip } from '../files/gzip.ts'
21
- import { loadTSV } from '../files/tsv.ts'
21
+ import { loadTSV, loadTSVGZ } from '../files/tsv.ts'
22
22
  import { parseTIFF } from '../files/tiff.ts'
23
23
  import { loadJSON } from '../files/json.ts'
24
24
  import { loadHeader } from '../files/nifti.ts'
@@ -209,14 +209,26 @@ export class BIDSContext implements Context {
209
209
  }
210
210
  }
211
211
  for (const file of sidecars) {
212
- const json = await loadJSON(file).catch((error) => {
213
- if (error.code) {
214
- this.dataset.issues.add({ ...error, location: file.path })
212
+ const json = await loadJSON(file).catch((error): Record<string, unknown> => {
213
+ if (error.key) {
214
+ this.dataset.issues.add({ code: error.key, location: file.path })
215
215
  return {}
216
216
  } else {
217
217
  throw error
218
218
  }
219
219
  })
220
+ const overrides = Object.keys(this.sidecar).filter((x) => Object.hasOwn(json, x))
221
+ for (const key of overrides) {
222
+ if (json[key] !== this.sidecar[key]) {
223
+ const overrideLocation = this.sidecarKeyOrigin[key]
224
+ this.dataset.issues.add({
225
+ code: 'SIDECAR_FIELD_OVERRIDE',
226
+ subCode: key,
227
+ location: overrideLocation,
228
+ issueMessage: `Sidecar key defined in ${file.path} overrides previous value (${json[key]}) from ${overrideLocation}`,
229
+ })
230
+ }
231
+ }
220
232
  this.sidecar = { ...json, ...this.sidecar }
221
233
  Object.keys(json).map((x) => this.sidecarKeyOrigin[x] ??= file.path)
222
234
  }
@@ -242,21 +254,40 @@ export class BIDSContext implements Context {
242
254
  }
243
255
 
244
256
  async loadColumns(): Promise<void> {
245
- if (this.extension !== '.tsv') {
246
- return
257
+ if (this.extension == '.tsv') {
258
+ this.columns = await loadTSV(this.file, this.dataset.options?.maxRows)
259
+ .catch((error) => {
260
+ if (error.code) {
261
+ this.dataset.issues.add({ ...error, location: this.file.path })
262
+ }
263
+ logger.warn(
264
+ `tsv file could not be opened by loadColumns '${this.file.path}'`,
265
+ )
266
+ logger.debug(error)
267
+ return new Map<string, string[]>() as ColumnsMap
268
+ }) as Record<string, string[]>
269
+ } else if (this.extension == '.tsv.gz') {
270
+ const headers = this.sidecar.Columns as string[];
271
+ if (!headers || this.size === 0) {
272
+ // Missing Columns will be caught by sidecar rules
273
+ // Note that these rules currently select for suffix, and will need to be generalized
274
+ // or duplicated for new .tsv.gz files
275
+ // `this.size === 0` will show as `EMPTY_FILE`, so do not add INVALID_GZIP
276
+ return
277
+ }
278
+ this.columns = await loadTSVGZ(this.file, headers, this.dataset.options?.maxRows)
279
+ .catch((error) => {
280
+ if (error.code) {
281
+ this.dataset.issues.add({ ...error, location: this.file.path })
282
+ }
283
+ logger.warn(
284
+ `tsv.gz file could not be opened by loadColumns '${this.file.path}'`,
285
+ )
286
+ logger.debug(error)
287
+ return new Map<string, string[]>() as ColumnsMap
288
+ }) as Record<string, string[]>
247
289
  }
248
290
 
249
- this.columns = await loadTSV(this.file, this.dataset.options?.maxRows)
250
- .catch((error) => {
251
- if (error.code) {
252
- this.dataset.issues.add({ ...error, location: this.file.path })
253
- }
254
- logger.warn(
255
- `tsv file could not be opened by loadColumns '${this.file.path}'`,
256
- )
257
- logger.debug(error)
258
- return new Map<string, string[]>() as ColumnsMap
259
- }) as Record<string, string[]>
260
291
  return
261
292
  }
262
293
 
@@ -325,37 +356,27 @@ export class BIDSContext implements Context {
325
356
  }) as Record<string, string[]>
326
357
  this.dataset.subjects.participant_id = participantsData['participant_id']
327
358
  }
328
-
329
- // Load phenotype from phenotype/*.tsv
330
- const phenotype_dir = this.dataset.tree.get('phenotype') as FileTree
331
- if (phenotype_dir) {
332
- const phenotypeFiles = phenotype_dir.files.filter((file) => file.name.endsWith('.tsv'))
333
- // Collect observed participant_ids
334
- const seen = new Set() as Set<string>
335
- for (const file of phenotypeFiles) {
336
- const phenotypeData = await loadTSV(file)
337
- .catch((error) => {
338
- return new Map()
339
- }) as Record<string, string[]>
340
- const participant_id = phenotypeData['participant_id']
341
- if (participant_id) {
342
- participant_id.forEach((id) => seen.add(id))
343
- }
344
- }
345
- this.dataset.subjects.phenotype = Array.from(seen)
346
- }
347
359
  }
348
360
 
349
361
  async asyncLoads() {
350
- await Promise.allSettled([
351
- this.loadSubjects(),
362
+ // loaders that may be depended on by other loaders
363
+ const initial = [
352
364
  this.loadSidecar(),
353
- this.loadColumns(),
354
365
  this.loadAssociations(),
366
+ ]
367
+ // loaders that do not depend on other loaders
368
+ const independent = [
369
+ this.loadSubjects(),
355
370
  this.loadNiftiHeader(),
356
371
  this.loadJSON(),
357
372
  this.loadGzip(),
358
373
  this.loadTIFF(),
359
- ])
374
+ ]
375
+
376
+ // Loaders with dependencies
377
+ await Promise.allSettled(initial)
378
+ await this.loadColumns()
379
+
380
+ await Promise.allSettled(independent)
360
381
  }
361
382
  }
@@ -69,7 +69,7 @@ Deno.test('tables eval* tests', async (t) => {
69
69
  const rule = schemaDefs.rules.tabular_data.modality_agnostic.Scans
70
70
  evalColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Scans')
71
71
  assertEquals(
72
- context.dataset.issues.get({ code: 'TSV_VALUE_INCORRECT_TYPE_NONREQUIRED' }).length,
72
+ context.dataset.issues.get({ code: 'TSV_VALUE_INCORRECT_TYPE' }).length,
73
73
  1,
74
74
  )
75
75
  })
@@ -159,13 +159,32 @@ Deno.test('tables eval* tests', async (t) => {
159
159
 
160
160
  // Overriding the default sex definition uses the provided values
161
161
  // Values in the default definition may raise issues
162
- issues = context.dataset.issues.get({ code: 'TSV_VALUE_INCORRECT_TYPE_NONREQUIRED' })
162
+ issues = context.dataset.issues.get({ code: 'TSV_VALUE_INCORRECT_TYPE' })
163
163
  assertEquals(issues.length, 1)
164
164
  assertEquals(issues[0].subCode, 'sex')
165
165
  assertEquals(issues[0].line, 4)
166
166
  assertEquals(issues[0].issueMessage, "'f'")
167
167
  })
168
168
 
169
+ await t.step('verify pseudo-age deprecation', () => {
170
+ const context = {
171
+ path: '/participants.tsv',
172
+ extension: '.tsv',
173
+ sidecar: {},
174
+ columns: {
175
+ participant_id: ['sub-01', 'sub-02', 'sub-03'],
176
+ age: ['10', '89+', '89+'],
177
+ },
178
+ dataset: { issues: new DatasetIssues() },
179
+ }
180
+ const rule = schemaDefs.rules.tabular_data.modality_agnostic.Participants
181
+ evalColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Participants')
182
+
183
+ // age gets a warning
184
+ let issues = context.dataset.issues.get({ code: 'TSV_PSEUDO_AGE_DEPRECATED' })
185
+ assertEquals(issues.length, 1)
186
+ })
187
+
169
188
  await t.step('verify column ordering', () => {
170
189
  const context = {
171
190
  path: '/sub-01/sub-01_scans.tsv',