votive 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -5
- package/lib/bundle.js +121 -55
- package/lib/createDatabase.js +895 -450
- package/lib/readAbstracts.js +12 -9
- package/lib/readBuffers.js +10 -10
- package/lib/readFilePaths.js +12 -12
- package/lib/readFolders.js +10 -17
- package/lib/readSources.js +107 -55
- package/lib/runJobs.js +1 -3
- package/lib/sqlite.js +12 -14
- package/lib/utils/index.js +8 -4
- package/lib/writeDestinations.js +23 -18
- package/package.json +2 -1
- package/tests/index.js +303 -263
- package/tests/markdown/prunee.md +1 -0
- package/tests/.votive.db +0 -0
package/lib/readAbstracts.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
|
|
2
2
|
/** @import {Database} from "./createDatabase.js" */
|
|
3
|
-
/** @import {
|
|
3
|
+
/** @import {ReadSourceFilesResult} from "./readSources.js"
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {object} ReadAbstractsResult
|
|
@@ -13,15 +13,17 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
|
-
* @param {
|
|
16
|
+
* @param {ReadSourceFilesResult} files
|
|
17
17
|
* @param {VotiveConfig} config
|
|
18
18
|
* @param {Database} database
|
|
19
19
|
* @param {FlatProcessors} processors
|
|
20
20
|
* @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
|
|
21
21
|
*/
|
|
22
|
-
function readAbstracts(
|
|
22
|
+
function readAbstracts(files, config, database, processors) {
|
|
23
23
|
|
|
24
|
-
const processed =
|
|
24
|
+
const processed = files.flatMap(file => {
|
|
25
|
+
|
|
26
|
+
const { abstract: unprocessedAbstract } = file
|
|
25
27
|
|
|
26
28
|
/**
|
|
27
29
|
* @param {FlatProcessors} processors
|
|
@@ -32,13 +34,14 @@ function readAbstracts(abstracts, config, database, processors) {
|
|
|
32
34
|
const [flatProcessor, ...rest] = processors
|
|
33
35
|
if (!flatProcessor) return { abstract, jobs }
|
|
34
36
|
const { processor, plugin } = flatProcessor
|
|
35
|
-
if (processor.syntax
|
|
36
|
-
|| !processor.
|
|
37
|
-
|| !processor.read.abstract
|
|
37
|
+
if (!processor.extensions.includes(file.syntax)
|
|
38
|
+
|| !processor.transformFile
|
|
38
39
|
) return recursiveProcess(rest, abstract, jobs)
|
|
39
40
|
|
|
40
|
-
const
|
|
41
|
-
|
|
41
|
+
const settings = database.setting.getByFolder(file.dir)
|
|
42
|
+
|
|
43
|
+
const processed = processor.transformFile(abstract, database, config)
|
|
44
|
+
processed.jobs && processed.jobs.forEach(job => job.syntax = file.syntax)
|
|
42
45
|
return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
|
|
43
46
|
}
|
|
44
47
|
|
package/lib/readBuffers.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
|
|
2
2
|
/** @import {Database} from "./createDatabase.js" */
|
|
3
|
-
/** @import {
|
|
3
|
+
/** @import {ReadSourceFilesResult} from "./readSources.js"
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {object} ReadAbstractsResult
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
|
-
* @param {
|
|
16
|
+
* @param {ReadSourceFilesResult} abstracts
|
|
17
17
|
* @param {VotiveConfig} config
|
|
18
18
|
* @param {Database} database
|
|
19
19
|
* @param {FlatProcessors} processors
|
|
@@ -21,8 +21,8 @@
|
|
|
21
21
|
*/
|
|
22
22
|
function readBuffers(abstracts, config, database, processors) {
|
|
23
23
|
|
|
24
|
-
const processed = abstracts.flatMap(
|
|
25
|
-
|
|
24
|
+
const processed = abstracts.flatMap(file => {
|
|
25
|
+
const { abstract: unprocessedAbstract, syntax } = file
|
|
26
26
|
/**
|
|
27
27
|
* @param {FlatProcessors} processors
|
|
28
28
|
* @param {Abstract} abstract
|
|
@@ -32,13 +32,13 @@ function readBuffers(abstracts, config, database, processors) {
|
|
|
32
32
|
const [flatProcessor, ...rest] = processors
|
|
33
33
|
if (!flatProcessor) return { abstract, jobs }
|
|
34
34
|
const { processor, plugin } = flatProcessor
|
|
35
|
-
if (processor.syntax
|
|
36
|
-
|| !processor.
|
|
37
|
-
|| !processor.
|
|
35
|
+
if (!processor.extensions.includes(syntax)
|
|
36
|
+
|| !processor.readFile
|
|
37
|
+
|| !processor.transformFile
|
|
38
38
|
) return recursiveProcess(rest, abstract, jobs)
|
|
39
39
|
|
|
40
|
-
const processed = processor.
|
|
41
|
-
processed.jobs && processed.jobs.forEach(job => job.syntax =
|
|
40
|
+
const processed = processor.transformFile(abstract, database, config)
|
|
41
|
+
processed.jobs && processed.jobs.forEach(job => job.syntax = file.syntax)
|
|
42
42
|
return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
|
|
43
43
|
}
|
|
44
44
|
|
|
@@ -56,4 +56,4 @@ function readBuffers(abstracts, config, database, processors) {
|
|
|
56
56
|
return { processedAbstracts, abstractsJobs }
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
export default
|
|
59
|
+
export default readBuffers
|
package/lib/readFilePaths.js
CHANGED
|
@@ -1,28 +1,28 @@
|
|
|
1
1
|
/** @import {VotiveConfig, FlatProcessors, Abstract, AbstractsWithSyntax, Jobs, FlatProcessor, Abstracts} from "./bundle.js" */
|
|
2
2
|
/** @import {Database} from "./createDatabase.js" */
|
|
3
|
-
/** @import {
|
|
3
|
+
/** @import {ReadSourceFilesResult} from "./readSources.js" */
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* @typedef {object} ReadAbstractsResult
|
|
7
7
|
* @property {Jobs} jobs
|
|
8
8
|
* @property {ReadAbstractResult[]} abstracts
|
|
9
9
|
*/
|
|
10
|
-
|
|
10
|
+
|
|
11
11
|
/**
|
|
12
12
|
* @typedef {object} ReadAbstractResult
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
|
-
* @param {
|
|
16
|
+
* @param {ReadSourceFilesResult} files
|
|
17
17
|
* @param {VotiveConfig} config
|
|
18
18
|
* @param {Database} database
|
|
19
19
|
* @param {FlatProcessors} processors
|
|
20
20
|
* @returns {{processedAbstracts: Abstracts, abstractsJobs: Jobs}}
|
|
21
21
|
*/
|
|
22
|
-
function readFilePaths(
|
|
23
|
-
|
|
24
|
-
const processed = abstracts.flatMap(({ abstract: unprocessedAbstract, syntax }) => {
|
|
22
|
+
function readFilePaths(files, config, database, processors) {
|
|
25
23
|
|
|
24
|
+
const processed = files.flatMap(file => {
|
|
25
|
+
const { abstract: unprocessedAbstract, syntax } = file
|
|
26
26
|
/**
|
|
27
27
|
* @param {FlatProcessors} processors
|
|
28
28
|
* @param {Abstract} abstract
|
|
@@ -31,14 +31,14 @@ function readFilePaths(abstracts, config, database, processors) {
|
|
|
31
31
|
function recursiveProcess(processors, abstract, jobs = []) {
|
|
32
32
|
const [flatProcessor, ...rest] = processors
|
|
33
33
|
if (!flatProcessor) return { abstract, jobs }
|
|
34
|
-
const { processor
|
|
35
|
-
if (processor.syntax
|
|
36
|
-
|| !processor.
|
|
37
|
-
|| !processor.
|
|
34
|
+
const { processor } = flatProcessor
|
|
35
|
+
if (!processor.extensions.includes(syntax)
|
|
36
|
+
|| !processor.readFile
|
|
37
|
+
|| !processor.transformFile
|
|
38
38
|
) return recursiveProcess(rest, abstract, jobs)
|
|
39
39
|
|
|
40
|
-
const processed = processor.
|
|
41
|
-
processed.jobs && processed.jobs.forEach(job => job.syntax =
|
|
40
|
+
const processed = processor.transformFile(abstract, database, config)
|
|
41
|
+
processed.jobs && processed.jobs.forEach(job => job.syntax = file.syntax)
|
|
42
42
|
return recursiveProcess(rest, processed.abstract, [...jobs, ...(processed.jobs || [])])
|
|
43
43
|
}
|
|
44
44
|
|
package/lib/readFolders.js
CHANGED
|
@@ -13,7 +13,7 @@ import path from "node:path"
|
|
|
13
13
|
*/
|
|
14
14
|
function readFolders(folders = [], config, database, processors) {
|
|
15
15
|
|
|
16
|
-
const folderProcessors = processors.filter(({ processor }) => processor.
|
|
16
|
+
const folderProcessors = processors.filter(({ processor }) => processor.readFolder)
|
|
17
17
|
|
|
18
18
|
const processed = folderProcessors.flatMap(({ processor, plugin }) => {
|
|
19
19
|
|
|
@@ -21,30 +21,23 @@ function readFolders(folders = [], config, database, processors) {
|
|
|
21
21
|
let folderPath = path.join(folder.parentPath, folder.name)
|
|
22
22
|
if(folderPath) folderPath += path.sep
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
/** @ts-ignore `.read` is throwing a warning, but it's guarded above */
|
|
26
|
-
const { destinations, jobs } = processor.read.folder(folderPath, database, config)
|
|
24
|
+
const { targets, jobs } = processor.readFolder(folderPath, database, config)
|
|
27
25
|
jobs.forEach(job => job.syntax = processor.syntax)
|
|
28
|
-
if (
|
|
29
|
-
|
|
30
|
-
database.
|
|
26
|
+
if (targets) {
|
|
27
|
+
targets.forEach(target => {
|
|
28
|
+
database.target.create(target)
|
|
31
29
|
})
|
|
32
30
|
return jobs
|
|
33
31
|
}
|
|
34
32
|
})
|
|
35
33
|
|
|
36
34
|
|
|
37
|
-
const
|
|
38
|
-
const
|
|
39
|
-
? path.format(rootInfo) + path.sep
|
|
40
|
-
: ""
|
|
41
|
-
|
|
42
|
-
// TODO: Add isRoot to type definition
|
|
43
|
-
const rootFolder = processor.read.folder(rootPath, database, config, true)
|
|
35
|
+
const rootPath = path.relative(config.sourceFolder, "")
|
|
36
|
+
const rootFolder = processor.readFolder(rootPath, database, config, true)
|
|
44
37
|
|
|
45
|
-
if (rootFolder.
|
|
46
|
-
rootFolder.
|
|
47
|
-
database.
|
|
38
|
+
if (rootFolder.targets) {
|
|
39
|
+
rootFolder.targets.forEach(target => {
|
|
40
|
+
database.target.create(target)
|
|
48
41
|
})
|
|
49
42
|
}
|
|
50
43
|
|
package/lib/readSources.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { decodeBuffer } from "encoding-sniffer"
|
|
2
2
|
import fs from "node:fs/promises"
|
|
3
3
|
import path from "node:path"
|
|
4
|
-
import
|
|
4
|
+
import pLimit from "p-limit"
|
|
5
5
|
|
|
6
6
|
/** @import {VotiveConfig, VotivePlugin, VotiveProcessor, FlatProcessors, Abstracts, Abstract, Jobs, ProcessorSyntax} from "./bundle.js" */
|
|
7
7
|
/** @import {Dirent} from "node:fs" */
|
|
@@ -10,7 +10,7 @@ import { visit } from "unist-util-visit"
|
|
|
10
10
|
/**
|
|
11
11
|
* @typedef {object} ReadSourcesResult
|
|
12
12
|
* @property {Dirent[]} folders
|
|
13
|
-
* @property {
|
|
13
|
+
* @property {ReadSourceFileResult[]} sources
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
/**
|
|
@@ -25,11 +25,28 @@ async function readSources(config, database, processors) {
|
|
|
25
25
|
recursive: true
|
|
26
26
|
})
|
|
27
27
|
|
|
28
|
-
const filteredDirents = (dirents || []).filter(fileFilter(config
|
|
28
|
+
const filteredDirents = (dirents || []).filter(fileFilter(config))
|
|
29
|
+
|
|
29
30
|
const { files, folders } = Object.groupBy(filteredDirents, (dirent) => dirent.isFile() ? "files" : "folders")
|
|
31
|
+
const loadingFiles = files.map(a => path.normalize(path.format({ name: a.name, dir: a.parentPath })))
|
|
30
32
|
if (!files) return { folders, sources: [] }
|
|
31
|
-
const
|
|
33
|
+
const limit = pLimit(5)
|
|
34
|
+
const readingSourceFiles = files.flatMap(readSourceFile(processors, database, config, limit))
|
|
35
|
+
// const sources = readingSourceFiles && readingSourceFiles.map(async file => {
|
|
36
|
+
// try {
|
|
37
|
+
// const loaded = await file
|
|
38
|
+
// // console.log(loaded)
|
|
39
|
+
// // const i = loadingFiles.find(a => a === loaded.sourceFilePath)
|
|
40
|
+
// // loadingFiles.splice(i,1)
|
|
41
|
+
// // console.log(loadingFiles)
|
|
42
|
+
// return loaded
|
|
43
|
+
// } catch (e) {
|
|
44
|
+
// console.error(e)
|
|
45
|
+
// }
|
|
46
|
+
// })
|
|
32
47
|
const sources = readingSourceFiles && (await Promise.all(readingSourceFiles)).filter(a => a)
|
|
48
|
+
|
|
49
|
+
|
|
33
50
|
return {
|
|
34
51
|
folders,
|
|
35
52
|
sources
|
|
@@ -38,9 +55,8 @@ async function readSources(config, database, processors) {
|
|
|
38
55
|
|
|
39
56
|
/**
|
|
40
57
|
* @param {VotiveConfig} config
|
|
41
|
-
* @param {Database} database
|
|
42
58
|
*/
|
|
43
|
-
function fileFilter(config
|
|
59
|
+
function fileFilter(config) {
|
|
44
60
|
/** @param {Dirent} dirent */
|
|
45
61
|
return (dirent) => {
|
|
46
62
|
const isDestinationFolder = !path.relative(config.destinationFolder, path.join(dirent.parentPath, dirent.name))
|
|
@@ -53,9 +69,9 @@ function fileFilter(config, database) {
|
|
|
53
69
|
return false // Ignore hidden files
|
|
54
70
|
} else if (dirent.parentPath.includes(path.sep + ".")) {
|
|
55
71
|
return false // Ignore hidden folders
|
|
56
|
-
} else if(dirent.parentPath.match(/^\.\w/)) {
|
|
72
|
+
} else if (dirent.parentPath.match(/^\.\w/)) {
|
|
57
73
|
return false // Ignore hidden folders
|
|
58
|
-
} if(isDestinationFolder) {
|
|
74
|
+
} if (isDestinationFolder) {
|
|
59
75
|
return false
|
|
60
76
|
}
|
|
61
77
|
return true
|
|
@@ -63,15 +79,18 @@ function fileFilter(config, database) {
|
|
|
63
79
|
}
|
|
64
80
|
|
|
65
81
|
/**
|
|
66
|
-
* @typedef {
|
|
82
|
+
* @typedef {ReadSourceFileResult[]} ReadSourceFilesResult
|
|
67
83
|
*/
|
|
68
84
|
|
|
69
85
|
/**
|
|
70
|
-
* @typedef {object}
|
|
86
|
+
* @typedef {object} ReadSourceFileResult
|
|
71
87
|
* @property {Abstract} abstract
|
|
72
88
|
* @property {ProcessorSyntax} syntax
|
|
89
|
+
* @property {object} [metadata]
|
|
90
|
+
* @property {string} [targetFilePath]
|
|
91
|
+
* @property {string} [dir]
|
|
73
92
|
* @property {Jobs} jobs
|
|
74
|
-
* @property {string}
|
|
93
|
+
* @property {string} sourceFilePath
|
|
75
94
|
*/
|
|
76
95
|
|
|
77
96
|
/**
|
|
@@ -79,101 +98,113 @@ function fileFilter(config, database) {
|
|
|
79
98
|
* @param {Database} database
|
|
80
99
|
* @param {VotiveConfig} config
|
|
81
100
|
*/
|
|
82
|
-
function readSourceFile(processors, database, config) {
|
|
101
|
+
function readSourceFile(processors, database, config, limit) {
|
|
83
102
|
/**
|
|
84
103
|
* @param {import("node:fs").Dirent} dirent
|
|
85
|
-
* @returns {Promise<
|
|
104
|
+
* @returns {Promise<ReadSourceFileResult>[]}
|
|
86
105
|
*/
|
|
87
106
|
return (dirent) => {
|
|
88
107
|
const { name, parentPath } = dirent
|
|
89
|
-
const
|
|
90
|
-
const
|
|
108
|
+
const sourceFilePath = path.join(parentPath, name)
|
|
109
|
+
const sourceFileInfo = path.parse(sourceFilePath)
|
|
91
110
|
|
|
92
|
-
const processing = processors.flatMap(({ plugin, processor }) => process(plugin, processor, config))
|
|
111
|
+
const processing = processors.flatMap(({ plugin, processor }) => process(plugin, processor, config, limit))
|
|
93
112
|
|
|
94
113
|
/**
|
|
95
114
|
* @param {VotivePlugin} plugin
|
|
96
115
|
* @param {VotiveProcessor} processor
|
|
97
116
|
* @param {VotiveConfig} config
|
|
98
117
|
*/
|
|
99
|
-
async function process(plugin, processor, config) {
|
|
100
|
-
const { read, filter,
|
|
101
|
-
if (filter
|
|
118
|
+
async function process(plugin, processor, config, limit) {
|
|
119
|
+
const { readFile: read, extensions: filter, format } = processor
|
|
120
|
+
if (filter.includes(sourceFileInfo.ext) && read) {
|
|
102
121
|
|
|
103
122
|
// Check modified time
|
|
104
|
-
const stat = await fs.stat(
|
|
105
|
-
const source = database.
|
|
106
|
-
|
|
123
|
+
const stat = await fs.stat(sourceFilePath)
|
|
124
|
+
const source = database.source.get(sourceFilePath)
|
|
125
|
+
const diff = source && source.lastModified - Number(Math.floor(stat.mtimeMs))
|
|
107
126
|
|
|
108
|
-
if (source &&
|
|
127
|
+
if (source && diff > -1) {
|
|
128
|
+
return null
|
|
129
|
+
}
|
|
109
130
|
|
|
110
|
-
database.
|
|
131
|
+
database.setting.deleteBySource(sourceFilePath)
|
|
111
132
|
|
|
112
|
-
|
|
113
|
-
const
|
|
133
|
+
const targetFilePath = route(sourceFilePath, plugin, config)
|
|
134
|
+
const targetFileExtension = path.extname(targetFilePath)
|
|
114
135
|
|
|
115
136
|
// Set URL if exists
|
|
116
137
|
const allJobs = []
|
|
117
|
-
const destination = {}
|
|
118
138
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
139
|
+
if (format === "buffer") {
|
|
140
|
+
|
|
141
|
+
// FIXME update this type
|
|
142
|
+
const data = read(sourceFilePath, database, config)
|
|
143
|
+
const { metadata, abstract, jobs } = data
|
|
122
144
|
|
|
123
|
-
database.
|
|
145
|
+
const target = database.target.create({
|
|
124
146
|
metadata,
|
|
125
147
|
abstract,
|
|
126
|
-
path:
|
|
127
|
-
syntax
|
|
148
|
+
path: targetFilePath,
|
|
128
149
|
})
|
|
129
150
|
|
|
130
|
-
|
|
151
|
+
|
|
152
|
+
if (Array.isArray(jobs)) {
|
|
131
153
|
allJobs.push(...jobs)
|
|
132
154
|
}
|
|
133
155
|
}
|
|
134
156
|
|
|
135
|
-
if (
|
|
136
|
-
const
|
|
137
|
-
const data =
|
|
157
|
+
if (format === "text") {
|
|
158
|
+
const stats = await fs.stat(sourceFilePath)
|
|
159
|
+
const data = await fs.readFile(sourceFilePath, { encoding: "utf-8" })
|
|
138
160
|
|
|
139
|
-
const { jobs, abstract, metadata } = read
|
|
161
|
+
const { jobs, abstract, metadata } = read(data, sourceFilePath, targetFilePath, database, config)
|
|
140
162
|
|
|
141
|
-
|
|
163
|
+
|
|
164
|
+
if (Array.isArray(jobs)) {
|
|
142
165
|
allJobs.push(...jobs)
|
|
143
166
|
}
|
|
144
167
|
|
|
145
|
-
database.
|
|
168
|
+
const target = database.target.create({
|
|
146
169
|
abstract,
|
|
147
|
-
path:
|
|
170
|
+
path: targetFilePath,
|
|
148
171
|
metadata,
|
|
149
|
-
syntax
|
|
150
172
|
})
|
|
151
173
|
|
|
152
|
-
//
|
|
153
|
-
allJobs && allJobs.forEach(job => job.syntax = processor.
|
|
174
|
+
// FIXME this won't work with the refactor
|
|
175
|
+
allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
|
|
154
176
|
updateSource()
|
|
155
|
-
return {
|
|
177
|
+
return {
|
|
178
|
+
abstract,
|
|
179
|
+
metadata,
|
|
180
|
+
syntax: targetFileExtension,
|
|
181
|
+
targetFilePath: target.path,
|
|
182
|
+
dir: target.dir,
|
|
183
|
+
sourceFilePath,
|
|
184
|
+
jobs: allJobs
|
|
185
|
+
}
|
|
156
186
|
}
|
|
157
187
|
|
|
158
|
-
//
|
|
159
|
-
allJobs && allJobs.forEach(job => job.syntax = processor.
|
|
188
|
+
// FIXME Job syntax
|
|
189
|
+
allJobs && allJobs.forEach(job => job.syntax = processor.extensions[0])
|
|
160
190
|
updateSource()
|
|
161
191
|
|
|
162
192
|
return {
|
|
163
193
|
jobs: allJobs,
|
|
164
194
|
abstract: null,
|
|
165
|
-
|
|
195
|
+
targetFilePath: null,
|
|
196
|
+
sourceFilePath: null,
|
|
166
197
|
metadata: null,
|
|
167
|
-
syntax
|
|
198
|
+
syntax: null
|
|
168
199
|
}
|
|
169
200
|
|
|
170
201
|
function updateSource() {
|
|
171
202
|
const timeStamp = stat.mtimeMs.toFixed()
|
|
172
203
|
|
|
173
204
|
if (source) {
|
|
174
|
-
database.
|
|
205
|
+
database.source.updateTimestamp(sourceFilePath, Number(timeStamp))
|
|
175
206
|
} else {
|
|
176
|
-
database.
|
|
207
|
+
database.source.create(sourceFilePath, targetFilePath, Number(timeStamp))
|
|
177
208
|
}
|
|
178
209
|
}
|
|
179
210
|
|
|
@@ -192,17 +223,38 @@ function readSourceFile(processors, database, config) {
|
|
|
192
223
|
function route(filePath, plugin, config) {
|
|
193
224
|
const { dir, ...parsedPath } = path.parse(filePath)
|
|
194
225
|
const rooty = !path.relative(config.sourceFolder, dir)
|
|
226
|
+
const segments = ["", ...dir.split(path.sep).filter(a => a)]
|
|
195
227
|
const pathInfo = {
|
|
196
228
|
inRootDir: rooty,
|
|
197
|
-
dir:
|
|
229
|
+
dir: segments,
|
|
198
230
|
...parsedPath
|
|
199
231
|
}
|
|
232
|
+
|
|
200
233
|
if (!plugin.router) return "0"
|
|
234
|
+
|
|
201
235
|
const routedPath = plugin.router(pathInfo)
|
|
202
|
-
|
|
203
|
-
|
|
236
|
+
|
|
237
|
+
if (!routedPath) return "0"
|
|
238
|
+
|
|
239
|
+
if (routedPath.hasOwnProperty("dir") && Array.isArray(routedPath.dir)) {
|
|
240
|
+
return path.normalize(path.format({
|
|
241
|
+
dir: path.join(...routedPath.dir),
|
|
242
|
+
root: routedPath.root || "",
|
|
243
|
+
base: routedPath.base,
|
|
244
|
+
name: routedPath.name,
|
|
245
|
+
ext: routedPath.ext
|
|
246
|
+
}))
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const routedInfo = {
|
|
250
|
+
dir: routedPath.dir || "",
|
|
251
|
+
root: routedPath.root || "",
|
|
252
|
+
base: routedPath.base,
|
|
253
|
+
name: routedPath.name,
|
|
254
|
+
ext: routedPath.ext
|
|
204
255
|
}
|
|
205
|
-
|
|
256
|
+
|
|
257
|
+
return path.normalize(path.format(routedInfo))
|
|
206
258
|
}
|
|
207
259
|
|
|
208
260
|
export default readSources
|
package/lib/runJobs.js
CHANGED
|
@@ -16,9 +16,6 @@ async function runJobs(jobs, config, database) {
|
|
|
16
16
|
if (processor.syntax === job.syntax) {
|
|
17
17
|
try {
|
|
18
18
|
const response = await fetch(job.data)
|
|
19
|
-
// TODO: Change name of "job" to "url"
|
|
20
|
-
// TODO: Change name of "runner" to "format"
|
|
21
|
-
// TODO: Probably get rid of syntax filter
|
|
22
19
|
if (response.status >= 200 && response.status < 300) {
|
|
23
20
|
const data = await response[job.runner]()
|
|
24
21
|
const processed = processor.fetcher(data)
|
|
@@ -28,6 +25,7 @@ async function runJobs(jobs, config, database) {
|
|
|
28
25
|
console.warn(`Error fetching URL: ${job.data}`)
|
|
29
26
|
}
|
|
30
27
|
} catch (e) {
|
|
28
|
+
console.error(e)
|
|
31
29
|
return
|
|
32
30
|
}
|
|
33
31
|
}
|
package/lib/sqlite.js
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// TODO: and, or, not, between, not in, order
|
|
2
|
-
|
|
3
1
|
const operators = {
|
|
4
2
|
glob: "GLOB",
|
|
5
3
|
like: "LIKE",
|
|
@@ -9,7 +7,7 @@ const operators = {
|
|
|
9
7
|
gte: ">=",
|
|
10
8
|
lte: "<=",
|
|
11
9
|
equal: "=",
|
|
12
|
-
notEqual: "
|
|
10
|
+
notEqual: "!="
|
|
13
11
|
}
|
|
14
12
|
|
|
15
13
|
const select = `SELECT * FROM destinations d`
|
|
@@ -39,16 +37,16 @@ const join = `LEFT JOIN metadata m ON d.path = m.destination`
|
|
|
39
37
|
*/
|
|
40
38
|
export default function createStatement(query) {
|
|
41
39
|
const segments = [select, join]
|
|
42
|
-
if(query.limit) segments.push(`LIMIT ${query.limit}`)
|
|
43
|
-
if(query.offset) segments.push(`OFFSET ${query.offset}`)
|
|
44
|
-
if(query.orderBy) segments.push(`ORDER BY ${query.orderBy}`)
|
|
45
|
-
if(query.filter) {
|
|
40
|
+
if (query.limit) segments.push(`LIMIT ${query.limit}`)
|
|
41
|
+
if (query.offset) segments.push(`OFFSET ${query.offset}`)
|
|
42
|
+
if (query.orderBy) segments.push(`ORDER BY ${query.orderBy}`)
|
|
43
|
+
if (query.filter) {
|
|
46
44
|
const conditions = query.filter.map(condition => {
|
|
47
45
|
const { property, operator, value } = condition
|
|
48
|
-
|
|
49
|
-
if(["dir", "abstract", "path"].includes(property)) {
|
|
50
|
-
|
|
51
|
-
|
|
46
|
+
|
|
47
|
+
if (["dir", "abstract", "path", "syntax"].includes(property)) {
|
|
48
|
+
|
|
49
|
+
return `d.${property} ${operators[operator]} ${formatValue(value)}`
|
|
52
50
|
}
|
|
53
51
|
return `m.label = '${property}' AND m.value ${operators[operator]} ${formatValue(value)}`
|
|
54
52
|
})
|
|
@@ -60,17 +58,17 @@ export default function createStatement(query) {
|
|
|
60
58
|
}
|
|
61
59
|
|
|
62
60
|
function formatProperty(property) {
|
|
63
|
-
if(property === "abstract") return `d.${property}`
|
|
61
|
+
if (property === "abstract") return `d.${property}`
|
|
64
62
|
else return `m.${property}`
|
|
65
63
|
}
|
|
66
64
|
|
|
67
65
|
/** @param {any} value */
|
|
68
66
|
function formatValue(value) {
|
|
69
|
-
if(Array.isArray(value)) {
|
|
67
|
+
if (Array.isArray(value)) {
|
|
70
68
|
return `(${value.map(x => `'${x}'`).join(', ')})`
|
|
71
69
|
}
|
|
72
70
|
|
|
73
|
-
if(typeof value === "number") return value
|
|
71
|
+
if (typeof value === "number") return value
|
|
74
72
|
|
|
75
73
|
return `'${value}'`
|
|
76
74
|
}
|
package/lib/utils/index.js
CHANGED
|
@@ -2,14 +2,18 @@ import { styleText } from "node:util"
|
|
|
2
2
|
import { statSync } from "node:fs"
|
|
3
3
|
import path from "path"
|
|
4
4
|
|
|
5
|
-
/**
|
|
6
|
-
|
|
5
|
+
/**
|
|
6
|
+
* @param {string} label
|
|
7
|
+
* @param {string} message
|
|
8
|
+
* @param {boolean} verbose
|
|
9
|
+
*/
|
|
10
|
+
export function stopwatch(label, message, verbose) {
|
|
7
11
|
const start = performance.now()
|
|
8
12
|
|
|
9
13
|
function stop() {
|
|
10
14
|
const end = performance.now()
|
|
11
15
|
const duration = end - start
|
|
12
|
-
console.info(`${label}
|
|
16
|
+
if(verbose) console.info(`${styleText("dim", label + ":")} ${styleText("magenta", message)} ${styleText("magenta", duration.toFixed(2) + "ms")}`)
|
|
13
17
|
}
|
|
14
18
|
|
|
15
19
|
return stop
|
|
@@ -22,7 +26,7 @@ export function splitURL(urlPath) {
|
|
|
22
26
|
const [urlFileName, ...urlDirSegmentsReversed] = urlPath.split("/").filter(a => a).reverse()
|
|
23
27
|
const segments = urlDirSegmentsReversed.reverse()
|
|
24
28
|
segments.push('')
|
|
25
|
-
const folder = segments.join(path.sep)
|
|
29
|
+
const folder = path.relative("", segments.join(path.sep))
|
|
26
30
|
return folder
|
|
27
31
|
}
|
|
28
32
|
|
package/lib/writeDestinations.js
CHANGED
|
@@ -12,41 +12,46 @@ import { checkFile } from "./utils/index.js"
|
|
|
12
12
|
*/
|
|
13
13
|
async function writeDestinations(config, database) {
|
|
14
14
|
const writeProcessors = config && config.plugins && config.plugins.flatMap(plugin => (
|
|
15
|
-
plugin.processors && plugin.processors.map(
|
|
15
|
+
plugin.processors && plugin.processors.map(processor => processor.writeFile && processor).filter(a => a)
|
|
16
16
|
)).filter(a => a)
|
|
17
17
|
|
|
18
18
|
if (!writeProcessors || !writeProcessors.length) throw "No write processor provided"
|
|
19
19
|
|
|
20
|
-
const
|
|
21
|
-
if (!destinations) return
|
|
20
|
+
const targets = database.target.getStale()
|
|
22
21
|
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
if (!targets) return
|
|
23
|
+
|
|
24
|
+
const writing = targets.filter(({ path }) => path !== "0").flatMap(destination => {
|
|
25
|
+
|
|
26
|
+
/* FIXME the following line doesn't make sense */
|
|
27
|
+
if (!destination.path) database.target.markFresh(destination.path)
|
|
28
|
+
if (!Object.keys(destination.abstract).length) return
|
|
25
29
|
const destinationPath = path.join(config.destinationFolder, String(destination.path))
|
|
26
30
|
const { dir } = path.parse(destinationPath)
|
|
27
31
|
return writeProcessors.map(async processor => {
|
|
28
|
-
if (processor.
|
|
29
|
-
const writeInfo = await processor.
|
|
30
|
-
if(!writeInfo) return
|
|
31
|
-
const { data,
|
|
32
|
+
if (processor.extensions.includes(destination.syntax)) {
|
|
33
|
+
const writeInfo = await processor.writeFile(destination, database, config)
|
|
34
|
+
if (!writeInfo) return
|
|
35
|
+
const { data, encoding = 'utf-8' } = writeInfo
|
|
32
36
|
|
|
33
37
|
async function write() {
|
|
34
38
|
const destinationExists = checkFile(dir)
|
|
35
39
|
|
|
36
|
-
|
|
37
|
-
if(!destinationExists) {
|
|
40
|
+
if (!destinationExists) {
|
|
38
41
|
await mkdir(dir, { recursive: true })
|
|
39
42
|
}
|
|
40
43
|
|
|
41
|
-
if(
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
44
|
+
if (data) {
|
|
45
|
+
if (processor.format === "text") {
|
|
46
|
+
await writeFile(destinationPath, data, "utf-8")
|
|
47
|
+
database.target.markFresh(destination.path)
|
|
48
|
+
} else {
|
|
49
|
+
await writeFile(destinationPath, data)
|
|
50
|
+
database.target.markFresh(destination.path)
|
|
51
|
+
}
|
|
45
52
|
}
|
|
46
|
-
|
|
47
|
-
database.freshenDependency(destination.path)
|
|
48
53
|
}
|
|
49
|
-
|
|
54
|
+
|
|
50
55
|
return write()
|
|
51
56
|
}
|
|
52
57
|
})
|