vis-chronicle 1.2.5 → 1.4.5

package/README.md

@@ -95,8 +95,6 @@ Item properties:
     * `general`: General query segment, usually for selecting the item and property.
 	* `start`: Query segment for selecting the start value.
 	* `end`: Query segment for selecting the end value.
-* `startPath`: Another way of writing Wikidata queries. TODO.
-* `endPath`: See `startPath`
 * `expectedDuration`: Describes the expected duration, for hinting if the start or end is missing.
     * `min`: The absolute minimum duration.
 	* `max`: The absolute maximum duration.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vis-chronicle",
-  "version": "1.2.5",
+  "version": "1.4.5",
   "description": "Generates JSON for populating a vis.js timeline from Wikidata queries.",
   "keywords": [
     "wikidata",

package/src/fetch.js

@@ -37,12 +37,11 @@ if (!outputFile)
 
 var metadataOutputFile = values["out-metadata"]
 
-const moment = require('moment')
 const fs = require('fs');
 const wikidata = require('./wikidata.js')
 const renderer = require('./render.js')
 const mypath = require("./mypath.js");
-const { flattenRelativeDate } = require('./index.js');
+const { flattenRelativeDate, rangeUnion, rangeUnionAdv } = require('./relativeDates.js');
 const wikidataToRange2 = require('./wikidataToRange.js')
 
 function wikidataToRange(param)
@@ -57,40 +56,6 @@ wikidata.verboseLogging = values["verbose"]
 wikidata.setLang(values["lang"])
 wikidata.initialize()
 
-function rangeUnionAdv(value, min, max)
-{
-	var aggregate = {}
-	if (min)
-	{
-		assert(min.min)
-		aggregate.min = min.min
-	}
-	else if (value && value.min)
-	{
-		aggregate.min = value.min
-	}
-	if (max)
-	{
-		assert(max.max)
-		aggregate.max = max.max
-	}
-	else if (value && value.max)
-	{
-		aggregate.max = value.max
-	}
-	return aggregate
-}
-
-function rangeUnion(a, b)
-{
-	if (!a) return b
-	if (!b) return a
-	return {
-		min: a.min && b.min ? moment.min(a.min, b.min) : a.min || b.min,
-		max: a.max && b.max ? moment.max(a.max, b.max) : a.max || b.max
-	}
-}
-
 async function entryPoint() {}
 
 entryPoint()
@@ -112,7 +77,6 @@ entryPoint()
 		var templateItem = wikidata.inputSpec.items[i]
 		if (templateItem.itemQuery || templateItem.items)
 		{
-			//TODO: caching for item queries
 			wikidata.inputSpec.items.splice(i, 1)
 			const newItems = await wikidata.createTemplateItems(templateItem)
 			for (const newItem of newItems)
@@ -200,8 +164,16 @@ entryPoint()
 	{
 		if (item.finished) continue
 
-		if (item.startPath) pathQueries.push(item.startPath)
-		if (item.endPath) pathQueries.push(item.endPath)
+		if (item.startPath)
+		{
+			item.startPath = wikidata.replaceQueryWildcards(item.startPath, item)
+			pathQueries.push(item.startPath)
+		}
+		if (item.endPath)
+		{
+			item.endPath = wikidata.replaceQueryWildcards(item.endPath, item)
+			pathQueries.push(item.endPath)
+		}
 
 		// the bundle key is the queries, as well as any wildcard parameters
 		const keyObject = {}

package/src/index.js

@@ -3,5 +3,5 @@ module.exports = {
 
 	SparqlBuilder: require('./sparql-builder'),
 	Wikidata: require('./wikidata.js'),
-	flattenRelativeDate: require('./relativeDates.js')
 }
+module.exports = Object.assign(module.exports, require('./relativeDates.js'))

package/src/relativeDates.js

@@ -2,6 +2,8 @@
 const moment = require('moment')
 const { toJewishDate, toGregorianDate, getIndexByJewishMonth } = require("jewish-date");
 const wikidata = require('./wikidata');
+const wikidataToRange = require('./wikidataToRange');
+const assert = require("node:assert/strict")
 
 function momentToHDate(inMoment)
 {
@@ -24,54 +26,182 @@ function durationToWikidataPrecision(duration)
 	else return 9
 }
 
-// Flattens a relative date string into a hard date string
-module.exports = function flattenRelativeDate(wikidataCache, dateString)
+/**
+ * Combines three {min, max} objects into a range containing all of them.
+ * @param {*} value Is overridden by min.min and max.max if they are present.
+ * @param {*} min 
+ * @param {*} max 
+ * @returns 
+ */
+function rangeUnionAdv(value, min, max)
 {
-	if (dateString == '') return null
+	var aggregate = {}
+	if (min)
+	{
+		assert(min.min)
+		aggregate.min = min.min
+	}
+	else if (value && value.min)
+	{
+		aggregate.min = value.min
+	}
+	if (max)
+	{
+		assert(max.max)
+		aggregate.max = max.max
+	}
+	else if (value && value.max)
+	{
+		aggregate.max = value.max
+	}
+	return aggregate
+}
+
+/**
+ * Combines two {min, max} objects into a range containing both of them.
+ * @param {*} a 
+ * @param {*} b 
+ * @returns 
+ */
+function rangeUnion(a, b)
+{
+	if (!a) return b
+	if (!b) return a
+	return {
+		min: a.min && b.min ? moment.min(a.min, b.min) : a.min || b.min,
+		max: a.max && b.max ? moment.max(a.max, b.max) : a.max || b.max
+	}
+}
+
+
+/**
+ * Breaks down a relative date path into its components.
+ * @param {*} dateString 
+ * @returns An array of strings, or null.
+ */
+function breakRelativeDate(dateString)
+{
+	if (!dateString) return null
 
 	// parse out relative date components
-	var relSplit
 	var match = dateString.match(wikidata.pathQueryRegex)
-	if (match)
+	if (!match)
 	{
-		relSplit = match.slice(1)
+		console.error(`Failed to parse relative date '${dateString}'.`)
+		return null
+	}
+
+	const dateComponents = [ match[1] ]
+	var opStartIndex = 0
+	const operatorString = match[2]
+	if (operatorString.length > 0)
+	{
+		for (var i = 1; i < operatorString.length; i++)
+		{
+			if (operatorString[i] == '+' || operatorString[i] == '>')
+			{
+				dateComponents.push(operatorString.substring(opStartIndex, i))
+				opStartIndex = i
+			}
+		}
+		dateComponents.push(operatorString.substring(opStartIndex, i))
+	}
+	return dateComponents
+}
+
+/**
+ * Flattens a relative date path string into a hard date.
+ * @param {*} wikidataCache 
+ * @param {*} dateString 
+ * @param {*} params { returnRange:BOOL }. Result will be a range { min:STRING, max:STRING } instead.
+ * @returns An object like { value:STRING, precision:INT }, or null
+ */
+function flattenRelativeDate(wikidataCache, dateString, params)
+{
+	var flatMoment = flattenRelativeDateToMoment(wikidataCache, dateString)
+
+	// if allowed and necessary, produce a range instead of a single value
+	if (params?.returnRange)
+	{
+		var parsedPath = breakRelativeDate(dateString)
+		if (!parsedPath) return { value: null }
+
+		const basePath = parsedPath[0]
+		const wdpk = basePath.substring(0, basePath.lastIndexOf(':'))
+		const qual = basePath.substring(basePath.lastIndexOf(':') + 1)
+		var minQuery
+		var maxQuery
+		switch (qual)
+		{
+			case "P580":
+			{
+				minQuery = `${wdpk}:P1319` // earliest date
+				maxQuery = `${wdpk}:P8555` // latest start date
+				break
+			}
+			case "P582":
+			{
+				minQuery = `${wdpk}:P8554` // earliest end date
+				maxQuery = `${wdpk}:P1326` // latest date
+				break
+			}
+		}
+		if (minQuery && maxQuery)
+		{
+			const minMoment = flattenRelativeDateToMoment(wikidataCache, minQuery + parsedPath.slice(1).join(''))
+			const maxMoment = flattenRelativeDateToMoment(wikidataCache, maxQuery + parsedPath.slice(1).join(''))
+			const value = wikidataToRange(flatMoment)
+			const minRange = wikidataToRange(minMoment)
+			const maxRange = wikidataToRange(maxMoment)
+			const aggregateRange = rangeUnionAdv(value, minRange, maxRange)
+			return {
+				min: aggregateRange.min ? aggregateRange.min.format('YYYYYY-MM-DDThh:mm:ss') : null,
+				max: aggregateRange.max ? aggregateRange.max.format('YYYYYY-MM-DDThh:mm:ss') : null
+			}
+		}
+		else
+		{
+			const flatRange = wikidataToRange(flatMoment)
+			return flatRange
+				? { min: flatRange.min.format('YYYYYY-MM-DDThh:mm:ss'), max: flatRange.max.format('YYYYYY-MM-DDThh:mm:ss') }
+				: { value: null }
+		}
 	}
 	else
 	{
-		console.error(`Failed to parse relative date '${dateString}'.`)
-		return null
+		if (flatMoment && flatMoment.value)
+		{
+			flatMoment.value = flatMoment.value.format('YYYYYY-MM-DDThh:mm:ss')
+		}
+		return flatMoment
 	}
+}
 
-	if (!relSplit[0])
+/**
+ * Flattens a relative date path string into a moment.
+ * @param {*} wikidataCache 
+ * @param {*} dateString 
+ * @returns An object like { value:MOMENT, precision:INT }, or null
+ */
+function flattenRelativeDateToMoment(wikidataCache, dateString)
+{
+	var parsedPath = breakRelativeDate(dateString)
+	if (!parsedPath)
 	{
 		return null
 	}
-	else if (!wikidataCache[relSplit[0]])
+	else if (!wikidataCache[parsedPath[0]])
 	{
-		console.error(`Date for '${relSplit[0]}' wasn't cached.`)
+		console.error(`Date for '${parsedPath[0]}' wasn't cached.`)
 		return null
 	}
 	else
 	{
-		const cacheEntry = wikidataCache[relSplit[0]]
-		if (cacheEntry.value && relSplit.length > 1)
+		const cacheEntry = wikidataCache[parsedPath[0]]
+		if (cacheEntry.value && parsedPath.length > 1)
 		{
 			// break up operators
-			const dateOperators = []
-			var opStartIndex = 0
-			const operatorString = relSplit[1]
-			if (operatorString.length > 0)
-			{
-				for (var i = 1; i < operatorString.length; i++)
-				{
-					if (operatorString[i] == '+' || operatorString[i] == '>')
-					{
-						dateOperators.push(operatorString.substring(opStartIndex, i))
-						opStartIndex = i
-					}
-				}
-				dateOperators.push(operatorString.substring(opStartIndex, i))
-			}
+			const dateOperators = parsedPath.slice(1)
 
 			// handle relative segments of date
 			// About precision:
@@ -146,14 +276,15 @@ module.exports = function flattenRelativeDate(wikidataCache, dateString)
 
 				//console.log(`${component} (${precision}): ${momentDate}`)
 			}
-			return {
-				value: momentDate.format('YYYYYY-MM-DDThh:mm:ss'),
-				precision: precision
-			}
+			return { value: momentDate, precision: precision }
 		}
 		else
 		{
-			return {...cacheEntry}
+			const cachedOut = {...cacheEntry}
+			if (cachedOut.value) cachedOut.value = moment(cachedOut.value)
+			return cachedOut
 		}
 	}
-}
+}
+
+module.exports = { breakRelativeDate, flattenRelativeDate, rangeUnion, rangeUnionAdv }

package/src/render.js

@@ -461,7 +461,12 @@ renderer.produceOutput = function(inputSpec, items)
 		{
 			const labelItem = {...outputItem}
 			labelItem.id += "-label"
-			labelItem.className = [labelItem.className, "visc-toplabel"].join(' '),
+
+			var classes = labelItem.className.split(' ')
+			classes = classes.filter(c => c != "visc-left-connection" && c != "visc-right-connection")
+			classes.push("visc-toplabel")
+			labelItem.className = classes.join(' ')
+
 			outputObject.items.push(labelItem)
 			outputItem.content = outputItem.content ? " " : ""
 		}

package/src/wikidata.js

@@ -2,6 +2,7 @@
 const mypath = require("./mypath.js")
 const fs = require('fs');
 const nodepath = require('node:path');
+const moment = require('moment')
 const globalData = require("./global-data.json")
 const assert = require('node:assert/strict')
 const SparqlBuilder = require("./sparql-builder.js")
@@ -12,9 +13,9 @@ const wikidata = module.exports = {
 	verboseLogging: false,
 
 	/**
-	 * 
+	 * Caches results for SPARQL queries, using the query string as the key.
 	 */
-	cache: {},
+	queryCache: {},
 
 	/**
 	 * Caches results for path queries, using the path as the key.
@@ -25,9 +26,9 @@ const wikidata = module.exports = {
 	cacheBuster: undefined,
 
 	/**
-	 * Relative path to the term cache file.
+	 * Relative path to the query cache file.
 	 */
-	termCacheFile: "intermediate/wikidata-term-cache.json",
+	queryCacheFile: "intermediate/sparql-query-cache.json",
 
 	/**
 	 * Relative path to the path cache file.
@@ -126,8 +127,8 @@ const wikidata = module.exports = {
 	{
 		try
 		{
-			const contents = await fs.promises.readFile(this.termCacheFile)
-			this.cache = JSON.parse(contents)
+			const contents = await fs.promises.readFile(this.queryCacheFile)
+			this.queryCache = JSON.parse(contents)
 		}
 		catch
 		{
@@ -147,11 +148,11 @@ const wikidata = module.exports = {
 
 	writeCache: async function()
 	{
-		await mypath.ensureDirectoryForFile(this.termCacheFile)
+		await mypath.ensureDirectoryForFile(this.queryCacheFile)
 
-		fs.writeFile(this.termCacheFile, JSON.stringify(this.cache), err => {
+		fs.writeFile(this.queryCacheFile, JSON.stringify(this.queryCache), err => {
 			if (err) {
-				console.error(`Error writing wikidata term cache:`)
+				console.error(`Error writing wikidata query cache:`)
 				console.error(err)
 			}
 		})
@@ -184,25 +185,32 @@ const wikidata = module.exports = {
 		}
 	},
 
-	postprocessQueryTerm: function(context, term, item)
+	replaceQueryWildcards: function(term, item, itemPrefix = "")
 	{
-		if (!term)
-		{
-			return term;
-		}
-
 		// replace query wildcards
 		for (const key in item)
 		{
 			var insertValue = item[key]
 			if (typeof insertValue === "string" && insertValue.startsWith("Q"))
-				insertValue = "wd:" + insertValue
+				insertValue = itemPrefix + insertValue
 			term = term.replaceAll(`{${key}}`, insertValue)
 		}
 
 		// detect unreplaced wildcards
 		//TODO:
 
+		return term
+	},
+
+	preprocessQueryTerm: function(context, term, item)
+	{
+		if (!term)
+		{
+			return term
+		}
+
+		term = this.replaceQueryWildcards(term, item, "wd:")
+
 		// terminate term
 		if (!term.trim().endsWith("."))
 		{
@@ -238,7 +246,7 @@ const wikidata = module.exports = {
 
 		//TODO: validate query has required wildcards
 		
-		queryTerm = this.postprocessQueryTerm(inQueryTerm, queryTerm, item)
+		queryTerm = this.preprocessQueryTerm(inQueryTerm, queryTerm, item)
 		return queryTerm
 	},
 
@@ -275,7 +283,7 @@ const wikidata = module.exports = {
 		if (typeof queryTerm === 'string' || queryTerm instanceof String)
 		{
 			return {
-				value: this.postprocessQueryTerm(inQueryTerm, queryTerm, item),
+				value: this.preprocessQueryTerm(inQueryTerm, queryTerm, item),
 				min: "?_prop pqv:P1319 ?_min_value.",
 				max: "?_prop pqv:P1326 ?_max_value."
 			}
@@ -285,7 +293,7 @@ const wikidata = module.exports = {
 			const result = {}
 			for (const key in queryTerm)
 			{
-				result[key] = this.postprocessQueryTerm(inQueryTerm, queryTerm[key], item)
+				result[key] = this.preprocessQueryTerm(inQueryTerm, queryTerm[key], item)
 			}
 			return result
 		}
@@ -393,15 +401,7 @@ const wikidata = module.exports = {
 		queryBuilder.addOptionalQueryTerm(`${propVar} wikibase:rank ${rankVar}.`)
 
 		const query = queryBuilder.build()
-
-		// read cache
-		const cacheKey = query
-		if (!this.skipCache && !item.skipCache && this.cache[cacheKey])
-		{
-			return this.cache[cacheKey]
-		}
-		
-		const data = await this.runQuery(query)
+		const data = await this.runQuery(query, item.skipCache)
 		console.log(`\tQuery for ${item.id} returned ${data.results.bindings.length} results.`)
 
 		const readBinding = function(binding)
@@ -487,10 +487,18 @@ const wikidata = module.exports = {
 			}
 		}
 
-		this.cache[cacheKey] = result;
 		return result;
 	},
 
+	/**
+	 * Formats the provided Wikidata date string as ISO 8601 with a six-digit year.
+	 */
+	normalizeWikidataDate: function(dateStr)
+	{
+		const date = moment(dateStr, 'Y-MM-DDThh:mm:ss')
+		return date.format('YYYYYY-MM-DDThh:mm:ss')
+	},
+
 	/**
 	 * Runs an unsorted list of path queries.
 	 */
@@ -557,7 +565,7 @@ const wikidata = module.exports = {
 			for (const binding of data.results.bindings)
 			{
 				const key = this.extractQidFromUrl(binding['item'].value)
-				this.pathCache[key] = { value: binding['date'].value, precision: binding['precision'].value }
+				this.pathCache[key] = { value: this.normalizeWikidataDate(binding['date'].value), precision: binding['precision'].value }
 				foundKeys.add(key)
 			}
 
@@ -600,7 +608,7 @@ const wikidata = module.exports = {
 				const qid = this.extractQidFromUrl(binding['item'].value)
 				const pid = this.extractQidFromUrl(binding['p'].value)
 				const key = `${qid}:${pid}`
-				this.pathCache[key] = { value: binding['date'].value, precision: binding['precision'].value }
+				this.pathCache[key] = { value: this.normalizeWikidataDate(binding['date'].value), precision: binding['precision'].value }
 				foundKeys.add(key)
 			}
 
@@ -620,6 +628,25 @@ const wikidata = module.exports = {
 		for (const wdProp of queries)
 		{
 			if (!this.pathCache[wdProp]) qualsToQuery.push(wdProp)
+
+			// grab corresponding min/max props as well
+			const split = wdProp.split(':')
+			const wdpk = split.slice(0, 3).join(':')
+			switch (split[3])
+			{
+				case "P580":
+					query = `${wdpk}:P1319` // earliest date
+					if (!this.pathCache[query]) qualsToQuery.push(query)
+					query = `${wdpk}:P8555` // latest start date
+					if (!this.pathCache[query]) qualsToQuery.push(query)
+					break;
+				case "P582":
+					query = `${wdpk}:P8554` // earliest end date
+					if (!this.pathCache[query]) qualsToQuery.push(query)
+					query = `${wdpk}:P1326` // latest date
+					if (!this.pathCache[query]) qualsToQuery.push(query)
+					break;
+			}
 		}
 		if (qualsToQuery.length > 0)
 		{
@@ -646,7 +673,7 @@ const wikidata = module.exports = {
 				const valId = this.extractQidFromUrl(binding['value'].value)
 				const qualId = this.extractQidFromUrl(binding['q'].value)
 				const key = `${itemId}:${propId}:${valId}:${qualId}`
-				this.pathCache[key] = { value: binding['date'].value, precision: binding['precision'].value }
+				this.pathCache[key] = { value: this.normalizeWikidataDate(binding['date'].value), precision: binding['precision'].value }
 				foundKeys.add(key)
 			}
 
@@ -695,7 +722,11 @@ const wikidata = module.exports = {
 		queryBuilder.addWikibaseLabel(this.lang)
 		
 		const query = queryBuilder.build()
-		const data = await this.runQuery(query)
+		const data = await this.runQuery(query, templateItem.skipCache)
+		if (!data)
+		{
+			throw 'No response data from Wikidata query.'
+		}
 
 		const newItems = []
 
@@ -723,8 +754,13 @@ const wikidata = module.exports = {
 	},
 
 	// runs a SPARQL query
-	runQuery: async function(query)
+	runQuery: async function(query, skipCache = false)
 	{
+		if (!skipCache && !this.skipCache && this.queryCache[query])
+		{
+			return this.queryCache[query]
+		}
+
 		if (this.verboseLogging) console.log(query)
 
 		assert(this.options)
@@ -739,6 +775,7 @@ const wikidata = module.exports = {
 		else
 		{
 			const data = await response.json()
+			this.queryCache[query] = data
 			return data
 		}
 	}