wikibase-cli 17.0.10 → 18.0.1

package/lib/get_entities_schemas.js

@@ -1,4 +1,3 @@
-import fetch from 'node-fetch'
 import { isEntitySchemaId } from 'wikibase-sdk'
 import { exitOnMissingInstance } from '#lib/exit_on_missing'
 import program from './program.js'

package/lib/get_ttl_entities.js

@@ -1,5 +1,4 @@
 import crypto from 'node:crypto'
-import fetch from 'node-fetch'
 import { isEntityId } from 'wikibase-sdk'
 import { exitOnMissingInstance } from '#lib/exit_on_missing'
 import program from '#lib/program'

package/lib/make_sparql_query.js

@@ -34,6 +34,7 @@ function makeRequest (url, format) {
       headers: {
         Accept: formatAcceptHeader[format],
       },
+      parseResponseAsJson: format === 'json',
     })
   } else {
     const [ postUrl, urlencodedSparql ] = url.split('?')
@@ -45,6 +46,7 @@ function makeRequest (url, format) {
         'Content-type': 'application/sparql-query',
         Accept: formatAcceptHeader[format],
       },
+      parseResponseAsJson: format === 'json',
     })
   }
 }
@@ -72,6 +74,9 @@ function parseResult (format) {
         results = minimizeSimplifiedSparqlResults(simplifySparqlResults(results))
         if (indexAttribute) results = indexBy(results, indexAttribute)
       }
+      if (format === 'csv') {
+        results = results.replace(/\r\n/g, '\n')
+      }
       return results
     } catch (err) {
       err.context = { results }

package/lib/request.js

@@ -1,26 +1,7 @@
-// Using a custom agent to set keepAlive=true
-// https://nodejs.org/api/http.html#http_class_http_agent
-// https://github.com/bitinn/node-fetch#custom-agent
-import http from 'node:http'
-import https from 'node:https'
-import { pick } from 'lodash-es'
-import fetch from 'node-fetch'
 import { debug } from '#lib/debug'
 import program from './program.js'
 import globalHeaders from './request_headers.js'
 
-const httpAgent = new http.Agent({ keepAlive: true })
-const httpsAgent = new https.Agent({ keepAlive: true })
-// Useful to:
-// - accept self-signed certificates
-// - accept certificates that would otherwise generate a UNABLE_TO_VERIFY_LEAF_SIGNATURE error
-const insecureHttpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: false })
-const tlsErrorsHosts = new Set([ 'datos.bne.es' ])
-
-const agent = ({ host, protocol }) => {
-  if (tlsErrorsHosts.has(host)) return protocol === 'http:' ? httpAgent : insecureHttpsAgent
-  else return protocol === 'http:' ? httpAgent : httpsAgent
-}
 const buildHeaders = customHeaders => {
   if (customHeaders) return Object.assign({}, globalHeaders, customHeaders)
   else return globalHeaders
@@ -29,41 +10,46 @@ const buildHeaders = customHeaders => {
 function request (url, options) {
   const { method = 'get', headers, body } = options
   debug('request', method.toUpperCase(), url, { headers, body })
+  options.keepalive = true
   return fetch(url, options)
 }
 
 export const get = async url => {
-  const res = await request(url, { headers: globalHeaders, agent })
+  const res = await request(url, { headers: globalHeaders })
   return handleResponse({ res, url })
 }
 
-export const customGet = async ({ url, headers }) => {
-  const res = await request(url, { headers: buildHeaders(headers), agent })
-  return handleResponse({ res, url })
+export const customGet = async ({ url, headers, parseResponseAsJson = true }) => {
+  const res = await request(url, { headers: buildHeaders(headers) })
+  return handleResponse({ res, url, parseResponseAsJson })
 }
 
-export const post = async ({ url, body, headers }) => {
+export const post = async ({ url, body, headers, parseResponseAsJson = true }) => {
   const method = 'post'
-  const res = await request(url, { method, body, headers: buildHeaders(headers), agent })
-  return handleResponse({ res, url, method })
+  const res = await request(url, { method, body, headers: buildHeaders(headers) })
+  return handleResponse({ res, url, method, parseResponseAsJson })
 }
 
-async function handleResponse ({ res, url, method }) {
+async function handleResponse ({ res, url, method, parseResponseAsJson = true }) {
   let body = await res.text()
   const { logResponseHeaders } = program
   if (logResponseHeaders) {
-    let headers = res.headers.raw()
+    const headers = {}
     if (typeof logResponseHeaders === 'string') {
       const headersSubset = logResponseHeaders.split(',')
-      headers = pick(headers, headersSubset)
+      for (const headerName of headersSubset) {
+        headers[headerName] = res.headers.get(headerName)?.split(';').map(val => val.trim())
+      }
     }
     console.error(JSON.stringify({ request: { url, method }, response: { headers } }))
   }
-  // When Wikibase crash it doesn't return JSON errors anymore
-  try {
-    body = JSON.parse(body)
-  } catch (err) {
-    console.error('could not parse response body as JSON', err, { body })
+  if (parseResponseAsJson) {
+    // When Wikibase crash it doesn't return JSON errors anymore
+    try {
+      body = JSON.parse(body)
+    } catch (err) {
+      console.error('could not parse response body as JSON', err, { body })
+    }
   }
   const { status: statusCode } = res
   if (statusCode >= 400) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wikibase-cli",
-  "version": "17.0.10",
+  "version": "18.0.1",
   "description": "A command-line interface to Wikibase",
   "type": "module",
   "main": "index.js",
@@ -14,7 +14,7 @@
     "lint": "eslint --config .eslintrc.cjs bin/* lib test",
     "lint-fix": "eslint --config .eslintrc.cjs --fix bin/* lib test",
     "lint-staged": "./scripts/lint_staged",
-    "prepublishOnly": "npm run lint && npm test",
+    "prepublishOnly": "export MOCHA_OPTIONS='--bail'; npm run lint && npm test",
     "postpublish": "npm run docker:publish && git push --tags",
     "test": "mocha --exit --timeout 20000 $MOCHA_OPTIONS",
     "update-toc": "./scripts/update_toc"
@@ -52,13 +52,12 @@
     "copy-paste": "^1.5.0",
     "lodash-es": "^4.17.21",
     "mkdirp": "^3.0.1",
-    "node-fetch": "^2.6.0",
     "open": "^9.1.0",
     "read": "^2.1.0",
     "shell-quote": "^1.8.1",
     "split": "^1.0.1",
     "through": "^2.3.8",
-    "wikibase-edit": "^6.0.4",
+    "wikibase-edit": "^7.0.2",
     "wikibase-sdk": "^10.0.2",
     "wikidata-lang": "^2.0.11"
   },
@@ -75,6 +74,6 @@
     "should": "^13.2.3"
   },
   "engines": {
-    "node": ">= 14.17.0"
+    "node": ">= 18"
   }
 }