wiki-entity 0.4.2 → 0.4.6

package/README.md

@@ -1,114 +1,117 @@
-# wiki-entity
-
-Wiki(pedia/data) entity extractor.
-
-## Usage
-
-```
-import { getEntities } from 'wiki-entity';
-
-// get Europe by title
-getEntities({ language: 'en', titles: ['Europe'] }).then(entities => {});
-
-// get Europe by id
-getEntities({ language: 'en', ids: ['Q46'] }).then(entities => {});
-```
-
-## API
-
-### getEntities(params): Promise<WikiEntity[]>
-
-Gets entities from wikidata and wikipedia.
-
-#### params
-
-Required. `ids` or `titles` require. Params properties:
-
-- **ids** :*string[]* - wikidata ids; (max 200)
-- **titles** :*string[]* - wikipedia article titles; (max 200)
-- **language** :*string* - language of `titles` param and of the result object info; default: `en`
-- **props** :*string[]* - entity props to get. Can by: `info`|`sitelinks`|`aliases`|`labels`|`descriptions`|`claims`|`datatype`;
-- **claims** :*string* - How to resolve the claims. Can be: `none`, `all`, `item`, `property`. Default: `none`. `all` resolves `item` and `property` types.
-- **extract** :*number* - Sentences in the extract. Default: `0`. Works only if `sitelinks` is present.
-- **types** :*boolean* | *string*[] - `true` to get entity types. Filter types by [prefixes](https://dbpedia.org/sparql?nsdecl). Example: [`dbo`, `schema`] will return only types defined by `dbpedia.org/ontology/` and `schema.org`. Default: `false`.
-- **redirects** : *boolean* - get wikipedia redirects titles. Default: `false`. Works only if `sitelinks` is present.
-- **categories** : *boolean* - get wikipedia article categories. Default: `false`. Works only if `sitelinks` is present.
-
-### convertToSimpleEntity(wikiEntity: WikiEntity): SimpleEntity
-
-Convert a complex WikiEntity object to SimpleEntity object.
-
-## WikiEntity
-
-`WikiEntity` is a simple version of an item returned by Wikidata API. It also may include some extra properties from Wikipadia API.
-
-```ts
-
-type WikiEntity = {
-    id: string;
-    label?: string;
-    description?: string;
-    extract?: string;
-    pageid?: number;
-    aliases?: string[];
-    sitelinks?: IIndexType<string>;
-    claims?: IIndexType<WikidataProperty>;
-    types?: string[];
-    redirects?: string[];
-    categories?: string[];
-}
-
-type WikidataProperty = {
-    id: string;
-    label?: string;
-    description?: string;
-    values: WikidataPropertyValue[];
-}
-
-type WikidataPropertyValue = {
-    datatype: string;
-    value: any;
-    pageid?: number;
-    value_string?: string;
-    label?: string;
-    description?: string;
-}
-
-type IIndexType<T> = {
-    [index: string]: T
-}
-
-```
-
-## SimpleEntity
-
-```ts
-
-export enum SimpleEntityType {
-    EVENT = 'E',
-    ORG = 'O',
-    PERSON = 'H',
-    PLACE = 'P',
-    PRODUCT = 'R',
-    WORK = 'W',
-}
-
-export type SimpleEntityData = { [prop: string]: string[] }
-
-export type SimpleEntity = {
-    lang?: string
-    wikiDataId?: string
-    name?: string
-    abbr?: string
-    description?: string
-    about?: string
-    wikiPageId?: number
-    wikiPageTitle?: string
-    type?: SimpleEntityType
-    types?: string[]
-    countryCodes?: string[]
-    data?: SimpleEntityData
-    categories?: string[]
-}
-
-```
+# wiki-entity
+
+Wiki(pedia/data) entity extractor.
+
+## Usage
+
+```
+import { getEntities } from 'wiki-entity';
+
+// get Europe by title
+getEntities({ language: 'en', titles: ['Europe'] }).then(entities => {});
+
+// get Europe by id
+getEntities({ language: 'en', ids: ['Q46'] }).then(entities => {});
+```
+
+## API
+
+### getEntities(params): Promise<WikiEntity[]>
+
+Gets entities from wikidata and wikipedia.
+
+#### params
+
+Required. `ids` or `titles` require. Params properties:
+
+- **ids** :*string[]* - wikidata ids; (max 200)
+- **titles** :*string[]* - wikipedia article titles; (max 200)
+- **language** :*string* - language of `titles` param and of the result object info; default: `en`
+- **props** :*string[]* - entity props to get. Can by: `info`|`sitelinks`|`aliases`|`labels`|`descriptions`|`claims`|`datatype`;
+- **claims** :*string* - How to resolve the claims. Can be: `none`, `all`, `item`, `property`. Default: `none`. `all` resolves `item` and `property` types.
+- **extract** :*number* - Sentences in the extract. Default: `0`. Works only if `sitelinks` is present.
+- **types** :*boolean* | *string*[] - `true` to get entity types. Filter types by [prefixes](https://dbpedia.org/sparql?nsdecl). Example: [`dbo`, `schema`] will return only types defined by `dbpedia.org/ontology/` and `schema.org`. Default: `false`.
+- **redirects** : *boolean* - get wikipedia redirects titles. Default: `false`. Works only if `sitelinks` is present.
+- **categories** : *boolean* - get wikipedia article categories. Default: `false`. Works only if `sitelinks` is present.
+- **httpTimeout**: *number* - http requests timeout
+- **languages**: "string"[] - languages for `WikiEntity.labels`
+
+### convertToSimpleEntity(wikiEntity: WikiEntity): SimpleEntity
+
+Convert a complex WikiEntity object to SimpleEntity object.
+
+## WikiEntity
+
+`WikiEntity` is a simple version of an item returned by Wikidata API. It also may include some extra properties from Wikipadia API.
+
+```ts
+
+type WikiEntity = {
+    id: string;
+    label?: string;
+    labels?: IIndexType<string>;
+    description?: string;
+    extract?: string;
+    pageid?: number;
+    aliases?: string[];
+    sitelinks?: IIndexType<string>;
+    claims?: IIndexType<WikidataProperty>;
+    types?: string[];
+    redirects?: string[];
+    categories?: string[];
+}
+
+type WikidataProperty = {
+    id: string;
+    label?: string;
+    description?: string;
+    values: WikidataPropertyValue[];
+}
+
+type WikidataPropertyValue = {
+    datatype: string;
+    value: any;
+    pageid?: number;
+    value_string?: string;
+    label?: string;
+    description?: string;
+}
+
+type IIndexType<T> = {
+    [index: string]: T
+}
+
+```
+
+## SimpleEntity
+
+```ts
+
+export enum SimpleEntityType {
+    EVENT = 'E',
+    ORG = 'O',
+    PERSON = 'H',
+    PLACE = 'P',
+    PRODUCT = 'R',
+    WORK = 'W',
+}
+
+export type SimpleEntityData = { [prop: string]: string[] }
+
+export type SimpleEntity = {
+    lang?: string
+    wikiDataId?: string
+    name?: string
+    abbr?: string
+    description?: string
+    about?: string
+    wikiPageId?: number
+    wikiPageTitle?: string
+    type?: SimpleEntityType
+    types?: string[]
+    countryCodes?: string[]
+    data?: SimpleEntityData
+    categories?: string[]
+}
+
+```

package/lib/index.js

@@ -13,16 +13,16 @@ exports.simplifyEntity = simplify_entity_1.simplifyEntity;
 __export(require("./simpleEntity"));
 __export(require("./types"));
 function getEntities(params) {
-    const lang = params.language || 'en';
+    const lang = params.language || "en";
     return wikidata_1.getEntities(params)
         .then(function (wikiDataEntities) {
         const entities = wikiDataEntities;
-        const ids = Object.keys(entities).filter(id => utils_1.isValidWikiId(id));
+        const ids = Object.keys(entities).filter((id) => utils_1.isValidWikiId(id));
         if (ids.length === 0) {
             return entities;
         }
         const tasks = [];
-        const wikiApi = new api_1.Api();
+        const wikiApi = new api_1.Api({}, params.httpTimeout);
         let callWikiApi = false;
         if (params.extract) {
             callWikiApi = true;
@@ -36,16 +36,26 @@ function getEntities(params) {
             callWikiApi = true;
             wikiApi.categories();
         }
-        if (callWikiApi && entities[ids[0]].sitelinks) {
-            const listEntities = ids.map(id => entities[id]);
+        if (callWikiApi && entities[ids[0]] && entities[ids[0]].sitelinks) {
             const titleIds = ids.reduce((prev, id) => {
-                if (entities[id].sitelinks[lang]) {
+                if (entities[id] &&
+                    entities[id].sitelinks &&
+                    entities[id].sitelinks[lang]) {
                     prev[entities[id].sitelinks[lang]] = id;
                 }
                 return prev;
             }, {});
-            tasks.push(wikiApi.query(lang, { titles: ids.map(id => entities[id].sitelinks[lang]).filter(item => !!item).join('|') })
-                .then(apiResults => apiResults.forEach(result => {
+            tasks.push(wikiApi
+                .query(lang, {
+                titles: ids
+                    .map((id) => (entities[id] &&
+                    entities[id].sitelinks &&
+                    entities[id].sitelinks[lang]) ||
+                    null)
+                    .filter((item) => !!item)
+                    .join("|")
+            })
+                .then((apiResults) => apiResults.forEach((result) => {
                 const entity = entities[titleIds[result.title]];
                 if (params.extract) {
                     entity.extract = result.extract;
@@ -59,12 +69,16 @@ function getEntities(params) {
             })));
         }
         if (params.types === true || Array.isArray(params.types)) {
-            const prefixes = Array.isArray(params.types) ? params.types : null;
-            ids.forEach(id => {
-                if (entities[id].sitelinks) {
-                    const enName = entities[id].sitelinks['en'];
+            const prefixes = Array.isArray(params.types)
+                ? params.types
+                : null;
+            ids.forEach((id) => {
+                if (entities[id] && entities[id].sitelinks) {
+                    const enName = entities[id].sitelinks["en"];
                     if (enName) {
-                        tasks.push(get_entity_types_1.getEntityTypesByName(enName, prefixes).then(types => { entities[id].types = types; }));
+                        tasks.push(get_entity_types_1.getEntityTypesByName(enName, prefixes).then((types) => {
+                            entities[id].types = types;
+                        }));
                     }
                 }
             });
@@ -72,7 +86,9 @@ function getEntities(params) {
         return Promise.all(tasks).then(() => entities);
     })
         .then(function (resultEntities) {
-        return Object.keys(resultEntities).map(id => resultEntities[id]);
+        return Object.keys(resultEntities)
+            .map((id) => resultEntities[id])
+            .filter((it) => !!it);
     });
 }
 exports.getEntities = getEntities;

package/lib/request.js

@@ -1,33 +1,18 @@
-'use strict';
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-const request = require('request');
-const debug = require('debug')('wiki-entity');
-function default_1(options) {
-    options = Object.assign({
-        method: 'GET',
-        json: true,
-        encoding: 'utf8',
-        headers: {
-            'User-Agent': 'entity-finder'
-        },
-        timeout: 5 * 1000
-    }, options);
-    if (options.qs) {
-        for (var prop in options.qs) {
-            if (~[null].indexOf(options.qs[prop])) {
-                delete options.qs[prop];
-            }
-        }
-    }
-    debug('request ' + (options.uri || options.url), options.qs);
-    return new Promise(function (resolve, reject) {
-        request(options, function (error, response, body) {
-            if (error || response.statusCode >= 400) {
-                return reject(error || new Error('Invalid statusCode=' + response.statusCode + (options.uri || options.url)));
-            }
-            resolve(body);
-        });
+const axios_1 = require("axios");
+function default_1(url, options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const response = yield axios_1.default(url, Object.assign({ method: "GET", responseType: "json", headers: { "User-Agent": "entity-finder" } }, options, { timeout: options.timeout || 10 * 1000 }));
+        return response.data;
     });
 }
 exports.default = default_1;
-;

package/lib/simpleEntity/getEntityData.js

@@ -8,11 +8,13 @@ function getEntityData(wikiEntity) {
     for (let key in wikiEntity.claims) {
         const values = wikiEntity.claims[key].values;
         data[key] = [];
-        values.forEach(value => {
+        values.forEach((value) => {
             if (!value.value_string && value.value === null) {
                 throw new Error(`WikiEntity claim value cannot be null: ${wikiEntity.id}:${key}`);
             }
-            const v = { value: value.value_string || value.value.toString() };
+            const v = {
+                value: value.value_string || value.value.toString()
+            };
             if (value.label && v.value !== value.label) {
                 v.label = value.label;
             }

package/lib/simpleEntity/getEntityType.js

@@ -13,6 +13,7 @@ const TYPES_MAP = {
     'dbo:Location': simpleEntity_1.SimpleEntityType.PLACE,
     'wikidata:Q515': simpleEntity_1.SimpleEntityType.PLACE,
     'wikidata:Q486972': simpleEntity_1.SimpleEntityType.PLACE,
+    'dbo:Company': simpleEntity_1.SimpleEntityType.ORG,
     'schema:Organization': simpleEntity_1.SimpleEntityType.ORG,
     'dbo:Organisation': simpleEntity_1.SimpleEntityType.ORG,
     'wikidata:Q43229': simpleEntity_1.SimpleEntityType.ORG,
@@ -29,10 +30,20 @@ const TYPES_MAP = {
     'dbo:Software': simpleEntity_1.SimpleEntityType.PRODUCT,
     'wikidata:Q7397': simpleEntity_1.SimpleEntityType.PRODUCT,
 };
+const TYPES_NAME_MAP = Object.keys(TYPES_MAP)
+    .reduce((map, name) => {
+    const type = TYPES_MAP[name];
+    map[type] = map[type] || [];
+    map[type].push(name);
+    return map;
+}, {});
 function getEntityType(wikiEntity) {
     if (!wikiEntity.types) {
         return null;
     }
+    if (containsType(wikiEntity.types, simpleEntity_1.SimpleEntityType.ORG) && containsType(wikiEntity.types, simpleEntity_1.SimpleEntityType.PERSON)) {
+        return simpleEntity_1.SimpleEntityType.ORG;
+    }
     for (var i = 0; i < wikiEntity.types.length; i++) {
         if (TYPES_MAP[wikiEntity.types[i]]) {
             return TYPES_MAP[wikiEntity.types[i]];
@@ -40,3 +51,11 @@ function getEntityType(wikiEntity) {
     }
 }
 exports.getEntityType = getEntityType;
+function containsType(types, type) {
+    for (const typeName of types) {
+        if (~TYPES_NAME_MAP[type].indexOf(typeName)) {
+            return true;
+        }
+    }
+    return false;
+}

package/lib/wikidata/api.js

@@ -1,43 +1,55 @@
 "use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const request_1 = require("../request");
 const types_1 = require("../types");
 const utils_1 = require("../utils");
-const API_URL = 'https://www.wikidata.org/w/api.php';
+const API_URL = "https://www.wikidata.org/w/api.php";
 function getEntities(params) {
-    const qs = {
-        action: 'wbgetentities',
-        ids: params.ids && params.ids.join('|'),
-        titles: params.titles && params.titles.join('|'),
-        props: (params.props || [
-            types_1.WikidataPropsParam.info,
-            types_1.WikidataPropsParam.sitelinks,
-            types_1.WikidataPropsParam.aliases,
-            types_1.WikidataPropsParam.labels,
-            types_1.WikidataPropsParam.descriptions,
-            types_1.WikidataPropsParam.claims,
-            types_1.WikidataPropsParam.datatype
-        ]).join('|'),
-        languages: getStringArrayParam(params.language, 'en'),
-        redirects: params.redirect || 'yes',
-        format: 'json',
-        sites: ''
-    };
-    if (params.titles) {
-        qs.sites = qs.languages.split('|')[0] + 'wiki';
-    }
-    else {
-        delete qs.sites;
-    }
-    return request_1.default({ qs: qs, url: API_URL, timeout: 10 * 1000 })
-        .then(data => {
-        if (hasError(data)) {
-            return Promise.reject(getError(data));
+    return __awaiter(this, void 0, void 0, function* () {
+        const qs = {
+            action: "wbgetentities",
+            ids: params.ids && params.ids.join("|"),
+            titles: params.titles && params.titles.join("|"),
+            props: (params.props || [
+                types_1.WikidataPropsParam.info,
+                types_1.WikidataPropsParam.sitelinks,
+                types_1.WikidataPropsParam.aliases,
+                types_1.WikidataPropsParam.labels,
+                types_1.WikidataPropsParam.descriptions,
+                types_1.WikidataPropsParam.claims,
+                types_1.WikidataPropsParam.datatype
+            ]).join("|"),
+            languages: params.languages
+                ? params.languages.join("|")
+                : params.language || "en",
+            redirects: params.redirect || "yes",
+            format: "json",
+            sites: ""
+        };
+        if (params.titles) {
+            qs.sites = qs.languages.split("|")[0] + "wiki";
+        }
+        else {
+            delete qs.sites;
         }
-        const entities = data && data.entities || {};
+        const data = yield request_1.default(API_URL, {
+            params: qs,
+            timeout: params.httpTimeout
+        });
+        if (hasError(data))
+            throw getError(data);
+        const entities = (data && data.entities) || {};
         for (let id of Object.keys(entities)) {
-            if (!utils_1.isValidWikiId(id)) {
-                entities[id] = null;
+            if (!utils_1.isValidWikiId(id) || ~Object.keys(entities[id]).indexOf("missing")) {
+                delete entities[id];
             }
         }
         return entities;
@@ -45,25 +57,21 @@ function getEntities(params) {
 }
 exports.getEntities = getEntities;
 function getManyEntities(params) {
-    try {
+    return __awaiter(this, void 0, void 0, function* () {
         validateParams(params);
-    }
-    catch (e) {
-        return Promise.reject(e);
-    }
-    const keyName = (!!params.ids) ? 'ids' : 'titles';
-    const keyValues = params[keyName];
-    const max = 50;
-    const countParts = keyValues.length / max + 1;
-    const parts = [];
-    for (var i = 0; i < countParts && i < 4; i++) {
-        const partParams = Object.assign({}, params);
-        partParams[keyName] = keyValues.slice(i * max, (i + 1) * max);
-        if (partParams[keyName].length > 0) {
-            parts.push(getEntities(partParams));
+        const keyName = !!params.ids ? "ids" : "titles";
+        const keyValues = params[keyName];
+        const max = 50;
+        const countParts = keyValues.length / max + 1;
+        const parts = [];
+        for (var i = 0; i < countParts && i < 4; i++) {
+            const partParams = Object.assign({}, params);
+            partParams[keyName] = keyValues.slice(i * max, (i + 1) * max);
+            if (partParams[keyName].length > 0) {
+                parts.push(getEntities(partParams));
+            }
         }
-    }
-    return Promise.all(parts).then(function (results) {
+        const results = yield Promise.all(parts);
         if (results.length === 0) {
             return null;
         }
@@ -78,12 +86,15 @@ function getManyEntities(params) {
 exports.getManyEntities = getManyEntities;
 function validateParams(params) {
     if (!params.ids && !params.titles) {
-        throw new Error('Invalid params: `ids` or `titles` are required');
+        throw new Error("Invalid params: `ids` or `titles` are required");
     }
 }
 exports.validateParams = validateParams;
 function getError(data) {
-    return data && data.error && new Error(data.error.info || 'Wikidata Api Error') || new Error('NO error');
+    return ((data &&
+        data.error &&
+        new Error(data.error.info || "Wikidata Api Error")) ||
+        new Error("NO error"));
 }
 function hasError(data) {
     return !!data.error;

package/lib/wikidata/get_entity_types.js

@@ -2,48 +2,51 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const request_1 = require("../request");
 const PREFIXES_MAP = {
-    'http://www.ontologydesignpatterns.org/ont/dul/DUL.owl#': 'dul',
-    'http://dbpedia.org/ontology/': 'dbo',
-    'http://www.w3.org/2002/07/owl#': 'owl',
-    'http://www.wikidata.org/entity/': 'wikidata',
-    'http://schema.org/': 'schema',
-    'http://xmlns.com/foaf/0.1/': 'foaf',
-    'http://www.w3.org/2003/01/geo/wgs84_pos#': 'geo'
+    "http://www.ontologydesignpatterns.org/ont/dul/DUL.owl#": "dul",
+    "http://dbpedia.org/ontology/": "dbo",
+    "http://www.w3.org/2002/07/owl#": "owl",
+    "http://www.wikidata.org/entity/": "wikidata",
+    "http://schema.org/": "schema",
+    "http://xmlns.com/foaf/0.1/": "foaf",
+    "http://www.w3.org/2003/01/geo/wgs84_pos#": "geo"
 };
-const PREFIXES = Object.keys(PREFIXES_MAP).map(key => PREFIXES_MAP[key]);
-const PREFIXES_REG = new RegExp('^(' + Object.keys(PREFIXES_MAP).join('|') + ')');
+const PREFIXES = Object.keys(PREFIXES_MAP).map((key) => PREFIXES_MAP[key]);
+const PREFIXES_REG = new RegExp("^(" + Object.keys(PREFIXES_MAP).join("|") + ")");
 function getEntityTypesByName(name, prefixes) {
     if (!prefixes || !prefixes.length) {
         prefixes = PREFIXES;
     }
     return dbpediaTypes(name)
-        .then(types => parseTypes(types))
-        .then(types => {
-        return types.filter(item => PREFIXES_REG.test(item))
-            .map(item => {
+        .then((types) => parseTypes(types))
+        .then((types) => {
+        return types
+            .filter((item) => PREFIXES_REG.test(item))
+            .map((item) => {
             const key = PREFIXES_REG.exec(item)[1];
-            return PREFIXES_MAP[key] + ':' + item.substr(key.length);
+            return PREFIXES_MAP[key] + ":" + item.substr(key.length);
         });
     })
-        .then(types => repairTypes(types));
+        .then((types) => repairTypes(types));
 }
 exports.getEntityTypesByName = getEntityTypesByName;
 function parseTypes(types) {
-    return types.map(item => item.type.value);
+    return types.map((item) => item.type.value);
 }
 function dbpediaTypes(name) {
-    const query = `
-SELECT ?type
-WHERE {
-    <http://dbpedia.org/resource/${name.replace(/\s+/g, '_')}> rdf:type ?type
+    const query = `
+SELECT ?type
+WHERE {
+    <http://dbpedia.org/resource/${name.replace(/\s+/g, "_")}> rdf:type ?type
 }`;
-    return request_1.default({ url: 'http://dbpedia.org/sparql', qs: { query: query }, timeout: 10 * 1000 })
-        .then(data => data.results && data.results.bindings);
+    return request_1.default("http://dbpedia.org/sparql", {
+        params: { query: query },
+        timeout: 10 * 1000
+    }).then((data) => data.results && data.results.bindings);
 }
 function repairTypes(types) {
-    const personIndex = types.findIndex(item => /:Person$/.test(item));
+    const personIndex = types.findIndex((item) => /:Person$/.test(item));
     if (personIndex > -1) {
-        const placeIndex = types.findIndex(item => /:Place$/.test(item));
+        const placeIndex = types.findIndex((item) => /:Place$/.test(item));
         if (placeIndex > -1) {
             types.splice(personIndex, 1);
         }

package/lib/wikidata/index.js

@@ -5,17 +5,18 @@ const api_1 = require("./api");
 const simplify_entity_1 = require("./simplify_entity");
 const utils_1 = require("../utils");
 function getEntities(params) {
-    const claims = params.claims || 'none';
-    const lang = params.language || 'en';
-    return api_1.getManyEntities(params)
-        .then(function (entities) {
-        const ids = Object.keys(entities).filter(id => utils_1.isValidWikiId(id));
-        ids.forEach(id => { entities[id] = simplify_entity_1.simplifyEntity(lang, entities[id]); });
+    const claims = params.claims || "none";
+    const lang = params.language || "en";
+    return api_1.getManyEntities(params).then(function (entities) {
+        const ids = Object.keys(entities).filter((id) => utils_1.isValidWikiId(id));
+        ids.forEach((id) => {
+            entities[id] = simplify_entity_1.simplifyEntity(lang, entities[id]);
+        });
         const tasks = [];
-        if (~['all', 'property'].indexOf(claims)) {
+        if (~["all", "property"].indexOf(claims)) {
             tasks.push(exploreEntitiesProperties(entities, lang));
         }
-        if (~['all', 'item'].indexOf(claims)) {
+        if (~["all", "item"].indexOf(claims)) {
             tasks.push(utils_1.eachSeries(ids, (id) => exploreEntityClaims(entities[id].claims, { language: lang })));
         }
         return Promise.all(tasks).then(() => entities);
@@ -24,9 +25,8 @@ function getEntities(params) {
 exports.getEntities = getEntities;
 function exploreEntitiesProperties(entities, lang) {
     let ids = [];
-    const paths = {};
     const entitiesIds = Object.keys(entities);
-    entitiesIds.forEach(entityId => {
+    entitiesIds.forEach((entityId) => {
         const entity = entities[entityId];
         if (entity.claims) {
             ids = ids.concat(Object.keys(entity.claims));
@@ -43,17 +43,17 @@ function exploreEntitiesProperties(entities, lang) {
             types_1.WikidataPropsParam.info,
             types_1.WikidataPropsParam.labels,
             types_1.WikidataPropsParam.descriptions,
-            types_1.WikidataPropsParam.datatype,
+            types_1.WikidataPropsParam.datatype
         ],
-        claims: 'none'
+        claims: "none"
     }).then(function (properties) {
-        Object.keys(properties).forEach(propertyId => {
-            entitiesIds.forEach(entityId => {
+        Object.keys(properties).forEach((propertyId) => {
+            entitiesIds.forEach((entityId) => {
                 const entity = entities[entityId];
                 if (entity.claims && entity.claims[propertyId]) {
                     if (entity.claims && entity.claims[propertyId]) {
                         for (var prop in properties[propertyId]) {
-                            if (~['label', 'description'].indexOf(prop)) {
+                            if (~["label", "description"].indexOf(prop)) {
                                 entity.claims[propertyId][prop] = properties[propertyId][prop];
                             }
                         }
@@ -70,9 +70,9 @@ function exploreEntityClaims(claims, params) {
     }
     const ids = [];
     const paths = {};
-    Object.keys(claims).forEach(property => {
+    Object.keys(claims).forEach((property) => {
         claims[property].values.forEach((propertyValue, index) => {
-            if (propertyValue.datatype === 'wikibase-item') {
+            if (propertyValue.datatype === "wikibase-item") {
                 const id = propertyValue.value;
                 paths[id] = paths[id] || [];
                 paths[id].push({ pid: property, value: propertyValue, index });
@@ -90,17 +90,17 @@ function exploreEntityClaims(claims, params) {
         types_1.WikidataPropsParam.info,
         types_1.WikidataPropsParam.labels,
         types_1.WikidataPropsParam.descriptions,
-        types_1.WikidataPropsParam.datatype,
+        types_1.WikidataPropsParam.datatype
     ];
-    params.claims = params.claims || 'none';
-    return getEntities(params).then(entities => {
-        Object.keys(entities).forEach(id => {
+    params.claims = params.claims || "none";
+    return getEntities(params).then((entities) => {
+        Object.keys(entities).forEach((id) => {
             const item = entities[id];
             const pa = paths[item.id] || [];
-            pa.forEach(pai => {
+            pa.forEach((pai) => {
                 const val = claims[pai.pid].values[pai.index];
                 for (var prop in item) {
-                    if (~['label', 'pageid', 'description'].indexOf(prop)) {
+                    if (~["label", "pageid", "description"].indexOf(prop)) {
                         val[prop] = item[prop];
                     }
                 }

package/lib/wikidata/simplify_entity.js

@@ -7,7 +7,8 @@ function simplifyEntity(lang, data, options = {}) {
         entity.pageid = data.pageid;
     }
     if (options.labels !== false && data.labels) {
-        entity.label = simplifyLabels(data.labels)[lang];
+        entity.labels = simplifyLabels(data.labels);
+        entity.label = entity.labels[lang];
     }
     if (options.descriptions !== false && data.descriptions) {
         entity.description = simplifyDescriptions(data.descriptions)[lang];
@@ -45,7 +46,9 @@ exports.simplifyLabels = simplifyLabels;
 function simplifySitelinks(data) {
     if (data) {
         const result = {};
-        Object.keys(data).forEach(site => { result[site.replace(/wiki$/, '')] = data[site].title; });
+        Object.keys(data).forEach((site) => {
+            result[site.replace(/wiki$/, "")] = data[site].title;
+        });
         return result;
     }
     return null;
@@ -54,7 +57,9 @@ exports.simplifySitelinks = simplifySitelinks;
 function getLangValue(data) {
     if (data) {
         const result = {};
-        Object.keys(data).forEach(lang => { result[lang] = data[lang].value; });
+        Object.keys(data).forEach((lang) => {
+            result[lang] = data[lang].value;
+        });
         return result;
     }
     return null;
@@ -62,7 +67,9 @@ function getLangValue(data) {
 function getManyLangValue(data) {
     if (data) {
         const result = {};
-        Object.keys(data).forEach(lang => { result[lang] = data[lang].map((item) => item.value); });
+        Object.keys(data).forEach((lang) => {
+            result[lang] = data[lang].map((item) => item.value);
+        });
         return result;
     }
     return null;

package/lib/wikipedia/api.js

@@ -1,133 +1,143 @@
 "use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const request_1 = require("../request");
-const API_URL = 'https://$lang.wikipedia.org/w/api.php';
+const API_URL = "https://$lang.wikipedia.org/w/api.php";
 class Api {
-    constructor(options = { action: 'query', format: 'json' }) {
-        this.options = options;
-        this.options.action = this.options.action || 'query';
-        this.options.format = 'json';
+    constructor(qs = { action: "query", format: "json" }, httpTimeout) {
+        this.qs = qs;
+        this.httpTimeout = httpTimeout;
+        this.qs.action = this.qs.action || "query";
+        this.qs.format = "json";
+        this.httpTimeout = httpTimeout || 10 * 1000;
     }
-    query(lang, options) {
-        if (options) {
-            this.options = Object.assign({}, this.options, options);
-        }
-        return request_1.default({ qs: this.options, url: API_URL.replace('$lang', lang), timeout: 10 * 1000 })
-            .then(data => {
-            if (hasError(data)) {
-                return Promise.reject(getError(data));
-            }
+    query(lang, qs) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (qs)
+                this.qs = Object.assign({}, this.qs, qs);
+            const data = yield request_1.default(API_URL.replace("$lang", lang), {
+                params: this.qs,
+                timeout: this.httpTimeout
+            });
+            if (hasError(data))
+                throw getError(data);
             if (data && data.query && data.query.pages) {
-                return Promise.resolve(Object.keys(data.query.pages)
-                    .map(id => data.query.pages[id])
-                    .map(data => {
-                    const item = { pageid: data['pageid'], title: data['title'] };
-                    if (data['categories']) {
-                        item.categories = data['categories'].map((item) => item.title);
+                return Object.keys(data.query.pages)
+                    .map((id) => data.query.pages[id])
+                    .map((data) => {
+                    const item = {
+                        pageid: data["pageid"],
+                        title: data["title"]
+                    };
+                    if (data["categories"]) {
+                        item.categories = data["categories"].map((item) => item.title);
                     }
-                    if (data['redirects']) {
-                        item.redirects = data['redirects'].map((item) => item.title);
+                    if (data["redirects"]) {
+                        item.redirects = data["redirects"].map((item) => item.title);
                     }
-                    if (data['extract']) {
-                        item.extract = data['extract'];
+                    if (data["extract"]) {
+                        item.extract = data["extract"];
                     }
                     return item;
-                }));
+                });
             }
-            return Promise.resolve([]);
+            return [];
         });
     }
     redirects() {
-        this.options['rdlimit'] = 'max';
-        this.addField('prop', 'redirects');
+        this.qs["rdlimit"] = "max";
+        this.addField("prop", "redirects");
         return this;
     }
     categories() {
-        this.options['cllimit'] = 'max';
-        this.options['clshow'] = '!hidden';
-        this.addField('prop', 'categories');
+        this.qs["cllimit"] = "max";
+        this.qs["clshow"] = "!hidden";
+        this.addField("prop", "categories");
         return this;
     }
     extract(sentences = 3) {
-        this.options['exsentences'] = sentences.toString();
-        this.options['explaintext'] = 'true';
-        this.addField('prop', 'extracts');
+        this.qs["exsentences"] = sentences.toString();
+        this.qs["explaintext"] = "true";
+        this.addField("prop", "extracts");
         return this;
     }
     addField(name, value) {
-        const vals = (this.options[name] || '').split('|').filter(item => !!item);
+        const vals = (this.qs[name] || "").split("|").filter((item) => !!item);
         vals.push(value);
-        this.options[name] = vals.join('|');
+        this.qs[name] = vals.join("|");
         return this;
     }
 }
 exports.Api = Api;
 function getExtracts(params) {
-    const qs = {
-        action: 'query',
-        titles: params.titles && params.titles.join('|'),
-        exsentences: params.sentences || 3,
-        prop: 'extracts',
-        explaintext: true,
-        exintro: true,
-        exlimit: 'max',
-        exsectionformat: 'plain',
-        format: 'json'
-    };
-    return request_1.default({ qs: qs, url: API_URL.replace('$lang', params.lang) })
-        .then(data => {
-        if (hasError(data)) {
-            return Promise.reject(getError(data));
-        }
+    return __awaiter(this, void 0, void 0, function* () {
+        const qs = {
+            action: "query",
+            titles: params.titles && params.titles.join("|"),
+            exsentences: params.sentences || 3,
+            prop: "extracts",
+            explaintext: true,
+            exintro: true,
+            exlimit: "max",
+            exsectionformat: "plain",
+            format: "json"
+        };
+        const data = yield request_1.default(API_URL.replace("$lang", params.lang), {
+            params: qs
+        });
+        if (hasError(data))
+            throw getError(data);
         if (data && data.query && data.query.pages) {
-            return Object.keys(data.query.pages).map(id => data.query.pages[id]);
+            return Object.keys(data.query.pages).map((id) => data.query.pages[id]);
         }
-        return Promise.resolve([]);
+        return [];
     });
 }
 exports.getExtracts = getExtracts;
-function getExtract(lang, title, sentences) {
-    return getExtracts({
-        lang: lang,
-        titles: [title],
-        sentences: sentences
-    }).then(function (extracts) {
-        if (extracts.length) {
-            return extracts[0];
-        }
-        return null;
-    });
-}
-exports.getExtract = getExtract;
+exports.getExtract = (lang, title, sentences) => getExtracts({
+    lang: lang,
+    titles: [title],
+    sentences: sentences
+}).then(function (extracts) {
+    if (extracts.length) {
+        return extracts[0];
+    }
+    return null;
+});
 function getRedirects(lang, title) {
-    const qs = {
-        action: 'query',
-        generator: 'redirects',
-        titles: title,
-        grdlimit: 'max',
-        format: 'json'
-    };
-    return request_1.default({ qs: qs, url: API_URL.replace('$lang', lang) })
-        .then(data => {
-        if (hasError(data)) {
-            return Promise.reject(getError(data));
-        }
+    return __awaiter(this, void 0, void 0, function* () {
+        const qs = {
+            action: "query",
+            generator: "redirects",
+            titles: title,
+            grdlimit: "max",
+            format: "json"
+        };
+        const data = yield request_1.default(API_URL.replace("$lang", lang), {
+            params: qs
+        });
+        if (hasError(data))
+            throw getError(data);
         if (data && data.query && data.query.pages) {
-            return Object.keys(data.query.pages).map(id => data.query.pages[id].title);
+            return Object.keys(data.query.pages).map((id) => data.query.pages[id].title);
         }
-        return Promise.resolve([]);
+        return [];
     });
 }
 exports.getRedirects = getRedirects;
 function getError(data) {
-    return data && data.error && new Error(data.error.info || 'Wikidata Api Error') || new Error('NO error');
+    return ((data &&
+        data.error &&
+        new Error(data.error.info || "Wikidata Api Error")) ||
+        new Error("NO error"));
 }
 function hasError(data) {
     return !!data.error;
 }
-function getStringArrayParam(value, def = null) {
-    if (!value || value.length === 0) {
-        return def;
-    }
-    return value;
-}

package/package.json

@@ -1,21 +1,20 @@
 {
   "name": "wiki-entity",
-  "version": "0.4.2",
+  "version": "0.4.6",
   "description": "Wiki entity fetcher",
   "main": "./lib/index.js",
   "types": "./types/index.d.ts",
   "scripts": {
     "pretest": "npm run tsc",
-    "test": "mocha -t 8000",
+    "test": "mocha -t 60000",
     "tsc": "tsc",
     "tscw": "tsc -w",
     "preversion": "npm run tsc",
     "postversion": "git push --follow-tags",
-    "prepublish": "npm run tsc",
-    "lint": "tslint ./src/**/*.ts"
+    "prepublish": "npm run tsc"
   },
   "engines": {
-    "node": ">=6.0.0"
+    "node": ">=8.0.0"
   },
   "repository": {
     "type": "git",
@@ -36,12 +35,12 @@
   },
   "homepage": "https://github.com/entitizer/wiki-entity#readme",
   "devDependencies": {
-    "@types/node": "^10.12.6",
-    "debug": "^4.1.0",
+    "@types/node": "^10.12.18",
+    "debug": "^4.1.1",
     "mocha": "^5.2.0",
-    "typescript": "^3.1.6"
+    "typescript": "^3.2.2"
   },
   "dependencies": {
-    "request": "^2.88.0"
+    "axios": "^0.21.1"
   }
 }

package/types/index.d.ts

@@ -1,7 +1,7 @@
-import { WikiEntity, WikiEntitiesParams } from './types';
-import { Api as WikipediaApi } from './wikipedia/api';
-export { simplifyEntity } from './wikidata/simplify_entity';
+import { WikiEntity, WikiEntitiesParams } from "./types";
+import { Api as WikipediaApi } from "./wikipedia/api";
+export { simplifyEntity } from "./wikidata/simplify_entity";
 export { WikipediaApi };
-export * from './simpleEntity';
-export * from './types';
+export * from "./simpleEntity";
+export * from "./types";
 export declare function getEntities(params: WikiEntitiesParams): Promise<WikiEntity[]>;

package/types/request.d.ts

@@ -1 +1,3 @@
-export default function <T>(options: any): Promise<T>;
+import { AxiosRequestConfig } from "axios";
+export declare type RequestOptions = AxiosRequestConfig;
+export default function <T>(url: string, options: RequestOptions): Promise<T>;

package/types/types.d.ts

@@ -26,6 +26,7 @@ export interface WikidataEntity extends WikidataBaseEntity {
     aliases?: string[];
     sitelinks?: PlainObject<string>;
     claims?: WikidataEntityClaims;
+    labels?: PlainObject<string>;
 }
 export declare type WikidataEntities = PlainObject<WikidataEntity>;
 export interface WikiEntity extends WikidataEntity {
@@ -35,7 +36,7 @@ export interface WikiEntity extends WikidataEntity {
     categories?: string[];
 }
 export declare type WikiEntities = PlainObject<WikiEntity>;
-export declare type ParamClaimsType = 'none' | 'all' | 'item' | 'property';
+export declare type ParamClaimsType = "none" | "all" | "item" | "property";
 export interface WikidataEntitiesParams {
     ids?: string[];
     titles?: string[];
@@ -43,6 +44,8 @@ export interface WikidataEntitiesParams {
     language?: string;
     redirect?: string;
     claims?: ParamClaimsType;
+    httpTimeout?: number;
+    languages?: string[];
 }
 export declare enum WikidataPropsParam {
     info = "info",

package/types/wikidata/api.d.ts

@@ -1,4 +1,4 @@
-import { WikidataEntities, WikidataEntitiesParams } from '../types';
+import { WikidataEntities, WikidataEntitiesParams } from "../types";
 export declare function getEntities(params: WikidataEntitiesParams): Promise<WikidataEntities>;
 export declare function getManyEntities(params: WikidataEntitiesParams): Promise<WikidataEntities>;
 export declare function validateParams(params: WikidataEntitiesParams): void;

package/types/wikidata/index.d.ts

@@ -1,3 +1,3 @@
-import { WikidataEntities, WikidataEntitiesParams, WikidataEntityClaims } from '../types';
+import { WikidataEntities, WikidataEntitiesParams, WikidataEntityClaims } from "../types";
 export declare function getEntities(params: WikidataEntitiesParams): Promise<WikidataEntities>;
 export declare function exploreEntityClaims(claims: WikidataEntityClaims, params: WikidataEntitiesParams): Promise<void>;

package/types/wikidata/simplify_entity.d.ts

@@ -1,4 +1,4 @@
-import { WikidataEntity } from '../types';
+import { WikidataEntity } from "../types";
 export declare type SimplifyEntityOptionsType = {
     labels?: boolean;
     descriptions?: boolean;

package/types/wikipedia/api.d.ts

@@ -1,4 +1,4 @@
-import { StringPlainObject } from '../utils';
+import { StringPlainObject } from "../utils";
 export declare type ExtractType = {
     pageid: number;
     title: string;
@@ -17,14 +17,15 @@ export declare type ApiResult = {
     redirects?: string[];
 };
 export declare class Api {
-    private options;
-    constructor(options?: StringPlainObject);
-    query(lang: string, options?: StringPlainObject): Promise<ApiResult[]>;
+    private qs;
+    private httpTimeout?;
+    constructor(qs?: StringPlainObject, httpTimeout?: number);
+    query(lang: string, qs?: StringPlainObject): Promise<ApiResult[]>;
     redirects(): this;
     categories(): this;
     extract(sentences?: number): this;
     private addField;
 }
 export declare function getExtracts(params: ExtractsParamsType): Promise<ExtractType[]>;
-export declare function getExtract(lang: string, title: string, sentences?: number): Promise<ExtractType>;
+export declare const getExtract: (lang: string, title: string, sentences?: number) => Promise<ExtractType>;
 export declare function getRedirects(lang: string, title: string): Promise<string[]>;