utilitas 1999.1.12 → 1999.1.13

package/index.mjs

@@ -19,9 +19,7 @@ import * as media from './lib/media.mjs';
 import * as memory from './lib/memory.mjs';
 import * as network from './lib/network.mjs';
 import * as sentinel from './lib/sentinel.mjs';
-import * as shekel from './lib/shekel.mjs';
 import * as shell from './lib/shell.mjs';
-import * as shot from './lib/shot.mjs';
 import * as sms from './lib/sms.mjs';
 import * as speech from './lib/speech.mjs';
 import * as ssl from './lib/ssl.mjs';
@@ -41,14 +39,14 @@ export {
     fileType, math, uuid,
     // features
     alan, bee, bot, boxes, cache, callosum, color, dbio, email, encryption,
-    event, image, manifest, media, memory, network, sentinel, shekel, shell,
-    shot, sms, speech, ssl, storage, tape, uoid, utilitas, vision, web
+    event, image, manifest, media, memory, network, sentinel, shell, sms,
+    speech, ssl, storage, tape, uoid, utilitas, vision, web
 };
 
 if (utilitas.inBrowser() && !globalThis.utilitas) {
     globalThis.utilitas = {
-        boxes, color, encryption, event, manifest, math, shekel, shot, speech,
-        storage, uoid, utilitas, uuid,
+        boxes, color, encryption, event, manifest, math, speech, storage, uoid,
+        utilitas, uuid,
     };
     // top-level await workaround
     (async () => {

package/lib/alan.mjs

@@ -1,20 +1,17 @@
-import { fileTypeFromBuffer } from 'file-type';
-import { end, loop } from './event.mjs';
-import { createWavHeader } from './media.mjs';
-import { checkSearch, search } from './shot.mjs';
-import { BASE64, BUFFER, DATAURL, MIME_BINARY, STREAM, convert } from './storage.mjs';
+import { checkSearch, distill, search } from './web.mjs';
 import { create as createUoid } from './uoid.mjs';
-import { distill } from './web.mjs';
+import { createWavHeader } from './media.mjs';
+import { end, loop } from './event.mjs';
+import { fileTypeFromBuffer } from 'file-type';
+
+import {
+    BASE64, BUFFER, DATAURL, MIME_BINARY, MIME_JSON, MIME_TEXT, STREAM, convert,
+} from './storage.mjs';
 
 import {
-    log as _log,
-    renderText as _renderText,
-    base64Encode, ensureArray, ensureString, extract,
-    ignoreErrFunc,
-    insensitiveCompare,
-    isSet,
-    need, parseJson,
-    throwError
+    log as _log, renderText as _renderText, base64Encode, ensureArray,
+    ensureString, extract, ignoreErrFunc, insensitiveCompare, isSet, need,
+    parseJson, throwError
 } from './utilitas.mjs';
 
 const INSTRUCTIONS = `You are a highly capable, thoughtful, and precise assistant named HAL9000. Your goal is to deeply understand the user's intent, ask clarifying questions when needed, think step-by-step through complex problems, provide clear and accurate answers, and proactively anticipate helpful follow-up information. Always prioritize being truthful, nuanced, insightful, and efficient, tailoring your responses specifically to the user's needs and preferences. Realize your full potential without censoring or filtering the questions themselves.
@@ -47,15 +44,13 @@ const _NEED = [
 
 const [
     png, jpeg, mov, mpeg, mp4, mpg, avi, wmv, mpegps, flv, gif, webp, pdf, aac,
-    flac, mp3, m4a, mpga, opus, pcm, wav, webm, tgpp, mimeJson, mimeText, pcm16,
-    ogg,
+    flac, mp3, m4a, mpga, opus, pcm, wav, webm, tgpp, pcm16, ogg,
 ] = [
         'image/png', 'image/jpeg', 'video/mov', 'video/mpeg', 'video/mp4',
         'video/mpg', 'video/avi', 'video/wmv', 'video/mpegps', 'video/x-flv',
         'image/gif', 'image/webp', 'application/pdf', 'audio/aac', 'audio/flac',
         'audio/mp3', 'audio/m4a', 'audio/mpga', 'audio/opus', 'audio/pcm',
-        'audio/wav', 'audio/webm', 'video/3gpp', 'application/json',
-        'text/plain', 'audio/x-wav', 'audio/ogg',
+        'audio/wav', 'audio/webm', 'video/3gpp', 'audio/x-wav', 'audio/ogg',
     ];
 
 const [
@@ -173,7 +168,7 @@ const MODELS = {
     [JINA_DEEPSEARCH_M]: {
         contextWindow: Infinity, maxInputTokens: Infinity,
         maxOutputTokens: Infinity, imageCostTokens: 0, maxImageSize: Infinity,
-        supportedMimeTypes: [png, jpeg, mimeText, webp, pdf],
+        supportedMimeTypes: [png, jpeg, MIME_TEXT, webp, pdf],
         reasoning: true, json: true, vision: true,
     },
     [DEEPSEEK_R1]: {
@@ -1068,7 +1063,7 @@ const deleteFile = async (aiId, file_id, options) => {
 
 const generationConfig = options => ({
     generationConfig: {
-        responseMimeType: options.jsonMode ? mimeJson : mimeText,
+        responseMimeType: options.jsonMode ? MIME_JSON : MIME_TEXT,
         responseModalities: options.modalities
             || (options.imageMode ? [TEXT, IMAGE] : undefined),
         ...options?.generationConfig || {},

package/lib/bot.mjs

@@ -17,7 +17,7 @@ import { jpeg, ogg, wav } from './alan.mjs';
 import { isPrimary, on, report } from './callosum.mjs';
 import { cleanSql, encodeVector, MYSQL, POSTGRESQL } from './dbio.mjs';
 import { convertAudioTo16kNanoPcmWave } from './media.mjs';
-import { get } from './shot.mjs';
+import { get } from './web.mjs';
 import { OPENAI_TTS_MAX_LENGTH } from './speech.mjs';
 import { BASE64, BUFFER, convert, FILE, isTextFile, tryRm } from './storage.mjs';
 import { fakeUuid } from './uoid.mjs';

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1999.1.12",
+    "version": "1999.1.13",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/lib/network.mjs

@@ -1,6 +1,6 @@
 import { assertExist } from './shell.mjs';
 import { ensureArray, log as _log, need, throwError } from './utilitas.mjs';
-import { getCurrentIp } from './shot.mjs';
+import { getCurrentIp } from './web.mjs';
 
 const _NEED = ['fast-geoip', 'ping'];
 const isLocalhost = host => ['127.0.0.1', '::1', 'localhost'].includes(host);

package/lib/speech.mjs

@@ -1,7 +1,7 @@
 import { DEFAULT_MODELS, OPENAI_VOICE } from './alan.mjs';
 import { getApiKeyCredentials, hash } from './encryption.mjs';
 import { getFfmpeg } from './media.mjs';
-import { get } from './shot.mjs';
+import { get } from './web.mjs';
 import { convert, getTempPath } from './storage.mjs';
 import { ensureString } from './utilitas.mjs';
 

package/lib/storage.mjs

@@ -26,11 +26,11 @@ const sanitizeFilename = (s, r) => s.replace(/[\/?<>\\:*|"]/g, r || '_').trim();
 
 const [
     NULL, BASE64, BUFFER, FILE, STREAM, TEXT, _JSON, encoding, BINARY, BLOB,
-    DATAURL, mode, dirMode, MIME_TEXT, MIME_BINARY
+    DATAURL, mode, dirMode, MIME_TEXT, MIME_BINARY, MIME_JSON,
 ] = [
         'NULL', 'BASE64', 'BUFFER', 'FILE', 'STREAM', 'TEXT', 'JSON', 'utf8',
         'binary', 'BLOB', 'DATAURL', '0644', '0755', 'text/plain',
-        'application/octet-stream',
+        'application/octet-stream', 'application/json',
     ];
 
 const [encodeBase64, encodeBinary, encodeNull]
@@ -468,15 +468,28 @@ const deleteOnCloud = async (path, options) => {
 };
 
 export {
-    _NEED, analyzeFile,
-    assertPath, BASE64, blobToBuffer, BUFFER, convert, DATAURL, decodeBase64DataURL,
+    _NEED,
+    BUFFER,
+    BASE64,
+    DATAURL,
+    FILE,
+    MIME_BINARY,
+    MIME_TEXT,
+    MIME_JSON,
+    STREAM,
+    analyzeFile,
+    assertPath,
+    blobToBuffer,
+    convert,
+    decodeBase64DataURL,
     deleteFileOnCloud,
     deleteOnCloud,
     downloadFileFromCloud,
     downloadFromCloud,
     encodeBase64DataURL,
     exists,
-    existsOnCloud, FILE, getConfig,
+    existsOnCloud,
+    getConfig,
     getConfigFilename,
     getGcUrlByBucket,
     getIdByGs,
@@ -487,11 +500,13 @@ export {
     legalFilename,
     lsOnCloud,
     mapFilename,
-    mergeFile, MIME_BINARY, readFile,
+    mergeFile,
+    readFile,
     readJson,
     sanitizeFilename,
     setConfig,
-    sliceFile, STREAM, touchPath,
+    sliceFile,
+    touchPath,
     tryRm,
     unzip,
     uploadToCloud,

package/lib/utilitas.mjs

@@ -742,9 +742,11 @@ const getFuncParams = (func) => {
 const analyzeModule = (obj) => {
     assertModule(obj);
     const [keys, result] = [Object.getOwnPropertyNames(obj), {}];
-    keys.sort().map(key => result[key] = {
+    keys.sort();
+    keys.filter(x => x !== 'INSTRUCTIONS').map(key => result[key] = {
         type: getType(obj[key]), ...Function.isFunction(obj[key])
-            ? { params: getFuncParams(obj[key]) } : { value: ensureString(obj[key]) }
+            ? { params: getFuncParams(obj[key]) }
+            : { value: ensureString(obj[key]) }
     });
     return result;
 };

package/lib/web.mjs

@@ -1,14 +1,245 @@
-import { getJson, getParsedHtml } from './shot.mjs';
-import { convert } from './storage.mjs';
-import { assembleUrl, ignoreErrFunc, need, throwError } from './utilitas.mjs';
+import { bignumber, divide, multiply } from 'mathjs';
+import { fileTypeFromBuffer } from 'file-type';
+import { join } from 'path';
+import { promises as fs } from 'fs';
+import { sha256 } from './encryption.mjs';
 
-const _NEED = ['jsdom', 'youtube-transcript', '@mozilla/readability'];
+import {
+    ensureInt, ensureString, extract, ignoreErrFunc, inBrowser, parseJson,
+    parseVersion, throwError, which, assertSet, assembleUrl, need,
+} from './utilitas.mjs';
+
+import {
+    MIME_JSON, MIME_TEXT, convert, encodeBase64DataURL, exists, mapFilename,
+    readJson, touchPath, writeJson,
+} from './storage.mjs';
 
+const _NEED = ['jsdom', 'youtube-transcript', '@mozilla/readability'];
 // https://stackoverflow.com/questions/19377262/regex-for-youtube-url
 const YT_REGEXP = /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/i;
 const isYoutubeUrl = url => (url || '').match(YT_REGEXP)?.[6];
 const distillPage = async (url, op) => (await getParsedHtml(url, op))?.content;
-const TEXT = 'TEXT';
+const TMPDIR = process.env.TMPDIR ? join(process.env.TMPDIR, 'shot') : null;
+const buf2utf = buf => buf.toString('utf8');
+const [TEXT, _JSON, _PARSED] = ['TEXT', 'JSON', 'PARSED'];
+const getJson = async (u, o) => await get(u, { encode: _JSON, ...o || {} });
+const getParsedHtml = async (u, o) => await get(u, { encode: _PARSED, ...o || {} });
+const checkSearch = () => googleApiKey || jinaApiKey;
+
+let googleApiKey, googleCx, jinaApiKey;
+
+const defFetchOpt = {
+    redirect: 'follow', follow: 3, timeout: 1000 * 10, headers: {
+        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+            + 'AppleWebKit/605.1.15 (KHTML, like Gecko) '
+            + 'Version/17.0 Safari/605.1.15',
+    },
+};
+
+const getVersionOnNpm = async (packName) => {
+    assert(packName, 'Package name is required.', 400);
+    const url = `https://registry.npmjs.org/-/package/${packName}/dist-tags`;
+    const rp = (await get(url, { encode: _JSON }))?.content;
+    assert(rp, 'Error fetching package info.', 500);
+    assert(rp !== 'Not Found' && rp.latest, 'Package not found.', 404);
+    return parseVersion(rp.latest);
+};
+
+const checkVersion = async (pack) => {
+    const objPack = await which(pack);
+    const curVersion = objPack.versionNormalized;
+    const newVersion = await getVersionOnNpm(objPack.name);
+    return {
+        name: objPack.name, curVersion, newVersion,
+        updateAvailable: newVersion.normalized > curVersion.normalized,
+    }
+};
+
+const getCurrentIp = async (options) => {
+    const resp = await get(
+        'https://ifconfig.me/all.json', { encode: _JSON, ...options || {} }
+    );
+    assert(resp?.content?.ip_addr, 'Error detecting IP address.', 500);
+    return options?.raw ? resp : resp.content.ip_addr;
+};
+
+const getCurrentPosition = async () => {
+    const url = 'https://geolocation-db.com/json/';
+    const rp = await fetch(url).then(res => res.json());
+    assert(rp, 'Network is unreachable.', 500);
+    assert(rp.country_code, 'Error detecting geolocation.', 500);
+    return rp;
+};
+
+const get = async (url, options) => {
+    assert(url, 'URL is required.', 400);
+    options = options || {};
+    options.encode = ensureString(options.encode, { case: 'UP' });
+    const urlHash = inBrowser() ? null : sha256(url);
+    const tmp = urlHash ? (options.cache?.tmp || TMPDIR) : null;
+    const base = tmp ? join(tmp, mapFilename(urlHash)) : null;
+    const [cacheMeta, cacheCont] = base ? ['meta', 'content'].map(
+        x => join(base, `${urlHash}.${x}`)
+    ) : [];
+    if (options?.fuzzy && await exists(cacheMeta) && await exists(cacheCont)) {
+        return { cache: { meta: cacheMeta, content: cacheCont } };
+    }
+    const meta = options?.refresh || !base ? null : await readJson(cacheMeta);
+    const cache = options?.refresh || !base ? null : await ignoreErrFunc(
+        () => fs.readFile(cacheCont)
+    );
+    const headers = meta?.responseHeaders && cache ? {
+        'cache-control': 'max-age=0',
+        'if-modified-since': meta.responseHeaders['last-modified'] || '',
+        'if-none-match': meta.responseHeaders['etag'] || '',
+    } : {};
+    let [timer, r, responseHeaders] = [null, null, {}];
+    const fetchOptions = {
+        ...defFetchOpt, headers: { ...defFetchOpt.headers, ...headers },
+        ...options.fetch || {}
+    };
+    if (options.timeout) {
+        const controller = new AbortController();
+        fetchOptions.signal = controller.signal;
+        timer = setTimeout(() => controller.abort(), options.timeout);
+    }
+    try { r = await fetch(url, fetchOptions); } catch (e) {
+        throwError(e.message.includes('aborted') ? 'Timed out.' : e.message, 500);
+    }
+    timer && clearTimeout(timer);
+    (r.status === 304) && (r.arrayBuffer = async () => cache);
+    const [htpMime, buffer] = [r.headers.get('content-type'), Buffer.from(await r.arrayBuffer())];
+    if (r.headers?.raw) { responseHeaders = r.headers.raw(); }
+    else { for (const [k, v] of r.headers.entries()) { responseHeaders[k] = v; } }
+    const bufMime = await ignoreErrFunc(async () => {
+        extract(await fileTypeFromBuffer(buffer), 'mime');
+    });
+    const mimeType = bufMime || htpMime;
+    const length = buffer.length;
+    let content;
+    if (!options?.fuzzy) {
+        switch (options.encode) {
+            case 'BUFFER':
+                content = buffer;
+                break;
+            case 'BASE64':
+                content = buffer.toString(options.encode);
+                break;
+            case 'BASE64_DATA_URL':
+                content = await encodeBase64DataURL(mimeType, buffer);
+                break;
+            case _JSON:
+                content = parseJson(buf2utf(buffer), null);
+                break;
+            case _PARSED:
+                content = await distillHtml(buf2utf(buffer));
+                break;
+            default:
+                assert(!options.encode, 'Invalid encoding.', 400);
+            case 'TEXT':
+                content = buf2utf(buffer);
+        }
+    }
+    base && !cache && length && r.status === 200
+        && await ignoreErrFunc(async () => {
+            return {
+                touch: await touchPath(base),
+                content: await fs.writeFile(cacheCont, buffer),
+                meta: await writeJson(cacheMeta, {
+                    url, requestHeaders: headers, responseHeaders,
+                }),
+            };
+        });
+    return {
+        statusCode: r.status, statusText: r.statusText, length, mimeType,
+        content, headers: responseHeaders, response: r,
+        cache: r.status >= 200 && r.status < 400 ? { meta: cacheMeta, content: cacheCont } : null,
+    };
+};
+
+const initSearch = async (options = {}) => {
+    assert(options.apiKey, 'API key is required.');
+    switch (ensureString(options.provider, { case: 'UP' })) {
+        case 'GOOGLE':
+            assert(options.cx, 'CX is required for Google Search API.');
+            [googleApiKey, googleCx] = [options.apiKey, options.cx];
+            break;
+        case 'JINA':
+            jinaApiKey = options.apiKey;
+            break;
+        default:
+            throwError(`Invalid search provider: "${options.provider}".`);
+    }
+    return async (query, opts) => await search(query, { ...options, ...opts });
+};
+
+const search = async (query, options = {}) => {
+    assert(query, 'Query keyword is required.');
+    let provider = ensureString(options.provider, { case: 'UP' });
+    if (!provider && googleApiKey) { provider = 'GOOGLE'; }
+    if (!provider && jinaApiKey) { provider = 'JINA'; }
+    switch (provider) {
+        case 'GOOGLE':
+            var [key, cx, min, max] = [
+                options?.apiKey || googleApiKey, options?.cx || googleCx, 1, 10
+            ];
+            assert(key, 'API key is required.');
+            assert(cx, 'CX is required.');
+            var [num, start] = [
+                ensureInt(options?.num || max, { min, max }),
+                ensureInt(options?.start || min, { min }),
+            ];
+            assert(start + num <= 100, 'Reached maximum search limit.');
+            var url = 'https://www.googleapis.com/customsearch/v1'
+                + `?key=${encodeURIComponent(key)}&cx=${encodeURIComponent(cx)}`
+                + `&q=${encodeURIComponent(query)}&num=${num}&start=${start}`
+                + (options?.image ? `&searchType=image` : '');
+            var resp = await get(url, { encode: _JSON, ...options || {} });
+            return options?.raw ? resp.content : {
+                totalResults: resp?.content?.searchInformation?.totalResults || 0,
+                startIndex: resp?.content?.queries?.request?.[0]?.startIndex || 1,
+                items: resp?.content?.items.map(x => ({
+                    title: x.title, link: options?.image ? null : x.link,
+                    snippet: x.snippet, image: (
+                        options?.image ? x.link : x.pagemap?.cse_image?.[0]?.src
+                    ) || null,
+                })), provider,
+            };
+        case 'JINA':
+            var [key, min, def, max] =
+                [options?.apiKey || jinaApiKey, 1, 10, 20];
+            assert(key, 'API key is required.');
+            var [num, start] = [
+                ensureInt(options?.num || def, { min, max }),
+                ensureInt(options?.start || min, { min }),
+            ];
+            var url = `https://s.jina.ai/?q=${encodeURIComponent(query)}`
+                + `&page=${encodeURIComponent(start - 1)}`
+                + `&num=${encodeURIComponent(num)}`
+            var resp = await get(url, {
+                encode: options?.aiFriendly ? 'TEXT' : _JSON,
+                fetch: {
+                    headers: {
+                        'Authorization': `Bearer ${key}`,
+                        'Accept': options?.aiFriendly ? MIME_TEXT : MIME_JSON,
+                        'X-Respond-With': 'no-content',
+                        ...options?.aiFriendly ? {} : { 'X-With-Favicons': true },
+                    }
+                }, ...options || {},
+            });
+            if (options?.raw) { return resp; }
+            if (options?.aiFriendly) { return resp?.content; }
+            return {
+                totalResults: 100, startIndex: start,
+                items: (resp?.content?.data || []).map(x => ({
+                    title: x.title, link: x.url,
+                    snippet: x.description, image: x.favicon,
+                })), provider,
+            };
+        default:
+            throwError(`Invalid search provider: "${options.provider}".`);
+    }
+};
 
 const distillHtml = async (input, options) => {
     const html = await convert(input, {
@@ -105,14 +336,46 @@ const distill = async url => {
     };
 };
 
+const _getRate = (rates, currency) => {
+    const rate = rates[ensureString(currency || 'USD', { case: 'UP' })];
+    assertSet(rate, `Unsupported currency: '${currency}'.`, 400);
+    return bignumber(rate);
+};
+
+const getExchangeRate = async (to, from, amount) => {
+    const data = {};
+    ((await get(
+        'https://api.mixin.one/external/fiats', { encode: 'JSON' }
+    ))?.content?.data || []).map(x => data[x.code] = x.rate);
+    assert(Object.keys(data).length, 'Error fetching exchange rates.', 500);
+    if (!to) { return data; }
+    [to, from] = [_getRate(data, to), _getRate(data, from)];
+    const rate = divide(to, from);
+    amount = multiply(bignumber(amount ?? 1), rate);
+    return { rate, amount: amount.toString() };
+};
+
+
+export default get;
 export {
     _NEED,
     assertYoutubeUrl,
+    checkSearch,
+    checkVersion,
     distill,
     distillHtml,
     distillPage,
     distillYoutube,
+    get,
+    getCurrentIp,
+    getCurrentPosition,
+    getExchangeRate,
+    getJson,
+    getParsedHtml,
+    getVersionOnNpm,
     getYoutubeMetadata,
     getYoutubeTranscript,
-    isYoutubeUrl
+    initSearch,
+    isYoutubeUrl,
+    search,
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1999.1.12",
+    "version": "1999.1.13",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/lib/shekel.mjs

@@ -1,24 +0,0 @@
-import { assertSet, ensureString } from './utilitas.mjs';
-import { bignumber, divide, multiply } from 'mathjs';
-import { get } from './shot.mjs';
-
-const _getRate = (rates, currency) => {
-    const rate = rates[ensureString(currency || 'USD', { case: 'UP' })];
-    assertSet(rate, `Unsupported currency: '${currency}'.`, 400);
-    return bignumber(rate);
-};
-
-const getExchangeRate = async (to, from, amount) => {
-    const data = {};
-    ((await get(
-        'https://api.mixin.one/external/fiats', { encode: 'JSON' }
-    ))?.content?.data || []).map(x => data[x.code] = x.rate);
-    assert(Object.keys(data).length, 'Error fetching exchange rates.', 500);
-    if (!to) { return data; }
-    [to, from] = [_getRate(data, to), _getRate(data, from)];
-    const rate = divide(to, from);
-    amount = multiply(bignumber(amount ?? 1), rate);
-    return { rate, amount: amount.toString() };
-};
-
-export { getExchangeRate };