tellegram 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Skoropad Aleksandr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,80 @@
+# teLLegraM
+
+[![Build](https://img.shields.io/github/actions/workflow/status/skoropadas/telegramify-markdown/release.yml?branch=master)](https://github.com/skoropadas/telegramify-markdown/actions)
+[![codecov](https://codecov.io/gh/skoropadas/telegramify-markdown/branch/master/graph/badge.svg?token=LxCmgGNUHl)](https://codecov.io/gh/skoropadas/telegramify-markdown)
+![License](https://img.shields.io/github/license/skoropadas/telegramify-markdown)
+
+teLLegraM is a library designed to format LLM (Large Language Model) generated text into [Telegram-specific-markdown (MarkdownV2)](https://core.telegram.org/bots/api#formatting-options), based on [Unified](https://github.com/unifiedjs/unified) and [Remark](https://github.com/remarkjs/remark/). It ensures that complex markdown from AI responses is perfectly interpreted by Telegram clients.
+
+## Install
+
+```bash
+npm install tellegram
+```
+
+## Usage
+
+```js
+const teLLegraM = require('tellegram');
+const markdown = `
+# Header
+## Subheader
+
+[1.0.0](http://version.com)
+
+* item 1
+* item 2
+* item 3
+
+And simple text with + some - symbols.
+`;
+
+teLLegraM(markdown);
+/*
+ *Header*
+ *Subheader*
+ 
+[1\.0\.0](http://version.com)
+
+ • item 1
+ • item 2
+ • item 3
+
+And simple text with \+ some \- symbols\.
+*/
+```
+
+## Possible options
+
+You can also add unsupported tags strategy as a second argument, which can be one of the following:
+
+- `escape` - escape unsupported symbols for unsupported tags
+- `remove` - remove unsupported tags
+- `keep` - ignore unsupported tags (default)
+
+```js
+const teLLegraM = require('teLLegraM');
+const markdown = `
+# Header
+
+> Blockquote
+
+<div>Text in div</div>
+`;
+
+teLLegraM(markdown, 'escape');
+/*
+*Header*
+
+\> Blockquote
+
+<div\>Text in div</div\>
+*/
+
+teLLegraM(markdown, 'remove');
+/*
+*Header*
+ */
+```
+
+[MIT Licence](LICENSE)

package/index.js

@@ -0,0 +1,2 @@
+import convert from './lib/convert.js';
+export default convert;

package/lib/convert.js

@@ -0,0 +1,25 @@
+import gfm from 'remark-gfm';
+import parse from 'remark-parse';
+import stringify from 'remark-stringify';
+import removeComments from 'remark-remove-comments';
+import unified from 'unified';
+
+import { collectDefinitions, removeDefinitions } from './definitions.js';
+import createTelegramifyOptions from './telegramify.js';
+
+export default (markdown, unsupportedTagsStrategy) => {
+	const definitions = {};
+
+	const telegramifyOptions = createTelegramifyOptions(definitions, unsupportedTagsStrategy);
+
+	return unified()
+		.use(parse)
+		.use(gfm)
+		.use(removeComments)
+		.use(collectDefinitions, definitions)
+		.use(removeDefinitions)
+		.use(stringify, telegramifyOptions)
+		.processSync(markdown)
+		.toString()
+		.replace(/<!---->\n/gi, '');
+};

package/lib/definitions.js

@@ -0,0 +1,27 @@
+import remove from 'unist-util-remove';
+import visit from 'unist-util-visit';
+
+/**
+ * Fills the provided record with `Definition`s contained in the mdast.
+ * They are keyed by identifier for subsequent `Reference` lookups.
+ *
+ * @param {Record<string, { title: null | string, url: string }>} definitions
+ */
+export const collectDefinitions = definitions => tree => {
+	visit(tree, 'definition', node => {
+		definitions[node.identifier] = {
+			title: node.title,
+			url: node.url,
+		};
+	});
+};
+
+/**
+ * Removes `Definition`s and their parent `Paragraph`s from the mdast.
+ * This avoids unwanted negative space in stringified output.
+ */
+export const removeDefinitions = () => tree => {
+	remove(tree, { cascade: true }, 'definition');
+};
+
+

package/lib/telegramify.js

@@ -0,0 +1,158 @@
+import defaultHandlers from 'mdast-util-to-markdown/lib/handle/index.js';
+import phrasing from 'mdast-util-to-markdown/lib/util/container-phrasing.js';
+import {toMarkdown as gfmTableToMarkdown} from 'mdast-util-gfm-table';
+
+import {wrap, isURL, escapeSymbols, processUnsupportedTags} from './utils.js';
+
+/**
+ * Creates custom `mdast-util-to-markdown` handlers that tailor the output for
+ * Telegram Markdown.
+ *
+ * @param {Readonly<Record<string, { title: null | string, url: string }>>} definitions
+ * Record of `Definition`s in the Markdown document, keyed by identifier.
+ *
+ * @returns {import('mdast-util-to-markdown').Handlers}
+ */
+const createHandlers = (definitions, unsupportedTagsStrategy) => ({
+	heading: (node, _parent, context) => {
+		// make headers to be just *strong*
+		const marker = '*';
+
+		const exit = context.enter('heading');
+		const value = phrasing(node, context, {before: marker, after: marker});
+		exit();
+
+		return wrap(value, marker);
+	},
+
+	strong: (node, _parent, context) => {
+		const marker = '*';
+
+		const exit = context.enter('strong');
+		const value = phrasing(node, context, {before: marker, after: marker});
+		exit();
+
+		return wrap(value, marker);
+	},
+
+	delete(node, _parent, context) {
+		const marker = '~';
+
+		const exit = context.enter('delete');
+		const value = phrasing(node, context, {before: marker, after: marker});
+		exit();
+
+		return wrap(value, marker);
+	},
+
+	emphasis: (node, _parent, context) => {
+		const marker = '_';
+
+		const exit = context.enter('emphasis');
+		const value = phrasing(node, context, {before: marker, after: marker});
+		exit();
+
+		return wrap(value, marker);
+	},
+
+	list: (...args) => defaultHandlers.list(...args).replace(/^(\d+)./gm, '$1\\.'),
+
+	listItem: (...args) => defaultHandlers.listItem(...args).replace(/^\*/, '•'),
+
+	code(node, _parent, context) {
+		const exit = context.enter('code');
+		// delete language prefix for deprecated markdown formatters (old Bitbucket Editor)
+		const content = node.value.replace(/^#![a-z]+\n/, ''); // ```\n#!javascript\ncode block\n```
+		exit();
+
+		const language = node.lang || '';
+		return `\`\`\`${language}\n${escapeSymbols(content, 'code')}\n\`\`\``;
+	},
+
+	link: (node, _parent, context) => {
+		const exit = context.enter('link');
+		const text = phrasing(node, context, {before: '|', after: '>'}) || escapeSymbols(node.title);
+		const isUrlEncoded = decodeURI(node.url) !== node.url;
+		const url = isUrlEncoded ? node.url : encodeURI(node.url);
+		exit();
+
+		if (!isURL(url)) return escapeSymbols(text) || escapeSymbols(url);
+
+		return text
+			? `[${text}](${escapeSymbols(url, 'link')})`
+			: `[${escapeSymbols(url)}](${escapeSymbols(url, 'link')})`;
+	},
+
+	linkReference: (node, _parent, context) => {
+		const exit = context.enter('linkReference');
+		const definition = definitions[node.identifier];
+		const text = phrasing(node, context, {before: '|', after: '>'}) || (definition ? definition.title : null);
+		exit();
+
+		if (!definition || !isURL(definition.url)) return escapeSymbols(text);
+
+		return text
+			? `[${text}](${escapeSymbols(definition.url, 'link')})`
+			: `[${escapeSymbols(definition.url)}](${escapeSymbols(definition.url, 'link')})`;
+	},
+
+	image: (node, _parent, context) => {
+		const exit = context.enter('image');
+		const text = node.alt || node.title;
+		const url = node.url
+		exit();
+
+		if (!isURL(url)) return escapeSymbols(text) || escapeSymbols(url);
+
+		return text
+			? `[${escapeSymbols(text)}](${escapeSymbols(url, 'link')})`
+			: `[${escapeSymbols(url)}](${escapeSymbols(url, 'link')})`;
+	},
+
+	imageReference: (node, _parent, context) => {
+		const exit = context.enter('imageReference');
+		const definition = definitions[node.identifier];
+		const text = node.alt || (definition ? definition.title : null);
+		exit();
+
+		if (!definition || !isURL(definition.url)) return escapeSymbols(text);
+
+		return text
+			? `[${escapeSymbols(text)}](${escapeSymbols(definition.url, 'link')})`
+			: `[${escapeSymbols(definition.url)}](${escapeSymbols(definition.url, 'link')})`;
+	},
+
+	text: (node, _parent, context) => {
+		const exit = context.enter('text');
+		const text = node.value;
+		exit();
+
+		return escapeSymbols(text);
+	},
+
+	blockquote: (node, _parent, context) =>
+		processUnsupportedTags(defaultHandlers.blockquote(node, _parent, context), unsupportedTagsStrategy),
+	html: (node, _parent, context) =>
+		processUnsupportedTags(defaultHandlers.html(node, _parent, context), unsupportedTagsStrategy),
+	table: (node, _parent, context) =>
+		processUnsupportedTags(gfmTableToMarkdown().handlers.table(node, _parent, context), unsupportedTagsStrategy),
+	thematicBreak: (_node, _parent, _context) =>
+		processUnsupportedTags('---', unsupportedTagsStrategy),
+});
+
+/**
+ * Creates options to be passed into a `remark-stringify` processor that tailor
+ * the output for Telegram Markdown.
+ *
+ * @param {Readonly<Record<string, { title: null | string, url: string }>>} definitions
+ * Record of `Definition`s in the Markdown document, keyed by identifier.
+ *
+ * @returns {import('remark-stringify').RemarkStringifyOptions}
+ */
+const createOptions = (definitions, unsupportedTagsStrategy) => ({
+	bullet: '*',
+	tightDefinitions: true,
+	handlers: createHandlers(definitions, unsupportedTagsStrategy),
+});
+
+export default createOptions;

package/lib/utils.js

@@ -0,0 +1,76 @@
+import { URL } from 'url';
+
+export function wrap(string, ...wrappers) {
+	return [
+		...wrappers,
+		string,
+		...wrappers.reverse(),
+	].join('');
+}
+
+export function isURL(string) {
+	try {
+		return Boolean(new URL(string));
+	} catch (error) {
+		return false;
+	}
+}
+
+export function escapeSymbols(text, textType = 'text') {
+	if (!text) {
+		return text;
+	}
+	switch (textType) {
+		case 'code':
+			return text
+				.replace(/\\/g, '\\\\')
+				.replace(/`/g, '\\`')
+		case 'link':
+			return text
+				.replace(/\\/g, '\\\\')
+				.replace(/\(/g, '\\(')
+				.replace(/\)/g, '\\)')
+		case 'ignore_escaped':
+			return text.replace(/(\\.)|([_*\[\]()~`>#+\-=|{}.!])/g, (match, escaped, char) => {
+				if (escaped) {
+					return escaped;
+				}
+				return '\\' + char;
+			});
+		default:
+			return text
+				.replace(/_/g, '\\_')
+				.replace(/\*/g, '\\*')
+				.replace(/\[/g, '\\[')
+				.replace(/]/g, '\\]')
+				.replace(/\(/g, '\\(')
+				.replace(/\)/g, '\\)')
+				.replace(/~/g, '\\~')
+				.replace(/`/g, '\\`')
+				.replace(/>/g, '\\>')
+				.replace(/#/g, '\\#')
+				.replace(/\+/g, '\\+')
+				.replace(/-/g, '\\-')
+				.replace(/=/g, '\\=')
+				.replace(/\|/g, '\\|')
+				.replace(/{/g, '\\{')
+				.replace(/}/g, '\\}')
+				.replace(/\./g, '\\.')
+				.replace(/!/g, '\\!');
+
+	}
+}
+
+export function processUnsupportedTags(content, strategy) {
+	switch (strategy) {
+		case 'escape':
+			return escapeSymbols(content, 'ignore_escaped');
+		case 'remove':
+			return '';
+		case 'keep':
+		default:
+			return content;
+	}
+}
+
+

package/package.json

@@ -0,0 +1,70 @@
+{
+	"name": "tellegram",
+	"version": "1.0.0",
+	"description": "Convert LLM-generated markdown into Telegram-specific markdown (MarkdownV2)",
+	"type": "module",
+	"main": "index.js",
+	"scripts": {
+		"test": "NODE_OPTIONS=--experimental-vm-modules npx jest --coverage",
+		"lint": "eslint",
+		"semantic-release": "semantic-release",
+		"prepare": "husky install",
+		"codecov": "codecov"
+	},
+	"files": [
+		"README.md",
+		"LICENSE",
+		"index.js",
+		"lib",
+		"types"
+	],
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/leask/tellegram.git"
+	},
+	"keywords": [
+		"telegram",
+		"markdown",
+		"telegramify",
+		"parser",
+		"remark",
+		"unified"
+	],
+	"types": "types/index.d.ts",
+	"author": "Leask Wong",
+	"license": "MIT",
+	"bugs": {
+		"url": "https://github.com/leask/tellegram/issues"
+	},
+	"homepage": "https://github.com/leask/tellegram#readme",
+	"dependencies": {
+		"mdast-util-gfm-table": "^0.1.6",
+		"mdast-util-to-markdown": "^0.6.2",
+		"remark-gfm": "^1.0.0",
+		"remark-parse": "^9.0.0",
+		"remark-remove-comments": "^0.2.0",
+		"remark-stringify": "^9.0.1",
+		"unified": "^9.0.0",
+		"unist-util-remove": "^2.0.1",
+		"unist-util-visit": "^2.0.3"
+	},
+	"devDependencies": {
+		"@commitlint/cli": "^12.1.1",
+		"@commitlint/config-conventional": "^12.1.1",
+		"codecov": "^3.8.3",
+		"eslint": "^7.24.0",
+		"husky": "^6.0.0",
+		"jest": "^29.3.1",
+		"lint-staged": "^10.5.4",
+		"prettier": "2.2.1",
+		"semantic-release": "^17.4.2"
+	},
+	"lint-staged": {
+		"*.{js,json,md}": [
+			"prettier --write",
+			"git add"
+		],
+		"*.{css,scss,less}": "stylelint --fix",
+		"*.js": "eslint --cache --fix"
+	}
+}

package/types/index.d.ts

@@ -0,0 +1,13 @@
+type UnsupportedTagsStrategy = 'escape' | 'remove' | 'keep'
+
+declare module 'tellegram' {
+
+  /**
+   * Converts markdown to Telegram's format.
+   * @param markdown The markdown to convert.
+   * @param unsupportedTagsStrategy The strategy to use for unsupported tags.
+   */
+  function convert(markdown: string, unsupportedTagsStrategy: UnsupportedTagsStrategy): string
+
+  export = convert
+}