vowel 0.0.2 → 0.1.0

package/src/lib/utilities/mutateMarkdownAST.js

@@ -0,0 +1,59 @@
+import urlMetadata from 'url-metadata';
+
+export default async function mutateMarkdownAST(ast, cache) {
+	const promises = ast.map(async (node) => {
+		// TODO: Improve this URL regex
+		if (node.type === 'paragraph') {
+			// URLs
+			if (
+				node.children?.length === 1 &&
+				node.children[0]?.value?.match(/^https?:\/\/\S+$/) &&
+				!node.children[0]?.children
+			) {
+				node.type = 'url';
+				const url = node.children[0].value;
+				if (!cache[node.value]) {
+					try {
+						const urlObject = new URL(url);
+						const response = await urlMetadata(urlObject.href, {
+							includeResponseBody: false,
+							ensureSecureImageRequest: true
+						});
+
+						const metadata = {
+							image: response['og:image'],
+							ogURL: response['og:url'],
+							canonicalURL: response.canonical,
+							title: response.title,
+							ogTitle: response['og:title'],
+							author: response.author,
+							description: response.description
+						};
+
+						cache[url] = metadata;
+						node.metadata = metadata;
+						node.value = url;
+
+						delete node.children;
+					} catch (error) {
+						node.value = url;
+						console.log(`Error on URL in content: ${url}`);
+					}
+				} else {
+					node.metadata = cache[node.value];
+				}
+			} else if (node.children[0].type === 'image') {
+				node.type = 'figure';
+				if (node?.children[1]?.type === 'text') {
+					node.children[1].type = 'figcaption';
+				}
+			}
+		}
+
+		if (node.children) {
+			await mutateMarkdownAST(node.children, cache);
+		}
+	});
+
+	await Promise.all(promises);
+}

package/src/lib/utilities/mutateMarkdownFrontmatter.js

@@ -0,0 +1,115 @@
+import urlMetadata from 'url-metadata';
+import { regexPatterns, isObject, parseDate } from '.';
+
+/**
+ * Description
+ * @param {any} value - Any frontmatter value
+ * @returns {any}
+ */
+function imputeType(value) {
+	if (value === null || value === undefined || Number.isNaN(value)) return 'nullish';
+	if (typeof value === 'boolean') return 'boolean';
+	if (typeof value === 'number') return 'number';
+	if (parseDate(value)) return 'date';
+	if (isObject(value)) return 'object';
+	if (Array.isArray(value)) return 'array';
+	if (typeof value === 'string') {
+		if (value.match(regexPatterns.img)) return 'image';
+		if (value.match(regexPatterns.pdf)) return 'pdf';
+		if (value.match(regexPatterns.url)) return 'url';
+		if (value.match(regexPatterns.path)) return 'path';
+		return 'string';
+	}
+	console.error('Unknown frontmatter data type: ' + value);
+	return 'other';
+}
+
+export default async function mutateMarkdownFrontmatter(frontmatter, cache) {
+	const keys = Object.keys(frontmatter);
+
+	const promises = keys.map(async (key) => {
+		const input = frontmatter[key];
+
+		const type = imputeType(input);
+
+		switch (type) {
+			case 'object': {
+				await mutateMarkdownFrontmatter(input, cache);
+				frontmatter[key] = {
+					type: 'object',
+					output: input
+				};
+				break;
+			}
+			case 'array': {
+				await mutateMarkdownFrontmatter(input, cache);
+				frontmatter[key] = {
+					type: 'array',
+					output: input
+				};
+				break;
+			}
+			case 'date': {
+				frontmatter[key] = {
+					type: 'date',
+					output: parseDate(input),
+					input
+				};
+				break;
+			}
+			case 'image': {
+				frontmatter[key] = {
+					type: 'image',
+					output: input,
+					input
+				};
+				break;
+			}
+			case 'pdf': {
+				frontmatter[key] = {
+					type: 'pdf',
+					output: input,
+					input
+				};
+				break;
+			}
+			case 'url': {
+				if (!cache[input]) {
+					try {
+						const url = new URL(input);
+						const metadata = await urlMetadata(url.href, {
+							includeResponseBody: false,
+							ensureSecureImageRequest: true
+						});
+						cache[input] = metadata;
+
+						frontmatter[key] = {
+							type: 'url',
+							output: metadata,
+							input
+						};
+					} catch (e) {
+						console.error(`Error on URL in frontmatter: ${frontmatter[key]}`);
+						frontmatter[key] = {
+							type: 'string',
+							output: input,
+							input
+						};
+					}
+				} else {
+					frontmatter[key] = {
+						type: 'url',
+						output: cache[input],
+						input
+					};
+				}
+				break;
+			}
+			case 'nullish': {
+				frontmatter[key] = undefined;
+			}
+		}
+	});
+
+	await Promise.all(promises);
+}

package/src/lib/utilities/parseDate.js

@@ -0,0 +1,43 @@
+import dates from 'any-date-parser';
+import ago from 'any-date-parser/src/formats/ago/ago.js';
+import chinese from 'any-date-parser/src/formats/chinese/chinese.js';
+import dayMonth from 'any-date-parser/src/formats/dayMonth/dayMonth.js';
+import dayMonthname from 'any-date-parser/src/formats/dayMonthname/dayMonthname.js';
+import monthDay from 'any-date-parser/src/formats/monthDay/monthDay.js';
+import monthnameDay from 'any-date-parser/src/formats/monthnameDay/monthnameDay.js';
+import today from 'any-date-parser/src/formats/today/today.js';
+
+dates.removeFormat(ago);
+dates.removeFormat(chinese);
+dates.removeFormat(dayMonth);
+dates.removeFormat(dayMonthname);
+dates.removeFormat(monthDay);
+dates.removeFormat(monthnameDay);
+dates.removeFormat(today);
+
+/*
+Left in the following formats:
+
+microsoftJson
+dayMonthYear
+dayMonthnameYear
+monthDayYear
+monthnameDayYear
+twitter
+yearMonthDay
+atSeconds
+time12Hours
+time24Hours
+*/
+
+export default function parseDate(maybeDate) {
+	if (!maybeDate) return false;
+	if (maybeDate instanceof Date) return maybeDate;
+	if (typeof maybeDate === 'number') return false;
+	if (typeof maybeDate === 'string' && maybeDate.match(/^\d+$/)) return false;
+	const parsedDate = dates.attempt(maybeDate);
+	if (parsedDate.invalid) return false;
+	const { month, day, year } = parsedDate;
+	const date = new Date(`${year}-${month}-${day}`);
+	return date;
+}

package/src/lib/utilities/processMarkdownFiles.js

@@ -0,0 +1,212 @@
+import fs from 'fs/promises';
+import path from 'path';
+// @ts-ignore
+import { buildURL, checkFileExists, readMarkdownFile } from '.';
+
+function DefaultFile(url) {
+	return {
+		ast: [
+			{
+				type: 'paragraph',
+				children: [
+					{
+						type: 'h1',
+						value: 'hey there',
+						// value: url + '?count=10',
+						position: {
+							start: {
+								line: 5,
+								column: 1,
+								offset: 25
+							},
+							end: {
+								line: 5,
+								column: 8,
+								offset: 32
+							}
+						}
+					}
+				],
+				position: {
+					start: {
+						line: 5,
+						column: 1,
+						offset: 25
+					},
+					end: {
+						line: 5,
+						column: 8,
+						offset: 32
+					}
+				}
+			}
+		],
+		url
+	};
+}
+
+function DefaultDirectory(path) {
+	return {
+		$: DefaultFile(path),
+		_: DefaultFile(path)
+	};
+}
+
+/** @typedef {Object} AdditionalFileProperties
+ * @property {string} [url] - URL to the page for the file.
+ * @property {number} [date] - Document date.
+ */
+
+/** @typedef {import("./readMarkdownFile").ParsedMarkdown & AdditionalFileProperties} MarkdownFile */
+
+/**
+ * @typedef {Object} Subdirectory
+ * @description A subdirectory of markdown files
+ * @property {MarkdownFile} $ A home file
+ * @property {MarkdownFile} _ A settingsfile
+ * @property {Object.<string, Object>} [additionalProps] Files or child directories.
+ */
+
+/**
+ * @typedef {Object.<string, MarkdownFile | Subdirectory>} Directory
+ * @description A directory of markdown files
+ * @property {MarkdownFile} $ A home file.
+ * @property {MarkdownFile} _ A settings file.
+ * @property {Object.<string, MarkdownFile | Subdirectory>} [additionalProps] Files or child directories.
+ */
+
+/**
+ * Recursively read a directory of markdown files and return an object of the files.
+ * @param {string} directoryPath - The directory to read.
+ * @param {string} parents - A string representing the parent directories.
+ * @param {import('./loadCache').Cache} cache
+ * @returns {Promise<Directory>}
+ */
+
+/**
+ * Recursively read a directory of markdown files and return an object of the files.
+ * @param {string} folderPath - The directory to read.
+ * @param {string} parents - A string representing the parent directories.
+ * @param {import('./loadCache').Cache} cache
+ * @returns {Promise<Directory>}
+ */
+async function readFolder(folderPath, parents, cache) {
+	function filterFiles(file) {
+		if (file.name.startsWith('.')) return false;
+		if (file.name.startsWith('README')) return false;
+		if (file.isFile() && !file.name.endsWith('.md')) return false;
+		if (file.name === 'node_modules') return false;
+		if (file.name === 'assets') return false;
+		if (file.name === 'templates') return false;
+		return true;
+	}
+
+	let files = (await fs.readdir(folderPath, { withFileTypes: true })).filter(filterFiles);
+
+	const promises = files.map(async (file) => await readFile(file, parents, cache, folderPath));
+
+	const folder = (await Promise.all(promises)).reduce((acc, obj) => ({ ...acc, ...obj }), {});
+
+	if (typeof folder === 'object' && !folder?.$) {
+		folder.$ = DefaultFile(parents || '/');
+		folder.$.imputedProperties = {
+			fileName: parents.split('/').at(-1) || 'Home'
+		};
+	}
+
+	if (typeof folder === 'object' && !folder?._) {
+		folder._ = DefaultFile(parents);
+	}
+
+	return folder;
+}
+
+async function readFile(file, parents, cache, folderPath) {
+	const route = path.parse(file.name).name;
+	const url = buildURL(parents, route);
+	const filePath = path.join(folderPath, file.name);
+
+	const extension = path.extname(file.name);
+
+	if (file.isFile() && extension === '.md') {
+		const startLoad = performance.now();
+		const shortPath = parents + '/' + file.name;
+
+		const { ast, frontmatter, imputedProperties } = await readMarkdownFile(filePath, cache);
+		const loadTime = (performance.now() - startLoad).toFixed(2);
+		console.log(`📄 ${shortPath} (${loadTime}ms)`);
+		if (frontmatter.published == false) return;
+		if (file.name === 'home.md' || file.name === 'settings.md') {
+			const key = file.name === 'home.md' ? '$' : '_';
+			return {
+				[key]: {
+					...frontmatter,
+					imputedProperties,
+					ast,
+					url
+				}
+			};
+		}
+		return {
+			[route]: {
+				$: {
+					...frontmatter,
+					imputedProperties,
+					ast,
+					url
+				}
+			}
+		};
+	}
+
+	/** @type Directory */
+	if (file.isDirectory()) {
+		const settingsPath = path.join(folderPath, file.name, 'settings.md');
+		const settingsFileExists = await checkFileExists(settingsPath);
+
+		/** @type {MarkdownFile} */
+		const _ = settingsFileExists ? await readMarkdownFile(settingsPath, cache) : DefaultFile(url);
+
+		const shortPath = parents + '/' + file.name;
+
+		const startLoad = performance.now();
+		const folder = await readFolder(path.join(folderPath, file.name), url, cache);
+
+		const loadTime = (performance.now() - startLoad).toFixed(2);
+		console.log(`📁 ${shortPath} (${loadTime}ms)`);
+
+		if (!folder.hasOwnProperty('$')) {
+			const $ = DefaultFile(url);
+			Object.assign(folder, { $ });
+		}
+		if (!folder.hasOwnProperty('_')) {
+			const _ = DefaultFile(url);
+			Object.assign(folder, { _ });
+		}
+
+		return {
+			[file.name]: folder
+		};
+	}
+}
+
+/**
+ * @typedef {Object} ProcessedFiles
+ * @description The return value of the processing function.
+ * @property {Directory} folder - The root directory.
+ * @property {Cache} finalCache - The application cache.
+ */
+
+/**
+ * Recursively reads the markdown files in a directory and returns a representative object.
+ * @param {import('./loadCache').Cache} cache
+ * @returns {Promise<ProcessedFiles>}
+ */
+export default async function processMarkdownFiles(cache) {
+	/** @type {Array<string>} */
+	// @ts-ignore
+	const [homeDir] = $home;
+	const folder = await readFolder(homeDir, '', cache);
+
+	return { folder, finalCache: cache };
+}

package/src/lib/utilities/readMarkdownFile.js

@@ -0,0 +1,134 @@
+import fs from 'fs/promises';
+import { remark } from 'remark';
+import remarkFrontmatter from 'remark-frontmatter';
+import remarkHTML from 'remark-html';
+import yaml from 'js-yaml';
+import { toString } from 'mdast-util-to-string';
+import path from 'path';
+
+import { mutateMarkdownAST, mutateMarkdownFrontmatter } from '.';
+
+/**
+ * @typedef {Object} FrontmatterNode
+ * @property {"yaml"} type - The type is yaml.
+ * @property {string} value - The value of the node.
+ * @property {Object} [position] - The position of the node.
+ */
+
+/**
+ * Parse a frontmatter node to a JS object.
+ * @param {FrontmatterNode | undefined} frontmatterNode - A markdown node of type "yaml".
+ * @returns {Object} A JS object containing the frontmatter
+ */
+function parseFrontmatter(frontmatterNode) {
+	if (!frontmatterNode) return {};
+	const frontmatter = yaml.load(frontmatterNode.value);
+	if (frontmatter && typeof frontmatter === 'object') {
+		return frontmatter;
+	}
+	return {};
+}
+
+function deduceDescriptionFromAST(ast) {
+	for (let i = 0; i < ast?.children?.length; i++) {
+		if (ast?.children?.[i]?.type === 'paragraph') {
+			const firstParagraph = toString(ast[i]);
+			if (firstParagraph.length > 150) {
+				return firstParagraph.slice(0, 150);
+			}
+			return firstParagraph;
+		}
+	}
+	return null;
+}
+
+function demoteHeadings(ast, start) {
+	for (let i = start; i < ast.length; i++) {
+		const node = ast[i];
+		if (node.type === 'heading') {
+			node.depth = node.depth + 1;
+		}
+	}
+}
+
+function checkHeadings(ast, start = 1) {
+	for (let i = start; i < ast.length; i++) {
+		if (ast[i]?.depth === 1) {
+			return true;
+		}
+	}
+	return false;
+}
+
+function extractTitleFromHeading(node) {
+	return node?.children?.[0].value;
+}
+
+function imputeTitleFromAST(ast) {
+	if (ast?.[0]?.depth === 1) {
+		return extractTitleFromHeading(ast[0]);
+	}
+
+	return null;
+}
+
+/**
+ * @typedef {Object} ParsedMarkdown
+ * @property {Object.<string, *>} [frontmatter] - Metadata for the document.
+ * @property {Array<Object>} ast - The content of the document.
+ * @property {Object} imputedProperties - Metadata imputed from content.
+ * @property {string} [url] - URL for the page.
+ * @description An object containing the parsed frontmatter and filtered AST.
+ */
+
+/**
+ * Reads a Markdown file and parses its content into frontmatter and abstract syntax tree (AST).
+ * It also filters out YAML frontmatter nodes from the AST.
+ *
+ * @async
+ * @function
+ * @param {string} filePath - The path to the Markdown file.
+ * @param {Object} cache - An optional cache object that can be passed to mutate the AST.
+ * @returns {Promise<ParsedMarkdown>}
+ * @throws {Error} Will throw an error if the file cannot be read or parsed.
+ */
+export default async function readMarkdownFile(filePath, cache) {
+	const fileName = path.basename(filePath.slice(0, -3));
+
+	const fileContent = await fs.readFile(filePath, 'utf8');
+	const markdownParser = remark().use(remarkFrontmatter, ['yaml']).use(remarkHTML);
+
+	const parsedMarkdown = markdownParser.parse(fileContent);
+	const frontmatterNode = parsedMarkdown.children.find((node) => node.type === 'yaml');
+	const frontmatter = frontmatterNode?.type === 'yaml' ? parseFrontmatter(frontmatterNode) : {};
+	const ast = remark().use(remarkFrontmatter).parse(fileContent);
+	const filteredAST = ast.children.filter((child) => child.type !== 'yaml');
+
+	const imputedTitle = imputeTitleFromAST(filteredAST);
+
+	if (imputedTitle) {
+		filteredAST.shift();
+	}
+
+	if (imputedTitle && checkHeadings(filteredAST, 1)) {
+		demoteHeadings(filteredAST, 1);
+	}
+
+	if (!frontmatter.title && imputedTitle) frontmatter.title = imputedTitle;
+
+	const imputedProperties = {
+		description: deduceDescriptionFromAST(ast),
+		title: imputedTitle,
+		fileName
+	};
+
+	const promises = [mutateMarkdownAST(ast.children, cache)];
+
+	// Don't mutate frontmatter on the settings pages
+	if (!filePath.endsWith('settings.md'))
+		promises.push(mutateMarkdownFrontmatter(frontmatter, cache));
+
+	await Promise.all(promises);
+
+	return { frontmatter, ast: filteredAST, imputedProperties };
+}

package/src/lib/utilities/regexPatterns.js

@@ -0,0 +1,12 @@
+const img = /^https?:\/\/\S+(avif|gif|heif|jpeg|jpg|png|tiff|webp)(\?\S+)?/;
+const pdf = /^https?:\/\/\S+(pdf)(\?\S+)?/;
+const url = /^https?:\/\/\S+$/;
+const path = /^\/[\/\S]+$/;
+// const humanDate = /^((Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec|January|February|March|April|May|June|July|August|September|October|November|December)[\d\040,\/(nd|rd|st|th)]{8,12}|[\d\040,\/(nd|rd|st|th)]{1,5}(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec|January|February|March|April|May|June|July|August|September|October|November|December),?\040\d{2,4})$/;
+
+export default {
+	url,
+	img,
+	path,
+	pdf
+};

package/src/lib/utilities/resolveHomeDirPath.js

@@ -0,0 +1,5 @@
+import path from 'path';
+
+export default function resolveHomeDirPath(filename, homeDir) {
+	return path.normalize(path.join(homeDir, filename));
+}

package/src/lib/utilities/writeCache.js

@@ -0,0 +1,14 @@
+import fs from 'fs/promises';
+import { resolveHomeDirPath } from '.';
+
+export default async function writeCache(data, homeDir) {
+	const cachePath = resolveHomeDirPath('.cache.json', homeDir);
+
+	try {
+		const jsonData = JSON.stringify(data, null, 2);
+		await fs.writeFile(cachePath, jsonData, 'utf8');
+		console.log('Cache updated');
+	} catch (error) {
+		console.error('Error updating cache data:', error);
+	}
+}

package/src/routes/[...path]/+layout.server.js

@@ -0,0 +1,71 @@
+import { loadCache, writeCache, processMarkdownFiles } from '$lib/utilities';
+import { access } from 'fs/promises';
+import { constants } from 'fs';
+import { normalize, join, basename } from 'path';
+
+export const prerender = true;
+// export const csr = false;
+// export const ssr = false;
+
+let contentCache;
+let contentCacheTime;
+
+async function checkFileExists(filePath, homeDir) {
+	try {
+		await access(join(homeDir, filePath), constants.F_OK);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/** @type {import('./$types').PageLoad} */
+export async function load() {
+	const startLoad = performance.now();
+	const now = Date.now();
+	const contentCacheDuration = 60000; // Cache for  1 minute
+	const initialURLCache = await loadCache($home[0]);
+
+	const cssExists = await checkFileExists(normalize('/assets/styles.css'), $home[0]);
+
+	if (cssExists) console.log('Found CSS file at /assets/styles.css');
+	else console.log('No CSS file found as /assets/styles.css');
+
+	const faviconExists = await checkFileExists(normalize('/assets/favicon.png'), $home[0]);
+
+	const files = {
+		css: {
+			exists: cssExists,
+			path: join($home[0], 'assets', 'styles.css')
+		},
+		favicon: {
+			exists: faviconExists,
+			path: join($home[0], 'assets', 'favicon.png')
+		}
+	};
+
+	let data = {};
+
+	if (contentCache && now - contentCacheTime < contentCacheDuration) {
+		console.log('Using content cache');
+		data = contentCache;
+	} else {
+		console.log('Not using content cache');
+		data = await processMarkdownFiles(initialURLCache);
+		contentCache = data;
+		contentCacheTime = now;
+	}
+
+	// const { folder: website, finalCache } = await processMarkdownFiles(initialCache);
+	const { folder: website, finalCache } = data;
+
+	if (JSON.stringify(initialURLCache) !== JSON.stringify(finalCache)) {
+		writeCache(finalCache, $home[0]);
+	}
+
+	console.log(`Load time: ${(performance.now() - startLoad).toFixed(2)} ms`);
+
+	const folderName = basename($home[0]);
+
+	return { website, homeDir: $home[0], folderName, files };
+}

package/src/routes/[...path]/+page.server.js

@@ -0,0 +1,22 @@
+import { getPagesByFolder, processMarkdownFiles, loadCache } from '../../lib/utilities';
+
+/** @type {import('./$types').PageLoad} */
+export async function load({ params }) {
+	const { path } = params;
+	const segments = path ? params.path.split('/') : [];
+	return { path, segments };
+}
+
+export const prerender = true;
+
+export async function entries() {
+	const initialCache = await loadCache($home[0]);
+
+	const { folder: website } = await processMarkdownFiles(initialCache);
+
+	const pages = getPagesByFolder(website, '/', false).map((page) => {
+		return { path: page.url?.replace('/', '') };
+	});
+
+	return pages;
+}