vowel 0.1.35 → 0.1.36

package/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## [0.1.36](https://github.com/samlfair/vowel/compare/v0.1.35...v0.1.36) (2024-10-17)
+
+
+### Features
+
+* send webmentions for urls with publish command ([e07abeb](https://github.com/samlfair/vowel/commit/e07abeb5911ab8079b4caec715eb0a463c4809cd))
+
 ## [0.1.35](https://github.com/samlfair/vowel/compare/v0.1.34...v0.1.35) (2024-10-17)
 
 ## [0.1.34](https://github.com/samlfair/vowel/compare/v0.1.33...v0.1.34) (2024-10-16)

package/bin.js

@@ -22,6 +22,10 @@ const homeDir = process.cwd();
 
 const spawnArgs = ['./server.js', '--directory', homeDir];
 if (args._.includes('build')) spawnArgs.push('--build');
+if (args._.includes('publish')) {
+	spawnArgs.push('--build')
+	spawnArgs.push('--publish')
+};
 
 if (!args.verbose) {
 	filterConsole(['jsconfig', 'vite', 'Vite', 'tsconfig', `--host`]);
@@ -50,6 +54,8 @@ child.stdout.on('data', (data) => {
 		console.log(`\n\n`);
 		processedFiles = 0;
 		foundFiles = 0;
+	} else if(false) { // Toggle to true to reveal all console output
+		console.log(message)
 	} else if (message.match('http://localhost:')) {
 		const url = message.match(/http:\/\/localhost:\S+/);
 		console.log(

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "vowel",
-	"version": "0.1.35",
+	"version": "0.1.36",
 	"homepage": "https://vowel.cc",
 	"author": "Sam Littlefair (https://littlefair.ca)",
 	"repository": {
@@ -33,6 +33,7 @@
 		"any-date-parser": "^1.5.4",
 		"change-case": "^5.4.1",
 		"commit-and-tag-version": "^12.4.4",
+		"fast-xml-parser": "^4.5.0",
 		"filter-console": "^1.0.0",
 		"js-yaml": "^4.1.0",
 		"loading-cli": "^1.1.2",

package/server.js

@@ -6,7 +6,7 @@ import { dirname, join } from 'path';
 import mri from 'mri';
 
 const args = mri(process.argv);
-const { build: isBuild, directory } = args;
+const { build: isBuild, directory, publish } = args;
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
 
@@ -18,7 +18,8 @@ const $home = isBuild ? process.cwd() : directory;
 
 const define = {
 	$home: [directory],
-	$build: [isBuild]
+	$build: [isBuild],
+	$publish: [publish]
 };
 
 const config = {

package/src/lib/components/Markdown/index.svelte

@@ -85,7 +85,8 @@
 		{:else if node.type === 'image'}
 			<Image {node} />
 		{:else if node.type === 'url'}
-			<LinkPreview url={node.value} metadata={node.metadata} {format} />
+			<!-- {@const dummy = console.log(node)} -->
+			<LinkPreview url={node.url} metadata={node.metadata} {format} />
 		{:else if node.type === 'listItem'}
 			{@const isChecklistItem = node.checked !== null}
 			{#if isChecklistItem}

package/src/lib/utilities/index.js

@@ -18,3 +18,4 @@ export { default as resolveHomeDirPath } from './resolveHomeDirPath';
 export { default as createPageClass } from './createPageClass';
 export { default as createFolderClass } from './createFolderClass';
 export { default as isActiveLink } from './isActiveLink';
+export { default as sendWebmention } from "./sendWebmention"

package/src/lib/utilities/mutateMarkdownAST.js

@@ -1,25 +1,52 @@
 import urlMetadata from 'url-metadata';
+import { sendWebmention } from './';
+
+function isURL(string) {
+	try {
+		new URL(string)
+		return true
+	} catch (e) {
+		return false
+	}
+}
+
+export default async function mutateMarkdownAST(ast, cache, webmentions) {
 
-export default async function mutateMarkdownAST(ast, cache) {
 	const promises = ast.map(async (node) => {
 		// TODO: Improve this URL regex
 		if (node.type === 'paragraph') {
+
 			// URLs
 			if (
 				node.children?.length === 1 &&
-				node.children[0]?.value?.match(/^https?:\/\/\S+$/) &&
-				!node.children[0]?.children
+				node.children?.[0]?.type === "link" &&
+				node.children?.[0]?.children?.length === 1
 			) {
 				node.type = 'url';
-				const url = node.children[0].value;
-				if (!cache[node.value]) {
+				const { url } = node.children[0];
+				node.url = url
+				if (!cache[node.url]) {
 					try {
 						const urlObject = new URL(url);
 						const response = await urlMetadata(urlObject.href, {
-							includeResponseBody: false,
+							includeResponseBody: true,
 							ensureSecureImageRequest: true
 						});
 
+						console.log("UR:")
+
+						
+						if(webmentions && !webmentions.find(webmention => webmention.target === url)) {
+							console.log("Sending webmention)")
+							const webmentionStatus = await sendWebmention(response.responseBody)
+							console.log({webmentionStatus})
+
+							webmentions.push({
+								target: url,
+								status: webmentionStatus
+							})
+						}
+
 						const metadata = {
 							image: response['og:image'],
 							ogURL: response['og:url'],
@@ -30,6 +57,8 @@ export default async function mutateMarkdownAST(ast, cache) {
 							description: response.description
 						};
 
+
+
 						cache[url] = metadata;
 						node.metadata = metadata;
 						node.value = url;
@@ -37,11 +66,11 @@ export default async function mutateMarkdownAST(ast, cache) {
 						delete node.children;
 					} catch (error) {
 						node.value = url;
-						// console.log(`Error on URL in content: ${url}`);
 					}
 				} else {
 					node.metadata = cache[node.value];
 				}
+
 			} else if (node.children[0].type === 'image') {
 				node.type = 'figure';
 				if (node?.children[1]?.type === 'text') {

package/src/lib/utilities/mutateMarkdownFrontmatter.js

@@ -1,5 +1,5 @@
 import urlMetadata from 'url-metadata';
-import { regexPatterns, isObject, parseDate } from '.';
+import { regexPatterns, isObject, parseDate, sendWebmention } from '.';
 
 /**
  * Description
@@ -24,7 +24,7 @@ function imputeType(value) {
 	return 'other';
 }
 
-export default async function mutateMarkdownFrontmatter(frontmatter, cache) {
+export default async function mutateMarkdownFrontmatter(frontmatter, cache, webmentions) {
 	const keys = Object.keys(frontmatter);
 
 	const promises = keys.map(async (key) => {
@@ -74,6 +74,7 @@ export default async function mutateMarkdownFrontmatter(frontmatter, cache) {
 				break;
 			}
 			case 'url': {
+
 				if (!cache[input]) {
 					try {
 						const url = new URL(input);
@@ -95,6 +96,15 @@ export default async function mutateMarkdownFrontmatter(frontmatter, cache) {
 							og_image: url['og:image']
 						};
 
+						if(webmentions && !webmentions.find(webmention => webmention.target === url)) {
+							const webmentionStatus = await sendWebmention(response.responseBody)
+							
+							webmentions.push({
+								target: input,
+								status: webmentionStatus
+							})
+						}
+
 						cache[input] = selectedMetadata;
 
 						frontmatter[key] = {

package/src/lib/utilities/processMarkdownFiles.js

@@ -90,7 +90,7 @@ function DefaultDirectory(path) {
  * @param {import('./loadCache').Cache} cache
  * @returns {Promise<Directory>}
  */
-async function readFolder(folderPath, parents, cache, hidden) {
+async function readFolder(folderPath, parents, cache, hidden, publishedData) {
 	function filterFiles(file) {
 		if (file.name.startsWith('.')) return false;
 		if (file.name.startsWith('README')) return false;
@@ -106,7 +106,7 @@ async function readFolder(folderPath, parents, cache, hidden) {
 	console.log(`filesfound:${files.length}`);
 
 	const promises = files.map(
-		async (file) => await readFile(file, parents, cache, folderPath, hidden)
+		async (file) => await readFile(file, parents, cache, folderPath, hidden, publishedData)
 	);
 
 	const folder = (await Promise.all(promises)).reduce((acc, obj) => ({ ...acc, ...obj }), {});
@@ -125,7 +125,7 @@ async function readFolder(folderPath, parents, cache, hidden) {
 	return folder;
 }
 
-async function readFile(file, parents, cache, folderPath, hidden) {
+async function readFile(file, parents, cache, folderPath, hidden, publishedData) {
 	const route = path.parse(file.name).name;
 	const url = buildURL(parents, route);
 	const filePath = path.join(folderPath, file.name);
@@ -138,7 +138,17 @@ async function readFile(file, parents, cache, folderPath, hidden) {
 		const startLoad = performance.now();
 		const shortPath = parents + '/' + file.name;
 
-		const { ast, frontmatter, imputedProperties } = await readMarkdownFile(filePath, cache);
+		let filePublishedData = publishedData?.find(item => item.path === url) 
+		if(publishedData && !filePublishedData) {
+			publishedData.push({
+				path: url,
+				webmentions: []
+			})
+			filePublishedData = publishedData.at(-1)
+		}
+		
+		const { ast, frontmatter, imputedProperties } = await readMarkdownFile(filePath, cache, filePublishedData);
+
 		const loadTime = (performance.now() - startLoad).toFixed(2);
 		// console.log(`📄 ${shortPath} (${loadTime}ms)`);
 		console.log('fileread');
@@ -179,7 +189,7 @@ async function readFile(file, parents, cache, folderPath, hidden) {
 		const shortPath = parents + '/' + file.name;
 
 		const startLoad = performance.now();
-		const folder = await readFolder(path.join(folderPath, file.name), url, cache, hide);
+		const folder = await readFolder(path.join(folderPath, file.name), url, cache, hide, publishedData);
 
 		const loadTime = (performance.now() - startLoad).toFixed(2);
 		// console.log(`📁 ${shortPath} (${loadTime}ms)`);
@@ -211,13 +221,11 @@ async function readFile(file, parents, cache, folderPath, hidden) {
  * @param {import('./loadCache').Cache} cache
  * @returns {Promise<ProcessedFiles>}
  */
-export default async function processMarkdownFiles(cache) {
+export default async function processMarkdownFiles(cache, publishedData) {
 	/** @type {Array<string>} */
 	// @ts-ignore
 	const [homeDir] = $home;
-	const folder = await readFolder(homeDir, '', cache);
-
-	console.log('loaded');
+	const folder = await readFolder(homeDir, '', cache, null, publishedData);
 
 	return { folder, finalCache: cache };
 }

package/src/lib/utilities/readMarkdownFile.js

@@ -110,7 +110,7 @@ function imputeTitleFromAST(ast) {
  * @returns {Promise<ParsedMarkdown>}
  * @throws {Error} Will throw an error if the file cannot be read or parsed.
  */
-export default async function readMarkdownFile(filePath, cache) {
+export default async function readMarkdownFile(filePath, cache, publishedData) {
 	const fileName = path.basename(filePath.slice(0, -3));
 
 	const fileContent = await fs.readFile(filePath, 'utf8');
@@ -167,11 +167,14 @@ export default async function readMarkdownFile(filePath, cache) {
 		fileName
 	};
 
-	const promises = [mutateMarkdownAST(ast.children, cache)];
+
+	const webmentions = publishedData?.webmentions
+
+	const promises = [mutateMarkdownAST(ast.children, cache, webmentions)];
 
 	// Don't mutate frontmatter on the settings pages
 	if (!filePath.endsWith('settings.md'))
-		promises.push(mutateMarkdownFrontmatter(frontmatter, cache));
+		promises.push(mutateMarkdownFrontmatter(frontmatter, cache, webmentions));
 
 	await Promise.all(promises);
 

package/src/lib/utilities/sendWebmention.js

@@ -0,0 +1,45 @@
+import { XMLParser } from "fast-xml-parser"
+
+
+export default async function sendWebmention(body) {
+    try {
+        const parser = new XMLParser({
+            unpairedTags: ["!doctype", "meta", "link", "hr", "br", "img"],
+            ignoreAttributes: false,
+            stopNodes: ["*.pre", "*.script"],
+            processEntities: true,
+            htmlEntities: true
+        })
+
+        let parsedData = parser.parse(body)
+        const webmentionEndpoint = parsedData?.html?.head?.link?.find(link => {
+            return link["@_rel"] === "webmention"
+        })?.["@_href"]
+
+        if (webmentionEndpoint) {
+            const params = new URLSearchParams()
+            params.append('source', "https://www.littlefair.ca")
+            params.append('target', 'https://www.vowel.cc')
+
+
+            const response = await fetch(webmentionEndpoint, {
+                method: "POST",
+                body: params,
+                headers: {
+                    'Content-Type': 'application/x-www-form-urlencoded'
+                }
+            })
+
+            if(response.status === 429) {
+                return "429"
+            } else if (response.status >= 200 && response.status < 300) {
+                return "success"
+            } else {
+                return "failure"
+            }
+        }
+    } catch (e) {
+        console.log(e)
+        return "error"
+    }
+}

package/src/routes/$vowel/published.json/+server.js

@@ -1,27 +1,54 @@
+import {
+	processMarkdownFiles,
+	loadCache,
+	checkFileExists,
+	readMarkdownFile
+} from '$lib/utilities';
+import { join } from 'path';
+
+
 export const prerender = true;
 
-export async function GET({}) {
-    const body = [
-        {
-            path: "/notes",
-            webmentions: [
-				{
-					target: "https://vowel.cc/about",
-					success: true,
-					error: undefined
+async function getPublishedData(domain) {
+	try {
+		console.log({domain})
+		const publishedURL = new URL("/$vowel/published.json", domain)
+		const publishedDataResponse = await fetch(publishedURL.href)
+		const publushedDataJSON = await publishedDataResponse.json()
+		return publushedDataJSON
+	} catch (e) {
+		console.log(e)
+		return undefined
+	}
+}
+
+export async function GET() {
+	// everything.request.url
+
+	let settings
+	
+	const settingsPath = join($home[0], "/settings.md")
+	const settingsExists = await checkFileExists(settingsPath);
+	if(settingsExists) {
+		settings = await readMarkdownFile(settingsPath)
+		console.log({settings})
+	}
 
-				}
-			]
-        }
-    ]
+	// Get published articles from live site
+	const publishedData = await getPublishedData(settings.frontmatter.domain)
 
+	if($publish[0]) {
+		const initialCache = await loadCache($home[0]);
+		
+		const { folder: website } = await processMarkdownFiles(initialCache, publishedData);
+	}
 
 	const headers = {
-		'Cache-Control': 'max-age=0, s-maxage=3600',
+		'Cache-Control': 'max-age=0, s-maxage=1',
 		'Content-Type': 'text/plain'
 	};
 
-	const response = new Response(JSON.stringify(body), {
+	const response = new Response(JSON.stringify(publishedData), {
 		headers
 	});
 	return response;

package/src/routes/{[...path]/+layout.server.js → +layout.server.js}

@@ -1,11 +1,9 @@
-import { loadCache, writeCache, processMarkdownFiles } from '$lib/utilities';
+import { loadCache, writeCache, processMarkdownFiles, readMarkdownFile } from '$lib/utilities';
 import { access } from 'fs/promises';
 import { constants } from 'fs';
 import { normalize, join, basename } from 'path';
 import { dev } from '$app/environment';
 import mri from 'mri';
-import { getPage } from '../../lib/utilities';
-import { error } from '@sveltejs/kit';
 
 const args = mri(process.argv);
 const isBuild = args._.includes('build');
@@ -25,6 +23,8 @@ async function checkFileExists(filePath, homeDir) {
 	}
 }
 
+
+
 /** @type {import('./$types').PageLoad} */
 export async function load() {
 	// const startLoad = performance.now();
@@ -51,6 +51,9 @@ export async function load() {
 		}
 	};
 
+
+
+
 	let data = {};
 
 	if (contentCache) {
@@ -69,8 +72,6 @@ export async function load() {
 		writeCache(hotURLCache, $home[0]);
 	}
 
-	// console.log(`Load time: ${(performance.now() - startLoad).toFixed(2)} ms`);
-
 	const folderName = basename($home[0]);
 
 	return { website, homeDir: $home[0], folderName, files, dev };