vowel 0.1.42 → 0.1.44

package/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## [0.1.44](https://github.com/samlfair/vowel/compare/v0.1.43...v0.1.44) (2024-10-21)
+
+
+### Bug Fixes
+
+* metadata retrieval bug ([566f7cf](https://github.com/samlfair/vowel/commit/566f7cf64be2749e30cddcb474f6385638c408a9))
+* webmentions bug ([2215ff5](https://github.com/samlfair/vowel/commit/2215ff5f5668d140535f6df3d30754fd685f22ae))
+
+## [0.1.43](https://github.com/samlfair/vowel/compare/v0.1.42...v0.1.43) (2024-10-18)
+
 ## [0.1.42](https://github.com/samlfair/vowel/compare/v0.1.41...v0.1.42) (2024-10-18)
 
 

package/bin.js

@@ -28,7 +28,7 @@ if (args._.includes('publish')) {
 };
 
 if (!args.verbose) {
-	filterConsole(['jsconfig', 'vite', 'Vite', 'tsconfig', `--host`]);
+	// filterConsole(['jsconfig', 'vite', 'Vite', 'tsconfig', `--host`]);
 	console.log(`\n\n`);
 }
 
@@ -54,7 +54,7 @@ child.stdout.on('data', (data) => {
 		console.log(`\n\n`);
 		processedFiles = 0;
 		foundFiles = 0;
-	} else if(false) { // Toggle to true to reveal all console output
+	} else if(true) { // Toggle to true to reveal all console output
 		console.log(message)
 	} else if (message.match('http://localhost:')) {
 		const url = message.match(/http:\/\/localhost:\S+/);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "vowel",
-	"version": "0.1.42",
+	"version": "0.1.44",
 	"homepage": "https://vowel.cc",
 	"author": "Sam Littlefair (https://littlefair.ca)",
 	"repository": {

package/src/lib/components/Page.svelte

@@ -13,8 +13,6 @@
 	// TODO: Add conditional logic for format = 'xml'
 
 	const headings = ast.filter((child) => child.type === 'heading');
-
-	console.log(webmentions)
 </script>
 
 <!-- Image -->

package/src/lib/utilities/getMetadata.js

@@ -0,0 +1,46 @@
+import { XMLParser } from "fast-xml-parser"
+import urlMetadata from 'url-metadata';
+
+export default async function getMetadata({ cache, url }) {
+	if (!cache[url]) {
+		try {
+			const urlObject = new URL(url);
+			const allMetadata = await urlMetadata(urlObject.href, {
+				includeResponseBody: true,
+				ensureSecureImageRequest: true
+			});
+
+			const parser = new XMLParser({
+				unpairedTags: ["!doctype", "meta", "link", "hr", "br", "img"],
+				ignoreAttributes: false,
+				stopNodes: ["*.pre", "*.script"],
+				processEntities: true,
+				htmlEntities: true
+			})
+	
+			let parsedData = parser.parse(allMetadata.responseBody)
+			const webmentionEndpoint = parsedData?.html?.head?.link?.find(link => {
+				return link["@_rel"] === "webmention"
+			})?.["@_href"]
+
+			const metadata = {
+				image: allMetadata['og:image'],
+				ogURL: allMetadata['og:url'],
+				canonicalURL: allMetadata.canonical,
+				title: allMetadata.title,
+				ogTitle: allMetadata['og:title'],
+				author: allMetadata.author,
+				description: allMetadata.description,
+				webmentionEndpoint
+			};
+
+			cache[url] = metadata;
+
+			return { metadata }
+		} catch (error) {
+			console.log({ fetchingMetadataError: error })
+			return { metadata: undefined }
+		}
+	}
+	return { metadata: cache[url] }
+}

package/src/lib/utilities/index.js

@@ -18,4 +18,5 @@ export { default as resolveHomeDirPath } from './resolveHomeDirPath';
 export { default as createPageClass } from './createPageClass';
 export { default as createFolderClass } from './createFolderClass';
 export { default as isActiveLink } from './isActiveLink';
-export { default as sendWebmention } from "./sendWebmention"
+export { default as sendWebmention } from "./sendWebmention"
+export { default as getMetadata } from "./getMetadata"

package/src/lib/utilities/mutateMarkdownAST.js

@@ -1,5 +1,4 @@
-import urlMetadata from 'url-metadata';
-import { sendWebmention } from './';
+import { sendWebmention, getMetadata } from './';
 
 function isURL(string) {
 	try {
@@ -10,8 +9,9 @@ function isURL(string) {
 	}
 }
 
-export default async function mutateMarkdownAST(ast, cache, webmentions, pageURL) {
 
+
+export default async function mutateMarkdownAST(ast, cache, webmentions, pageURL) {
 	const promises = ast.map(async (node) => {
 		// TODO: Improve this URL regex
 		if (node.type === 'paragraph') {
@@ -25,55 +25,32 @@ export default async function mutateMarkdownAST(ast, cache, webmentions, pageURL
 				node.type = 'url';
 				const { url } = node.children[0];
 				node.url = url
-				if (!cache[node.url]) {
-					try {
-						const urlObject = new URL(url);
-						const response = await urlMetadata(urlObject.href, {
-							includeResponseBody: true,
-							ensureSecureImageRequest: true
-						});
 
-						console.log(!!webmentions)
-						console.log({ webmentions })
-						console.log(webmentions.find(webmention => webmention.target === url))
+				const { metadata } = await getMetadata({ cache, url })
 
+				node.metadata = metadata;
+				node.value = url;
+				delete node.children;
 
-						if (webmentions && !webmentions.find(webmention => webmention.target === url)) {
-							webmentions.push({
-								target: url,
-								status: "new"
-							})
-						} else if (webmentions) {
-							const webmention = webmentions.find(webmention => webmention.target === url)
-							if (webmention.status === "new") {
-								webmention.status = await sendWebmention({ body: response.responseBody, target: webmention.target, source: pageURL })
-							}
+				try {
+					console.log("Looking for webmentions")
+					if (webmentions && !webmentions.find(webmention => webmention.target === url)) {
+						console.log("Found new page")
+						webmentions.push({
+							target: url,
+							status: "new"
+						})
+					} else if (webmentions) {
+						console.log("Found new webmention")
+						const webmention = webmentions.find(webmention => webmention.target === url)
+						if (webmention.status === "new" || webmention.status === "failure" || webmention.status === "429") {
+							console.log("Sending webmention")
+							webmention.status = await sendWebmention({ endpoint: metadata.webmentionEndpoint, target: webmention.target, source: pageURL })
 						}
-
-						const metadata = {
-							image: response['og:image'],
-							ogURL: response['og:url'],
-							canonicalURL: response.canonical,
-							title: response.title,
-							ogTitle: response['og:title'],
-							author: response.author,
-							description: response.description
-						};
-
-
-
-						cache[url] = metadata;
-						node.metadata = metadata;
-						node.value = url;
-
-						delete node.children;
-					} catch (error) {
-						node.value = url;
 					}
-				} else {
-					node.metadata = cache[url];
+				} catch (error) {
+					console.log({ webmentionError: error })
 				}
-
 			} else if (node.children[0].type === 'image') {
 				node.type = 'figure';
 				if (node?.children[1]?.type === 'text') {

package/src/lib/utilities/mutateMarkdownFrontmatter.js

@@ -1,5 +1,5 @@
 import urlMetadata from 'url-metadata';
-import { regexPatterns, isObject, parseDate, sendWebmention } from '.';
+import { regexPatterns, isObject, parseDate, sendWebmention, getMetadata } from '.';
 
 /**
  * Description
@@ -74,62 +74,32 @@ export default async function mutateMarkdownFrontmatter(frontmatter, cache, webm
 				break;
 			}
 			case 'url': {
+				const { metadata } = await getMetadata({
+					cache,
+					url: input
+				})
 
-				if (!cache[input]) {
-					try {
-						const url = new URL(input);
-						const metadata = await urlMetadata(url.href, {
-							includeResponseBody: true,
-							ensureSecureImageRequest: true
-						});
-
-						const selectedMetadata = {
-							url: metadata.url,
-							canonical: metadata.canonical,
-							title: metadata.title,
-							image: metadata.image,
-							favicons: url.favicons,
-							og_url: url['og:url'],
-							og_title: url['og:title'],
-							og_description: url['og:description'],
-							og_site_name: url['og:side_name'],
-							og_image: url['og:image']
-						};
+				frontmatter[key] = {
+					type: 'url',
+					output: metadata || input,
+					input
+				};
 
-						if (webmentions && !webmentions.find(webmention => webmention.target === input)) {
-							webmentions.push({
-								target: input,
-								status: "new"
-							})
-						} else if (webmentions) {
-							const webmention = webmentions.find(webmention => webmention.target === input)
-							if (webmention.status === "new") {
-								webmention.status = await sendWebmention({ body: metadata.responseBody, target: webmention.target, source: pageURL })
-							}
+				try {
+					if (webmentions && !webmentions.find(webmention => webmention.target === input)) {
+						webmentions.push({
+							target: input,
+							status: "new"
+						})
+					} else if (webmentions) {
+						const webmention = webmentions.find(webmention => webmention.target === input)
+						if (webmention.status === "new") {
+							webmention.status = await sendWebmention({ endpoint: metadata.webmentionEndpoint, target: webmention.target, source: pageURL })
 						}
-
-
-						cache[input] = selectedMetadata;
-
-						frontmatter[key] = {
-							type: 'url',
-							output: selectedMetadata,
-							input
-						};
-					} catch (e) {
-						console.error(`Error on URL in frontmatter: ${frontmatter[key]}`);
-						frontmatter[key] = {
-							type: 'string',
-							output: input,
-							input
-						};
 					}
-				} else {
-					frontmatter[key] = {
-						type: 'url',
-						output: cache[input],
-						input
-					};
+
+				} catch (error) {
+					console.log({ webmentionsError: error })
 				}
 				break;
 			}

package/src/lib/utilities/processMarkdownFiles.js

@@ -90,7 +90,7 @@ function DefaultDirectory(path) {
  * @param {import('./loadCache').Cache} cache
  * @returns {Promise<Directory>}
  */
-async function readFolder(folderPath, parents, cache, hidden, publishedData) {
+async function readFolder(folderPath, parents, cache, hidden, publishedData, domain) {
 	function filterFiles(file) {
 		if (file.name.startsWith('.')) return false;
 		if (file.name.startsWith('README')) return false;
@@ -106,7 +106,7 @@ async function readFolder(folderPath, parents, cache, hidden, publishedData) {
 	console.log(`filesfound:${files.length}`);
 
 	const promises = files.map(
-		async (file) => await readFile(file, parents, cache, folderPath, hidden, publishedData)
+		async (file) => await readFile(file, parents, cache, folderPath, hidden, publishedData, domain)
 	);
 
 	const folder = (await Promise.all(promises)).reduce((acc, obj) => ({ ...acc, ...obj }), {});
@@ -125,16 +125,28 @@ async function readFolder(folderPath, parents, cache, hidden, publishedData) {
 	return folder;
 }
 
-async function readFile(file, parents, cache, folderPath, hidden, publishedData) {
+function buildHREF(url, domain) {
+	try {
+		const urlObj = new URL(domain)
+		urlObj.pathname = url
+		return urlObj.href
+	} catch (e) {
+		return undefined
+	
+	}
+}
+
+async function readFile(file, parents, cache, folderPath, hidden, publishedData, domain) {
 	const route = path.parse(file.name).name;
 	const url = buildURL(parents, route);
 	const filePath = path.join(folderPath, file.name);
-
+	
 	const extension = path.extname(file.name);
-
+	
 	const hide = (file.name.startsWith('$') && file.name.length > 1) || hidden;
-
+	
 	if (file.isFile() && extension === '.md') {
+		const href = buildHREF(url, domain)
 		const startLoad = performance.now();
 		const shortPath = parents + '/' + file.name;
 
@@ -147,7 +159,7 @@ async function readFile(file, parents, cache, folderPath, hidden, publishedData)
 			filePublishedData = publishedData.at(-1)
 		}
 		
-		const { ast, frontmatter, imputedProperties } = await readMarkdownFile(filePath, cache, filePublishedData, url);
+		const { ast, frontmatter, imputedProperties } = await readMarkdownFile(filePath, cache, filePublishedData, href);
 
 		const loadTime = (performance.now() - startLoad).toFixed(2);
 		// console.log(`📄 ${shortPath} (${loadTime}ms)`);
@@ -189,7 +201,7 @@ async function readFile(file, parents, cache, folderPath, hidden, publishedData)
 		const shortPath = parents + '/' + file.name;
 
 		const startLoad = performance.now();
-		const folder = await readFolder(path.join(folderPath, file.name), url, cache, hide, publishedData);
+		const folder = await readFolder(path.join(folderPath, file.name), url, cache, hide, publishedData, domain);
 
 		const loadTime = (performance.now() - startLoad).toFixed(2);
 		// console.log(`📁 ${shortPath} (${loadTime}ms)`);
@@ -221,11 +233,11 @@ async function readFile(file, parents, cache, folderPath, hidden, publishedData)
  * @param {import('./loadCache').Cache} cache
  * @returns {Promise<ProcessedFiles>}
  */
-export default async function processMarkdownFiles(cache, publishedData) {
+export default async function processMarkdownFiles(cache, publishedData, domain) {
 	/** @type {Array<string>} */
 	// @ts-ignore
 	const [homeDir] = $home;
-	const folder = await readFolder(homeDir, '', cache, null, publishedData);
+	const folder = await readFolder(homeDir, '', cache, null, publishedData, domain);
 
 	return { folder, finalCache: cache };
 }

package/src/lib/utilities/sendWebmention.js

@@ -1,28 +1,11 @@
-import { XMLParser } from "fast-xml-parser"
-
-
-export default async function sendWebmention({ body, source, target }) {
+export default async function sendWebmention({ endpoint, source, target }) {
     try {
-        const parser = new XMLParser({
-            unpairedTags: ["!doctype", "meta", "link", "hr", "br", "img"],
-            ignoreAttributes: false,
-            stopNodes: ["*.pre", "*.script"],
-            processEntities: true,
-            htmlEntities: true
-        })
-
-        let parsedData = parser.parse(body)
-        const webmentionEndpoint = parsedData?.html?.head?.link?.find(link => {
-            return link["@_rel"] === "webmention"
-        })?.["@_href"]
-
-        if (webmentionEndpoint) {
+        if (endpoint) {
             const params = new URLSearchParams()
             params.append('source', source)
             params.append('target', target)
 
-
-            const response = await fetch(webmentionEndpoint, {
+            const response = await fetch(endpoint, {
                 method: "POST",
                 body: params,
                 headers: {
@@ -30,16 +13,22 @@ export default async function sendWebmention({ body, source, target }) {
                 }
             })
 
+            console.log({
+                source, target,
+                webmentionResponse: await response.json()
+            })
+
             if(response.status === 429) {
                 return "429"
             } else if (response.status >= 200 && response.status < 300) {
                 return "success"
             } else {
+                console.log({ failure: await response.json() })
                 return "failure"
             }
         }
-    } catch (e) {
-        console.log(e)
+    } catch (error) {
+        console.log({ error })
         return "error"
     }
 }

package/src/routes/$vowel/published.json/+server.js

@@ -25,22 +25,22 @@ export async function GET() {
 	// everything.request.url
 
 	let settings
-	
+
 	const settingsPath = join($home[0], "/settings.md")
 	const settingsExists = await checkFileExists(settingsPath);
-	if(settingsExists) {
+	if (settingsExists) {
 		settings = await readMarkdownFile(settingsPath)
 	}
 
 	// Get published articles from live site
 	const publishedData = await getPublishedData(settings.frontmatter.domain)
 
-	console.log({publish: $publish[0]})
-
-	if($publish[0]) {
+	// if($publish[0]) {
+	if (true) {
 		const initialCache = await loadCache($home[0]);
-		
-		const { folder: website } = await processMarkdownFiles(initialCache, publishedData);
+
+
+		const { folder: website } = await processMarkdownFiles(initialCache, publishedData, settings.frontmatter.domain);
 	}
 
 	const headers = {
@@ -48,8 +48,6 @@ export async function GET() {
 		'Content-Type': 'text/plain'
 	};
 
-	console.log({ publishedData })
-
 	const response = new Response(JSON.stringify(publishedData), {
 		headers
 	});