replaywebpage 1.6.5 → 1.7.0

package/assets/chevron-double-down.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-chevron-double-down" viewBox="0 0 16 16">
+  <path fill-rule="evenodd" d="M1.646 6.646a.5.5 0 0 1 .708 0L8 12.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/>
+  <path fill-rule="evenodd" d="M1.646 2.646a.5.5 0 0 1 .708 0L8 8.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/>
+</svg>

package/assets/chevron-double-up.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-chevron-double-up" viewBox="0 0 16 16">
+  <path fill-rule="evenodd" d="M7.646 2.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 3.707 2.354 9.354a.5.5 0 1 1-.708-.708l6-6z"/>
+  <path fill-rule="evenodd" d="M7.646 6.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1-.708.708L8 7.707l-5.646 5.647a.5.5 0 0 1-.708-.708l6-6z"/>
+</svg>

package/assets/globe.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-globe" viewBox="0 0 16 16">
+  <path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
+</svg>

package/assets/main.scss

@@ -113,7 +113,7 @@ a:focus {
 
 // Add html and body styles to :host for webcomponents
 :host {
-  background-color: $body-background-color;
+  //background-color: $body-background-color;
   font-size: $body-size;
   -moz-osx-font-smoothing: grayscale;
   -webkit-font-smoothing: antialiased;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "replaywebpage",
   "productName": "ReplayWeb.page",
-  "version": "1.6.5",
+  "version": "1.7.0",
   "description": "Serverless Web Archive Replay",
   "repository": "https://github.com/webrecorder/replayweb.page",
   "homepage": "https://replayweb.page/",
@@ -9,13 +9,13 @@
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.4",
-    "@webrecorder/wabac": "^2.12.5",
+    "@webrecorder/wabac": "^2.13.2",
+    "auto-js-ipfs": "^1.3.9",
     "bulma": "^0.9.3",
     "electron-log": "^4.4.1",
     "electron-updater": "^4.3.9",
     "fetch-ndjson": "^1.1.6",
     "flexsearch": "^0.7.21",
-    "ipfs-core": "^0.10.8",
     "keyword-mark-element": "^0.2.0",
     "lit": "^2.1.1",
     "marked": "^4.0.10",
@@ -31,9 +31,10 @@
   "devDependencies": {
     "copy-webpack-plugin": "^9.0.1",
     "css-loader": "^6.2.0",
-    "electron": "^19.0.4",
+    "electron": "^20.3.0",
     "electron-builder": "^23.0.3",
     "electron-notarize": "^1.1.1",
+    "eslint": "^8.23.1",
     "http-server": "^13.0.2",
     "mini-css-extract-plugin": "^2.3.0",
     "node-sass": "^7.0.1",
@@ -117,7 +118,7 @@
       "extraResources": [
         "plugins-mac"
       ],
-      "singleArchFiles": "dist/prebuilds/**/*"
+      "singleArchFiles": "{dist/prebuilds,dist/prebuilds/**}"
     },
     "linux": {
       "category": "Archiving;Utility;",

package/src/appmain.js

@@ -1,11 +1,10 @@
 import { LitElement, html, css } from "lit";
 import { wrapCss, rwpLogo, IS_APP, VERSION, clickOnSpacebarPress } from "./misc";
 
-import { registerSW } from "./pageutils";
-
 import fasHelp from "@fortawesome/fontawesome-free/svgs/solid/question-circle.svg";
 import fasArrowLeft from "@fortawesome/fontawesome-free/svgs/solid/arrow-left.svg";
 import fasArrowRight from "@fortawesome/fontawesome-free/svgs/solid/arrow-right.svg";
+import { SWManager } from "./swmanager";
 
 
 // ===========================================================================
@@ -28,9 +27,8 @@ class ReplayWebApp extends LitElement
 
     this.loadInfo = null;
 
-    if (swName) {
-      registerSW(swName);
-    }
+    this.swmanager = new SWManager({name: swName, appName: this.appName});
+    this.swmanager.register().catch(() => this.swErrorMsg = this.swmanager.renderErrorReport(this.mainLogo));
 
     this.safariKeyframes();
   }
@@ -56,7 +54,9 @@ class ReplayWebApp extends LitElement
       collPageUrl: { type: String },
       pageTitle: { type: String },
       pageReplay: { type: Boolean },
-      source: { type: String }
+      source: { type: String },
+
+      swErrorMsg: { type: Object },
     };
   }
 
@@ -248,6 +248,10 @@ class ReplayWebApp extends LitElement
       return html``;
     }
 
+    if (this.embed && this.swErrorMsg) {
+      return this.swErrorMsg;
+    }
+
     return html`
       ${!this.embed || this.embed === "full" ? this.renderNavBar() : ""}
 
@@ -351,10 +355,13 @@ class ReplayWebApp extends LitElement
       this.showAbout = true;
     }
 
-    if (this.pageParams.get("config")) {
+    if (this.pageParams.has("embed")) {
       if (!this.loadInfo) {
         this.loadInfo = {};
       }
+    }
+
+    if (this.pageParams.get("config")) {
       try {
         this.loadInfo.extraConfig = JSON.parse(this.pageParams.get("config"));
       } catch (e) {
@@ -362,38 +369,33 @@ class ReplayWebApp extends LitElement
       }
     }
 
+    if (this.pageParams.get("baseUrlSourcePrefix")) {
+      this.loadInfo.extraConfig = this.loadInfo.extraConfig || {};
+      this.loadInfo.extraConfig.baseUrlSourcePrefix = this.pageParams.get("baseUrlSourcePrefix");
+    }
+
     if (this.pageParams.get("basePageUrl")) {
-      if (!this.loadInfo) {
-        this.loadInfo = {extraConfig: {}};
-      }
-      if (!this.loadInfo.extraConfig) {
-        this.loadInfo.extraConfig = {};
-      }
+      this.loadInfo.extraConfig = this.loadInfo.extraConfig || {};
       this.loadInfo.extraConfig.baseUrl = this.pageParams.get("basePageUrl");
     }
 
     if (this.pageParams.get("customColl")) {
-      if (!this.loadInfo) {
-        this.loadInfo = {};
-      }
       this.loadInfo.customColl = this.pageParams.get("customColl");
     }
 
     if (this.pageParams.get("noWebWorker") === "1") {
-      if (!this.loadInfo) {
-        this.loadInfo = {};
-      }
       this.loadInfo.noWebWorker = true;
     }
 
 
     if (this.pageParams.get("noCache") === "1") {
-      if (!this.loadInfo) {
-        this.loadInfo = {};
-      }
       this.loadInfo.noCache = true;
     }
 
+    if (this.pageParams.get("hideOffscreen") === "1") {
+      this.loadInfo.hideOffscreen = true;
+    }
+
     if (IS_APP && this.sourceUrl.startsWith("file://")) {
       // eslint-disable-next-line no-undef
       const url = new URL(__APP_FILE_SERVE_PREFIX__);

package/src/coll-index.js

@@ -404,44 +404,88 @@ class CollInfo extends LitElement
     `;
   }
 
-  render() {
+  renderSource(coll) {
+    return html`
+  <div class="column is-4">
+    <span class="source-text"><p class="minihead">Source</p>
+    ${coll.sourceUrl && (coll.sourceUrl.startsWith("http://") || coll.sourceUrl.startsWith("https://")) ? html`
+    <a href="${coll.sourceUrl}">${coll.sourceUrl}&nbsp;</a>` : html`
+    ${coll.sourceUrl}&nbsp;`}
+    </span>
+
+    <a @click="${(e) => this.onCopy(e, coll.sourceUrl)}" class="copy"><fa-icon .svg="${fasCopy}"/></a>
+    ${coll.sourceUrl && coll.sourceUrl.startsWith("googledrive://") ? html`
+      <p><i>(${coll.filename})</i></p>` : ""}
+  </div>
+  <div class="column is-2"><p class="minihead">Date Loaded</p>${coll.ctime ? new Date(coll.ctime).toLocaleString() : ""}</div>
+  <div class="column is-2"><p class="minihead">Total Size</p>${prettyBytes(Number(coll.size || 0))}</div>
+  `;
+  }
+
+  renderSummaryView() {
+    const coll = this.coll;
+
+    return html`
+    <div class="columns">
+      <div class="column col-title is-4">
+        <span class="subtitle has-text-weight-bold">
+          <a href="?source=${encodeURIComponent(coll.sourceUrl)}">${coll.title || coll.filename}</a>
+        </span>
+      </div>
+      ${this.renderSource(coll)}
+    </div>`;
+  }
+
+  renderDetailed() {
     const coll = this.coll;
-    const detailed = this.detailed;
+
+    let {numValid, numInvalid, domain, certFingerprint, datapackageHash, software} = this.coll.verify || {};
+    numValid = numValid || 0;
+    numInvalid = numInvalid || 0;
+
+    const certFingerprintUrl = certFingerprint ? `https://search.censys.io/certificates/${certFingerprint}` : "";
 
     return html`
       <div class="columns">
         <div class="column col-title is-4">
           <span class="subtitle has-text-weight-bold">
-            ${detailed ? html`
             ${coll.title || coll.filename}
-            ` : html`
-            <a href="?source=${encodeURIComponent(coll.sourceUrl)}">${coll.title || coll.filename}</a>`}
           </span>
         </div>
-        ${detailed && coll.desc ? html`
+        ${coll.desc ? html`
           <div class="column">
             <p class="minihead">Description</p>
             ${coll.desc}
           </div>` : html`
         `}
-        <div class="column is-4">
-          <span class="source-text"><p class="minihead">Source</p>
-          ${coll.sourceUrl && (coll.sourceUrl.startsWith("http://") || coll.sourceUrl.startsWith("https://")) ? html`
-          <a href="${coll.sourceUrl}">${coll.sourceUrl}&nbsp;</a>` : html`
-          ${coll.sourceUrl}&nbsp;`}
-          </span>
+        <div class="column"><p class="minihead">Filename</p>${coll.filename}</div>
+        ${this.renderSource(coll)}
+
+        ${domain ? html`
+        <div class="column">
+          <p class="minihead">Observed By</p>
+          <p>${domain}</p>
+          ${certFingerprintUrl ? html`<span><a target="_blank" href="${certFingerprintUrl}">View Certificate</a></span>` : ""}
+        </div>
+        ` : software ? html`
+        <div class="column">
+          <p class="minihead">Created With</p>
+          ${software || "Unknown"}
+        </div>
+        ` : ""}
 
-          <a @click="${(e) => this.onCopy(e, coll.sourceUrl)}" class="copy"><fa-icon .svg="${fasCopy}"/></a>
-          ${coll.sourceUrl && coll.sourceUrl.startsWith("googledrive://") ? html`
-            <p><i>(${coll.filename})</i></p>` : ""}
+        <div class="column">
+          <p class="minihead">Validation</p>
+          ${numValid > 0 || numInvalid > 0 ? html`
+          <p>${numValid} hashes verified${numInvalid ? html`, ${numInvalid} invalid` : ""}</p>` : html`
+          Not Available`}
         </div>
-        ${detailed ? html`
-        <div class="column"><p class="minihead">Filename</p>${coll.filename}</div>` : html``}
 
-        <div class="column is-2"><p class="minihead">Date Loaded</p>${coll.ctime ? new Date(coll.ctime).toLocaleString() : ""}</div>
-        <div class="column is-2"><p class="minihead">Total Size</p>${prettyBytes(Number(coll.size || 0))}</div>
+        <div class="column">
+          <p class="minihead">Package Hash</p>
+        ${datapackageHash || "Not Available"}
+        </div>
 
-        ${detailed ? html`
         <div class="column">
           <p class="minihead">Loading Mode</p>
           ${coll.onDemand ? "Download On-Demand" : "Fully Local"}
@@ -450,22 +494,12 @@ class CollInfo extends LitElement
           <p class="minihead">Collection id</p>
           ${coll.coll}
         </div>
-        ` : html``}
-
       </div>`;
   }
 
-  // purge + delete buttons
-  //
-  // <div class="column">
-  //   <button @click="${(e) => this.onPurge(true)}" class="button">Purge Cache + Full Reload</button>
-  // </div>
-
-  // ${this.canDelete ? html`
-  // <div class="column">
-  //   <button @click="${(e) => this.onPurge(false)}" class="button is-outlined is-danger">Delete Archive</button>
-  // </div>` : html``}
-  //
+  render() {
+    return this.detailed ? this.renderDetailed() : this.renderSummaryView();
+  }
 
   onCopy(event, sourceUrl) {
     event.preventDefault();

package/src/coll.js

@@ -24,6 +24,7 @@ import fasMenuV from "@fortawesome/fontawesome-free/svgs/solid/ellipsis-v.svg";
 import fasAngleLeft from "@fortawesome/fontawesome-free/svgs/solid/angle-left.svg";
 import fasAngleRight from "@fortawesome/fontawesome-free/svgs/solid/angle-right.svg";
 
+import { RWPEmbedReceipt } from "./embed-receipt.js";
 import Split from "split.js";
 
 
@@ -61,6 +62,7 @@ class Coll extends LitElement
     };
 
     this.menuActive = false;
+    this.embedDropdownActive = false;
 
     this.hasStory = false;
 
@@ -107,6 +109,8 @@ class Coll extends LitElement
       menuActive: { type: Boolean },
 
       embed: { type: String },
+      embedDropdownActive: { type: Boolean },
+
       editable: { type: Boolean },
       browsable: { type: Boolean },
       clearable: { type: Boolean },
@@ -131,7 +135,7 @@ class Coll extends LitElement
       this.isFullscreen = !!document.fullscreenElement;
     });
 
-    if (this.embed) {
+    if (this.embed && this.loadInfo && this.loadInfo.hideOffscreen) {
       this.observer = new IntersectionObserver((entries/*, observer*/) => {
         this.isVisible = entries[0].isIntersecting;
       });
@@ -289,7 +293,7 @@ class Coll extends LitElement
       this.collInfo.title = this.collInfo.filename;
     }
 
-    if (this.embed === "replayonly") {
+    if (this.embed === "replayonly" || this.embed === "replay-with-info") {
       this.showSidebar = false;
     }
 
@@ -475,6 +479,13 @@ class Coll extends LitElement
       flex: auto;
     }
 
+    rwp-embed-receipt {
+      flex-direction: column;
+      display: flex;
+    }
+
+    ${RWPEmbedReceipt.embedStyles}
+
     ${Coll.replayBarStyles}`;
   }
 
@@ -657,6 +668,7 @@ class Coll extends LitElement
     } else if (this.collInfo) {
       return html`
       ${this.renderLocationBar()}
+      ${this.renderVerifyInfo()}
       <div id="tabContents">
         <div id="contents" class="is-light ${isSidebar ? "sidebar" : (isReplay ? "is-hidden" : "full-pages")}"
              role="${isSidebar ? "complementary" : ""}" aria-label="${isSidebar ? "Browse Contents" : ""}">
@@ -744,7 +756,7 @@ class Coll extends LitElement
   }
 
   renderLocationBar() {
-    if (this.embed === "replayonly") {
+    if (this.embed === "replayonly" || this.embed == "replay-with-info") {
       return "";
     }
 
@@ -880,6 +892,20 @@ class Coll extends LitElement
     </nav><p id="skip-replay-target" tabindex="-1" class="is-sr-only">Skipped</p>`;
   }
 
+  renderVerifyInfo() {
+    if (this.embed !== "replay-with-info") {
+      return "";
+    }
+
+    return html`<rwp-embed-receipt
+            .collInfo=${this.collInfo}
+            url=${this.url}
+            ts=${this.ts}
+            .appLogo=${this.appLogo}
+            >
+            </rwp-embed-receipt>`;
+  }
+
   dragStart() {
     const replay = this.renderRoot.querySelector("wr-coll-replay");
     if (replay) {

package/src/electron-ipfs-handler.js

@@ -0,0 +1,44 @@
+import { protocol, session } from "electron";
+
+import fetchToHandler from "fetch-to-electron-protocol-handler";
+import makeIPFSFetch from "js-ipfs-fetch";
+import * as IPFS from "ipfs-core";
+
+export const PROTOCOL_PERMISSIONS = {
+  standard: true,
+  secure: true,
+  allowServiceWorkers: true,
+  supportFetchAPI: true,
+  bypassCSP: false,
+  corsEnabled: true,
+  stream: true
+};
+
+export function registerPrivileges(privileges = PROTOCOL_PERMISSIONS) {
+  protocol.registerSchemesAsPrivileged([
+    {scheme: "ipfs", privileges},
+    {scheme: "ipns", privileges},
+    {scheme: "ipld", privileges}
+  ]);
+}
+
+export function registerHandler(ipfsRepoPath) {
+  const handler = createHandler(ipfsRepoPath);
+  protocol.registerStreamProtocol("ipfs", handler);
+  protocol.registerStreamProtocol("ipns", handler);
+  protocol.registerStreamProtocol("ipld", handler);
+}
+
+export function createHandler (ipfsRepoPath) {
+  const {handler} = fetchToHandler(async () => {
+    const ipfs = await IPFS.create({
+      repo: ipfsRepoPath
+    });
+
+    const fetch = await makeIPFSFetch({ ipfs });
+
+    return fetch;
+  }, session.defaultSession);
+
+  return handler;
+}

package/src/electron-replay-app.js

@@ -9,7 +9,7 @@ import path from "path";
 import fs from "fs";
 
 import { ArchiveResponse, Rewriter } from "@webrecorder/wabac/src/rewrite";
-import { IPFSClient } from "@webrecorder/wabac/src/ipfs";
+import {create as createIPFS} from "auto-js-ipfs";
 
 import { PassThrough, Readable } from "stream";
 
@@ -27,33 +27,6 @@ const REPLAY_PREFIX = STATIC_PREFIX + "w/";
 
 const URL_RX = /([^/]+)\/([\d]+)(?:\w\w_)?\/(.*)$/;
 
-
-let IPFS = null;
-
-// ============================================================================
-class NativeIPFSClient extends IPFSClient
-{
-  constructor(repoPath) {
-    super();
-    this.repoPath = repoPath;
-  }
-
-  async _doInitIPFS() {
-    if (!IPFS) {
-      IPFS = require("ipfs-core");
-    }
-
-    this.ipfs = await IPFS.create({
-      repo: this.repoPath,
-      init: {emptyRepo: true},
-      //preload: {enabled: false},
-    });
-
-    this.resetGC();
-  }
-}
-
-
 // ============================================================================
 class ElectronReplayApp
 {
@@ -145,11 +118,6 @@ class ElectronReplayApp
       app.setPath("userData", path.join(app.getPath("appData"), this.profileName));
     }
 
-    const ipfsRepoPath = path.join(app.getPath("userData"), "js-ipfs");
-    console.log("ipfs path", ipfsRepoPath);
-
-    this.ipfsClient = new NativeIPFSClient(ipfsRepoPath);
-
     app.on("will-finish-launching", () => {
       app.on("open-file", (event, filePath) => {
         this.openNextFile = filePath;
@@ -185,6 +153,9 @@ class ElectronReplayApp
   }
 
   onAppReady() {
+    const ipfsRepoPath = path.join(app.getPath("userData"), "js-ipfs");
+    console.log("ipfs path", ipfsRepoPath);
+
     this.checkUpdates();
 
     this.screenSize = screen.getPrimaryDisplay().workAreaSize;
@@ -197,6 +168,12 @@ class ElectronReplayApp
           return { action: "deny"};
         }
 
+        // load external URLs in native browser
+        if (!url.startsWith(STATIC_PREFIX)) {
+          shell.openExternal(url);
+          return { action: "deny"};
+        }
+
         return { action: "allow"};
       });
     });
@@ -293,13 +270,16 @@ class ElectronReplayApp
         return;
 
       } else if (ipfsCID) {
+        const ipfsURL = `ipfs://${ipfsCID}}`;
+
         console.log("ipfs serve:", ipfsCID);
 
-        await this.ipfsClient.initIPFS();
+        // TODO: Pass in config?
+        this.ipfsClient = await createIPFS();
 
         console.log("inited");
 
-        let size = await this.ipfsClient.getFileSize(ipfsCID);
+        let size = await this.ipfsClient.getSize(ipfsURL);
 
         console.log("got size", size);
 
@@ -314,7 +294,10 @@ class ElectronReplayApp
         if (request.method === "GET") {
           const offset = start || 0;
           const length = end ? end - start + 1 : size;
-          data = Readable.from(await this.ipfsClient.cat(ipfsCID, {offset, length}));
+          data = Readable.from(await this.ipfsClient.get(ipfsURL, {
+            start: offset,
+            end: offset +length
+          }));
         }
 
         callback({statusCode, headers, data});