Theseus: add-on stores live in memory — no more 16 s "Not Responding" at launch

An add-on's storage.get read and parsed its whole store file on every call,
and storage.set read, parsed and rewrote it — synchronously, on the main
thread. Traced on a real profile (installed 0.3.70): with a 7.5 MB Aegis
store, 30 of the first 35 s of main-thread time went to storage.get, the
window sat in "Not Responding" from 3 s to 19 s, and the first page showed at
19 s. One get cost ~73 ms; Aegis does dozens per state update.

lib/addon-store.cjs keeps one in-memory copy per store, shared by the
add-on's api.storage (addons-host.js) and its pages (addon-storage-* IPC in
main.js). After a one-time load a get costs microseconds; values are copied
in and out (structuredClone), so callers keep the old semantics. Writes are
coalesced (100 ms) and land as temp-file + rename, and are flushed on quit;
a store that doesn't parse is moved aside instead of being replaced by {}.

Measured on copies of the same profile, dev build:
  first page 16.9-17.6 s -> 1.5-1.7 s; main thread blocked 24.7-25.8 s of
  30 -> 1.0-1.1 s; longest freeze 13.7-15.0 s -> 0.6 s.

The Aegis side (capping its unbounded txCache) ships separately through
Aegis's own update channel. The boot tracer gains total/longest block columns.

addons-host.js

@@ -16,6 +16,7 @@
 //   <userData>/extensions-data/<id>.json — per-add-on kv store (api.storage)
 
 const fs = require("node:fs");
+const { storeFor } = require("./lib/addon-store.cjs");
 const path = require("node:path");
 
 // Extension points the framework understands. Extending this list means also
@@ -446,22 +447,18 @@ class AddonHost {
       // than guessing the path from its own folder.
       dataDir: this.dataDir,
       log: (...a) => this.log(`[${manifest.id}]`, ...a),
-      // Persistent per-add-on storage. Small kv JSON on disk.
-      storage: {
-        get: (key, fallback = null) => {
-          try {
-            const raw = JSON.parse(fs.readFileSync(storageFile, "utf8"));
-            return key in raw ? raw[key] : fallback;
-          } catch { return fallback; }
-        },
-        set: (key, value) => {
-          let store = {};
-          try { store = JSON.parse(fs.readFileSync(storageFile, "utf8")); } catch {}
-          store[key] = value;
-          try { fs.writeFileSync(storageFile, JSON.stringify(store)); } catch (e) { this.log(`[${manifest.id}] storage.set failed:`, e?.message); }
-        },
-        all: () => { try { return JSON.parse(fs.readFileSync(storageFile, "utf8")); } catch { return {}; } },
-      },
+      // Persistent per-add-on storage: kv JSON on disk, held in memory by
+      // lib/addon-store.cjs (shared with the add-on's pages via main.js) —
+      // re-reading the whole file on every get froze the browser once a
+      // store grew to megabytes.
+      storage: (() => {
+        const store = storeFor(storageFile, (...a) => this.log(`[${manifest.id}]`, ...a));
+        return {
+          get: (key, fallback = null) => store.get(key, fallback),
+          set: (key, value) => store.set(key, value),
+          all: () => store.all(),
+        };
+      })(),
       // Register a sidebar panel — a right-side WebContentsView that hosts
       // one of the add-on's HTML pages. `page` is a path RELATIVE to the
       // add-on folder. `title` shows in the sidebar tab strip. `icon` is

lib/addon-store.cjs

@@ -0,0 +1,96 @@
+// Per-add-on key/value stores (<userData>/extensions-data/<id>.json), shared
+// by the add-on's own API (addons-host.js) and its pages (main.js IPC).
+//
+// The store used to be read and parsed from disk on EVERY get, and read +
+// parsed + rewritten on every set — synchronously, on the Electron main
+// thread. With a 7.5 MB Aegis store that was ~60 ms per get, and Aegis does
+// many per state update: measured 2026-10-03 on a real profile, 30 s of the
+// first 35 s of the main thread went to storage.get, and the window sat in
+// "Not Responding" for 16 s after launch.
+//
+// Now each store is read once and kept in memory. A get costs only the value
+// it returns; values are copied in and out (structuredClone), so a caller
+// that mutates what it got — or what it passed to set — never changes the
+// store behind its back, exactly as with the old parse-per-call. Writes are
+// coalesced (WRITE_DELAY_MS) and land as an atomic temp-file + rename, so a
+// crash mid-write can't leave a truncated file; flushAll() runs on quit.
+// A store file that exists but doesn't parse is moved aside before the
+// first write instead of being silently replaced by {} (which lost every key).
+"use strict";
+const fs = require("fs");
+const path = require("path");
+
+const WRITE_DELAY_MS = 100;
+const stores = new Map(); // file -> Store
+
+class Store {
+  constructor(file, log) {
+    this.file = file;
+    this.log = log || (() => {});
+    this.data = null;    // loaded lazily
+    this.timer = null;
+    this.dirty = false;
+  }
+  _load() {
+    if (this.data) return this.data;
+    let text = null;
+    try { text = fs.readFileSync(this.file, "utf8"); } catch (e) { if (e.code !== "ENOENT") this.log(`store read failed (${path.basename(this.file)}):`, e.message); }
+    if (text == null) { this.data = {}; return this.data; }
+    try {
+      const parsed = JSON.parse(text);
+      this.data = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
+    } catch (e) {
+      // Keep the unreadable file for recovery rather than overwriting it.
+      const aside = `${this.file}.corrupt-${Date.now()}`;
+      try { fs.renameSync(this.file, aside); } catch {}
+      this.log(`store ${path.basename(this.file)} did not parse (${e.message}); kept as ${path.basename(aside)}`);
+      this.data = {};
+    }
+    return this.data;
+  }
+  get(key, fallback = null) {
+    const d = this._load();
+    return key in d ? structuredClone(d[key]) : fallback;
+  }
+  set(key, value) {
+    const d = this._load();
+    // JSON drops undefined: the old write-then-reparse made such a key vanish,
+    // so a later get returned the fallback. Keep that.
+    if (value === undefined) delete d[key]; else d[key] = structuredClone(value);
+    this._schedule();
+  }
+  all() { return structuredClone(this._load()); }
+  _schedule() {
+    this.dirty = true;
+    if (this.timer) return;
+    this.timer = setTimeout(() => { this.timer = null; this.flush(); }, WRITE_DELAY_MS);
+  }
+  flush() {
+    if (this.timer) { clearTimeout(this.timer); this.timer = null; }
+    if (!this.dirty || !this.data) return true;
+    const tmp = `${this.file}.tmp`;
+    try {
+      fs.mkdirSync(path.dirname(this.file), { recursive: true });
+      fs.writeFileSync(tmp, JSON.stringify(this.data));
+      fs.renameSync(tmp, this.file);
+      this.dirty = false;
+      return true;
+    } catch (e) {
+      this.log(`store write failed (${path.basename(this.file)}):`, e.message);
+      try { fs.rmSync(tmp, { force: true }); } catch {}
+      return false;
+    }
+  }
+}
+
+// The one Store for a file — every reader and writer of an add-on's store
+// must go through this, or the in-memory copy and the file drift apart.
+function storeFor(file, log) {
+  const key = path.resolve(file).toLowerCase();
+  let s = stores.get(key);
+  if (!s) { s = new Store(path.resolve(file), log); stores.set(key, s); }
+  return s;
+}
+function flushAll() { for (const s of stores.values()) s.flush(); }
+
+module.exports = { storeFor, flushAll, WRITE_DELAY_MS };

main.js

@@ -2122,6 +2122,7 @@ let addonHost = null;
 // call so the current credentials always match the current proxy.
 let proxyAuth = null;
 const { AddonHost } = require("./addons-host.js");
+const { storeFor, flushAll: flushAddonStores } = require("./lib/addon-store.cjs");
 const addonUpdater = require("./addon-updater.js");
 const { PUBKEYS_HEX: ADDON_UPDATE_PUBKEYS } = require("./addon-update-pubkeys.js");
 // Extensions live under <userData>\extensions (installed copies), with their
@@ -5674,30 +5675,24 @@ ipcMain.handle("approval-pick", (e, reqId, action, checked, extra) => {
 // Add-on HTML pages get storage.get/set/all via sidebar-preload.js. Main
 // derives the add-on id from the sender's file:// URL so a page can only
 // touch its own store; any file:// outside addons/ returns nothing.
+// Same in-memory store as the add-on's own api.storage (lib/addon-store.cjs).
+const addonStore = (id) => storeFor(path.join(addonsDataDir(), id + ".json"), (...a) => console.warn(`[addons] [${id}]`, ...a));
 ipcMain.handle("addon-storage-get", (e, key, fallback) => {
   const id = addonIdForSender(e.sender);
   if (!id) return fallback ?? null;
-  try {
-    const raw = JSON.parse(fs.readFileSync(path.join(addonsDataDir(), id + ".json"), "utf8"));
-    return key in raw ? raw[key] : (fallback ?? null);
-  } catch { return fallback ?? null; }
+  return addonStore(id).get(key, fallback ?? null);
 });
 ipcMain.handle("addon-storage-set", (e, key, value) => {
   const id = addonIdForSender(e.sender);
   if (!id) return false;
   if (typeof key !== "string" || key.length > 128) return false;
-  const file = path.join(addonsDataDir(), id + ".json");
-  let store = {};
-  try { store = JSON.parse(fs.readFileSync(file, "utf8")); } catch {}
-  store[key] = value;
-  try { fs.mkdirSync(addonsDataDir(), { recursive: true }); fs.writeFileSync(file, JSON.stringify(store)); return true; }
-  catch (err) { console.warn(`[addons] storage.set failed for ${id}:`, err?.message); return false; }
+  addonStore(id).set(key, value);
+  return true;
 });
 ipcMain.handle("addon-storage-all", (e) => {
   const id = addonIdForSender(e.sender);
   if (!id) return {};
-  try { return JSON.parse(fs.readFileSync(path.join(addonsDataDir(), id + ".json"), "utf8")); }
-  catch { return {}; }
+  return addonStore(id).all();
 });
 // Error-page actions. All origin-gated to error.html so a third-party page
 // that happens to see the API shape (home-preload exposes it on every tab)
@@ -7709,6 +7704,7 @@ if (!process.env.THESEUS_NO_AUTOSTART && !app.requestSingleInstanceLock()) {
     e.preventDefault();
     if (quitClearing) return;
     quitClearing = true;
+    flushAddonStores(); // add-on stores write coalesced; land them now
     // Auto-clear per user settings. saveSession() runs first so restoreSession
     // still works UNLESS the user asked to drop history — in which case we
     // wipe the session file too so the next launch is genuinely blank.

scripts/boot-trace/run.mjs

@@ -55,6 +55,8 @@ function summarise(file) {
     toolbar: at(/loaded #\d+ file:chrome\.html/),
     firstPage: at(/loaded #\d+ file:home\.html/),
     blocks6s: ev.filter((e) => e.kind === "block" && e.endedAt < 6000).reduce((a, e) => a + e.ms, 0),
+    blockedAll: ev.filter((e) => e.kind === "block").reduce((a, e) => a + e.ms, 0),
+    longestBlock: ev.filter((e) => e.kind === "block").reduce((a, e) => Math.max(a, e.ms), 0),
     addons: ev.filter((e) => e.kind === "addon").reduce((a, e) => a + e.ms, 0),
     overlaysLoaded: ev.filter((e) => e.kind === "wc" && /loaded #\d+ file:(popover|engine-picker|downloads|address-picker|pw-fill|link-status|approval|js-dialog)\.html/.test(e.label)).length,
     fetches6s: ev.filter((e) => e.kind === "fetch" && e.start < 6000).length,
@@ -72,7 +74,7 @@ for (let i = 1; i <= RUNS; i++) {
   const r = summarise(file); rows.push(r);
   console.log(r.failed.length ? `INVALID — ${r.failed.length} page load(s) failed: ${r.failed[0]}` : "done");
 }
-const cols = [["ready", "ready"], ["toolbar", "toolbar"], ["firstPage", "1st page"], ["blocks6s", "blocked<6s"], ["addons", "addons"], ["overlaysLoaded", "overlays"], ["fetches6s", "fetch<6s"], ["proc15", "procs@15s"], ["mem15", "MB@15s"]];
+const cols = [["ready", "ready"], ["toolbar", "toolbar"], ["firstPage", "1st page"], ["blocks6s", "blocked<6s"], ["blockedAll", "blocked all"], ["longestBlock", "longest"], ["addons", "addons"], ["overlaysLoaded", "overlays"], ["fetches6s", "fetch<6s"], ["proc15", "procs@15s"], ["mem15", "MB@15s"]];
 console.log("\n" + ["run", ...cols.map((c) => c[1])].map((h) => h.padStart(11)).join(""));
 rows.forEach((r, i) => console.log((r.failed.length ? "!" : " ") + [String(i + 1), ...cols.map(([k]) => String(r[k] ?? "—"))].map((v) => v.padStart(11)).join("")));
 console.log(`\ntimes in ms since process start; MB = private memory of all Theseus processes. Traces: ${traceDir}`);