feat(theseus): search-engine icons ship in the build; custom engines cache theirs on disk

Every engine icon was an <img> pointing at Google's favicon service, fetched
again each time the picker, the toolbar or Settings rendered. Offline the
whole list collapsed to the emoji fallbacks, and each open told Google
which engines the user has configured. The catalog's icons now live in
engine-icons/<id>.png inside the app; a custom engine's icon is fetched
once (its own /favicon.ico first, the favicon service as fallback), cached
under the profile, and removed with the engine. Settings no longer falls
through to DuckDuckGo's icon service either. Phind ships no icon: its site
serves none through the bot wall.

main.js

@@ -4,7 +4,7 @@
 // h, Sia s3, direct ip, redirect u). Tabs, nav controls, a search box, a home
 // page, and optional Tor onion routing. No system daemon; the app is the trust
 // boundary.
-const { app, BrowserWindow, WebContentsView, ipcMain, protocol, session, Menu, clipboard, nativeTheme, shell, dialog } = require("electron");
+const { app, BrowserWindow, WebContentsView, ipcMain, protocol, session, Menu, clipboard, nativeTheme, shell, dialog, net } = require("electron");
 const path = require("path");
 const url = require("url");
 const http = require("http");
@@ -162,16 +162,86 @@ const SEARCH_ENGINES = {
 // Engines enabled by default (shown in the toolbar dropdown). The rest are in the
 // catalog and can be turned on from Settings. Custom + detected engines are always on.
 const DEFAULT_ENABLED = ["startpage", "duckduckgo", "google", "brave", "bing"];
-// DuckDuckGo's icon service reliably returns a favicon for ANY domain from one
-// privacy-respecting host — far more robust than guessing /favicon.ico per site.
-// Search-engine favicon source. DuckDuckGo's icons.duckduckgo.com/ip3/…
-// service was returning inconsistent results (Brave, Bing, Yandex etc.
-// came back 404 → the settings row fell through to an emoji). Google's
-// /s2/favicons service is materially more reliable, returns a real 32×32
-// PNG for essentially every host, and doesn't require login. Kept as a
-// single point so the fallback source can be swapped again in one place.
-const faviconUrl = (domain) => (domain ? `https://www.google.com/s2/favicons?domain=${domain}&sz=32` : null);
-function customFavicon(url) { try { return faviconUrl(new URL(String(url).replace("%s", "x")).hostname); } catch { return null; } }
+// Search-engine icons. The catalog's icons ship inside the build
+// (engine-icons/<id>.png) so the toolbar, the picker and Settings look the
+// same offline as online, and opening the picker no longer tells a favicon
+// service which engines the user has configured — until 0.3.58 every icon
+// was an <img> pointing at Google's /s2/favicons, re-fetched on each open
+// and falling through to the emoji whenever the network was away. A custom
+// engine's icon is fetched once (the site's own /favicon.ico first, Google's
+// service as the fallback) and cached under the profile; it shows the emoji
+// until that lands. A catalog engine without a bundled file (Phind: its
+// site is behind a bot wall and serves no icon) keeps the emoji too.
+const ENGINE_ICON_DIR = path.join(__dirname, "engine-icons");
+const bundledEngineIcons = new Set((() => { try { return fs.readdirSync(ENGINE_ICON_DIR); } catch { return []; } })());
+const bundledIcon = (id) => (bundledEngineIcons.has(id + ".png") ? url.pathToFileURL(path.join(ENGINE_ICON_DIR, id + ".png")).href : null);
+const customIconDir = () => path.join(app.getPath("userData"), "engine-icons");
+const ICON_EXTS = ["png", "ico", "jpg", "gif", "webp", "svg"];
+function engineHost(u) { try { return new URL(String(u).replace("%s", "x")).hostname.toLowerCase(); } catch { return ""; } }
+function cachedIconFile(host) {
+  if (!host) return null;
+  for (const ext of ICON_EXTS) { const f = path.join(customIconDir(), `${host}.${ext}`); if (fs.existsSync(f)) return f; }
+  return null;
+}
+function customFavicon(u) {
+  const host = engineHost(u);
+  const f = cachedIconFile(host);
+  if (f) return url.pathToFileURL(f).href;
+  if (host) fetchEngineIcon(host).catch(() => {});
+  return null;
+}
+// Drop a removed custom engine's cached icon unless another custom engine on
+// the same host still uses it.
+function dropCustomIcon(id) {
+  const eng = (settings.customEngines || []).find((e) => e.id === id);
+  const host = eng ? engineHost(eng.url) : "";
+  if (!host || (settings.customEngines || []).some((e) => e.id !== id && engineHost(e.url) === host)) return;
+  const f = cachedIconFile(host);
+  if (f) try { fs.unlinkSync(f); } catch {}
+}
+const iconFetches = new Map(); // host → in-flight Promise, or the time the last attempt failed
+const ICON_RETRY_MS = 60 * 60 * 1000;
+function sniffImage(buf) {
+  if (buf.length < 8) return null;
+  if (buf.slice(1, 4).toString() === "PNG") return "png";
+  if (buf.readUInt32BE(0) === 0x00000100) return "ico";
+  if (buf[0] === 0xff && buf[1] === 0xd8) return "jpg";
+  if (buf.slice(0, 4).toString() === "GIF8") return "gif";
+  if (buf.slice(0, 4).toString() === "RIFF" && buf.slice(8, 12).toString() === "WEBP") return "webp";
+  if (/^\s*(<\?xml[^>]*>\s*)?(<!--[\s\S]*?-->\s*)*<svg[\s>]/i.test(buf.slice(0, 512).toString("utf8"))) return "svg";
+  return null;
+}
+async function fetchIconBytes(u) {
+  const r = await net.fetch(u, { redirect: "follow", signal: AbortSignal.timeout(8000), headers: { accept: "image/*,*/*;q=0.5" } });
+  if (!r.ok) throw new Error(`http ${r.status}`);
+  const buf = Buffer.from(await r.arrayBuffer());
+  if (buf.length < 64 || buf.length > 512 * 1024) throw new Error(`size ${buf.length}`);
+  const ext = sniffImage(buf);
+  if (!ext) throw new Error("not an image");
+  return { buf, ext };
+}
+function fetchEngineIcon(host) {
+  if (!app.isReady()) return Promise.resolve(null); // net.fetch needs the app; the startup pass retries
+  const cur = iconFetches.get(host);
+  if (cur && typeof cur.then === "function") return cur;
+  if (typeof cur === "number" && Date.now() - cur < ICON_RETRY_MS) return Promise.resolve(null);
+  const p = (async () => {
+    let got = null;
+    for (const src of [`https://${host}/favicon.ico`, `https://www.google.com/s2/favicons?domain=${host}&sz=64`]) {
+      try { got = await fetchIconBytes(src); break; } catch {}
+    }
+    if (!got) { iconFetches.set(host, Date.now()); return null; }
+    const dir = customIconDir();
+    fs.mkdirSync(dir, { recursive: true });
+    const f = path.join(dir, `${host}.${got.ext}`);
+    fs.writeFileSync(f + ".tmp", got.buf); fs.renameSync(f + ".tmp", f);
+    iconFetches.delete(host);
+    emitEngines(); // the toolbar and the picker repaint with the real icon
+    return f;
+  })();
+  iconFetches.set(host, p);
+  return p;
+}
 function isEnabled(id) { return (settings.enabledEngines || DEFAULT_ENABLED).includes(id); }
 // Two-tier state: an engine is INSTALLED if it's in the user's Additional
 // list (visible in Settings), and ENABLED if it's currently toggled on
@@ -183,7 +253,7 @@ function isInstalled(id) {
 }
 function allEngines() {
   const list = Object.entries(SEARCH_ENGINES).map(([id, e]) =>
-    ({ id, name: e.name, sym: e.sym, favicon: faviconUrl(e.fav), kind: e.kind || "search", tier: e.tier || "catalog", builtin: true, installed: isInstalled(id), enabled: isEnabled(id) }));
+    ({ id, name: e.name, sym: e.sym, favicon: bundledIcon(id), kind: e.kind || "search", tier: e.tier || "catalog", builtin: true, installed: isInstalled(id), enabled: isEnabled(id) }));
   for (const c of settings.customEngines || [])
     list.push({ id: c.id, name: c.name, sym: c.sym || "🔍", favicon: customFavicon(c.url), kind: c.kind || "search", tier: "custom", builtin: false, installed: true, enabled: isEnabled(c.id) });
   // Apply the user's custom order; ids not in engineOrder keep their natural order (stable sort).
@@ -5626,7 +5696,7 @@ ipcMain.handle("set-search-engine", (_e, id) => {
   if (allEngines().some((e) => e.id === id)) { settings.searchEngine = id; saveSettings(); emitEngines(); }
   return settings.searchEngine;
 });
-ipcMain.handle("add-engine", (_e, eng) => {
+ipcMain.handle("add-engine", async (_e, eng) => {
   // A custom engine needs a name and a URL template containing "%s". Adding
   // installs it in both the settings list AND the toolbar dropdown.
   if (eng && eng.name && eng.url && String(eng.url).includes("%s")) {
@@ -5637,12 +5707,18 @@ ipcMain.handle("add-engine", (_e, eng) => {
     settings.enabledEngines = [...new Set([...(settings.enabledEngines || DEFAULT_ENABLED), id])];
     settings.searchEngine = id; // select the one just added
     saveSettings(); emitEngines();
+    // Settings pulls the list once on return, so give the icon fetch a moment
+    // to land; offline or slow, the row shows the emoji and the toolbar
+    // repaints on its own when the icon arrives.
+    const host = engineHost(eng.url);
+    if (host && !cachedIconFile(host)) await Promise.race([fetchEngineIcon(host).catch(() => null), new Promise((r) => setTimeout(r, 4000))]);
   }
   return { engines: allEngines(), current: settings.searchEngine };
 });
 ipcMain.handle("remove-engine", (_e, id) => {
   // Drop a custom engine entirely — from customEngines and any list that
   // referenced it.
+  dropCustomIcon(id);
   settings.customEngines = (settings.customEngines || []).filter((e) => e.id !== id);
   settings.enabledEngines = (settings.enabledEngines || DEFAULT_ENABLED).filter((x) => x !== id);
   if (settings.searchEngine === id) settings.searchEngine = enabledEnginesList()[0]?.id || "duckduckgo";
@@ -5654,6 +5730,7 @@ ipcMain.handle("remove-engine", (_e, id) => {
 // aliases to remove-engine (they don't live in installedEngines).
 ipcMain.handle("remove-from-list", (_e, id) => {
   if ((settings.customEngines || []).some((e) => e.id === id)) {
+    dropCustomIcon(id);
     settings.customEngines = (settings.customEngines || []).filter((e) => e.id !== id);
   } else if (SEARCH_ENGINES[id]) {
     settings.installedEngines = (settings.installedEngines || DEFAULT_ENABLED).filter((x) => x !== id);
@@ -6532,6 +6609,9 @@ if (!process.env.THESEUS_NO_AUTOSTART && !app.requestSingleInstanceLock()) {
     installRequestFilter();
     migrateExtensionDirs();
     initAddons();
+    // Custom engines saved before icons were cached (or whose fetch never
+    // landed) get theirs once the startup burst is over.
+    setTimeout(() => { for (const c of settings.customEngines || []) customFavicon(c.url); }, 15000);
     webapps.init({
       parentWindow: () => (win && !win.isDestroyed() ? win : undefined),
       ask: askSheet,

package.json

@@ -87,6 +87,7 @@
       "webapps.js",
       "js-dialog.html",
       "js-dialog-preload.js",
+      "engine-icons/**/*",
       "lib/**/*",
       "package.json",
       "node_modules/**/*",

settings.html

@@ -878,22 +878,14 @@
     // search engines: default picker + custom-engine list + add form
     const sel = document.getElementById("searchEngine");
     const esc = (s) => String(s || "").replace(/</g, "&lt;");
-    // Try the primary favicon URL (Google's /s2/favicons — set in main.js);
-    // fall back to DuckDuckGo's icons.duckduckgo.com service if that 404s,
-    // then to the emoji sym as final fallback so the row is never blank.
+    // e.favicon is a file:// URL (a bundled catalog icon, or a custom engine's
+    // icon cached under the profile — see main.js); null means "not cached
+    // yet" or "no icon known", and the emoji stands in. No network fetch
+    // happens from here.
     const engIcon = (e) => {
-      let alt = "";
-      try {
-        const u = new URL(e.favicon || "https://x");
-        const domain = u.searchParams.get("domain") || u.hostname;
-        if (domain && domain !== "x") alt = `https://icons.duckduckgo.com/ip3/${domain}.ico`;
-      } catch {}
       const sym = (e.sym || "🔍").replace(/'/g, "&#39;");
-      const fallbackChain = alt
-        ? `this.onerror=null;this.src='${alt}';this.setAttribute('data-tried','1');this.onerror=function(){this.replaceWith(Object.assign(document.createElement('span'),{className:'es',textContent:'${sym}'}))};`
-        : `this.replaceWith(Object.assign(document.createElement('span'),{className:'es',textContent:'${sym}'}))`;
       return e.favicon
-        ? `<img class="ei" src="${esc(e.favicon)}" onerror="${fallbackChain}">`
+        ? `<img class="ei" src="${esc(e.favicon)}" onerror="this.replaceWith(Object.assign(document.createElement('span'),{className:'es',textContent:'${sym}'}))">`
         : `<span class="es">${sym}</span>`;
     };
     const ENGINE_KINDS = [