Pithos: list waiting files that lost their copy, with Upload again and Remove

When s3d still lists a file as waiting but its buffered copy is gone,
Upload now fails with a raw admin error and nothing says which files are
behind it. Overview and Drives now list them (waiting, copy missing in
two checks a moment apart, no upload running, older than two minutes),
each with Upload again… (replace it from the original) or Remove, and
Upload now explains the cause instead of passing s3d's error through.

core/stuck.js

@@ -0,0 +1,41 @@
+// Files s3d still lists as waiting for Sia whose buffered copy is gone.
+//
+// s3d keeps every not-yet-uploaded object as uploads/<filename> and marks it
+// uploaded by setting sia_object_id. An s3d stopped in the middle of an
+// upload has been seen to delete the copies without recording the upload;
+// every flush after that fails on them ("failed to open upload"). They hold
+// no data any more: not on this computer, never on Sia. Pithos lists them so
+// the user can upload the file again or remove the entry.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+async function candidates(dataDir, olderThanMs) {
+  const file = path.join(dataDir, 's3d.db');
+  if (!fs.existsSync(file)) return [];
+  const { DatabaseSync } = await import('node:sqlite');
+  const db = new DatabaseSync(file, { readOnly: true });
+  try {
+    const rows = db.prepare(`SELECT b.name AS bucket, u.name AS user, o.name AS key, o.size AS size, o.filename AS filename, o.updated_at AS updated
+      FROM objects o JOIN buckets b ON b.id = o.bucket_id LEFT JOIN users u ON u.id = b.user_id
+      WHERE o.sia_object_id IS NULL AND o.is_latest = 1 AND o.is_delete_marker = 0
+        AND o.filename IS NOT NULL AND o.filename != ''`).all();
+    const uploads = path.join(dataDir, 'uploads');
+    const cutoff = Date.now() - olderThanMs;
+    return rows
+      // updated_at is Unix seconds
+      .filter((r) => { const n = Number(r.updated); const t = Number.isFinite(n) ? (n < 1e12 ? n * 1000 : n) : Date.parse(r.updated); return !Number.isFinite(t) || t < cutoff; })
+      .filter((r) => !fs.existsSync(path.join(uploads, path.basename(String(r.filename)))))
+      .map((r) => ({ bucket: r.bucket, user: r.user, key: r.key, size: Number(r.size), filename: String(r.filename) }));
+  } finally { db.close(); }
+}
+
+// Only entries missing twice, a moment apart: s3d removes a copy just before
+// it records the upload, and a list taken in between would be wrong.
+export async function stuckUploads(dataDir, { olderThanMs = 120_000, settleMs = 2000 } = {}) {
+  const first = await candidates(dataDir, olderThanMs);
+  if (!first.length) return [];
+  await new Promise((r) => setTimeout(r, settleMs));
+  const again = new Set((await candidates(dataDir, olderThanMs)).map((r) => `${r.bucket}\n${r.key}`));
+  return first.filter((r) => again.has(`${r.bucket}\n${r.key}`));
+}

test/stuck.test.js

@@ -0,0 +1,35 @@
+// Waiting files whose buffered copy is gone are listed; uploaded files,
+// files with their copy, and very recent changes are not.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { stuckUploads } from '../core/stuck.js';
+
+test('stuck uploads: missing copy, still waiting, not brand new', async (t) => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pithos-stuck-'));
+  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+  fs.mkdirSync(path.join(dir, 'uploads'));
+  const db = new DatabaseSync(path.join(dir, 's3d.db'));
+  db.exec(`CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);
+    CREATE TABLE buckets (id INTEGER PRIMARY KEY, name TEXT, user_id INTEGER);
+    CREATE TABLE objects (bucket_id INTEGER, name TEXT, is_latest INTEGER, is_delete_marker INTEGER, size INTEGER, updated_at INTEGER, filename TEXT, sia_object_id BLOB);
+    INSERT INTO users VALUES (1, 'game.x');
+    INSERT INTO buckets VALUES (1, 'house-music', 1);`);
+  const old = Math.floor(Date.now() / 1000) - 3600;
+  const now = Math.floor(Date.now() / 1000);
+  const add = db.prepare('INSERT INTO objects VALUES (1, ?, 1, 0, ?, ?, ?, ?)');
+  add.run('lost.mp3', 5000, old, 'aaaa.obj', null);          // stuck
+  add.run('waiting.mp3', 4000, old, 'bbbb.obj', null);       // copy present
+  add.run('uploaded.mp3', 3000, old, null, Buffer.from([1])); // on Sia
+  add.run('just-now.mp3', 2000, now, 'cccc.obj', null);      // too recent to judge
+  db.prepare('INSERT INTO objects VALUES (1, ?, 0, 0, ?, ?, ?, NULL)').run('old-version.mp3', 1, old, 'dddd.obj'); // not latest
+  db.close();
+  fs.writeFileSync(path.join(dir, 'uploads', 'bbbb.obj'), 'x');
+
+  const items = await stuckUploads(dir, { settleMs: 10 });
+  assert.deepEqual(items, [{ bucket: 'house-music', user: 'game.x', key: 'lost.mp3', size: 5000, filename: 'aaaa.obj' }]);
+  assert.deepEqual(await stuckUploads(path.join(dir, 'nope'), { settleMs: 10 }), []);
+});

ui/app.css

@@ -475,3 +475,9 @@ a.brand:hover > span:first-of-type { color: var(--accent); }
 .link-card .row { gap: 6px; flex-wrap: wrap; }
 .link-make { gap: 10px; align-items: center; flex-wrap: wrap; }
 .link-card a.btn { text-decoration: none; }
+
+/* Waiting files that lost their copy (stuck uploads) */
+.banner.stuck .stack { gap: 8px; width: 100%; }
+.stuck-list { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 6px; }
+.stuck-list li { display: flex; gap: 8px; align-items: center; justify-content: space-between; flex-wrap: wrap; }
+.stuck-name { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex: 1 1 220px; }

ui/app.js

@@ -294,7 +294,7 @@ function overview(main) {
   main.append(
     h('div', { class: 'page-head' }, h('div', {}, h('h1', {}, 'Overview'), h('p', { class: 'muted' }, 'Your s3d gateway and its upload pipeline to Sia.'))),
     banners,
-    pendingBanner(),
+    stuckBanner(), pendingBanner(),
     h('div', { class: 'grid two' }, daemonCard, statsCard),
   );
 
@@ -1424,6 +1424,60 @@ function siaAccountBlock(a, onRefresh) {
 // "N files are waiting to upload to Sia" with Upload now. s3d holds small
 // uploads until about 40 MB has built up; until then they exist only on this
 // computer, which is easy to miss.
+// Files s3d still lists as waiting whose copy on this computer is gone (an
+// upload was cut off): each can be uploaded again from the original, or removed.
+function stuckBanner() {
+  const el = h('div', { hidden: true });
+  async function paint() {
+    let items = [];
+    try { items = (await api('GET', '/api/uploads/stuck')).items || []; } catch {}
+    if (!items.length) { el.hidden = true; return; }
+    const remove = async (i) => {
+      try { await api('POST', '/api/uploads/stuck/remove', { bucket: i.bucket, key: i.key }); } catch (e) { fail(e); }
+    };
+    const again = (i) => {
+      const input = h('input', { type: 'file', hidden: true, onchange: async () => {
+        const file = input.files[0];
+        if (!file) return;
+        const path = `/api/s3/${encodeURIComponent(i.user)}/buckets/${encodeURIComponent(i.bucket)}/object?key=${encodeURIComponent(i.key)}`;
+        try {
+          if (bridge) {
+            const r = await bridge.invoke('upload', { path, bytes: await file.arrayBuffer(), type: file.type });
+            if (r.status >= 300) throw new Error(r.data?.error || `HTTP ${r.status}`);
+          } else {
+            const r = await fetch(path, { method: 'PUT', headers: { 'x-pithos': '1', ...(file.type ? { 'content-type': file.type } : {}) }, body: file });
+            if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || `HTTP ${r.status}`);
+          }
+          toast(`Uploaded ${i.key.split('/').pop()} again`);
+          paint();
+        } catch (e) { fail(e); }
+      } });
+      document.body.append(input);
+      input.click();
+      setTimeout(() => input.remove(), 60_000);
+    };
+    put(el, h('div', { class: 'banner warn stuck' },
+      h('div', { class: 'stack' },
+        h('span', {}, h('strong', {}, `${items.length} file${items.length === 1 ? '' : 's'} lost ${items.length === 1 ? 'its' : 'their'} copy before reaching Sia. `),
+          h('span', { class: 'small' }, items.length === 1
+            ? 'An upload was cut off, and it was removed from this computer before it reached Sia. Upload it again from the original, or remove it; until then Upload now stops on it.'
+            : 'An upload was cut off, and they were removed from this computer before they reached Sia. Upload each one again from the original, or remove it; until then Upload now stops on them.')),
+        h('ul', { class: 'stuck-list' }, items.map((i) => h('li', {},
+          h('span', { class: 'stuck-name', title: `${i.bucket}/${i.key}` }, h('span', { class: 'muted' }, `${i.bucket} / `), i.key, h('span', { class: 'muted small' }, ` · ${fmtBytes(i.size)}`)),
+          h('span', { class: 'row' },
+            h('button', { class: 'btn small', onclick: () => again(i) }, 'Upload again…'),
+            h('button', { class: 'btn small danger', onclick: async () => { await remove(i); paint(); } }, 'Remove'))))),
+        items.length > 1 && h('div', {}, h('button', { class: 'btn small danger', onclick: async () => {
+          if (!(await confirmModal(`Remove ${items.length} entries?`, 'Their files are already gone from this computer and never reached Sia; this only removes the names.', 'Remove all'))) return;
+          for (const i of items) await remove(i);
+          paint();
+        } }, 'Remove all')))));
+    el.hidden = false;
+  }
+  paint();
+  return el;
+}
+
 function pendingBanner() {
   const el = h('div', { hidden: true });
   const st = daemonState();
@@ -1494,7 +1548,7 @@ async function bucketList(head, body, user, userSel) {
     h('div', { class: 'row' }, userPicker(userSel), h('button', { class: 'btn primary', onclick: create }, 'New drive')));
   const card = h('div', { class: 'card' }, h('p', { class: 'muted' }, 'Loading…'));
   const bar = h('div', { class: 'view-bar' });
-  put(body, pendingBanner(), bar, card);
+  put(body, stuckBanner(), pendingBanner(), bar, card);
   const base = `/api/s3/${encodeURIComponent(user)}/buckets`;
   let mode = viewMode('drives', 'icons');
   let list = [];