57df93fa97
When s3d still lists a file as waiting but its buffered copy is gone, Upload now fails with a raw admin error and nothing says which files are behind it. Overview and Drives now list them (waiting, copy missing in two checks a moment apart, no upload running, older than two minutes), each with Upload again… (replace it from the original) or Remove, and Upload now explains the cause instead of passing s3d's error through.
41 lines
2.3 KiB
JavaScript
41 lines
2.3 KiB
JavaScript
// Files s3d still lists as waiting for Sia whose buffered copy is gone.
|
|
//
|
|
// s3d keeps every not-yet-uploaded object as uploads/<filename> and marks it
|
|
// uploaded by setting sia_object_id. An s3d stopped in the middle of an
|
|
// upload has been seen to delete the copies without recording the upload;
|
|
// every flush after that fails on them ("failed to open upload"). They hold
|
|
// no data any more: not on this computer, never on Sia. Pithos lists them so
|
|
// the user can upload the file again or remove the entry.
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
async function candidates(dataDir, olderThanMs) {
|
|
const file = path.join(dataDir, 's3d.db');
|
|
if (!fs.existsSync(file)) return [];
|
|
const { DatabaseSync } = await import('node:sqlite');
|
|
const db = new DatabaseSync(file, { readOnly: true });
|
|
try {
|
|
const rows = db.prepare(`SELECT b.name AS bucket, u.name AS user, o.name AS key, o.size AS size, o.filename AS filename, o.updated_at AS updated
|
|
FROM objects o JOIN buckets b ON b.id = o.bucket_id LEFT JOIN users u ON u.id = b.user_id
|
|
WHERE o.sia_object_id IS NULL AND o.is_latest = 1 AND o.is_delete_marker = 0
|
|
AND o.filename IS NOT NULL AND o.filename != ''`).all();
|
|
const uploads = path.join(dataDir, 'uploads');
|
|
const cutoff = Date.now() - olderThanMs;
|
|
return rows
|
|
// updated_at is Unix seconds
|
|
.filter((r) => { const n = Number(r.updated); const t = Number.isFinite(n) ? (n < 1e12 ? n * 1000 : n) : Date.parse(r.updated); return !Number.isFinite(t) || t < cutoff; })
|
|
.filter((r) => !fs.existsSync(path.join(uploads, path.basename(String(r.filename)))))
|
|
.map((r) => ({ bucket: r.bucket, user: r.user, key: r.key, size: Number(r.size), filename: String(r.filename) }));
|
|
} finally { db.close(); }
|
|
}
|
|
|
|
// Only entries missing twice, a moment apart: s3d removes a copy just before
|
|
// it records the upload, and a list taken in between would be wrong.
|
|
export async function stuckUploads(dataDir, { olderThanMs = 120_000, settleMs = 2000 } = {}) {
|
|
const first = await candidates(dataDir, olderThanMs);
|
|
if (!first.length) return [];
|
|
await new Promise((r) => setTimeout(r, settleMs));
|
|
const again = new Set((await candidates(dataDir, olderThanMs)).map((r) => `${r.bucket}\n${r.key}`));
|
|
return first.filter((r) => again.has(`${r.bucket}\n${r.key}`));
|
|
}
|