42 lines
2.3 KiB
JavaScript
42 lines
2.3 KiB
JavaScript
|
// Files s3d still lists as waiting for Sia whose buffered copy is gone.
|
||
|
|
//
|
||
|
|
// s3d keeps every not-yet-uploaded object as uploads/<filename> and marks it
|
||
|
|
// uploaded by setting sia_object_id. An s3d stopped in the middle of an
|
||
|
|
// upload has been seen to delete the copies without recording the upload;
|
||
|
|
// every flush after that fails on them ("failed to open upload"). They hold
|
||
|
|
// no data any more: not on this computer, never on Sia. Pithos lists them so
|
||
|
|
// the user can upload the file again or remove the entry.
|
||
|
|
|
||
|
|
import fs from 'node:fs';
|
||
|
|
import path from 'node:path';
|
||
|
|
|
||
|
|
async function candidates(dataDir, olderThanMs) {
|
||
|
|
const file = path.join(dataDir, 's3d.db');
|
||
|
|
if (!fs.existsSync(file)) return [];
|
||
|
|
const { DatabaseSync } = await import('node:sqlite');
|
||
|
|
const db = new DatabaseSync(file, { readOnly: true });
|
||
|
|
try {
|
||
|
|
const rows = db.prepare(`SELECT b.name AS bucket, u.name AS user, o.name AS key, o.size AS size, o.filename AS filename, o.updated_at AS updated
|
||
|
|
FROM objects o JOIN buckets b ON b.id = o.bucket_id LEFT JOIN users u ON u.id = b.user_id
|
||
|
|
WHERE o.sia_object_id IS NULL AND o.is_latest = 1 AND o.is_delete_marker = 0
|
||
|
|
AND o.filename IS NOT NULL AND o.filename != ''`).all();
|
||
|
|
const uploads = path.join(dataDir, 'uploads');
|
||
|
|
const cutoff = Date.now() - olderThanMs;
|
||
|
|
return rows
|
||
|
|
// updated_at is Unix seconds
|
||
|
|
.filter((r) => { const n = Number(r.updated); const t = Number.isFinite(n) ? (n < 1e12 ? n * 1000 : n) : Date.parse(r.updated); return !Number.isFinite(t) || t < cutoff; })
|
||
|
|
.filter((r) => !fs.existsSync(path.join(uploads, path.basename(String(r.filename)))))
|
||
|
|
.map((r) => ({ bucket: r.bucket, user: r.user, key: r.key, size: Number(r.size), filename: String(r.filename) }));
|
||
|
|
} finally { db.close(); }
|
||
|
|
}
|
||
|
|
|
||
|
|
// Only entries missing twice, a moment apart: s3d removes a copy just before
|
||
|
|
// it records the upload, and a list taken in between would be wrong.
|
||
|
|
export async function stuckUploads(dataDir, { olderThanMs = 120_000, settleMs = 2000 } = {}) {
|
||
|
|
const first = await candidates(dataDir, olderThanMs);
|
||
|
|
if (!first.length) return [];
|
||
|
|
await new Promise((r) => setTimeout(r, settleMs));
|
||
|
|
const again = new Set((await candidates(dataDir, olderThanMs)).map((r) => `${r.bucket}\n${r.key}`));
|
||
|
|
return first.filter((r) => again.has(`${r.bucket}\n${r.key}`));
|
||
|
|
}
|