Pithos: run your drives on Silent Mode with your own Sia account
s3d binds one Sia account per process, so the server runs one s3d per person (pithos-hosted), started on demand and flushed and stopped when idle. Only the connected s3d data folder moves between PC and server; the Sia recovery phrase never leaves the PC and nothing is re-uploaded to Sia. Because the server holds the connection key, local Pithos encrypts names and contents before they leave the machine, with keys from the Theseus vault (or a local key file elsewhere). What s3d cannot do on its own, the gateway now enforces: public folders, expiring share links and S3 keys for other people limited to one folder with read, add-only or edit rights. Files those reach are stored readable, since visitors have no key, and go back to encrypted when the rule is removed. Sign-up needs a BNS name signature, a deposit or an invite, with a per-IP limit.
This commit is contained in:
Local Dev
parent
14f5ad6d74
commit
d00bd369b7
19 changed files with 2117 additions and 13 deletions
|
|
@ -15,7 +15,10 @@ const ARCHIVE_PREFIX = 'previous-account-';
|
|||
const META = 'pithos-archive.json';
|
||||
const LABEL_FILE = 'pithos.json';
|
||||
|
||||
const keep = (name) => name === 's3d.yml' || name.startsWith(ARCHIVE_PREFIX);
|
||||
// Files that belong to this computer, not to a Sia account. The hosted key
|
||||
// file above all: moving or deleting it would make encrypted files unreadable.
|
||||
const LOCAL = new Set(['s3d.yml', 'pithos-prefs.json', 'pithos-hosted-secret.json']);
|
||||
const keep = (name) => LOCAL.has(name) || name.startsWith(ARCHIVE_PREFIX);
|
||||
|
||||
function stamp(d = new Date()) {
|
||||
const p = (n) => String(n).padStart(2, '0');
|
||||
|
|
@ -48,6 +51,7 @@ export function listArchives(dataDir) {
|
|||
indexerUrl: meta.indexerUrl || null,
|
||||
label: meta.label || readLabel(dir),
|
||||
users: meta.users || [],
|
||||
movedTo: meta.movedTo || null,
|
||||
hasData: fs.existsSync(path.join(dir, 's3d.db')),
|
||||
};
|
||||
});
|
||||
|
|
@ -76,6 +80,11 @@ export function restoreArchive(dataDir, name, metaForCurrent = {}) {
|
|||
if (!name.startsWith(ARCHIVE_PREFIX) || name.includes('/') || name.includes('\\') || name.includes('..')) throw new Error('not an account archive');
|
||||
const src = path.join(dataDir, name);
|
||||
if (!fs.existsSync(path.join(src, 's3d.db'))) throw new Error('that archive has no s3d data');
|
||||
let meta = {};
|
||||
try { meta = JSON.parse(fs.readFileSync(path.join(src, META), 'utf8')); } catch {}
|
||||
// The copy left behind when drives moved to Silent Mode is out of date the
|
||||
// moment the server s3d writes anything; running it would fork the account.
|
||||
if (meta.movedTo) throw new Error(`these drives now live on ${meta.movedTo}; move them back from Settings instead`);
|
||||
const archivedCurrent = fs.existsSync(path.join(dataDir, 's3d.db')) ? archiveCurrent(dataDir, metaForCurrent).name : null;
|
||||
// Anything left (tmp folders from a fresh, unconnected s3d) is clutter.
|
||||
for (const entry of fs.readdirSync(dataDir)) {
|
||||
|
|
|
|||
382
core/hosted-routes.js
Normal file
382
core/hosted-routes.js
Normal file
|
|
@ -0,0 +1,382 @@
|
|||
// "On Silent Mode" inside the control server. In hosted mode the UI talks to
|
||||
// this server exactly as before; this module answers by:
|
||||
// - forwarding daemon, users, keys, stats and status calls to the gateway,
|
||||
// - giving the object routes an S3 client that encrypts and decrypts here,
|
||||
// - turning "Access" and "Share link" into gateway rules (and converting the
|
||||
// files they cover to readable copies, since visitors have no key),
|
||||
// - moving the s3d data folder to the server and back.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import * as admin from './admin.js';
|
||||
import * as accounts from './accounts.js';
|
||||
import { readPrefs, writePrefs } from './prefs.js';
|
||||
import { ensureConfig } from './config.js';
|
||||
import {
|
||||
fileSecrets, signIn, gateway, HostedError, EncryptingS3Client, plainCovered,
|
||||
dataFiles, uploadData, downloadData,
|
||||
} from './hosted.js';
|
||||
|
||||
export const DEFAULT_HOSTED_URL = 'https://s3.silentmode.st';
|
||||
|
||||
// Calls forwarded as they are. The gateway allows the same set.
|
||||
const FORWARD = [
|
||||
['GET', /^\/api\/status$/], ['GET', /^\/api\/logs$/], ['GET', /^\/api\/stats$/],
|
||||
['POST', /^\/api\/flush$/], ['GET', /^\/api\/autoflush$/],
|
||||
['POST', /^\/api\/daemon\/(start|restart)$/],
|
||||
['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/],
|
||||
['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/],
|
||||
['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/],
|
||||
];
|
||||
// Things that act on this computer's s3d and make no sense while drives live
|
||||
// on the server.
|
||||
const LOCAL_ONLY = [
|
||||
[/^\/api\/binary\/install$/, 'There is nothing to install while your drives are on Silent Mode.'],
|
||||
[/^\/api\/login/, 'Your drives are on Silent Mode, already connected to your Sia account.'],
|
||||
[/^\/api\/account\/(switch|restore)$/, 'Move your drives back to this computer before switching Sia accounts.'],
|
||||
[/^\/api\/daemon\/stop$/, 'On Silent Mode, s3d stops by itself when nobody uses it.'],
|
||||
];
|
||||
|
||||
export function installHosted(ctx) {
|
||||
const { route, configFile, cfg, daemon, broadcast, registration, resetRegistration, HttpError } = ctx;
|
||||
const secrets = ctx.secrets || fileSecrets(configFile);
|
||||
const prefs = () => readPrefs(configFile);
|
||||
const hp = () => prefs().hosted || null;
|
||||
const isHosted = () => prefs().mode === 'hosted' && !!hp()?.token;
|
||||
const gw = () => {
|
||||
const h = hp();
|
||||
if (!h?.token) throw new HttpError(401, 'sign in to Silent Mode first');
|
||||
return gateway(h.url, h.token);
|
||||
};
|
||||
const wrap = async (fn) => {
|
||||
try { return await fn(); } catch (e) {
|
||||
if (e instanceof HostedError) {
|
||||
if (e.status === 401) { writePrefs(configFile, { hosted: { ...hp(), token: null } }); }
|
||||
throw new HttpError(e.status, e.message);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
|
||||
// ---- rules, cached briefly ----
|
||||
let rulesCache = null;
|
||||
async function rules() {
|
||||
if (!isHosted()) return [];
|
||||
if (rulesCache && Date.now() - rulesCache.at < 30_000) return rulesCache.list;
|
||||
const list = await wrap(() => gw().call('GET', '/hosted/v1/rules'));
|
||||
rulesCache = { at: Date.now(), list };
|
||||
return list;
|
||||
}
|
||||
const dropRules = () => { rulesCache = null; };
|
||||
|
||||
// ---- the S3 client for object routes ----
|
||||
const keyCache = new Map();
|
||||
async function hostedKeys(user) {
|
||||
const c = keyCache.get(user);
|
||||
if (c && Date.now() - c.at < 60_000) return c.keys;
|
||||
const keys = await wrap(() => gw().call('GET', `/api/keys?user=${encodeURIComponent(user)}`));
|
||||
keyCache.set(user, { at: Date.now(), keys });
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function s3For(user, localKeys) {
|
||||
if (isHosted()) {
|
||||
const keys = await hostedKeys(user);
|
||||
if (!keys.length) throw new HttpError(409, `user "${user}" has no access keys yet - create one first`);
|
||||
return new EncryptingS3Client({
|
||||
endpoint: hp().url, accessKeyId: keys[0].accessKeyId, secretKey: keys[0].secretKey,
|
||||
master: () => secrets.encryptionKey(), rules,
|
||||
});
|
||||
}
|
||||
// Back on this computer, files encrypted while hosted still decrypt.
|
||||
if (prefs().encryptedData && localKeys?.length) {
|
||||
return new EncryptingS3Client({
|
||||
endpoint: `http://${cfg().apiAddress}`, accessKeyId: localKeys[0].accessKeyId, secretKey: localKeys[0].secretKey,
|
||||
master: () => secrets.encryptionKey(), encryptWrites: false,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- status, logs and events from the server ----
|
||||
let remote = null; // last /api/status from the gateway
|
||||
async function hostedStatus(local) {
|
||||
const h = hp();
|
||||
try {
|
||||
remote = await wrap(() => gw().call('GET', '/api/status'));
|
||||
return { ...remote, host: local.host, mode: 'hosted', login: local.login, hosted: { ...remote.hosted, url: h.url } };
|
||||
} catch (e) {
|
||||
return { ...local, mode: 'hosted', daemon: { state: 'stopped', error: e.message }, hosted: { url: h.url, error: e.message }, registration: { registered: true } };
|
||||
}
|
||||
}
|
||||
let lastSeq = 0;
|
||||
let lastDaemon = '';
|
||||
async function pollRemote() {
|
||||
if (!isHosted() || !ctx.hasSubscribers()) return;
|
||||
try {
|
||||
const st = await gw().call('GET', '/api/status');
|
||||
remote = st;
|
||||
const d = JSON.stringify(st.daemon);
|
||||
if (d !== lastDaemon) { lastDaemon = d; broadcast('status', st.daemon); }
|
||||
const logs = await gw().call('GET', `/api/logs?since=${lastSeq}`);
|
||||
for (const l of logs || []) { lastSeq = Math.max(lastSeq, l.seq); broadcast('log', l); }
|
||||
} catch { /* the next poll tries again */ }
|
||||
}
|
||||
const poller = setInterval(() => { pollRemote(); }, 5000);
|
||||
poller.unref?.();
|
||||
|
||||
// ---- intercept: runs before the local routes in hosted mode ----
|
||||
async function intercept(req, res, url, send) {
|
||||
if (!isHosted()) return false;
|
||||
const p = url.pathname;
|
||||
for (const [re, msg] of LOCAL_ONLY) if (re.test(p)) throw new HttpError(409, msg);
|
||||
if (p === '/api/config' && req.method === 'PUT') throw new HttpError(409, 'Server settings are managed by Silent Mode while your drives are there.');
|
||||
if (p === '/api/status') return false; // answered by the local route, which asks us
|
||||
if (!FORWARD.some(([m, re]) => m === req.method && re.test(p))) return false;
|
||||
const body = req.method === 'GET' ? undefined : await readJson(req);
|
||||
const out = await wrap(() => gw().call(req.method, p + url.search, { body }));
|
||||
if (/^\/api\/(users|keys)/.test(p) && req.method !== 'GET') keyCache.clear();
|
||||
send(out);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---- routes ----
|
||||
route('GET', '/api/hosted', async () => {
|
||||
const h = hp();
|
||||
const out = { mode: isHosted() ? 'hosted' : 'local', url: h?.url || DEFAULT_HOSTED_URL, id: h?.id || null, signedIn: !!h?.token, secrets: secrets.kind || 'vault' };
|
||||
if (h?.token) {
|
||||
try { out.me = await gw().call('GET', '/hosted/v1/me'); } catch (e) { out.error = e.message; if (e.status === 401) out.signedIn = false; }
|
||||
}
|
||||
if (!out.signedIn) {
|
||||
try { out.info = await gateway(out.url).call('GET', '/hosted/v1/info'); } catch (e) { out.error = `Silent Mode is not reachable: ${e.message}`; }
|
||||
}
|
||||
return out;
|
||||
});
|
||||
|
||||
route('POST', '/api/hosted/signin', async (req) => {
|
||||
const { url = DEFAULT_HOSTED_URL, signup } = await readJson(req);
|
||||
if (!/^https:\/\/[^\s/]+$|^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/.test(url.replace(/\/$/, ''))) throw new HttpError(400, 'the server address must be https');
|
||||
const r = await wrap(() => signIn(url.replace(/\/$/, ''), secrets, signup));
|
||||
if (r.needsSignup) return r;
|
||||
writePrefs(configFile, { hosted: { url: url.replace(/\/$/, ''), id: r.id, token: r.token } });
|
||||
return { ok: true, id: r.id, state: r.state };
|
||||
});
|
||||
|
||||
route('POST', '/api/hosted/signout', async () => {
|
||||
const h = hp();
|
||||
if (h?.token) await gw().call('DELETE', '/hosted/v1/session').catch(() => {});
|
||||
writePrefs(configFile, { hosted: h ? { ...h, token: null } : null, mode: 'local' });
|
||||
dropRules();
|
||||
keyCache.clear();
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
// Use drives already on Silent Mode from this computer (a second device).
|
||||
route('POST', '/api/hosted/mode', async (req) => {
|
||||
const { mode } = await readJson(req);
|
||||
if (mode === 'local') { writePrefs(configFile, { mode: 'local' }); return { mode: 'local' }; }
|
||||
if (mode !== 'hosted') throw new HttpError(400, 'mode is local or hosted');
|
||||
const me = await wrap(() => gw().call('GET', '/hosted/v1/me'));
|
||||
if (me.state !== 'active') throw new HttpError(409, 'Silent Mode has no drives for you yet. Move them there first.');
|
||||
writePrefs(configFile, { mode: 'hosted', encryptedData: true });
|
||||
return { mode: 'hosted' };
|
||||
});
|
||||
|
||||
// Local -> Silent Mode. Nothing is re-uploaded to Sia: s3d's database and
|
||||
// any objects still waiting for Sia move, then s3d runs on the server.
|
||||
route('POST', '/api/hosted/move-up', async (req) => {
|
||||
const { force } = await readJson(req);
|
||||
const g = gw();
|
||||
const progress = (p) => broadcast('move', { direction: 'up', ...p });
|
||||
const me = await wrap(() => g.call('GET', '/hosted/v1/me'));
|
||||
if (me.state !== 'empty') throw new HttpError(409, 'Silent Mode already holds drives for you. Use them from here, or move them back first.');
|
||||
const c = cfg();
|
||||
const reg = await registration();
|
||||
if (!reg?.registered) throw new HttpError(409, 'Connect this computer to your Sia account first; then move the connection to Silent Mode.');
|
||||
const wasRunning = !!daemon.child;
|
||||
if (wasRunning) {
|
||||
progress({ phase: 'flush' });
|
||||
try { await admin.flush(c); } catch (e) {
|
||||
if (!force) throw new HttpError(409, `Some uploads could not reach Sia yet (${e.message}). Try again later, or move anyway: they will finish on the server.`);
|
||||
}
|
||||
}
|
||||
progress({ phase: 'stop' });
|
||||
await daemon.stop();
|
||||
let committed = false;
|
||||
try {
|
||||
const files = dataFiles(c.directory);
|
||||
if (!files.some((f) => f.path === 's3d.db')) throw new HttpError(409, 'there is no s3d database to move');
|
||||
await uploadData(g, c.directory, files, progress);
|
||||
progress({ phase: 'start' });
|
||||
const r = await wrap(() => g.call('POST', '/hosted/v1/data/commit', { body: { files } }));
|
||||
committed = true;
|
||||
if (!r.registered) throw new HttpError(502, 'the server could not use the moved connection');
|
||||
// Keep the local copy, set aside like an account switch, never deleted.
|
||||
const meta = { indexerUrl: reg.indexerUrl, users: r.users, movedTo: hp().url };
|
||||
accounts.archiveCurrent(c.directory, meta);
|
||||
resetRegistration();
|
||||
writePrefs(configFile, { mode: 'hosted', encryptedData: true, autostart: false });
|
||||
keyCache.clear();
|
||||
dropRules();
|
||||
progress({ phase: 'done' });
|
||||
return { ok: true, ...r };
|
||||
} catch (e) {
|
||||
if (!committed) await g.call('POST', '/hosted/v1/data/abort').catch(() => {});
|
||||
if (wasRunning && !committed) { try { ensureConfig(configFile); daemon.start(); } catch {} }
|
||||
progress({ phase: 'failed', error: e.message });
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
|
||||
// Silent Mode -> local. The server flushes and stops its s3d, the folder
|
||||
// comes down, the local s3d starts on it, then the server copy is deleted.
|
||||
route('POST', '/api/hosted/move-down', async (req) => {
|
||||
const { force } = await readJson(req);
|
||||
const g = gw();
|
||||
const progress = (p) => broadcast('move', { direction: 'down', ...p });
|
||||
const c = cfg();
|
||||
if (daemon.child) await daemon.stop();
|
||||
let archived = null;
|
||||
if (fs.existsSync(path.join(c.directory, 's3d.db'))) {
|
||||
archived = accounts.archiveCurrent(c.directory, { note: 'set aside to bring drives back from Silent Mode' }).name;
|
||||
}
|
||||
progress({ phase: 'flush' });
|
||||
const man = await wrap(() => g.call('POST', `/hosted/v1/data/checkout${force ? '?force=1' : ''}`));
|
||||
try {
|
||||
await downloadData(g, c.directory, man.files, progress);
|
||||
progress({ phase: 'start' });
|
||||
ensureConfig(configFile);
|
||||
writePrefs(configFile, { mode: 'local', autostart: true, encryptedData: true });
|
||||
resetRegistration();
|
||||
daemon.start();
|
||||
const until = Date.now() + 30_000;
|
||||
while (daemon.state === 'starting' && Date.now() < until) await new Promise((r) => setTimeout(r, 200));
|
||||
if (daemon.state !== 'running') throw new Error('s3d did not start on the moved folder');
|
||||
const reg = await registration();
|
||||
if (!reg?.registered) throw new Error('the moved folder has no Sia connection');
|
||||
await wrap(() => g.call('POST', '/hosted/v1/data/release'));
|
||||
keyCache.clear();
|
||||
dropRules();
|
||||
progress({ phase: 'done' });
|
||||
return { ok: true };
|
||||
} catch (e) {
|
||||
await daemon.stop().catch(() => {});
|
||||
for (const f of man.files) fs.rmSync(path.join(c.directory, ...f.path.split('/')), { force: true });
|
||||
if (archived) { try { accounts.restoreArchive(c.directory, archived, {}); } catch {} }
|
||||
writePrefs(configFile, { mode: 'hosted' });
|
||||
await g.call('POST', '/hosted/v1/data/cancel-checkout').catch(() => {});
|
||||
progress({ phase: 'failed', error: e.message });
|
||||
throw new HttpError(502, `Moving back failed and nothing changed: ${e.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Rules: public folders, shared keys, links. Files they cover become
|
||||
// readable copies; removing a rule encrypts them again.
|
||||
route('GET', '/api/hosted/rules', async () => rules());
|
||||
|
||||
route('POST', '/api/hosted/rules', async (req) => {
|
||||
const body = await readJson(req);
|
||||
const client = await s3For(body.user);
|
||||
if (!client) throw new HttpError(409, 'not on Silent Mode');
|
||||
const rule = await wrap(() => gw().call('POST', '/hosted/v1/rules', { body }));
|
||||
dropRules();
|
||||
await client.convert(rule.bucket, rule.prefix, { toPlain: true, onProgress: (p) => broadcast('convert', { rule: rule.id, ...p }) });
|
||||
return rule;
|
||||
});
|
||||
|
||||
route('DELETE', '/api/hosted/rules', async (req, url) => {
|
||||
const id = url.searchParams.get('id');
|
||||
const user = url.searchParams.get('user');
|
||||
const rule = (await rules()).find((r) => r.id === id);
|
||||
if (!rule) throw new HttpError(404, 'no such rule');
|
||||
await wrap(() => gw().call('DELETE', `/hosted/v1/rules?id=${encodeURIComponent(id)}`));
|
||||
dropRules();
|
||||
const remaining = await rules();
|
||||
const client = await s3For(user);
|
||||
if (client) {
|
||||
await client.convert(rule.bucket, rule.prefix, {
|
||||
toPlain: false,
|
||||
skip: (key) => plainCovered(remaining, rule.bucket, key),
|
||||
onProgress: (p) => broadcast('convert', { rule: rule.id, ...p }),
|
||||
});
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
// The whole-drive "Access" dialog, answered with a public rule on "".
|
||||
async function drivePolicy(bucket) {
|
||||
const r = (await rules()).find((x) => x.type === 'public' && x.bucket === bucket && x.prefix === '');
|
||||
return { policy: null, public: r ? r.rights : 'private', hosted: true };
|
||||
}
|
||||
async function setDrivePolicy(user, bucket, mode) {
|
||||
const current = (await rules()).filter((x) => x.type === 'public' && x.bucket === bucket && x.prefix === '');
|
||||
const client = await s3For(user);
|
||||
for (const r of current) await wrap(() => gw().call('DELETE', `/hosted/v1/rules?id=${encodeURIComponent(r.id)}`));
|
||||
dropRules();
|
||||
if (mode === 'private') {
|
||||
const remaining = await rules();
|
||||
await client.convert(bucket, '', { toPlain: false, skip: (key) => plainCovered(remaining, bucket, key), onProgress: (p) => broadcast('convert', p) });
|
||||
return;
|
||||
}
|
||||
await wrap(() => gw().call('POST', '/hosted/v1/rules', { body: { type: 'public', bucket, prefix: '', rights: mode } }));
|
||||
dropRules();
|
||||
await client.convert(bucket, '', { toPlain: true, onProgress: (p) => broadcast('convert', p) });
|
||||
}
|
||||
|
||||
// "Share link" for one file: a link rule, and the file stored readable.
|
||||
async function shareLink(user, bucket, key, seconds) {
|
||||
const client = await s3For(user);
|
||||
const expires = Date.now() + Math.min(Math.max(60, seconds | 0), 30 * 86400) * 1000;
|
||||
const rule = await wrap(() => gw().call('POST', '/hosted/v1/rules', { body: { type: 'link', bucket, prefix: key, expires } }));
|
||||
dropRules();
|
||||
await client.convert(bucket, key, { toPlain: true, skip: (k) => k !== key });
|
||||
const links = (prefs().hostedLinks || []).filter((l) => l.expires > Date.now());
|
||||
links.push({ user, bucket, key, expires });
|
||||
writePrefs(configFile, { hostedLinks: links });
|
||||
return `${hp().url}/l/${rule.token}`;
|
||||
}
|
||||
|
||||
// Files shared by a link that has run out go back to being encrypted.
|
||||
async function sweepLinks() {
|
||||
if (!isHosted()) return;
|
||||
const all = prefs().hostedLinks || [];
|
||||
const expired = all.filter((l) => l.expires <= Date.now());
|
||||
if (!expired.length) return;
|
||||
dropRules();
|
||||
const remaining = await rules();
|
||||
for (const l of expired) {
|
||||
if (plainCovered(remaining, l.bucket, l.key)) continue;
|
||||
const client = await s3For(l.user).catch(() => null);
|
||||
await client?.convert(l.bucket, l.key, { toPlain: false, skip: (k) => k !== l.key }).catch(() => {});
|
||||
}
|
||||
writePrefs(configFile, { hostedLinks: all.filter((l) => l.expires > Date.now()) });
|
||||
}
|
||||
const sweeper = setInterval(() => { sweepLinks().catch(() => {}); }, 10 * 60_000);
|
||||
sweeper.unref?.();
|
||||
|
||||
// Key file export/import for hosts without a vault.
|
||||
route('GET', '/api/hosted/secret', () => {
|
||||
if (!secrets.exportSecret) throw new HttpError(400, 'your keys come from the Theseus vault; back up the vault instead');
|
||||
return secrets.exportSecret();
|
||||
});
|
||||
route('POST', '/api/hosted/secret', async (req) => {
|
||||
if (!secrets.importSecret) throw new HttpError(400, 'your keys come from the Theseus vault');
|
||||
if (isHosted() || hp()?.token) throw new HttpError(409, 'sign out of Silent Mode before replacing your key');
|
||||
secrets.importSecret(await readJson(req));
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
return {
|
||||
isHosted, intercept, s3For, hostedStatus, drivePolicy, setDrivePolicy, shareLink,
|
||||
remoteDaemon: () => remote?.daemon || { state: 'stopped' },
|
||||
close() { clearInterval(poller); clearInterval(sweeper); },
|
||||
};
|
||||
}
|
||||
|
||||
async function readJson(req) {
|
||||
const parts = [];
|
||||
for await (const c of req) parts.push(c);
|
||||
const text = Buffer.concat(parts).toString('utf8');
|
||||
if (!text) return {};
|
||||
try { return JSON.parse(text); } catch { return {}; }
|
||||
}
|
||||
369
core/hosted.js
Normal file
369
core/hosted.js
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
// The client half of "On Silent Mode": signing in to pithos-hosted, the S3
|
||||
// client that encrypts on the way out and decrypts on the way in, converting
|
||||
// folders between private (encrypted) and readable when they are shared, and
|
||||
// moving the s3d data folder between this computer and the server.
|
||||
//
|
||||
// Nothing secret leaves this process: the identity key signs a nonce, the
|
||||
// encryption key never goes anywhere, and the Sia recovery phrase is not
|
||||
// involved at all (only the connected s3d data folder moves).
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { S3Client, readBody, objectPath } from './s3.js';
|
||||
import {
|
||||
driveKeys, encryptPath, decryptPath, encryptBody, decryptBody, cipherRange, plainSize, parseRange, HEADER_LEN,
|
||||
} from './crypt.js';
|
||||
|
||||
// ---- secrets ---------------------------------------------------------------
|
||||
|
||||
const PKCS8_ED25519 = Buffer.from('302e020100300506032b657004220420', 'hex');
|
||||
|
||||
// Hosts with a vault (Theseus) pass their own; everyone else gets a random
|
||||
// secret in a file beside s3d.yml. Losing that file loses the files it
|
||||
// encrypted, so the UI offers to export it.
|
||||
export function fileSecrets(configFile) {
|
||||
const file = path.join(path.dirname(configFile), 'pithos-hosted-secret.json');
|
||||
let root = null;
|
||||
function load() {
|
||||
if (root) return root;
|
||||
try { root = Buffer.from(JSON.parse(fs.readFileSync(file, 'utf8')).secret, 'base64'); } catch {}
|
||||
if (!root || root.length !== 32) {
|
||||
root = crypto.randomBytes(32);
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify({ v: 1, secret: root.toString('base64'), note: 'Pithos hosted identity and encryption key. Anyone with this file can read your encrypted files; without it they are lost.' }, null, 2), { mode: 0o600 });
|
||||
}
|
||||
return root;
|
||||
}
|
||||
const derive = (label) => Buffer.from(crypto.hkdfSync('sha256', load(), Buffer.alloc(0), label, 32));
|
||||
return {
|
||||
kind: 'file',
|
||||
file,
|
||||
identitySeed: async () => derive('pithos/hosted-identity/v1'),
|
||||
encryptionKey: async () => derive('pithos/hosted-encryption/v1'),
|
||||
exportSecret: () => JSON.parse(fs.readFileSync(file, 'utf8')),
|
||||
importSecret(obj) {
|
||||
const b = Buffer.from(String(obj?.secret || ''), 'base64');
|
||||
if (b.length !== 32) throw new Error('this is not a Pithos key file');
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify({ v: 1, secret: b.toString('base64'), note: 'Pithos hosted identity and encryption key.' }, null, 2), { mode: 0o600 });
|
||||
root = b;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function identityFromSeed(seed) {
|
||||
const key = crypto.createPrivateKey({ key: Buffer.concat([PKCS8_ED25519, seed]), format: 'der', type: 'pkcs8' });
|
||||
const pub = crypto.createPublicKey(key).export({ format: 'der', type: 'spki' }).subarray(-32);
|
||||
return { pubkey: pub.toString('hex'), sign: (msg) => crypto.sign(null, Buffer.from(msg, 'utf8'), key).toString('base64') };
|
||||
}
|
||||
|
||||
// ---- gateway calls -------------------------------------------------------------
|
||||
|
||||
export class HostedError extends Error {
|
||||
constructor(status, message, data) { super(message); this.status = status; this.data = data; }
|
||||
}
|
||||
|
||||
export function gateway(url, token) {
|
||||
const base = url.replace(/\/$/, '');
|
||||
async function call(method, p, { body, headers = {}, raw, stream } = {}) {
|
||||
const h = { ...headers };
|
||||
if (token) h.authorization = `Bearer ${token}`;
|
||||
let payload = body;
|
||||
if (body !== undefined && !stream && !Buffer.isBuffer(body)) { h['content-type'] = 'application/json'; payload = JSON.stringify(body); }
|
||||
const res = await fetch(base + p, { method, headers: h, body: payload, duplex: stream ? 'half' : undefined });
|
||||
if (raw) return res;
|
||||
const text = await res.text();
|
||||
let data = null;
|
||||
try { data = text ? JSON.parse(text) : null; } catch { data = { error: text.slice(0, 300) }; }
|
||||
if (!res.ok) throw new HostedError(res.status, data?.error || `HTTP ${res.status}`, data);
|
||||
return data;
|
||||
}
|
||||
return { base, call, auth: token ? `Bearer ${token}` : null };
|
||||
}
|
||||
|
||||
export async function signIn(url, secrets, signup) {
|
||||
const g = gateway(url);
|
||||
const id = identityFromSeed(await secrets.identitySeed());
|
||||
const { nonce, message } = await g.call('GET', '/hosted/v1/challenge');
|
||||
try {
|
||||
return await g.call('POST', '/hosted/v1/session', { body: { pubkey: id.pubkey, nonce, signature: id.sign(message), signup } });
|
||||
} catch (e) {
|
||||
if (e.status === 404 && e.data?.needsSignup) return { needsSignup: true, ...e.data };
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- which files are readable ------------------------------------------------------
|
||||
|
||||
// Files anyone else can reach (public folders, shared keys, share links) must
|
||||
// be stored readable; everything else is encrypted.
|
||||
export function plainCovered(rules, bucket, key) {
|
||||
const now = Date.now();
|
||||
return rules.some((r) => r.bucket === bucket && (!r.expires || r.expires > now) && (
|
||||
r.type === 'link' ? r.prefix === key : key.startsWith(r.prefix)
|
||||
));
|
||||
}
|
||||
|
||||
const MIME = {
|
||||
html: 'text/html', htm: 'text/html', txt: 'text/plain', md: 'text/markdown', css: 'text/css', js: 'text/javascript',
|
||||
json: 'application/json', xml: 'application/xml', csv: 'text/csv', pdf: 'application/pdf', zip: 'application/zip',
|
||||
png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', svg: 'image/svg+xml', avif: 'image/avif', ico: 'image/x-icon',
|
||||
mp3: 'audio/mpeg', ogg: 'audio/ogg', oga: 'audio/ogg', wav: 'audio/wav', flac: 'audio/flac', m4a: 'audio/mp4', opus: 'audio/opus',
|
||||
mp4: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', mkv: 'video/x-matroska',
|
||||
};
|
||||
export const guessType = (key) => MIME[String(key).split('.').pop().toLowerCase()] || 'application/octet-stream';
|
||||
|
||||
// ---- the encrypting S3 client ----------------------------------------------------------
|
||||
|
||||
// Same surface as S3Client for everything the control server's object routes
|
||||
// use. Keys and sizes it returns are the plain ones. `encryptWrites` is false
|
||||
// on this computer (local s3d already keeps files private) but reads still
|
||||
// decrypt, so drives brought back from Silent Mode stay readable.
|
||||
export class EncryptingS3Client {
|
||||
constructor({ endpoint, accessKeyId, secretKey, master, rules = async () => [], encryptWrites = true }) {
|
||||
this.inner = new S3Client({ endpoint, accessKeyId, secretKey });
|
||||
this.master = master; // async () => Buffer(32)
|
||||
this.rules = rules;
|
||||
this.encryptWrites = encryptWrites;
|
||||
this.keys = new Map();
|
||||
}
|
||||
|
||||
async dk(bucket) {
|
||||
if (!this.keys.has(bucket)) this.keys.set(bucket, driveKeys(await this.master(), bucket));
|
||||
return this.keys.get(bucket);
|
||||
}
|
||||
|
||||
async storeAsPlain(bucket, key) {
|
||||
return !this.encryptWrites || plainCovered(await this.rules(), bucket, key);
|
||||
}
|
||||
|
||||
listBuckets() { return this.inner.listBuckets(); }
|
||||
createBucket(b) { return this.inner.createBucket(b); }
|
||||
deleteBucket(b) { return this.inner.deleteBucket(b); }
|
||||
presign(...a) { return this.inner.presign(...a); }
|
||||
getPolicy(b) { return this.inner.getPolicy(b); }
|
||||
putPolicy(b, p) { return this.inner.putPolicy(b, p); }
|
||||
deletePolicy(b) { return this.inner.deletePolicy(b); }
|
||||
|
||||
// A folder can hold both forms (files from before hosting are plain), so a
|
||||
// listing below the root asks for both prefixes and merges them.
|
||||
async listObjects(bucket, { prefix = '', delimiter = '/', token, max = 1000 } = {}) {
|
||||
const dk = await this.dk(bucket);
|
||||
const cut = prefix.lastIndexOf('/') + 1;
|
||||
const folder = prefix.slice(0, cut);
|
||||
const encFolder = folder ? encryptPath(dk, folder) : '';
|
||||
const state = token ? JSON.parse(Buffer.from(token, 'base64url').toString('utf8')) : {};
|
||||
const sources = folder ? [['e', encFolder], ['p', folder]] : [['p', '']];
|
||||
const folders = new Map();
|
||||
const objects = new Map();
|
||||
const next = {};
|
||||
for (const [name, pfx] of sources) {
|
||||
if (state[name] === null) continue;
|
||||
const page = await this.inner.listObjects(bucket, { prefix: pfx, delimiter, token: state[name] || undefined, max });
|
||||
next[name] = page.truncated ? page.nextToken : null;
|
||||
for (const f of page.folders) {
|
||||
const d = decryptPath(dk, f);
|
||||
if (d.path.startsWith(prefix)) folders.set(d.path, { path: d.path, encrypted: d.encrypted });
|
||||
}
|
||||
for (const o of page.objects) {
|
||||
const d = decryptPath(dk, o.key);
|
||||
if (!d.path.startsWith(prefix)) continue;
|
||||
const size = d.encrypted && !d.path.endsWith('/') ? plainSize(o.size) : o.size;
|
||||
// Same name in both forms (re-uploaded after hosting): the encrypted one is newer.
|
||||
if (objects.has(d.path) && !d.encrypted) continue;
|
||||
objects.set(d.path, { key: d.path, size: size ?? o.size, modified: o.modified, etag: o.etag, encrypted: d.encrypted, storageKey: o.key });
|
||||
}
|
||||
}
|
||||
const more = Object.values(next).some((v) => v);
|
||||
const order = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
||||
return {
|
||||
folders: [...folders.keys()].sort(order),
|
||||
objects: [...objects.values()].sort((a, b) => order(a.key, b.key)),
|
||||
truncated: more,
|
||||
nextToken: more ? Buffer.from(JSON.stringify(next)).toString('base64url') : null,
|
||||
};
|
||||
}
|
||||
|
||||
async putObject(bucket, key, body, { contentType, contentLength } = {}) {
|
||||
if (await this.storeAsPlain(bucket, key)) return this.inner.putObject(bucket, key, body, { contentType, contentLength });
|
||||
const dk = await this.dk(bucket);
|
||||
const ek = encryptPath(dk, key);
|
||||
if (key.endsWith('/')) return this.inner.putObject(bucket, ek, Buffer.alloc(0), { contentLength: 0 });
|
||||
const len = Number(contentLength ?? (Buffer.isBuffer(body) ? body.length : NaN));
|
||||
if (!Number.isFinite(len)) throw new Error('encrypted uploads need a Content-Length');
|
||||
const { stream, size } = encryptBody(dk, key, body, len);
|
||||
const r = await this.inner.putObject(bucket, ek, stream, { contentType: 'application/octet-stream', contentLength: size });
|
||||
// An older readable copy under the same name would shadow nothing but
|
||||
// would still be readable on the server: remove it.
|
||||
if (ek !== key) await this.inner.deleteObject(bucket, key).catch(() => {});
|
||||
return r;
|
||||
}
|
||||
|
||||
// Resolves to a readable stream carrying statusCode and headers, like the
|
||||
// http.IncomingMessage S3Client.getObject returns.
|
||||
async getObject(bucket, key, { range } = {}) {
|
||||
const dk = await this.dk(bucket);
|
||||
const ek = encryptPath(dk, key);
|
||||
const head = await this.inner.request('HEAD', objectPath(bucket, ek));
|
||||
head.resume();
|
||||
if (head.statusCode === 404 || head.statusCode === 403) return this.inner.getObject(bucket, key, { range });
|
||||
if (head.statusCode >= 300) return this.inner.getObject(bucket, ek, { range });
|
||||
const total = Number(head.headers['content-length']);
|
||||
const r = cipherRange(total, range ? parseRange(range, plainSize(total) ?? 0) : null);
|
||||
const base = { 'content-type': guessType(key), 'accept-ranges': 'bytes' };
|
||||
if (head.headers.etag) base.etag = head.headers.etag;
|
||||
if (head.headers['last-modified']) base['last-modified'] = head.headers['last-modified'];
|
||||
if (r.unsatisfiable) return withStatus(Readable.from([]), 416, { ...base, 'content-range': `bytes */${r.plain}` });
|
||||
const headerRes = await this.inner.getObject(bucket, ek, { range: `bytes=0-${r.first === 0 ? r.to : HEADER_LEN - 1}` });
|
||||
if (headerRes.statusCode >= 300) return headerRes;
|
||||
// The header arrives first; when the wanted chunks follow it directly the
|
||||
// same response carries them too.
|
||||
const iter = headerRes[Symbol.asyncIterator]();
|
||||
let first = Buffer.alloc(0);
|
||||
while (first.length < HEADER_LEN) {
|
||||
const { value, done } = await iter.next();
|
||||
if (done) throw new Error('encrypted file is shorter than its header');
|
||||
first = Buffer.concat([first, value]);
|
||||
}
|
||||
const header = first.subarray(0, HEADER_LEN);
|
||||
let body;
|
||||
if (r.first === 0) {
|
||||
const rest = first.subarray(HEADER_LEN);
|
||||
body = Readable.from((async function* () { if (rest.length) yield rest; for (;;) { const { value, done } = await iter.next(); if (done) return; yield value; } })());
|
||||
} else {
|
||||
iter.return?.();
|
||||
headerRes.destroy();
|
||||
body = await this.inner.getObject(bucket, ek, { range: `bytes=${r.from}-${r.to}` });
|
||||
if (body.statusCode >= 300) return body;
|
||||
}
|
||||
const out = decryptBody(dk, key, header, r.plain ? body : Readable.from([]), r);
|
||||
const headers = { ...base, 'content-length': String(r.plain ? r.end - r.start + 1 : 0) };
|
||||
let status = 200;
|
||||
if (range) { status = 206; headers['content-range'] = `bytes ${r.start}-${r.end}/${r.plain}`; }
|
||||
return withStatus(out, status, headers);
|
||||
}
|
||||
|
||||
async deleteObject(bucket, key) {
|
||||
const dk = await this.dk(bucket);
|
||||
const ek = encryptPath(dk, key);
|
||||
await this.inner.deleteObject(bucket, ek);
|
||||
if (ek !== key) await this.inner.deleteObject(bucket, key);
|
||||
}
|
||||
|
||||
async deletePrefix(bucket, prefix) {
|
||||
let n = 0;
|
||||
let token;
|
||||
do {
|
||||
const page = await this.listObjects(bucket, { prefix, delimiter: '', token });
|
||||
for (const o of page.objects) { await this.inner.deleteObject(bucket, o.storageKey); n++; }
|
||||
token = page.truncated ? page.nextToken : null;
|
||||
} while (token);
|
||||
return n;
|
||||
}
|
||||
|
||||
// Re-stores every file under prefix in the wanted form: readable (shared)
|
||||
// or encrypted (private). Rules must already say what the prefix should be.
|
||||
async convert(bucket, prefix, { toPlain, onProgress = () => {}, skip = () => false }) {
|
||||
const dk = await this.dk(bucket);
|
||||
let done = 0;
|
||||
let token;
|
||||
const todo = [];
|
||||
do {
|
||||
const page = await this.listObjects(bucket, { prefix, delimiter: '', token });
|
||||
for (const o of page.objects) if (o.encrypted === toPlain && !skip(o.key)) todo.push(o);
|
||||
token = page.truncated ? page.nextToken : null;
|
||||
} while (token);
|
||||
for (const o of todo) {
|
||||
onProgress({ done, total: todo.length, key: o.key });
|
||||
const target = toPlain ? o.key : encryptPath(dk, o.key);
|
||||
if (o.key.endsWith('/')) {
|
||||
await this.inner.putObject(bucket, target, Buffer.alloc(0), { contentLength: 0 });
|
||||
} else if (toPlain) {
|
||||
const src = await this.getObject(bucket, o.key);
|
||||
if (src.statusCode >= 300) throw new Error(`could not read ${o.key}: HTTP ${src.statusCode}`);
|
||||
await this.inner.putObject(bucket, target, src, { contentType: guessType(o.key), contentLength: o.size });
|
||||
} else {
|
||||
const src = await this.inner.getObject(bucket, o.storageKey);
|
||||
if (src.statusCode >= 300) throw new Error(`could not read ${o.key}: HTTP ${src.statusCode}`);
|
||||
const { stream, size } = encryptBody(dk, o.key, src, o.size);
|
||||
await this.inner.putObject(bucket, target, stream, { contentType: 'application/octet-stream', contentLength: size });
|
||||
}
|
||||
await this.inner.deleteObject(bucket, o.storageKey);
|
||||
done++;
|
||||
}
|
||||
onProgress({ done, total: todo.length });
|
||||
return { converted: done };
|
||||
}
|
||||
}
|
||||
|
||||
function withStatus(stream, statusCode, headers) {
|
||||
stream.statusCode = statusCode;
|
||||
stream.headers = headers;
|
||||
return stream;
|
||||
}
|
||||
|
||||
// ---- moving the data folder ----------------------------------------------------------------
|
||||
|
||||
// What travels: the database and objects still waiting for Sia. Logs, temp
|
||||
// files, the local config and earlier accounts stay on this computer.
|
||||
const SKIP_TOP = new Set(['s3d.yml', 'pithos-prefs.json', 'pithos-hosted-secret.json', 'pithos-archive.json', 'tmp', 's3d.log']);
|
||||
export function dataFiles(dataDir) {
|
||||
const out = [];
|
||||
const rec = (rel) => {
|
||||
for (const e of fs.readdirSync(path.join(dataDir, rel), { withFileTypes: true })) {
|
||||
const r = rel ? `${rel}/${e.name}` : e.name;
|
||||
if (!rel && (SKIP_TOP.has(e.name) || e.name.startsWith('previous-account-') || e.name.endsWith('.log') || e.name.endsWith('.tmp'))) continue;
|
||||
if (e.isDirectory()) rec(r);
|
||||
else if (e.isFile()) {
|
||||
const data = fs.readFileSync(path.join(dataDir, r));
|
||||
out.push({ path: r, size: data.length, sha256: crypto.createHash('sha256').update(data).digest('hex') });
|
||||
}
|
||||
}
|
||||
};
|
||||
rec('');
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function uploadData(g, dataDir, files, onProgress = () => {}) {
|
||||
let sent = 0;
|
||||
const total = files.reduce((n, f) => n + f.size, 0);
|
||||
for (const f of files) {
|
||||
onProgress({ phase: 'upload', file: f.path, sent, total });
|
||||
const res = await fetch(`${g.base}/hosted/v1/data/file?path=${encodeURIComponent(f.path)}`, {
|
||||
method: 'PUT',
|
||||
headers: { authorization: g.auth, 'content-length': String(f.size) },
|
||||
body: fs.createReadStream(path.join(dataDir, f.path)),
|
||||
duplex: 'half',
|
||||
});
|
||||
const r = await res.json().catch(() => ({}));
|
||||
if (!res.ok) throw new HostedError(res.status, r.error || `upload of ${f.path} failed`);
|
||||
if (r.sha256 !== f.sha256) throw new Error(`${f.path} changed while it was being sent`);
|
||||
sent += f.size;
|
||||
}
|
||||
onProgress({ phase: 'upload', sent, total });
|
||||
}
|
||||
|
||||
export async function downloadData(g, dataDir, files, onProgress = () => {}) {
|
||||
let got = 0;
|
||||
const total = files.reduce((n, f) => n + f.size, 0);
|
||||
for (const f of files) {
|
||||
onProgress({ phase: 'download', file: f.path, got, total });
|
||||
const target = path.join(dataDir, ...f.path.split('/'));
|
||||
if (!target.startsWith(path.resolve(dataDir))) throw new Error('bad path from server');
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
const res = await fetch(`${g.base}/hosted/v1/data/file?path=${encodeURIComponent(f.path)}`, { headers: { authorization: g.auth } });
|
||||
if (!res.ok) throw new HostedError(res.status, (await res.json().catch(() => ({}))).error || `download of ${f.path} failed`);
|
||||
const hash = crypto.createHash('sha256');
|
||||
const src = Readable.fromWeb(res.body);
|
||||
src.on('data', (c) => hash.update(c));
|
||||
await pipeline(src, fs.createWriteStream(target));
|
||||
if (hash.digest('hex') !== f.sha256) throw new Error(`${f.path} arrived damaged`);
|
||||
got += f.size;
|
||||
}
|
||||
onProgress({ phase: 'download', got, total });
|
||||
}
|
||||
|
||||
export { readBody };
|
||||
21
core/s3.js
21
core/s3.js
|
|
@ -4,16 +4,17 @@
|
|||
|
||||
import crypto from 'node:crypto';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
const REGION = 'us-east-1';
|
||||
const SERVICE = 's3';
|
||||
export const REGION = 'us-east-1';
|
||||
export const SERVICE = 's3';
|
||||
|
||||
const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex');
|
||||
const hmac = (k, s) => crypto.createHmac('sha256', k).update(s).digest();
|
||||
|
||||
// RFC 3986 encoding as SigV4 wants it; '/' kept for object keys in paths.
|
||||
function uriEncode(str, keepSlash) {
|
||||
export function uriEncode(str, keepSlash) {
|
||||
const out = encodeURIComponent(str)
|
||||
.replace(/[!'()*]/g, (c) => '%' + c.charCodeAt(0).toString(16).toUpperCase());
|
||||
// (A "never matches" regex like /$^/ is not one: it matches the empty
|
||||
|
|
@ -21,17 +22,17 @@ function uriEncode(str, keepSlash) {
|
|||
return keepSlash ? out.replace(/%2F/g, '/') : out;
|
||||
}
|
||||
|
||||
function canonicalQuery(query) {
|
||||
export function canonicalQuery(query) {
|
||||
return Object.keys(query).sort()
|
||||
.map((k) => `${uriEncode(k)}=${uriEncode(String(query[k] ?? ''))}`)
|
||||
.join('&');
|
||||
}
|
||||
|
||||
function amzDate(d = new Date()) {
|
||||
export function amzDate(d = new Date()) {
|
||||
return d.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '');
|
||||
}
|
||||
|
||||
function signingKey(secret, date) {
|
||||
export function signingKey(secret, date) {
|
||||
return hmac(hmac(hmac(hmac('AWS4' + secret, date), REGION), SERVICE), 'aws4_request');
|
||||
}
|
||||
|
||||
|
|
@ -84,11 +85,13 @@ export class S3Client {
|
|||
|
||||
// Low-level request. body: Buffer | string | Readable | undefined.
|
||||
// Resolves the raw http.IncomingMessage so callers can stream it.
|
||||
request(method, path, { query = {}, headers = {}, body } = {}) {
|
||||
const signedHeaders = this.sign(method, path, query, headers);
|
||||
request(method, path, { query = {}, headers = {}, body, payloadHash } = {}) {
|
||||
const signedHeaders = this.sign(method, path, query, headers, payloadHash ? { payloadHash } : undefined);
|
||||
const qs = canonicalQuery(query);
|
||||
// The hosted gateway is https; a local s3d is plain http on loopback.
|
||||
const transport = this.endpoint.protocol === 'https:' ? https : http;
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = http.request({
|
||||
const req = transport.request({
|
||||
protocol: this.endpoint.protocol,
|
||||
hostname: this.endpoint.hostname,
|
||||
port: this.endpoint.port,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import { installS3d, S3D_VERSION, assetForPlatform } from './binary.js';
|
|||
import * as accounts from './accounts.js';
|
||||
import { readPrefs, writePrefs } from './prefs.js';
|
||||
import { createAutoFlush, clampMinutes, DEFAULT_MINUTES } from './autoflush.js';
|
||||
import { installHosted } from './hosted-routes.js';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const UI_DIR = path.join(HERE, '..', 'ui');
|
||||
|
|
@ -49,6 +50,7 @@ export async function createPithos(opts = {}) {
|
|||
hostName = 'web', // 'web' | 'desktop' | 'theseus' — UI tweaks only
|
||||
openExternal, // (url) => void, for hosts that can open the system browser
|
||||
showPanel, // () => void, for hosts with a side panel to switch back to (Theseus)
|
||||
secrets, // hosted identity + encryption keys; the Theseus vault, else a local key file
|
||||
} = opts;
|
||||
|
||||
const configFile = resolveConfigPath(configOpt);
|
||||
|
|
@ -67,6 +69,7 @@ export async function createPithos(opts = {}) {
|
|||
const subscribers = new Set();
|
||||
let versionCache = null;
|
||||
|
||||
let hosted = null; // "On Silent Mode", installed below
|
||||
const broadcast = (event, data) => {
|
||||
const msg = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
|
||||
for (const res of subscribers) res.write(msg);
|
||||
|
|
@ -74,7 +77,8 @@ export async function createPithos(opts = {}) {
|
|||
daemon.on('status', (s) => {
|
||||
// A registration done outside Pithos (s3d login by hand) shows up on the next start.
|
||||
if (s.state === 'starting' && !regCache?.registered) regCache = null;
|
||||
broadcast('status', s);
|
||||
// While drives are hosted the UI follows the server's s3d instead.
|
||||
if (!hosted?.isHosted()) broadcast('status', s);
|
||||
});
|
||||
daemon.on('log', (l) => broadcast('log', l));
|
||||
login.on('state', (s) => {
|
||||
|
|
@ -119,8 +123,11 @@ export async function createPithos(opts = {}) {
|
|||
}
|
||||
|
||||
async function s3For(user) {
|
||||
if (hosted.isHosted()) return hosted.s3For(user);
|
||||
const keys = await cli.listKeys(user);
|
||||
if (!keys.length) throw new HttpError(409, `user "${user}" has no access keys yet - create one first`);
|
||||
const decrypting = await hosted.s3For(user, keys);
|
||||
if (decrypting) return decrypting;
|
||||
const c = cfg();
|
||||
return new S3Client({ endpoint: `http://${c.apiAddress}`, accessKeyId: keys[0].accessKeyId, secretKey: keys[0].secretKey });
|
||||
}
|
||||
|
|
@ -143,7 +150,7 @@ export async function createPithos(opts = {}) {
|
|||
autostart('Pithos was opened');
|
||||
}
|
||||
const v = await version();
|
||||
return {
|
||||
const local = {
|
||||
host: hostName,
|
||||
daemon: daemon.status(),
|
||||
external: await probeExternal(c),
|
||||
|
|
@ -152,6 +159,7 @@ export async function createPithos(opts = {}) {
|
|||
login: login.snapshot(),
|
||||
registration: await registration(),
|
||||
};
|
||||
return hosted.isHosted() ? hosted.hostedStatus(local) : local;
|
||||
});
|
||||
|
||||
// Start and Stop also record what the user wants, so s3d comes back by
|
||||
|
|
@ -356,6 +364,8 @@ export async function createPithos(opts = {}) {
|
|||
// Objects are user content: never let them run script on this origin.
|
||||
headers['content-security-policy'] = "sandbox; default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'";
|
||||
res.writeHead(up.statusCode, headers);
|
||||
// A decryption failure mid-file must not look like a complete download.
|
||||
up.on('error', () => res.destroy());
|
||||
up.pipe(res);
|
||||
return STREAMED;
|
||||
});
|
||||
|
|
@ -404,15 +414,22 @@ export async function createPithos(opts = {}) {
|
|||
|
||||
route('GET', '/api/s3/:user/buckets/:bucket/presign', async (req, url, p) => {
|
||||
const q = url.searchParams;
|
||||
if (hosted.isHosted()) return { url: await hosted.shareLink(p.user, p.bucket, requireKey(url), Number(q.get('expires') || 3600)) };
|
||||
const client = await s3For(p.user);
|
||||
return { url: client.presign(p.bucket, requireKey(url), Number(q.get('expires') || 3600), q.get('endpoint') || undefined) };
|
||||
});
|
||||
route('GET', '/api/s3/:user/buckets/:bucket/policy', async (req, url, p) => {
|
||||
if (hosted.isHosted()) return hosted.drivePolicy(p.bucket);
|
||||
const policy = await (await s3For(p.user)).getPolicy(p.bucket);
|
||||
return { policy, public: describePolicy(policy) };
|
||||
});
|
||||
route('PUT', '/api/s3/:user/buckets/:bucket/policy', async (req, url, p) => {
|
||||
const { mode } = await jsonBody(req); // 'private' | 'read' | 'read-list'
|
||||
if (hosted.isHosted()) {
|
||||
if (!['private', 'read', 'read-list'].includes(mode)) throw new HttpError(400, 'mode must be private, read or read-list');
|
||||
await hosted.setDrivePolicy(p.user, p.bucket, mode);
|
||||
return { ok: true };
|
||||
}
|
||||
const client = await s3For(p.user);
|
||||
if (mode === 'private') { await client.deletePolicy(p.bucket).catch((e) => { if (e.status !== 404) throw e; }); }
|
||||
else if (mode === 'read' || mode === 'read-list') await client.putPolicy(p.bucket, publicReadPolicy(p.bucket, { list: mode === 'read-list' }));
|
||||
|
|
@ -420,6 +437,12 @@ export async function createPithos(opts = {}) {
|
|||
return { ok: true };
|
||||
});
|
||||
|
||||
hosted = installHosted({
|
||||
route, configFile, cfg, daemon, broadcast, registration, HttpError, secrets,
|
||||
resetRegistration: () => { regCache = null; },
|
||||
hasSubscribers: () => subscribers.size > 0,
|
||||
});
|
||||
|
||||
// ---- server ------------------------------------------------------------
|
||||
|
||||
const STREAMED = Symbol('streamed');
|
||||
|
|
@ -493,9 +516,11 @@ export async function createPithos(opts = {}) {
|
|||
if (!viaTicket && !cookieSession(req)) throw new HttpError(401, 'not signed in');
|
||||
if (req.method !== 'GET') requireCsrf(req);
|
||||
|
||||
if (await hosted.intercept(req, res, url, (out) => sendJson(res, 200, out))) return;
|
||||
|
||||
if (url.pathname === '/api/events') {
|
||||
res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-store', connection: 'keep-alive' });
|
||||
res.write(`event: status\ndata: ${JSON.stringify(daemon.status())}\n\n`);
|
||||
res.write(`event: status\ndata: ${JSON.stringify(hosted.isHosted() ? hosted.remoteDaemon() : daemon.status())}\n\n`);
|
||||
res.write(`event: login\ndata: ${JSON.stringify(login.snapshot())}\n\n`);
|
||||
subscribers.add(res);
|
||||
const ping = setInterval(() => res.write(': ping\n\n'), 25_000);
|
||||
|
|
@ -590,6 +615,7 @@ export async function createPithos(opts = {}) {
|
|||
},
|
||||
async close() {
|
||||
clearInterval(autoFlushTimer);
|
||||
hosted.close();
|
||||
login.cancel();
|
||||
for (const s of subscribers) s.end();
|
||||
await daemon.stop();
|
||||
|
|
|
|||
65
hosted/auth.js
Normal file
65
hosted/auth.js
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
// Sign-in: a person is an Ed25519 key (from their Theseus vault, or a key file
|
||||
// on hosts without one). The gateway hands out a nonce, the client signs it,
|
||||
// and gets a session token back. Only token hashes are stored.
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { readJson, writeJson } from './store.js';
|
||||
|
||||
const SPKI_ED25519 = Buffer.from('302a300506032b6570032100', 'hex');
|
||||
const SESSION_DAYS = 90;
|
||||
|
||||
export const accountId = (pubkeyHex) => crypto.createHash('sha256').update(Buffer.from(pubkeyHex, 'hex')).digest('hex').slice(0, 32);
|
||||
|
||||
export function challengeText(publicUrl, nonce) {
|
||||
return `pithos-hosted-v1\n${publicUrl}\n${nonce}`;
|
||||
}
|
||||
|
||||
export function verifyEd25519(pubkeyHex, message, signatureB64) {
|
||||
if (!/^[0-9a-f]{64}$/i.test(pubkeyHex || '')) return false;
|
||||
try {
|
||||
const key = crypto.createPublicKey({ key: Buffer.concat([SPKI_ED25519, Buffer.from(pubkeyHex, 'hex')]), format: 'der', type: 'spki' });
|
||||
return crypto.verify(null, Buffer.from(message, 'utf8'), key, Buffer.from(String(signatureB64 || ''), 'base64'));
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
export function createAuth(root) {
|
||||
const file = path.join(root, 'sessions.json');
|
||||
const nonces = new Map(); // nonce -> expiry
|
||||
const sessions = readJson(file, {});
|
||||
|
||||
const hash = (t) => crypto.createHash('sha256').update(t).digest('hex');
|
||||
|
||||
return {
|
||||
challenge() {
|
||||
const now = Date.now();
|
||||
for (const [n, exp] of nonces) if (exp < now) nonces.delete(n);
|
||||
if (nonces.size > 10_000) throw Object.assign(new Error('too many sign-ins in progress, try again shortly'), { status: 503 });
|
||||
const nonce = crypto.randomBytes(24).toString('base64url');
|
||||
nonces.set(nonce, now + 5 * 60_000);
|
||||
return nonce;
|
||||
},
|
||||
// Consumes the nonce whether or not the signature holds.
|
||||
takeNonce(nonce) {
|
||||
const exp = nonces.get(nonce);
|
||||
nonces.delete(nonce);
|
||||
return !!exp && exp > Date.now();
|
||||
},
|
||||
issue(id) {
|
||||
const token = crypto.randomBytes(32).toString('base64url');
|
||||
const now = Date.now();
|
||||
for (const [h, s] of Object.entries(sessions)) if (s.exp < now) delete sessions[h];
|
||||
sessions[hash(token)] = { id, exp: now + SESSION_DAYS * 86_400_000 };
|
||||
writeJson(file, sessions);
|
||||
return token;
|
||||
},
|
||||
lookup(token) {
|
||||
const s = token && sessions[hash(token)];
|
||||
return s && s.exp > Date.now() ? s.id : null;
|
||||
},
|
||||
revoke(token) {
|
||||
delete sessions[hash(token)];
|
||||
writeJson(file, sessions);
|
||||
},
|
||||
};
|
||||
}
|
||||
31
hosted/deploy/nginx-s3.silentmode.st.conf
Normal file
31
hosted/deploy/nginx-s3.silentmode.st.conf
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# s3.silentmode.st -> pithos-hosted. Bodies stream straight through: uploads
|
||||
# can be gigabytes, and SigV4 signs the exact bytes and headers.
|
||||
server {
|
||||
listen 443 ssl;
|
||||
http2 on;
|
||||
server_name s3.silentmode.st;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/s3.silentmode.st/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/s3.silentmode.st/privkey.pem;
|
||||
|
||||
client_max_body_size 5g;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 1h;
|
||||
proxy_send_timeout 1h;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8720;
|
||||
proxy_http_version 1.1;
|
||||
# s3d checks the signature against the Host the client signed.
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
proxy_set_header Connection "";
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name s3.silentmode.st;
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
19
hosted/deploy/pithos-hosted.example.json
Normal file
19
hosted/deploy/pithos-hosted.example.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"listen": "127.0.0.1:8720",
|
||||
"publicUrl": "https://s3.silentmode.st",
|
||||
"root": "/var/lib/pithos-hosted",
|
||||
"s3dBinary": "/opt/pithos/bin/s3d",
|
||||
"portBase": 21000,
|
||||
"maxRunning": 30,
|
||||
"idleMinutes": 30,
|
||||
"bufferLimitBytes": 2147483648,
|
||||
"maxDataBytes": 4294967296,
|
||||
"downloadCapBytes": 53687091200,
|
||||
"trustProxy": true,
|
||||
"gate": {
|
||||
"bns": { "api": "https://navigate.st" },
|
||||
"deposit": null,
|
||||
"invites": [],
|
||||
"perIpPerDay": 3
|
||||
}
|
||||
}
|
||||
22
hosted/deploy/pithos-hosted.service
Normal file
22
hosted/deploy/pithos-hosted.service
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
[Unit]
|
||||
Description=Pithos on Silent Mode (hosted s3d gateway)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
User=pithos
|
||||
Group=pithos
|
||||
WorkingDirectory=/opt/pithos
|
||||
ExecStart=/usr/bin/node /opt/pithos/hosted/main.js --config /etc/pithos-hosted.json
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
# Stopping flushes every s3d's leftovers to Sia first.
|
||||
TimeoutStopSec=300
|
||||
KillMode=mixed
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/pithos-hosted
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
30
hosted/electrum.js
Normal file
30
hosted/electrum.js
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
// The one Electrum call the deposit gate needs: a verbose transaction.
|
||||
|
||||
import tls from 'node:tls';
|
||||
import net from 'node:net';
|
||||
|
||||
export function electrumCall({ host, port, tls: useTls = true, timeoutMs = 15_000 }, method, params) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const sock = useTls ? tls.connect({ host, port, servername: host }) : net.connect({ host, port });
|
||||
let buf = '';
|
||||
const timer = setTimeout(() => { sock.destroy(); reject(new Error('Electrum server did not answer')); }, timeoutMs);
|
||||
const done = (err, val) => { clearTimeout(timer); sock.end(); if (err) reject(err); else resolve(val); };
|
||||
sock.setEncoding('utf8');
|
||||
sock.on('error', (e) => done(e));
|
||||
sock.write(JSON.stringify({ jsonrpc: '2.0', id: 0, method: 'server.version', params: ['pithos-hosted', '1.4'] }) + '\n');
|
||||
sock.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }) + '\n');
|
||||
sock.on('data', (d) => {
|
||||
buf += d;
|
||||
let i;
|
||||
while ((i = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, i);
|
||||
buf = buf.slice(i + 1);
|
||||
let msg;
|
||||
try { msg = JSON.parse(line); } catch { continue; }
|
||||
if (msg.id !== 1) continue;
|
||||
if (msg.error) return done(new Error(msg.error.message || 'Electrum error'));
|
||||
return done(null, msg.result);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
126
hosted/gate.js
Normal file
126
hosted/gate.js
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
// Who may open a hosted slot. A slot costs us memory, disk and traffic, so
|
||||
// sign-up asks for one of:
|
||||
// bns a signature from the wallet that owns a BNS name
|
||||
// (one slot per name)
|
||||
// deposit a BCH payment of at least `minSats` to the operator's deposit
|
||||
// address carrying OP_RETURN "pithos:<account id>" (one slot per
|
||||
// tx; refunds are an operator decision)
|
||||
// invite a code the operator handed out (one use each)
|
||||
// plus a per-IP limit on sign-ups per day.
|
||||
|
||||
import path from 'node:path';
|
||||
import {
|
||||
sha256, hash160, secp256k1, utf8ToBin, binToHex, base64ToBin, cashAddressToLockingBytecode,
|
||||
} from '@bitauth/libauth';
|
||||
import { readJson, writeJson } from './store.js';
|
||||
import { electrumCall } from './electrum.js';
|
||||
|
||||
export const bnsMessage = (id) => `pithos-hosted signup ${id}`;
|
||||
export const depositMemo = (id) => `pithos:${id}`;
|
||||
|
||||
function varStr(bytes) {
|
||||
const n = bytes.length;
|
||||
const prefix = n < 0xfd ? [n] : [0xfd, n & 0xff, (n >> 8) & 0xff];
|
||||
return Uint8Array.from([...prefix, ...bytes]);
|
||||
}
|
||||
|
||||
// BIP-137: recover the signer, then compare its locking bytecode with the
|
||||
// address we expected. Recovery alone proves nothing: any signature recovers
|
||||
// to *some* key.
|
||||
export function signedByAddress(message, signatureB64, expectedAddress) {
|
||||
let sig;
|
||||
try { sig = base64ToBin(String(signatureB64).trim()); } catch { return false; }
|
||||
if (!(sig instanceof Uint8Array) || sig.length !== 65 || sig[0] < 27 || sig[0] > 34) return false;
|
||||
const compressed = sig[0] >= 31;
|
||||
const digest = sha256.hash(sha256.hash(Uint8Array.from([
|
||||
...varStr(utf8ToBin('Bitcoin Signed Message:\n')), ...varStr(utf8ToBin(message)),
|
||||
])));
|
||||
const recover = compressed ? secp256k1.recoverPublicKeyCompressed : secp256k1.recoverPublicKeyUncompressed;
|
||||
const pub = recover(sig.slice(1), (sig[0] - 27) & 3, digest);
|
||||
if (typeof pub === 'string') return false;
|
||||
const signer = binToHex(Uint8Array.from([0x76, 0xa9, 0x14, ...hash160(pub), 0x88, 0xac]));
|
||||
const expected = cashAddressToLockingBytecode(expectedAddress);
|
||||
return typeof expected !== 'string' && binToHex(expected.bytecode) === signer;
|
||||
}
|
||||
|
||||
export function createGate(root, cfg, { fetchImpl = fetch, electrum = electrumCall } = {}) {
|
||||
const file = path.join(root, 'gate.json');
|
||||
const state = readJson(file, { names: {}, txids: {}, invites: {}, ip: {} });
|
||||
const save = () => writeJson(file, state);
|
||||
const today = () => new Date().toISOString().slice(0, 10);
|
||||
|
||||
async function bns({ name, signature }, id) {
|
||||
if (!cfg.bns) throw gateError('BNS sign-up is not enabled here');
|
||||
const n = String(name || '').trim().toLowerCase();
|
||||
if (!/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(n)) throw gateError('enter a BNS name like yourname.bch');
|
||||
if (state.names[n] && state.names[n] !== id) throw gateError(`${n} already opened a hosted slot`);
|
||||
const r = await fetchImpl(`${cfg.bns.api.replace(/\/$/, '')}/api/name/${encodeURIComponent(n)}`);
|
||||
const info = await r.json().catch(() => ({}));
|
||||
if (!r.ok || !info.registered || !info.owner) throw gateError(`${n} is not a registered BNS name`);
|
||||
if (!signedByAddress(bnsMessage(id), signature, info.owner)) {
|
||||
throw gateError(`the signature is not from the wallet that owns ${n} (${info.owner})`);
|
||||
}
|
||||
return { type: 'bns', name: n, owner: info.owner };
|
||||
}
|
||||
|
||||
async function deposit({ txid }, id) {
|
||||
const d = cfg.deposit;
|
||||
if (!d) throw gateError('deposit sign-up is not enabled here');
|
||||
const t = String(txid || '').trim().toLowerCase();
|
||||
if (!/^[0-9a-f]{64}$/.test(t)) throw gateError('enter the 64-character transaction id');
|
||||
if (state.txids[t] && state.txids[t] !== id) throw gateError('this deposit already opened a hosted slot');
|
||||
const tx = await electrum(d.electrum, 'blockchain.transaction.get', [t, true]);
|
||||
if ((tx.confirmations || 0) < (d.confirmations ?? 1)) throw gateError('the deposit has no confirmation yet; try again in about ten minutes');
|
||||
const want = cashAddressToLockingBytecode(d.address);
|
||||
if (typeof want === 'string') throw new Error('deposit address in the gateway config is invalid');
|
||||
const wantHex = binToHex(want.bytecode);
|
||||
const memoHex = Buffer.from(depositMemo(id), 'utf8').toString('hex');
|
||||
let paid = 0;
|
||||
let memo = false;
|
||||
for (const o of tx.vout || []) {
|
||||
const hex = String(o.scriptPubKey?.hex || '');
|
||||
if (hex === wantHex) paid += Math.round(Number(o.value) * 1e8);
|
||||
if (hex.startsWith('6a') && hex.includes(memoHex)) memo = true;
|
||||
}
|
||||
if (paid < d.minSats) throw gateError(`the deposit must send at least ${d.minSats} sats to ${d.address}`);
|
||||
if (!memo) throw gateError(`the deposit must carry the memo "${depositMemo(id)}" (OP_RETURN)`);
|
||||
return { type: 'deposit', txid: t, sats: paid };
|
||||
}
|
||||
|
||||
function invite({ code }, id) {
|
||||
const c = String(code || '').trim();
|
||||
if (!(cfg.invites || []).includes(c)) throw gateError('this invite code is not valid');
|
||||
if (state.invites[c] && state.invites[c] !== id) throw gateError('this invite code was already used');
|
||||
return { type: 'invite', code: c };
|
||||
}
|
||||
|
||||
return {
|
||||
describe() {
|
||||
return {
|
||||
bns: !!cfg.bns,
|
||||
deposit: cfg.deposit ? { address: cfg.deposit.address, minSats: cfg.deposit.minSats } : null,
|
||||
invite: !!(cfg.invites && cfg.invites.length),
|
||||
};
|
||||
},
|
||||
async admit(proof, id, ip) {
|
||||
const day = today();
|
||||
for (const key of Object.keys(state.ip)) if (!key.startsWith(day)) delete state.ip[key];
|
||||
const ipKey = `${day} ${ip}`;
|
||||
if ((state.ip[ipKey] || 0) >= (cfg.perIpPerDay ?? 3)) throw gateError('too many sign-ups from this network today', 429);
|
||||
const kind = proof?.type;
|
||||
let r;
|
||||
if (kind === 'bns') r = await bns(proof, id);
|
||||
else if (kind === 'deposit') r = await deposit(proof, id);
|
||||
else if (kind === 'invite') r = invite(proof, id);
|
||||
else throw gateError('choose how to open your slot: BNS name, deposit or invite', 400);
|
||||
if (r.name) state.names[r.name] = id;
|
||||
if (r.txid) state.txids[r.txid] = id;
|
||||
if (r.code) state.invites[r.code] = id;
|
||||
state.ip[ipKey] = (state.ip[ipKey] || 0) + 1;
|
||||
save();
|
||||
return r;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function gateError(message, status = 403) { return Object.assign(new Error(message), { status }); }
|
||||
39
hosted/main.js
Normal file
39
hosted/main.js
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env node
|
||||
// pithos-hosted entry: node hosted/main.js --config /etc/pithos-hosted.json
|
||||
|
||||
import fs from 'node:fs';
|
||||
import { createHosted } from './server.js';
|
||||
|
||||
const DEFAULTS = {
|
||||
listen: '127.0.0.1:8720',
|
||||
publicUrl: 'https://s3.silentmode.st',
|
||||
root: '/var/lib/pithos-hosted',
|
||||
s3dBinary: null,
|
||||
portBase: 21000,
|
||||
maxRunning: 30,
|
||||
idleMinutes: 30,
|
||||
bufferLimitBytes: 2 * 1024 ** 3,
|
||||
maxDataBytes: 4 * 1024 ** 3,
|
||||
downloadCapBytes: 50 * 1024 ** 3,
|
||||
trustProxy: false,
|
||||
gate: { bns: { api: 'https://navigate.st' }, deposit: null, invites: [], perIpPerDay: 3 },
|
||||
};
|
||||
|
||||
const i = process.argv.indexOf('--config');
|
||||
const file = i > 0 ? process.argv[i + 1] : process.env.PITHOS_HOSTED_CONFIG;
|
||||
if (!file) { console.error('usage: pithos-hosted --config <file.json>'); process.exit(2); }
|
||||
const cfg = { ...DEFAULTS, ...JSON.parse(fs.readFileSync(file, 'utf8')) };
|
||||
if (!cfg.s3dBinary || !fs.existsSync(cfg.s3dBinary)) { console.error(`s3dBinary not found: ${cfg.s3dBinary}`); process.exit(2); }
|
||||
cfg.publicUrl = cfg.publicUrl.replace(/\/$/, '');
|
||||
|
||||
const hosted = await createHosted(cfg);
|
||||
let stopping = false;
|
||||
for (const sig of ['SIGTERM', 'SIGINT']) {
|
||||
process.on(sig, async () => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
console.log(`${sig}: flushing and stopping every s3d`);
|
||||
await hosted.close().catch((e) => console.error(e));
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
18
hosted/package.json
Normal file
18
hosted/package.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"name": "pithos-hosted",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "Pithos on Silent Mode: one s3d per person, on their own Sia account, behind a gateway that adds sign-in, per-folder access and client-side encryption.",
|
||||
"license": "MPL-2.0",
|
||||
"type": "module",
|
||||
"main": "main.js",
|
||||
"scripts": {
|
||||
"start": "node main.js --config /etc/pithos-hosted.json"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"dependencies": {
|
||||
"@bitauth/libauth": "3.1.0-next.8"
|
||||
}
|
||||
}
|
||||
93
hosted/rules.js
Normal file
93
hosted/rules.js
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
// Access rules for one hosted user: what s3d cannot express on its own.
|
||||
//
|
||||
// public anyone may read a drive or a folder (rights: read | read-list)
|
||||
// link anyone holding the link may read one file until it expires
|
||||
// grant whoever holds a gateway-minted S3 key may act on a drive or a
|
||||
// folder with the rights given: read, add (new files only) or edit
|
||||
//
|
||||
// Prefixes are plain paths. Files under a rule's prefix are stored readable
|
||||
// (the owner's Pithos converts them), because the people a rule is for have
|
||||
// no decryption key.
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
export const RIGHTS = {
|
||||
public: ['read', 'read-list'],
|
||||
link: ['read'],
|
||||
grant: ['read', 'add', 'edit'],
|
||||
};
|
||||
|
||||
const id = (n = 12) => crypto.randomBytes(n).toString('base64url');
|
||||
|
||||
export function validateRule(input, buckets) {
|
||||
const type = input.type;
|
||||
if (!RIGHTS[type]) throw new Error('rule type must be public, link or grant');
|
||||
const rights = input.rights || RIGHTS[type][0];
|
||||
if (!RIGHTS[type].includes(rights)) throw new Error(`a ${type} rule allows ${RIGHTS[type].join(', ')}`);
|
||||
const bucket = String(input.bucket || '');
|
||||
if (buckets && !buckets.includes(bucket)) throw new Error(`no drive named "${bucket}"`);
|
||||
const prefix = String(input.prefix || '');
|
||||
if (prefix.startsWith('/') || prefix.includes('//')) throw new Error('bad folder path');
|
||||
if (type === 'link' && (!prefix || prefix.endsWith('/'))) throw new Error('a link is for one file');
|
||||
if (type !== 'link' && prefix && !prefix.endsWith('/')) throw new Error('rules apply to a drive or a folder (end it with /)');
|
||||
let expires = null;
|
||||
if (input.expires != null) {
|
||||
expires = Number(input.expires);
|
||||
if (!Number.isFinite(expires) || expires < Date.now()) throw new Error('the expiry must be in the future');
|
||||
}
|
||||
if (type === 'link' && !expires) throw new Error('links need an expiry');
|
||||
const rule = { id: id(), type, bucket, prefix, rights, expires, label: String(input.label || '').slice(0, 120), createdAt: Date.now() };
|
||||
if (type === 'link') rule.token = id(24);
|
||||
if (type === 'grant') { rule.accessKeyId = 'PG' + crypto.randomBytes(9).toString('hex').toUpperCase(); rule.secretKey = id(30); }
|
||||
return rule;
|
||||
}
|
||||
|
||||
export const live = (r, now = Date.now()) => !r.expires || r.expires > now;
|
||||
|
||||
// What an S3 request wants to do, from method, query and key.
|
||||
export function classify(method, query, key, headers = {}) {
|
||||
if (key == null || key === '') {
|
||||
if (method === 'GET' && !hasAny(query, ['policy', 'acl', 'cors', 'lifecycle', 'versioning', 'tagging', 'uploads'])) return { op: 'list' };
|
||||
return { op: 'bucket-admin' };
|
||||
}
|
||||
if (headers['x-amz-copy-source']) return { op: 'copy' };
|
||||
if (hasAny(query, ['acl', 'tagging', 'retention', 'legal-hold'])) return { op: 'object-admin' };
|
||||
if (method === 'GET' || method === 'HEAD') return { op: 'read' };
|
||||
if (method === 'PUT' || method === 'POST') return { op: 'write' };
|
||||
if (method === 'DELETE') return { op: hasAny(query, ['uploadId']) ? 'write' : 'delete' };
|
||||
return { op: 'object-admin' };
|
||||
}
|
||||
|
||||
function hasAny(query, names) { return names.some((n) => Object.prototype.hasOwnProperty.call(query, n)); }
|
||||
|
||||
// Decides a grant-key request. Returns { allow, needsAbsent } or { allow:false, reason }.
|
||||
// needsAbsent: an "add" grant may only create keys that do not exist yet.
|
||||
export function decideGrant(rule, bucket, op, key, listPrefix = '') {
|
||||
if (!live(rule)) return { allow: false, reason: 'this key has expired' };
|
||||
if (bucket !== rule.bucket) return { allow: false, reason: 'this key is for another drive' };
|
||||
if (op === 'list') {
|
||||
return listPrefix.startsWith(rule.prefix) ? { allow: true } : { allow: false, reason: `this key may only list ${rule.prefix || 'the drive'}` };
|
||||
}
|
||||
if (!String(key).startsWith(rule.prefix)) return { allow: false, reason: `this key only reaches ${rule.prefix || 'the drive'}` };
|
||||
if (op === 'read') return { allow: true };
|
||||
if (op === 'write') {
|
||||
if (rule.rights === 'edit') return { allow: true };
|
||||
if (rule.rights === 'add') return { allow: true, needsAbsent: true };
|
||||
return { allow: false, reason: 'this key is read-only' };
|
||||
}
|
||||
if (op === 'delete') return rule.rights === 'edit' ? { allow: true } : { allow: false, reason: 'this key may not delete' };
|
||||
return { allow: false, reason: 'not allowed with a shared key' };
|
||||
}
|
||||
|
||||
// Longest live public rule covering bucket/key, or null.
|
||||
export function publicRuleFor(rules, bucket, key) {
|
||||
let best = null;
|
||||
for (const r of rules) {
|
||||
if (r.type !== 'public' || r.bucket !== bucket || !live(r) || !key.startsWith(r.prefix)) continue;
|
||||
if (!best || r.prefix.length > best.prefix.length) best = r;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
// Rules as the owner sees them; grant secrets stay in (the owner made them).
|
||||
export function forOwner(rules) { return rules.filter((r) => live(r)); }
|
||||
531
hosted/server.js
Normal file
531
hosted/server.js
Normal file
|
|
@ -0,0 +1,531 @@
|
|||
// pithos-hosted: s3d on Silent Mode with each person's own Sia account.
|
||||
//
|
||||
// One HTTP listener (behind nginx/TLS) serves four kinds of traffic:
|
||||
//
|
||||
// /hosted/v1/... sign-in, sign-up gate, access rules, moving a data
|
||||
// folder in and out (Bearer session token)
|
||||
// /api/... the owner's own Pithos API, forwarded to their
|
||||
// per-account control server (Bearer session token)
|
||||
// /p/<id>/<drive>/.. public files, /l/<token> share links (no auth)
|
||||
// anything else S3, path-style. Owner keys pass straight through to
|
||||
// their s3d (which checks the signature itself); keys
|
||||
// minted for other people are verified and limited here,
|
||||
// then re-signed with the owner's key.
|
||||
|
||||
import http from 'node:http';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { readConfig } from '../core/config.js';
|
||||
import { objectPath } from '../core/s3.js';
|
||||
import { createAuth, accountId, challengeText, verifyEd25519 } from './auth.js';
|
||||
import { createGate, bnsMessage, depositMemo } from './gate.js';
|
||||
import { createSupervisor, httpError } from './supervisor.js';
|
||||
import { parseAuth, verify, SigError } from './sigv4.js';
|
||||
import { validateRule, classify, decideGrant, publicRuleFor, live, forOwner } from './rules.js';
|
||||
import { readJson, writeJson } from './store.js';
|
||||
|
||||
// Owner API calls the gateway forwards. Everything that would act on the
|
||||
// server itself (binary, s3d.yml, Sia login, account switching) is the
|
||||
// gateway's business, and object routes are refused so nothing can bypass
|
||||
// the client-side encryption.
|
||||
const FORWARD = [
|
||||
['GET', /^\/api\/status$/], ['GET', /^\/api\/logs$/], ['GET', /^\/api\/stats$/],
|
||||
['POST', /^\/api\/flush$/], ['GET', /^\/api\/autoflush$/],
|
||||
['POST', /^\/api\/daemon\/(start|restart)$/],
|
||||
['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/],
|
||||
['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/],
|
||||
['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/],
|
||||
];
|
||||
const KEY_CHANGES = /^\/api\/(users|keys)/;
|
||||
|
||||
export async function createHosted(cfg, log = console.log) {
|
||||
fs.mkdirSync(cfg.root, { recursive: true });
|
||||
const auth = createAuth(cfg.root);
|
||||
const gate = createGate(cfg.root, cfg.gate || {});
|
||||
const sup = createSupervisor(cfg, log);
|
||||
const grantsFile = path.join(cfg.root, 'grants.json');
|
||||
const linksFile = path.join(cfg.root, 'links.json');
|
||||
const grants = readJson(grantsFile, {}); // accessKeyId -> { id, rule }
|
||||
const links = readJson(linksFile, {}); // token -> { id, rule }
|
||||
|
||||
const rulesFile = (id) => path.join(sup.dir(id), 'rules.json');
|
||||
const rulesOf = (id) => readJson(rulesFile(id), []);
|
||||
function saveRules(id, rules) {
|
||||
writeJson(rulesFile(id), rules);
|
||||
for (const [k, v] of Object.entries(grants)) if (v.id === id) delete grants[k];
|
||||
for (const [k, v] of Object.entries(links)) if (v.id === id) delete links[k];
|
||||
for (const r of rules) {
|
||||
if (r.type === 'grant') grants[r.accessKeyId] = { id, rule: r.id };
|
||||
if (r.type === 'link') links[r.token] = { id, rule: r.id };
|
||||
}
|
||||
writeJson(grantsFile, grants);
|
||||
writeJson(linksFile, links);
|
||||
}
|
||||
const ruleById = (id, ruleId) => rulesOf(id).find((r) => r.id === ruleId) || null;
|
||||
|
||||
// ---- traffic meter -------------------------------------------------------
|
||||
const month = () => new Date().toISOString().slice(0, 7);
|
||||
const usage = new Map(); // id -> { month, down, up }
|
||||
function usageOf(id) {
|
||||
let u = usage.get(id);
|
||||
if (!u) { u = readJson(path.join(sup.dir(id), 'usage.json'), null) || { month: month(), down: 0, up: 0 }; usage.set(id, u); }
|
||||
if (u.month !== month()) { u.month = month(); u.down = 0; u.up = 0; }
|
||||
return u;
|
||||
}
|
||||
const meterTimer = setInterval(() => {
|
||||
for (const [id, u] of usage) writeJson(path.join(sup.dir(id), 'usage.json'), u);
|
||||
}, 30_000);
|
||||
meterTimer.unref?.();
|
||||
function checkCap(id) {
|
||||
if (cfg.downloadCapBytes && usageOf(id).down >= cfg.downloadCapBytes) {
|
||||
throw httpError(429, `this account has used its ${fmtBytes(cfg.downloadCapBytes)} of downloads this month`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------
|
||||
const clientIp = (req) => (cfg.trustProxy && String(req.headers['x-forwarded-for'] || '').split(',')[0].trim()) || req.socket.remoteAddress || '';
|
||||
function bearer(req) {
|
||||
const m = /^Bearer\s+(\S+)$/.exec(req.headers.authorization || '');
|
||||
const id = m && auth.lookup(m[1]);
|
||||
if (!id) throw httpError(401, 'sign in first');
|
||||
return { id, token: m[1] };
|
||||
}
|
||||
|
||||
async function forwardApi(req, res, url, id) {
|
||||
const a = sup.account(id);
|
||||
if (url.pathname === '/api/status' && a.state !== 'active') {
|
||||
return json(res, 200, hostedStatus(a, { daemon: { state: 'stopped' }, registration: { registered: false } }));
|
||||
}
|
||||
const inst = await sup.ensure(id);
|
||||
const release = sup.hold(inst);
|
||||
try {
|
||||
const target = new URL(url.pathname + url.search, inst.pithos.url);
|
||||
const up = await new Promise((resolve, reject) => {
|
||||
const r = http.request(target, {
|
||||
method: req.method,
|
||||
headers: { cookie: inst.cookie, 'x-pithos': '1', 'content-type': req.headers['content-type'] || 'application/json' },
|
||||
}, resolve);
|
||||
r.on('error', reject);
|
||||
req.pipe(r);
|
||||
});
|
||||
const text = await readAll(up);
|
||||
if (req.method !== 'GET' && KEY_CHANGES.test(url.pathname)) await sup.indexKeys(inst);
|
||||
let body = text;
|
||||
if (up.statusCode === 200 && url.pathname === '/api/status') body = JSON.stringify(hostedStatus(a, JSON.parse(text)));
|
||||
if (up.statusCode === 200 && url.pathname === '/api/account') {
|
||||
const acc = JSON.parse(text);
|
||||
body = JSON.stringify({ indexerUrl: acc.indexerUrl, registered: acc.registered, label: acc.label, archives: [], hosted: true });
|
||||
}
|
||||
res.writeHead(up.statusCode, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' });
|
||||
res.end(body);
|
||||
} finally { release(); }
|
||||
}
|
||||
|
||||
// Server paths stay on the server.
|
||||
function hostedStatus(a, st) {
|
||||
return {
|
||||
...st,
|
||||
host: 'hosted',
|
||||
config: st.config ? { apiAddress: null, adminAddress: null, file: null, directory: null, https: null, exists: true } : undefined,
|
||||
hosted: { id: a.id, state: a.state, publicUrl: cfg.publicUrl, usage: usageOf(a.id), downloadCapBytes: cfg.downloadCapBytes || null },
|
||||
};
|
||||
}
|
||||
|
||||
// ---- data folder in and out ------------------------------------------------
|
||||
const incomingDir = (id) => path.join(sup.dir(id), 'incoming');
|
||||
const incomingManifest = (id) => path.join(sup.dir(id), 'incoming.json');
|
||||
|
||||
function safeRel(rel) {
|
||||
const r = String(rel || '').replace(/\\/g, '/');
|
||||
if (!r || r.startsWith('/') || r.split('/').some((s) => s === '..' || s === '' || s === '.') || /^[A-Za-z]:/.test(r)) throw httpError(400, 'bad file path');
|
||||
return r;
|
||||
}
|
||||
|
||||
async function receiveFile(req, id, rel) {
|
||||
const a = sup.account(id);
|
||||
if (a.state !== 'empty') throw httpError(409, 'drives are already hosted here; move them back to your computer first');
|
||||
const file = path.join(incomingDir(id), safeRel(rel));
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const hash = crypto.createHash('sha256');
|
||||
let size = 0;
|
||||
const man = readJson(incomingManifest(id), {});
|
||||
const total = Object.values(man).reduce((n, f) => n + f.size, 0);
|
||||
const limit = cfg.maxDataBytes || 4 * 1024 ** 3;
|
||||
req.on('data', (c) => { hash.update(c); size += c.length; if (total + size > limit) req.destroy(httpError(413, 'the data folder is larger than this server accepts')); });
|
||||
await pipeline(req, fs.createWriteStream(file));
|
||||
man[safeRel(rel)] = { size, sha256: hash.digest('hex') };
|
||||
writeJson(incomingManifest(id), man);
|
||||
return man[safeRel(rel)];
|
||||
}
|
||||
|
||||
async function commit(id, files) {
|
||||
const a = sup.account(id);
|
||||
if (a.state !== 'empty') throw httpError(409, 'drives are already hosted here');
|
||||
const man = readJson(incomingManifest(id), {});
|
||||
if (!Array.isArray(files) || !files.length) throw httpError(400, 'no files listed');
|
||||
for (const f of files) {
|
||||
const got = man[safeRel(f.path)];
|
||||
if (!got || got.sha256 !== f.sha256 || got.size !== f.size) throw httpError(400, `${f.path} did not arrive intact; send it again`);
|
||||
}
|
||||
if (Object.keys(man).length !== files.length) throw httpError(400, 'the server has files the list does not mention; start the move again');
|
||||
if (!fs.existsSync(path.join(incomingDir(id), 's3d.db'))) throw httpError(400, 'this is not an s3d data folder (no s3d.db)');
|
||||
await sup.close(id, 'new data folder');
|
||||
fs.rmSync(sup.dataDir(id), { recursive: true, force: true });
|
||||
fs.renameSync(incomingDir(id), sup.dataDir(id));
|
||||
fs.rmSync(incomingManifest(id), { force: true });
|
||||
sup.setState(id, 'active');
|
||||
let inst;
|
||||
try {
|
||||
inst = await sup.ensure(id);
|
||||
} catch (e) {
|
||||
sup.setState(id, 'empty');
|
||||
throw httpError(502, `s3d could not start on the moved folder: ${e.message}`);
|
||||
}
|
||||
const reg = await inst.cli.registration().catch(() => ({ registered: false }));
|
||||
const keys = await sup.indexKeys(inst);
|
||||
return { registered: reg.registered, indexerUrl: reg.indexerUrl || null, users: [...new Set(keys.map((k) => k.user))] };
|
||||
}
|
||||
|
||||
function walk(base) {
|
||||
const out = [];
|
||||
const rec = (d) => {
|
||||
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
|
||||
const p = path.join(d, e.name);
|
||||
if (e.isDirectory()) rec(p);
|
||||
else if (e.isFile()) out.push(p);
|
||||
}
|
||||
};
|
||||
rec(base);
|
||||
return out.map((p) => {
|
||||
const data = fs.readFileSync(p);
|
||||
return { path: path.relative(base, p).split(path.sep).join('/'), size: data.length, sha256: crypto.createHash('sha256').update(data).digest('hex') };
|
||||
});
|
||||
}
|
||||
|
||||
async function checkout(id, force) {
|
||||
const a = sup.account(id);
|
||||
if (a.state === 'moving-out') return { files: walk(sup.dataDir(id)) };
|
||||
if (a.state !== 'active') throw httpError(409, 'nothing is hosted here');
|
||||
const inst = await sup.ensure(id);
|
||||
const flushed = await sup.flushQuietly(inst, 10 * 60_000);
|
||||
if (!flushed && !force) throw httpError(409, 'some uploads could not be sent to Sia yet; try again later, or move anyway and they will finish on your computer');
|
||||
await sup.close(id, 'moving to the owner\'s computer');
|
||||
sup.setState(id, 'moving-out');
|
||||
return { files: walk(sup.dataDir(id)) };
|
||||
}
|
||||
|
||||
function release(id) {
|
||||
const a = sup.account(id);
|
||||
if (a.state !== 'moving-out') throw httpError(409, 'start the move first');
|
||||
fs.rmSync(sup.dataDir(id), { recursive: true, force: true });
|
||||
fs.mkdirSync(sup.dataDir(id), { recursive: true });
|
||||
saveRules(id, []);
|
||||
sup.setState(id, 'empty');
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
// ---- S3 ----------------------------------------------------------------------
|
||||
function splitS3Path(rawPath) {
|
||||
const parts = rawPath.split('/').slice(1);
|
||||
const dec = (s) => { try { return decodeURIComponent(s); } catch { return s; } };
|
||||
const bucket = dec(parts[0] || '');
|
||||
const key = parts.length > 1 ? parts.slice(1).map(dec).join('/') : null;
|
||||
return { bucket, key };
|
||||
}
|
||||
|
||||
async function s3(req, res, url) {
|
||||
const a = parseAuth(req, url);
|
||||
if (!a) return s3Error(res, 403, 'AccessDenied', 'sign requests with a key; public files are at /p/<account>/<drive>/<path>');
|
||||
const ownerId = sup.ownerOfKey(a.akid);
|
||||
if (ownerId) {
|
||||
checkReadCap(req, ownerId);
|
||||
const inst = await sup.ensure(ownerId);
|
||||
return rawProxy(req, res, inst);
|
||||
}
|
||||
const g = grants[a.akid];
|
||||
const rule = g && ruleById(g.id, g.rule);
|
||||
if (!rule) return s3Error(res, 403, 'InvalidAccessKeyId', 'The AWS Access Key Id you provided does not exist in our records.');
|
||||
verify(req, url, a, rule.secretKey);
|
||||
const raw = req.url.split('?')[0];
|
||||
const { bucket, key } = splitS3Path(raw);
|
||||
const query = Object.fromEntries(url.searchParams);
|
||||
const { op } = classify(req.method, query, key, req.headers);
|
||||
const d = decideGrant(rule, bucket, op, key, query.prefix || '');
|
||||
if (!d.allow) return s3Error(res, 403, 'AccessDenied', d.reason);
|
||||
const payload = req.headers['x-amz-content-sha256'] || 'UNSIGNED-PAYLOAD';
|
||||
if (payload === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
|
||||
return s3Error(res, 501, 'NotImplemented', 'signed streaming uploads are not supported with shared keys; turn off payload signing in your S3 client');
|
||||
}
|
||||
checkReadCap(req, g.id);
|
||||
const inst = await sup.ensure(g.id);
|
||||
const client = await sup.bucketClient(inst, bucket);
|
||||
if (!client) return s3Error(res, 404, 'NoSuchBucket', 'The specified bucket does not exist');
|
||||
if (d.needsAbsent) {
|
||||
const head = await client.request('HEAD', objectPath(bucket, key));
|
||||
head.resume();
|
||||
if (head.statusCode === 200) return s3Error(res, 403, 'AccessDenied', 'this key may add files but not replace them');
|
||||
}
|
||||
const pass = {};
|
||||
for (const [k, v] of Object.entries(req.headers)) {
|
||||
if (/^(content-type|content-length|content-md5|content-encoding|content-disposition|cache-control|range|if-(match|none-match|modified-since|unmodified-since))$/.test(k) || (k.startsWith('x-amz-') && !/^x-amz-(date|content-sha256|security-token)$/.test(k))) pass[k] = v;
|
||||
}
|
||||
const release = sup.hold(inst);
|
||||
try {
|
||||
const up = await client.request(req.method, key == null ? objectPath(bucket) : objectPath(bucket, key), { query, headers: pass, body: req, payloadHash: payload });
|
||||
relay(up, res, g.id, req.method);
|
||||
await done(res);
|
||||
} finally { release(); }
|
||||
}
|
||||
|
||||
function checkReadCap(req, id) {
|
||||
if (req.method === 'GET') checkCap(id);
|
||||
}
|
||||
|
||||
async function rawProxy(req, res, inst) {
|
||||
const c = readConfig(sup.configFile(inst.id));
|
||||
const [host, port] = c.apiAddress.split(':');
|
||||
const release = sup.hold(inst);
|
||||
const u = usageOf(inst.id);
|
||||
try {
|
||||
const up = await new Promise((resolve, reject) => {
|
||||
const headers = { ...req.headers };
|
||||
delete headers['x-forwarded-for']; delete headers['x-real-ip']; delete headers.connection;
|
||||
const r = http.request({ host, port: Number(port), method: req.method, path: req.url, headers }, resolve);
|
||||
r.on('error', reject);
|
||||
req.on('data', (chunk) => { u.up += chunk.length; });
|
||||
req.pipe(r);
|
||||
});
|
||||
relay(up, res, inst.id, req.method);
|
||||
await done(res);
|
||||
} finally { release(); }
|
||||
}
|
||||
|
||||
function relay(up, res, id, method) {
|
||||
const u = usageOf(id);
|
||||
res.writeHead(up.statusCode, up.headers);
|
||||
if (method === 'GET') up.on('data', (c) => { u.down += c.length; });
|
||||
up.pipe(res);
|
||||
}
|
||||
|
||||
// ---- public files and links ----------------------------------------------------
|
||||
async function serveObject(req, res, id, bucket, key, { inline = true } = {}) {
|
||||
checkCap(id);
|
||||
const inst = await sup.ensure(id);
|
||||
const client = await sup.bucketClient(inst, bucket);
|
||||
if (!client) throw httpError(404, 'not found');
|
||||
const release = sup.hold(inst);
|
||||
try {
|
||||
const up = await client.getObject(bucket, key, { range: req.headers.range });
|
||||
if (up.statusCode >= 300 && up.statusCode !== 304) { up.resume(); throw httpError(up.statusCode === 403 ? 404 : up.statusCode, 'not found'); }
|
||||
const headers = {};
|
||||
for (const h of ['content-type', 'content-length', 'content-range', 'accept-ranges', 'etag', 'last-modified']) if (up.headers[h]) headers[h] = up.headers[h];
|
||||
const name = key.split('/').pop() || 'download';
|
||||
headers['content-disposition'] = `${inline ? 'inline' : 'attachment'}; filename*=UTF-8''${encodeURIComponent(name)}`;
|
||||
headers['x-content-type-options'] = 'nosniff';
|
||||
headers['content-security-policy'] = "sandbox; default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'";
|
||||
headers['cache-control'] = 'public, max-age=60';
|
||||
const u = usageOf(id);
|
||||
res.writeHead(up.statusCode, headers);
|
||||
if (req.method === 'HEAD') { up.resume(); return res.end(); }
|
||||
up.on('data', (c) => { u.down += c.length; });
|
||||
up.pipe(res);
|
||||
await done(res);
|
||||
} finally { release(); }
|
||||
}
|
||||
|
||||
async function servePublic(req, res, url) {
|
||||
const m = /^\/p\/([0-9a-f]{32})\/([^/]+)\/?(.*)$/.exec(url.pathname);
|
||||
if (!m) throw httpError(404, 'not found');
|
||||
const [, id, bucket] = m;
|
||||
const key = decodeURIComponent(m[3]);
|
||||
const rule = publicRuleFor(rulesOf(id), bucket, key);
|
||||
if (!rule) throw httpError(404, 'not found');
|
||||
if (key === '' || key.endsWith('/')) {
|
||||
if (rule.rights !== 'read-list') throw httpError(404, 'not found');
|
||||
const inst = await sup.ensure(id);
|
||||
const client = await sup.bucketClient(inst, bucket);
|
||||
if (!client) throw httpError(404, 'not found');
|
||||
const page = await client.listObjects(bucket, { prefix: key, token: url.searchParams.get('token') || undefined });
|
||||
return json(res, 200, {
|
||||
folders: page.folders.map((f) => f.slice(key.length)),
|
||||
files: page.objects.filter((o) => o.key !== key).map((o) => ({ name: o.key.slice(key.length), size: o.size, modified: o.modified })),
|
||||
next: page.truncated ? page.nextToken : null,
|
||||
});
|
||||
}
|
||||
return serveObject(req, res, id, bucket, key);
|
||||
}
|
||||
|
||||
async function serveLink(req, res, url) {
|
||||
const token = url.pathname.slice(3);
|
||||
const l = links[token];
|
||||
const rule = l && ruleById(l.id, l.rule);
|
||||
if (!rule || !live(rule)) throw httpError(410, 'this link has expired');
|
||||
return serveObject(req, res, l.id, rule.bucket, rule.prefix, { inline: !url.searchParams.has('download') });
|
||||
}
|
||||
|
||||
// ---- routing -------------------------------------------------------------------
|
||||
async function handle(req, res) {
|
||||
const url = new URL(req.url, cfg.publicUrl);
|
||||
const isS3 = /^AWS4-HMAC-SHA256 /.test(req.headers.authorization || '') || url.searchParams.has('X-Amz-Algorithm');
|
||||
if (isS3) return s3(req, res, url);
|
||||
const p = url.pathname;
|
||||
|
||||
if (p.startsWith('/p/') && (req.method === 'GET' || req.method === 'HEAD')) return servePublic(req, res, url);
|
||||
if (p.startsWith('/l/') && (req.method === 'GET' || req.method === 'HEAD')) return serveLink(req, res, url);
|
||||
|
||||
if (p === '/hosted/v1/info' && req.method === 'GET') {
|
||||
return json(res, 200, { publicUrl: cfg.publicUrl, gate: gate.describe(), downloadCapBytes: cfg.downloadCapBytes || null, s3Endpoint: cfg.publicUrl });
|
||||
}
|
||||
if (p === '/hosted/v1/challenge' && req.method === 'GET') {
|
||||
const nonce = auth.challenge();
|
||||
return json(res, 200, { nonce, message: challengeText(cfg.publicUrl, nonce) });
|
||||
}
|
||||
if (p === '/hosted/v1/session' && req.method === 'POST') {
|
||||
const body = await jsonBody(req);
|
||||
if (!auth.takeNonce(body.nonce)) throw httpError(401, 'the sign-in request expired; try again');
|
||||
if (!verifyEd25519(body.pubkey, challengeText(cfg.publicUrl, body.nonce), body.signature)) throw httpError(401, 'the signature does not match');
|
||||
const id = accountId(body.pubkey);
|
||||
let a = sup.account(id);
|
||||
if (!a) {
|
||||
if (!body.signup) {
|
||||
return json(res, 404, { error: 'no hosted account for this key yet', needsSignup: true, id, gate: gate.describe(), bnsMessage: bnsMessage(id), depositMemo: depositMemo(id) });
|
||||
}
|
||||
const g = await gate.admit(body.signup, id, clientIp(req));
|
||||
a = sup.createAccount({ id, pubkey: body.pubkey.toLowerCase(), gate: g });
|
||||
log(`[${id.slice(0, 8)}] new account via ${g.type}`);
|
||||
}
|
||||
return json(res, 200, { token: auth.issue(id), id, state: a.state });
|
||||
}
|
||||
|
||||
if (p.startsWith('/hosted/v1/') || p.startsWith('/api/')) {
|
||||
const { id, token } = bearer(req);
|
||||
if (p.startsWith('/api/')) {
|
||||
if (!FORWARD.some(([m, re]) => m === req.method && re.test(p))) throw httpError(403, 'not available on a hosted account');
|
||||
return forwardApi(req, res, url, id);
|
||||
}
|
||||
const a = sup.account(id);
|
||||
const q = url.searchParams;
|
||||
switch (`${req.method} ${p}`) {
|
||||
case 'GET /hosted/v1/me':
|
||||
return json(res, 200, { id, state: a.state, gate: { type: a.gate?.type, name: a.gate?.name || null }, createdAt: a.createdAt, usage: usageOf(id), downloadCapBytes: cfg.downloadCapBytes || null, publicUrl: cfg.publicUrl });
|
||||
case 'DELETE /hosted/v1/session':
|
||||
auth.revoke(token);
|
||||
return json(res, 200, { ok: true });
|
||||
case 'GET /hosted/v1/rules':
|
||||
return json(res, 200, forOwner(rulesOf(id)));
|
||||
case 'POST /hosted/v1/rules': {
|
||||
const inst = await sup.ensure(id);
|
||||
const rule = validateRule(await jsonBody(req), await sup.mapBuckets(inst));
|
||||
const rules = rulesOf(id).filter((r) => live(r));
|
||||
if (rules.length >= 500) throw httpError(400, 'too many rules on one account');
|
||||
rules.push(rule);
|
||||
saveRules(id, rules);
|
||||
return json(res, 200, rule);
|
||||
}
|
||||
case 'DELETE /hosted/v1/rules': {
|
||||
const rules = rulesOf(id);
|
||||
const next = rules.filter((r) => r.id !== q.get('id'));
|
||||
if (next.length === rules.length) throw httpError(404, 'no such rule');
|
||||
saveRules(id, next);
|
||||
return json(res, 200, { ok: true });
|
||||
}
|
||||
case 'PUT /hosted/v1/data/file':
|
||||
return json(res, 200, await receiveFile(req, id, q.get('path')));
|
||||
case 'POST /hosted/v1/data/commit':
|
||||
return json(res, 200, await commit(id, (await jsonBody(req)).files));
|
||||
case 'POST /hosted/v1/data/abort':
|
||||
if (a.state !== 'empty') throw httpError(409, 'nothing to abort');
|
||||
fs.rmSync(incomingDir(id), { recursive: true, force: true });
|
||||
fs.rmSync(incomingManifest(id), { force: true });
|
||||
return json(res, 200, { ok: true });
|
||||
case 'POST /hosted/v1/data/checkout':
|
||||
return json(res, 200, await checkout(id, q.get('force') === '1'));
|
||||
case 'GET /hosted/v1/data/file': {
|
||||
if (a.state !== 'moving-out') throw httpError(409, 'start the move first');
|
||||
const file = path.join(sup.dataDir(id), safeRel(q.get('path')));
|
||||
if (!fs.existsSync(file)) throw httpError(404, 'no such file');
|
||||
res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-length': fs.statSync(file).size });
|
||||
await pipeline(fs.createReadStream(file), res);
|
||||
return;
|
||||
}
|
||||
case 'POST /hosted/v1/data/cancel-checkout':
|
||||
if (a.state !== 'moving-out') throw httpError(409, 'no move in progress');
|
||||
sup.setState(id, 'active');
|
||||
return json(res, 200, { ok: true });
|
||||
case 'POST /hosted/v1/data/release':
|
||||
return json(res, 200, release(id));
|
||||
default:
|
||||
throw httpError(404, 'no such endpoint');
|
||||
}
|
||||
}
|
||||
throw httpError(404, 'not found');
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
handle(req, res).catch((err) => {
|
||||
const status = err instanceof SigError ? err.status : err.status || 500;
|
||||
if (status >= 500) log('[hosted]', err.stack || err.message);
|
||||
if (res.headersSent) return res.destroy();
|
||||
const isS3 = /^AWS4-HMAC-SHA256 /.test(req.headers.authorization || '');
|
||||
if (isS3) return s3Error(res, status, err.code || (status === 404 ? 'NoSuchKey' : 'InternalError'), err.message);
|
||||
json(res, status, { error: status >= 500 && !err.status ? 'internal error' : err.message });
|
||||
});
|
||||
});
|
||||
server.requestTimeout = 0; // large uploads and downloads stream for a long time
|
||||
|
||||
const [host, port] = cfg.listen.split(':');
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(Number(port), host, resolve); });
|
||||
log(`pithos-hosted on ${cfg.listen} for ${cfg.publicUrl}`);
|
||||
|
||||
return {
|
||||
server,
|
||||
supervisor: sup,
|
||||
async close() {
|
||||
clearInterval(meterTimer);
|
||||
for (const [id, u] of usage) writeJson(path.join(sup.dir(id), 'usage.json'), u);
|
||||
await new Promise((r) => server.close(r));
|
||||
await sup.closeAll();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function json(res, status, body) {
|
||||
const data = JSON.stringify(body);
|
||||
res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' });
|
||||
res.end(data);
|
||||
}
|
||||
|
||||
function s3Error(res, status, code, message) {
|
||||
const esc = (s) => String(s).replace(/[<>&'"]/g, (c) => `&#${c.charCodeAt(0)};`);
|
||||
res.writeHead(status, { 'content-type': 'application/xml' });
|
||||
res.end(`<?xml version="1.0" encoding="UTF-8"?><Error><Code>${esc(code)}</Code><Message>${esc(message)}</Message></Error>`);
|
||||
}
|
||||
|
||||
function readAll(stream) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const parts = [];
|
||||
stream.on('data', (c) => parts.push(c));
|
||||
stream.on('end', () => resolve(Buffer.concat(parts).toString('utf8')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function jsonBody(req) {
|
||||
let size = 0;
|
||||
const parts = [];
|
||||
for await (const c of req) { size += c.length; if (size > 1024 * 1024) throw httpError(413, 'request too large'); parts.push(c); }
|
||||
try { return JSON.parse(Buffer.concat(parts).toString('utf8') || '{}'); } catch { throw httpError(400, 'invalid JSON'); }
|
||||
}
|
||||
|
||||
function done(res) {
|
||||
return new Promise((resolve) => { if (res.writableFinished) return resolve(); res.on('finish', resolve); res.on('close', resolve); });
|
||||
}
|
||||
|
||||
function fmtBytes(n) {
|
||||
const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
let i = 0;
|
||||
while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; }
|
||||
return `${n.toFixed(n < 10 && i ? 1 : 0)} ${u[i]}`;
|
||||
}
|
||||
96
hosted/sigv4.js
Normal file
96
hosted/sigv4.js
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
// SigV4 on the receiving side. The gateway reads the access key out of every
|
||||
// S3 request to know whose s3d it is for. Keys it minted itself (grants to
|
||||
// other people) are fully verified here, because s3d never sees them; an
|
||||
// owner's own s3d keys are verified again by s3d, so for those the gateway
|
||||
// only needs the key id.
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import { uriEncode, canonicalQuery, signingKey, REGION, SERVICE } from '../core/s3.js';
|
||||
|
||||
const MAX_SKEW_MS = 15 * 60_000;
|
||||
|
||||
// { akid, presigned, signedHeaders, signature, date, scope } or null.
|
||||
export function parseAuth(req, url) {
|
||||
const h = req.headers.authorization;
|
||||
if (h) {
|
||||
const m = /^AWS4-HMAC-SHA256\s+Credential=([^/,\s]+)\/(\d{8})\/([^/]+)\/([^/]+)\/aws4_request,\s*SignedHeaders=([^,\s]+),\s*Signature=([0-9a-f]{64})\s*$/i.exec(h);
|
||||
if (!m) return null;
|
||||
return { akid: m[1], day: m[2], region: m[3], service: m[4], signedHeaders: m[5].toLowerCase(), signature: m[6].toLowerCase(), date: req.headers['x-amz-date'], presigned: false };
|
||||
}
|
||||
const q = url.searchParams;
|
||||
if (q.get('X-Amz-Algorithm') === 'AWS4-HMAC-SHA256') {
|
||||
const cred = /^([^/]+)\/(\d{8})\/([^/]+)\/([^/]+)\/aws4_request$/.exec(q.get('X-Amz-Credential') || '');
|
||||
if (!cred) return null;
|
||||
return {
|
||||
akid: cred[1], day: cred[2], region: cred[3], service: cred[4],
|
||||
signedHeaders: (q.get('X-Amz-SignedHeaders') || '').toLowerCase(),
|
||||
signature: (q.get('X-Amz-Signature') || '').toLowerCase(),
|
||||
date: q.get('X-Amz-Date'), expires: Number(q.get('X-Amz-Expires') || 0), presigned: true,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseAmzDate(s) {
|
||||
const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(s || '');
|
||||
return m ? Date.UTC(+m[1], m[2] - 1, +m[3], +m[4], +m[5], +m[6]) : NaN;
|
||||
}
|
||||
|
||||
// The raw path as the client sent it, decoded and re-encoded the one way
|
||||
// SigV4 for S3 defines, so clients that over- or under-escape still match.
|
||||
export function canonicalPath(rawPath) {
|
||||
const decoded = rawPath.split('/').map((s) => { try { return decodeURIComponent(s); } catch { return s; } });
|
||||
return decoded.map((s) => uriEncode(s)).join('/');
|
||||
}
|
||||
|
||||
function rawQuery(req) {
|
||||
const i = req.url.indexOf('?');
|
||||
const out = {};
|
||||
if (i < 0) return out;
|
||||
for (const part of req.url.slice(i + 1).split('&')) {
|
||||
if (!part) continue;
|
||||
const eq = part.indexOf('=');
|
||||
const dec = (s) => { try { return decodeURIComponent(s.replace(/\+/g, '%20')); } catch { return s; } };
|
||||
const k = dec(eq < 0 ? part : part.slice(0, eq));
|
||||
out[k] = eq < 0 ? '' : dec(part.slice(eq + 1));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Throws with an S3-style code when the signature does not hold.
|
||||
export function verify(req, url, auth, secretKey, now = Date.now()) {
|
||||
if (auth.region !== REGION || auth.service !== SERVICE) throw sigError('AuthorizationHeaderMalformed', `use region ${REGION}`);
|
||||
const t = parseAmzDate(auth.date);
|
||||
if (!Number.isFinite(t) || auth.date.slice(0, 8) !== auth.day) throw sigError('AccessDenied', 'missing or inconsistent X-Amz-Date');
|
||||
if (auth.presigned) {
|
||||
if (!(auth.expires > 0 && auth.expires <= 604800)) throw sigError('AuthorizationQueryParametersError', 'bad X-Amz-Expires');
|
||||
if (now > t + auth.expires * 1000) throw sigError('AccessDenied', 'Request has expired');
|
||||
if (t - now > MAX_SKEW_MS) throw sigError('RequestTimeTooSkewed', 'request date is in the future');
|
||||
} else if (Math.abs(now - t) > MAX_SKEW_MS) {
|
||||
throw sigError('RequestTimeTooSkewed', 'the difference between the request time and the server time is too large');
|
||||
}
|
||||
const query = rawQuery(req);
|
||||
if (auth.presigned) delete query['X-Amz-Signature'];
|
||||
const names = auth.signedHeaders.split(';').filter(Boolean);
|
||||
if (!names.includes('host')) throw sigError('AccessDenied', 'host must be signed');
|
||||
const headerLines = names.map((n) => {
|
||||
const v = req.headers[n];
|
||||
if (v == null) throw sigError('AccessDenied', `signed header ${n} is missing`);
|
||||
return `${n}:${String(Array.isArray(v) ? v.join(',') : v).trim().replace(/\s+/g, ' ')}\n`;
|
||||
}).join('');
|
||||
const payload = auth.presigned ? 'UNSIGNED-PAYLOAD' : (req.headers['x-amz-content-sha256'] || 'UNSIGNED-PAYLOAD');
|
||||
const canonical = [req.method, canonicalPath(url.pathname === '' ? '/' : req.url.split('?')[0]), canonicalQuery(query), headerLines, auth.signedHeaders, payload].join('\n');
|
||||
const scope = `${auth.day}/${REGION}/${SERVICE}/aws4_request`;
|
||||
const toSign = ['AWS4-HMAC-SHA256', auth.date, scope, crypto.createHash('sha256').update(canonical).digest('hex')].join('\n');
|
||||
const want = crypto.createHmac('sha256', signingKey(secretKey, auth.day)).update(toSign).digest();
|
||||
const got = Buffer.from(auth.signature, 'hex');
|
||||
if (got.length !== want.length || !crypto.timingSafeEqual(got, want)) {
|
||||
throw sigError('SignatureDoesNotMatch', 'The request signature we calculated does not match the signature you provided');
|
||||
}
|
||||
}
|
||||
|
||||
export class SigError extends Error {
|
||||
constructor(code, message, status = 403) { super(message); this.code = code; this.status = status; }
|
||||
}
|
||||
|
||||
function sigError(code, message) { return new SigError(code, message, code === 'AuthorizationHeaderMalformed' || code === 'AuthorizationQueryParametersError' ? 400 : 403); }
|
||||
16
hosted/store.js
Normal file
16
hosted/store.js
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
// Small JSON files written atomically: the gateway's whole state is files
|
||||
// under its root, so a backup is a copy of one folder.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
export function readJson(file, fallback) {
|
||||
try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return fallback; }
|
||||
}
|
||||
|
||||
export function writeJson(file, value) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const tmp = `${file}.${process.pid}.tmp`;
|
||||
fs.writeFileSync(tmp, JSON.stringify(value, null, 2), { mode: 0o600 });
|
||||
fs.renameSync(tmp, file);
|
||||
}
|
||||
206
hosted/supervisor.js
Normal file
206
hosted/supervisor.js
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
// One s3d per hosted account, each wrapped in the same createPithos() the
|
||||
// desktop runs, so the server speaks exactly the /api the Pithos UI already
|
||||
// knows. Accounts live under <root>/users/<id>/:
|
||||
//
|
||||
// account.json who (public key), how they got in (gate), port slot, state
|
||||
// s3d.yml loopback-only addresses from the slot, random admin password
|
||||
// data/ the s3d data folder moved here from the user's PC
|
||||
// rules.json public folders, links and shared keys
|
||||
// usage.json bytes served this month
|
||||
//
|
||||
// An s3d starts on the account's first request and is stopped after
|
||||
// `idleMinutes` without one, flushing leftovers to Sia first. At most
|
||||
// `maxRunning` run at once; the least recently used idle one makes room.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createPithos } from '../core/server.js';
|
||||
import { makeCli } from '../core/cli.js';
|
||||
import { readConfig, writeConfig, randomPassword } from '../core/config.js';
|
||||
import { writePrefs } from '../core/prefs.js';
|
||||
import { S3Client } from '../core/s3.js';
|
||||
import * as admin from '../core/admin.js';
|
||||
import { readJson, writeJson } from './store.js';
|
||||
|
||||
export function createSupervisor(cfg, log = console.log) {
|
||||
const usersDir = path.join(cfg.root, 'users');
|
||||
const slotsFile = path.join(cfg.root, 'slots.json');
|
||||
const keysFile = path.join(cfg.root, 'keys.json');
|
||||
const running = new Map(); // id -> instance
|
||||
const loading = new Map(); // id -> Promise<instance>
|
||||
const keyIndex = readJson(keysFile, {}); // accessKeyId -> account id (owners' s3d keys)
|
||||
|
||||
const dir = (id) => path.join(usersDir, id);
|
||||
const accountFile = (id) => path.join(dir(id), 'account.json');
|
||||
const configFile = (id) => path.join(dir(id), 's3d.yml');
|
||||
const dataDir = (id) => path.join(dir(id), 'data');
|
||||
|
||||
function account(id) {
|
||||
if (!/^[0-9a-f]{32}$/.test(id)) return null;
|
||||
return readJson(accountFile(id), null);
|
||||
}
|
||||
|
||||
function saveAccount(a) { writeJson(accountFile(a.id), a); }
|
||||
|
||||
function createAccount({ id, pubkey, gate }) {
|
||||
const slots = readJson(slotsFile, { next: 0 });
|
||||
const slot = slots.next++;
|
||||
writeJson(slotsFile, slots);
|
||||
const a = { id, pubkey, gate, slot, state: 'empty', createdAt: Date.now() };
|
||||
fs.mkdirSync(dataDir(id), { recursive: true });
|
||||
const port = cfg.portBase + slot * 2;
|
||||
writeConfig(configFile(id), {
|
||||
apiAddress: `127.0.0.1:${port}`,
|
||||
adminAddress: `127.0.0.1:${port + 1}`,
|
||||
adminPassword: randomPassword(),
|
||||
directory: dataDir(id),
|
||||
'sia.diskUsageLimit': cfg.bufferLimitBytes,
|
||||
});
|
||||
// Leftovers always go to Sia in hosted mode: an idle s3d gets stopped.
|
||||
writePrefs(configFile(id), { autostart: false, autoFlush: true, autoFlushMinutes: 10 });
|
||||
saveAccount(a);
|
||||
return a;
|
||||
}
|
||||
|
||||
function setState(id, state) {
|
||||
const a = account(id);
|
||||
a.state = state;
|
||||
saveAccount(a);
|
||||
writePrefs(configFile(id), { autostart: state === 'active' });
|
||||
return a;
|
||||
}
|
||||
|
||||
async function load(id) {
|
||||
if (running.has(id)) return running.get(id);
|
||||
if (loading.has(id)) return loading.get(id);
|
||||
const p = (async () => {
|
||||
await makeRoom();
|
||||
const pithos = await createPithos({
|
||||
host: '127.0.0.1', port: 0, hostName: 'hosted',
|
||||
configFile: configFile(id), binary: cfg.s3dBinary,
|
||||
});
|
||||
const inst = { id, pithos, cli: makeCli(pithos.daemon), cookie: pithos.internalSession(), lastUsed: Date.now(), busy: 0, buckets: new Map() };
|
||||
running.set(id, inst);
|
||||
pithos.daemon.on('log', (l) => { if (l.stream !== 'out') log(`[${id.slice(0, 8)}] ${l.line}`); });
|
||||
return inst;
|
||||
})().finally(() => loading.delete(id));
|
||||
loading.set(id, p);
|
||||
return p;
|
||||
}
|
||||
|
||||
// The instance with s3d up, for anything that talks S3.
|
||||
async function ensure(id) {
|
||||
const a = account(id);
|
||||
if (!a) throw httpError(404, 'no such account');
|
||||
if (a.state === 'empty') throw httpError(409, 'nothing is hosted yet: move your drives here from Pithos first');
|
||||
if (a.state === 'moving-out') throw httpError(409, 'this account is being moved to its owner\'s computer');
|
||||
const inst = await load(id);
|
||||
inst.lastUsed = Date.now();
|
||||
const d = inst.pithos.daemon;
|
||||
if (d.state !== 'running') {
|
||||
if (d.state !== 'starting') d.start();
|
||||
const until = Date.now() + 30_000;
|
||||
while (d.state === 'starting' && Date.now() < until) await sleep(200);
|
||||
if (d.state !== 'running') throw httpError(503, 'your s3d did not start; try again in a minute');
|
||||
await indexKeys(inst);
|
||||
}
|
||||
return inst;
|
||||
}
|
||||
|
||||
async function indexKeys(inst) {
|
||||
const keys = await inst.cli.listKeys().catch(() => []);
|
||||
let changed = false;
|
||||
for (const [k, v] of Object.entries(keyIndex)) if (v === inst.id && !keys.some((x) => x.accessKeyId === k)) { delete keyIndex[k]; changed = true; }
|
||||
for (const k of keys) if (keyIndex[k.accessKeyId] !== inst.id) { keyIndex[k.accessKeyId] = inst.id; changed = true; }
|
||||
if (changed) writeJson(keysFile, keyIndex);
|
||||
inst.keys = keys;
|
||||
inst.buckets.clear();
|
||||
return keys;
|
||||
}
|
||||
|
||||
// Every drive on the account, mapped to a key of the s3d user owning it.
|
||||
async function mapBuckets(inst) {
|
||||
inst.buckets.clear();
|
||||
for (const k of inst.keys || await indexKeys(inst)) {
|
||||
const list = await clientFor(inst, k).listBuckets().catch(() => []);
|
||||
for (const b of list) if (!inst.buckets.has(b.name)) inst.buckets.set(b.name, k);
|
||||
}
|
||||
return [...inst.buckets.keys()];
|
||||
}
|
||||
|
||||
// An owner key that can act on `bucket`.
|
||||
async function bucketClient(inst, bucket) {
|
||||
if (!inst.buckets.has(bucket)) await mapBuckets(inst);
|
||||
const k = inst.buckets.get(bucket);
|
||||
return k ? clientFor(inst, k) : null;
|
||||
}
|
||||
|
||||
function clientFor(inst, k) {
|
||||
return new S3Client({ endpoint: `http://${readConfig(configFile(inst.id)).apiAddress}`, accessKeyId: k.accessKeyId, secretKey: k.secretKey });
|
||||
}
|
||||
|
||||
async function close(id, reason) {
|
||||
const inst = running.get(id);
|
||||
if (!inst) return;
|
||||
running.delete(id);
|
||||
log(`[${id.slice(0, 8)}] stopping s3d (${reason})`);
|
||||
await inst.pithos.close().catch((e) => log(`[${id.slice(0, 8)}] close failed: ${e.message}`));
|
||||
}
|
||||
|
||||
async function flushQuietly(inst, timeoutMs = 120_000) {
|
||||
const c = readConfig(configFile(inst.id));
|
||||
try {
|
||||
const st = await admin.uploadStats(c);
|
||||
if (!st.pendingObjects) return true;
|
||||
await Promise.race([admin.flush(c), sleep(timeoutMs).then(() => { throw new Error('timed out'); })]);
|
||||
return true;
|
||||
} catch (e) {
|
||||
log(`[${inst.id.slice(0, 8)}] flush before stop failed: ${e.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function makeRoom() {
|
||||
if (running.size < cfg.maxRunning) return;
|
||||
const idle = [...running.values()].filter((i) => !i.busy).sort((a, b) => a.lastUsed - b.lastUsed);
|
||||
if (!idle.length) throw httpError(503, 'the server is busy; try again in a minute');
|
||||
await flushQuietly(idle[0]);
|
||||
await close(idle[0].id, 'making room');
|
||||
}
|
||||
|
||||
async function sweep() {
|
||||
const cutoff = Date.now() - cfg.idleMinutes * 60_000;
|
||||
for (const inst of [...running.values()]) {
|
||||
if (inst.busy || inst.lastUsed > cutoff) continue;
|
||||
await flushQuietly(inst);
|
||||
if (!inst.busy && inst.lastUsed <= cutoff) await close(inst.id, 'idle');
|
||||
}
|
||||
}
|
||||
const sweeper = setInterval(() => { sweep().catch((e) => log('sweep failed:', e.message)); }, 60_000);
|
||||
sweeper.unref?.();
|
||||
|
||||
// Holds an instance "in use" for the duration of a streamed request, so the
|
||||
// sweeper does not stop s3d under a long download.
|
||||
function hold(inst) {
|
||||
inst.busy++;
|
||||
let done = false;
|
||||
return () => { if (!done) { done = true; inst.busy--; inst.lastUsed = Date.now(); } };
|
||||
}
|
||||
|
||||
return {
|
||||
dir, dataDir, configFile, account, createAccount, saveAccount, setState,
|
||||
load, ensure, close, flushQuietly, bucketClient, mapBuckets, indexKeys, hold,
|
||||
ownerOfKey: (akid) => keyIndex[akid] || null,
|
||||
async closeAll() {
|
||||
clearInterval(sweeper);
|
||||
for (const id of [...running.keys()]) {
|
||||
await flushQuietly(running.get(id), 30_000);
|
||||
await close(id, 'shutdown');
|
||||
}
|
||||
},
|
||||
runningCount: () => running.size,
|
||||
};
|
||||
}
|
||||
|
||||
export function httpError(status, message) { return Object.assign(new Error(message), { status }); }
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
|
@ -36,6 +36,15 @@ module.exports = {
|
|||
openExternal: (url) => api.openTab(url),
|
||||
// "Back to sidebar" from the full-tab view.
|
||||
showPanel: () => api.revealSidebar("main"),
|
||||
// "On Silent Mode": sign-in identity and file encryption keys come
|
||||
// from the vault, so another computer with the same vault reaches
|
||||
// and reads the same drives. NEVER CHANGE THESE PATHS: encrypted
|
||||
// files would become unreadable.
|
||||
secrets: {
|
||||
kind: "vault",
|
||||
identitySeed: () => vaultKey("pithos/hosted-identity/v1", "Sign in to your drives on Silent Mode."),
|
||||
encryptionKey: () => vaultKey("pithos/hosted-encryption/v1", "Open your encrypted files on Silent Mode."),
|
||||
},
|
||||
});
|
||||
cookie = p.internalSession();
|
||||
pithos = p;
|
||||
|
|
@ -138,6 +147,20 @@ module.exports = {
|
|||
|
||||
api.onMessage("vault-status", () => vaultStatus());
|
||||
|
||||
async function vaultKey(purpose, reason) {
|
||||
const st = await vaultStatus();
|
||||
if (!st.setup) throw new Error("Set up the password vault in Settings › Passwords first.");
|
||||
if (!st.unlocked) {
|
||||
if (!canPrompt) throw new Error("Unlock the password vault in Settings › Passwords first.");
|
||||
const r = await vault.requestUnlock({ reason });
|
||||
if (!r.ok) throw new Error("The vault stayed locked.");
|
||||
}
|
||||
const bytes = await vault.derive(purpose);
|
||||
const key = Buffer.from(bytes.subarray(0, 32));
|
||||
bytes.fill(0);
|
||||
return key;
|
||||
}
|
||||
|
||||
// Watch for the vault locking (Settings › Lock now, or a lock from another
|
||||
// extension) and tell the panel at once. This runs in the main process:
|
||||
// a hidden panel's own timers are throttled to about once a minute.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue