From d00bd369b7075d4ef398c6a8fbbfac3daa514152 Mon Sep 17 00:00:00 2001 From: Local Dev Date: Sun, 4 Oct 2026 01:33:20 +0200 Subject: [PATCH] Pithos: run your drives on Silent Mode with your own Sia account s3d binds one Sia account per process, so the server runs one s3d per person (pithos-hosted), started on demand and flushed and stopped when idle. Only the connected s3d data folder moves between PC and server; the Sia recovery phrase never leaves the PC and nothing is re-uploaded to Sia. Because the server holds the connection key, local Pithos encrypts names and contents before they leave the machine, with keys from the Theseus vault (or a local key file elsewhere). What s3d cannot do on its own, the gateway now enforces: public folders, expiring share links and S3 keys for other people limited to one folder with read, add-only or edit rights. Files those reach are stored readable, since visitors have no key, and go back to encrypted when the rule is removed. Sign-up needs a BNS name signature, a deposit or an invite, with a per-IP limit. --- core/accounts.js | 11 +- core/hosted-routes.js | 382 ++++++++++++++++ core/hosted.js | 369 +++++++++++++++ core/s3.js | 21 +- core/server.js | 32 +- hosted/auth.js | 65 +++ hosted/deploy/nginx-s3.silentmode.st.conf | 31 ++ hosted/deploy/pithos-hosted.example.json | 19 + hosted/deploy/pithos-hosted.service | 22 + hosted/electrum.js | 30 ++ hosted/gate.js | 126 +++++ hosted/main.js | 39 ++ hosted/package.json | 18 + hosted/rules.js | 93 ++++ hosted/server.js | 531 ++++++++++++++++++++++ hosted/sigv4.js | 96 ++++ hosted/store.js | 16 + hosted/supervisor.js | 206 +++++++++ theseus/index.js | 23 + 19 files changed, 2117 insertions(+), 13 deletions(-) create mode 100644 core/hosted-routes.js create mode 100644 core/hosted.js create mode 100644 hosted/auth.js create mode 100644 hosted/deploy/nginx-s3.silentmode.st.conf create mode 100644 hosted/deploy/pithos-hosted.example.json create mode 100644 hosted/deploy/pithos-hosted.service create mode 100644 hosted/electrum.js create mode 100644 hosted/gate.js create mode 100644 hosted/main.js create mode 100644 hosted/package.json create mode 100644 hosted/rules.js create mode 100644 hosted/server.js create mode 100644 hosted/sigv4.js create mode 100644 hosted/store.js create mode 100644 hosted/supervisor.js diff --git a/core/accounts.js b/core/accounts.js index b2c925f..ba83434 100644 --- a/core/accounts.js +++ b/core/accounts.js @@ -15,7 +15,10 @@ const ARCHIVE_PREFIX = 'previous-account-'; const META = 'pithos-archive.json'; const LABEL_FILE = 'pithos.json'; -const keep = (name) => name === 's3d.yml' || name.startsWith(ARCHIVE_PREFIX); +// Files that belong to this computer, not to a Sia account. The hosted key +// file above all: moving or deleting it would make encrypted files unreadable. +const LOCAL = new Set(['s3d.yml', 'pithos-prefs.json', 'pithos-hosted-secret.json']); +const keep = (name) => LOCAL.has(name) || name.startsWith(ARCHIVE_PREFIX); function stamp(d = new Date()) { const p = (n) => String(n).padStart(2, '0'); @@ -48,6 +51,7 @@ export function listArchives(dataDir) { indexerUrl: meta.indexerUrl || null, label: meta.label || readLabel(dir), users: meta.users || [], + movedTo: meta.movedTo || null, hasData: fs.existsSync(path.join(dir, 's3d.db')), }; }); @@ -76,6 +80,11 @@ export function restoreArchive(dataDir, name, metaForCurrent = {}) { if (!name.startsWith(ARCHIVE_PREFIX) || name.includes('/') || name.includes('\\') || name.includes('..')) throw new Error('not an account archive'); const src = path.join(dataDir, name); if (!fs.existsSync(path.join(src, 's3d.db'))) throw new Error('that archive has no s3d data'); + let meta = {}; + try { meta = JSON.parse(fs.readFileSync(path.join(src, META), 'utf8')); } catch {} + // The copy left behind when drives moved to Silent Mode is out of date the + // moment the server s3d writes anything; running it would fork the account. + if (meta.movedTo) throw new Error(`these drives now live on ${meta.movedTo}; move them back from Settings instead`); const archivedCurrent = fs.existsSync(path.join(dataDir, 's3d.db')) ? archiveCurrent(dataDir, metaForCurrent).name : null; // Anything left (tmp folders from a fresh, unconnected s3d) is clutter. for (const entry of fs.readdirSync(dataDir)) { diff --git a/core/hosted-routes.js b/core/hosted-routes.js new file mode 100644 index 0000000..d5a97c8 --- /dev/null +++ b/core/hosted-routes.js @@ -0,0 +1,382 @@ +// "On Silent Mode" inside the control server. In hosted mode the UI talks to +// this server exactly as before; this module answers by: +// - forwarding daemon, users, keys, stats and status calls to the gateway, +// - giving the object routes an S3 client that encrypts and decrypts here, +// - turning "Access" and "Share link" into gateway rules (and converting the +// files they cover to readable copies, since visitors have no key), +// - moving the s3d data folder to the server and back. + +import fs from 'node:fs'; +import path from 'node:path'; +import * as admin from './admin.js'; +import * as accounts from './accounts.js'; +import { readPrefs, writePrefs } from './prefs.js'; +import { ensureConfig } from './config.js'; +import { + fileSecrets, signIn, gateway, HostedError, EncryptingS3Client, plainCovered, + dataFiles, uploadData, downloadData, +} from './hosted.js'; + +export const DEFAULT_HOSTED_URL = 'https://s3.silentmode.st'; + +// Calls forwarded as they are. The gateway allows the same set. +const FORWARD = [ + ['GET', /^\/api\/status$/], ['GET', /^\/api\/logs$/], ['GET', /^\/api\/stats$/], + ['POST', /^\/api\/flush$/], ['GET', /^\/api\/autoflush$/], + ['POST', /^\/api\/daemon\/(start|restart)$/], + ['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/], + ['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/], + ['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/], +]; +// Things that act on this computer's s3d and make no sense while drives live +// on the server. +const LOCAL_ONLY = [ + [/^\/api\/binary\/install$/, 'There is nothing to install while your drives are on Silent Mode.'], + [/^\/api\/login/, 'Your drives are on Silent Mode, already connected to your Sia account.'], + [/^\/api\/account\/(switch|restore)$/, 'Move your drives back to this computer before switching Sia accounts.'], + [/^\/api\/daemon\/stop$/, 'On Silent Mode, s3d stops by itself when nobody uses it.'], +]; + +export function installHosted(ctx) { + const { route, configFile, cfg, daemon, broadcast, registration, resetRegistration, HttpError } = ctx; + const secrets = ctx.secrets || fileSecrets(configFile); + const prefs = () => readPrefs(configFile); + const hp = () => prefs().hosted || null; + const isHosted = () => prefs().mode === 'hosted' && !!hp()?.token; + const gw = () => { + const h = hp(); + if (!h?.token) throw new HttpError(401, 'sign in to Silent Mode first'); + return gateway(h.url, h.token); + }; + const wrap = async (fn) => { + try { return await fn(); } catch (e) { + if (e instanceof HostedError) { + if (e.status === 401) { writePrefs(configFile, { hosted: { ...hp(), token: null } }); } + throw new HttpError(e.status, e.message); + } + throw e; + } + }; + + // ---- rules, cached briefly ---- + let rulesCache = null; + async function rules() { + if (!isHosted()) return []; + if (rulesCache && Date.now() - rulesCache.at < 30_000) return rulesCache.list; + const list = await wrap(() => gw().call('GET', '/hosted/v1/rules')); + rulesCache = { at: Date.now(), list }; + return list; + } + const dropRules = () => { rulesCache = null; }; + + // ---- the S3 client for object routes ---- + const keyCache = new Map(); + async function hostedKeys(user) { + const c = keyCache.get(user); + if (c && Date.now() - c.at < 60_000) return c.keys; + const keys = await wrap(() => gw().call('GET', `/api/keys?user=${encodeURIComponent(user)}`)); + keyCache.set(user, { at: Date.now(), keys }); + return keys; + } + + async function s3For(user, localKeys) { + if (isHosted()) { + const keys = await hostedKeys(user); + if (!keys.length) throw new HttpError(409, `user "${user}" has no access keys yet - create one first`); + return new EncryptingS3Client({ + endpoint: hp().url, accessKeyId: keys[0].accessKeyId, secretKey: keys[0].secretKey, + master: () => secrets.encryptionKey(), rules, + }); + } + // Back on this computer, files encrypted while hosted still decrypt. + if (prefs().encryptedData && localKeys?.length) { + return new EncryptingS3Client({ + endpoint: `http://${cfg().apiAddress}`, accessKeyId: localKeys[0].accessKeyId, secretKey: localKeys[0].secretKey, + master: () => secrets.encryptionKey(), encryptWrites: false, + }); + } + return null; + } + + // ---- status, logs and events from the server ---- + let remote = null; // last /api/status from the gateway + async function hostedStatus(local) { + const h = hp(); + try { + remote = await wrap(() => gw().call('GET', '/api/status')); + return { ...remote, host: local.host, mode: 'hosted', login: local.login, hosted: { ...remote.hosted, url: h.url } }; + } catch (e) { + return { ...local, mode: 'hosted', daemon: { state: 'stopped', error: e.message }, hosted: { url: h.url, error: e.message }, registration: { registered: true } }; + } + } + let lastSeq = 0; + let lastDaemon = ''; + async function pollRemote() { + if (!isHosted() || !ctx.hasSubscribers()) return; + try { + const st = await gw().call('GET', '/api/status'); + remote = st; + const d = JSON.stringify(st.daemon); + if (d !== lastDaemon) { lastDaemon = d; broadcast('status', st.daemon); } + const logs = await gw().call('GET', `/api/logs?since=${lastSeq}`); + for (const l of logs || []) { lastSeq = Math.max(lastSeq, l.seq); broadcast('log', l); } + } catch { /* the next poll tries again */ } + } + const poller = setInterval(() => { pollRemote(); }, 5000); + poller.unref?.(); + + // ---- intercept: runs before the local routes in hosted mode ---- + async function intercept(req, res, url, send) { + if (!isHosted()) return false; + const p = url.pathname; + for (const [re, msg] of LOCAL_ONLY) if (re.test(p)) throw new HttpError(409, msg); + if (p === '/api/config' && req.method === 'PUT') throw new HttpError(409, 'Server settings are managed by Silent Mode while your drives are there.'); + if (p === '/api/status') return false; // answered by the local route, which asks us + if (!FORWARD.some(([m, re]) => m === req.method && re.test(p))) return false; + const body = req.method === 'GET' ? undefined : await readJson(req); + const out = await wrap(() => gw().call(req.method, p + url.search, { body })); + if (/^\/api\/(users|keys)/.test(p) && req.method !== 'GET') keyCache.clear(); + send(out); + return true; + } + + // ---- routes ---- + route('GET', '/api/hosted', async () => { + const h = hp(); + const out = { mode: isHosted() ? 'hosted' : 'local', url: h?.url || DEFAULT_HOSTED_URL, id: h?.id || null, signedIn: !!h?.token, secrets: secrets.kind || 'vault' }; + if (h?.token) { + try { out.me = await gw().call('GET', '/hosted/v1/me'); } catch (e) { out.error = e.message; if (e.status === 401) out.signedIn = false; } + } + if (!out.signedIn) { + try { out.info = await gateway(out.url).call('GET', '/hosted/v1/info'); } catch (e) { out.error = `Silent Mode is not reachable: ${e.message}`; } + } + return out; + }); + + route('POST', '/api/hosted/signin', async (req) => { + const { url = DEFAULT_HOSTED_URL, signup } = await readJson(req); + if (!/^https:\/\/[^\s/]+$|^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/.test(url.replace(/\/$/, ''))) throw new HttpError(400, 'the server address must be https'); + const r = await wrap(() => signIn(url.replace(/\/$/, ''), secrets, signup)); + if (r.needsSignup) return r; + writePrefs(configFile, { hosted: { url: url.replace(/\/$/, ''), id: r.id, token: r.token } }); + return { ok: true, id: r.id, state: r.state }; + }); + + route('POST', '/api/hosted/signout', async () => { + const h = hp(); + if (h?.token) await gw().call('DELETE', '/hosted/v1/session').catch(() => {}); + writePrefs(configFile, { hosted: h ? { ...h, token: null } : null, mode: 'local' }); + dropRules(); + keyCache.clear(); + return { ok: true }; + }); + + // Use drives already on Silent Mode from this computer (a second device). + route('POST', '/api/hosted/mode', async (req) => { + const { mode } = await readJson(req); + if (mode === 'local') { writePrefs(configFile, { mode: 'local' }); return { mode: 'local' }; } + if (mode !== 'hosted') throw new HttpError(400, 'mode is local or hosted'); + const me = await wrap(() => gw().call('GET', '/hosted/v1/me')); + if (me.state !== 'active') throw new HttpError(409, 'Silent Mode has no drives for you yet. Move them there first.'); + writePrefs(configFile, { mode: 'hosted', encryptedData: true }); + return { mode: 'hosted' }; + }); + + // Local -> Silent Mode. Nothing is re-uploaded to Sia: s3d's database and + // any objects still waiting for Sia move, then s3d runs on the server. + route('POST', '/api/hosted/move-up', async (req) => { + const { force } = await readJson(req); + const g = gw(); + const progress = (p) => broadcast('move', { direction: 'up', ...p }); + const me = await wrap(() => g.call('GET', '/hosted/v1/me')); + if (me.state !== 'empty') throw new HttpError(409, 'Silent Mode already holds drives for you. Use them from here, or move them back first.'); + const c = cfg(); + const reg = await registration(); + if (!reg?.registered) throw new HttpError(409, 'Connect this computer to your Sia account first; then move the connection to Silent Mode.'); + const wasRunning = !!daemon.child; + if (wasRunning) { + progress({ phase: 'flush' }); + try { await admin.flush(c); } catch (e) { + if (!force) throw new HttpError(409, `Some uploads could not reach Sia yet (${e.message}). Try again later, or move anyway: they will finish on the server.`); + } + } + progress({ phase: 'stop' }); + await daemon.stop(); + let committed = false; + try { + const files = dataFiles(c.directory); + if (!files.some((f) => f.path === 's3d.db')) throw new HttpError(409, 'there is no s3d database to move'); + await uploadData(g, c.directory, files, progress); + progress({ phase: 'start' }); + const r = await wrap(() => g.call('POST', '/hosted/v1/data/commit', { body: { files } })); + committed = true; + if (!r.registered) throw new HttpError(502, 'the server could not use the moved connection'); + // Keep the local copy, set aside like an account switch, never deleted. + const meta = { indexerUrl: reg.indexerUrl, users: r.users, movedTo: hp().url }; + accounts.archiveCurrent(c.directory, meta); + resetRegistration(); + writePrefs(configFile, { mode: 'hosted', encryptedData: true, autostart: false }); + keyCache.clear(); + dropRules(); + progress({ phase: 'done' }); + return { ok: true, ...r }; + } catch (e) { + if (!committed) await g.call('POST', '/hosted/v1/data/abort').catch(() => {}); + if (wasRunning && !committed) { try { ensureConfig(configFile); daemon.start(); } catch {} } + progress({ phase: 'failed', error: e.message }); + throw e; + } + }); + + // Silent Mode -> local. The server flushes and stops its s3d, the folder + // comes down, the local s3d starts on it, then the server copy is deleted. + route('POST', '/api/hosted/move-down', async (req) => { + const { force } = await readJson(req); + const g = gw(); + const progress = (p) => broadcast('move', { direction: 'down', ...p }); + const c = cfg(); + if (daemon.child) await daemon.stop(); + let archived = null; + if (fs.existsSync(path.join(c.directory, 's3d.db'))) { + archived = accounts.archiveCurrent(c.directory, { note: 'set aside to bring drives back from Silent Mode' }).name; + } + progress({ phase: 'flush' }); + const man = await wrap(() => g.call('POST', `/hosted/v1/data/checkout${force ? '?force=1' : ''}`)); + try { + await downloadData(g, c.directory, man.files, progress); + progress({ phase: 'start' }); + ensureConfig(configFile); + writePrefs(configFile, { mode: 'local', autostart: true, encryptedData: true }); + resetRegistration(); + daemon.start(); + const until = Date.now() + 30_000; + while (daemon.state === 'starting' && Date.now() < until) await new Promise((r) => setTimeout(r, 200)); + if (daemon.state !== 'running') throw new Error('s3d did not start on the moved folder'); + const reg = await registration(); + if (!reg?.registered) throw new Error('the moved folder has no Sia connection'); + await wrap(() => g.call('POST', '/hosted/v1/data/release')); + keyCache.clear(); + dropRules(); + progress({ phase: 'done' }); + return { ok: true }; + } catch (e) { + await daemon.stop().catch(() => {}); + for (const f of man.files) fs.rmSync(path.join(c.directory, ...f.path.split('/')), { force: true }); + if (archived) { try { accounts.restoreArchive(c.directory, archived, {}); } catch {} } + writePrefs(configFile, { mode: 'hosted' }); + await g.call('POST', '/hosted/v1/data/cancel-checkout').catch(() => {}); + progress({ phase: 'failed', error: e.message }); + throw new HttpError(502, `Moving back failed and nothing changed: ${e.message}`); + } + }); + + // Rules: public folders, shared keys, links. Files they cover become + // readable copies; removing a rule encrypts them again. + route('GET', '/api/hosted/rules', async () => rules()); + + route('POST', '/api/hosted/rules', async (req) => { + const body = await readJson(req); + const client = await s3For(body.user); + if (!client) throw new HttpError(409, 'not on Silent Mode'); + const rule = await wrap(() => gw().call('POST', '/hosted/v1/rules', { body })); + dropRules(); + await client.convert(rule.bucket, rule.prefix, { toPlain: true, onProgress: (p) => broadcast('convert', { rule: rule.id, ...p }) }); + return rule; + }); + + route('DELETE', '/api/hosted/rules', async (req, url) => { + const id = url.searchParams.get('id'); + const user = url.searchParams.get('user'); + const rule = (await rules()).find((r) => r.id === id); + if (!rule) throw new HttpError(404, 'no such rule'); + await wrap(() => gw().call('DELETE', `/hosted/v1/rules?id=${encodeURIComponent(id)}`)); + dropRules(); + const remaining = await rules(); + const client = await s3For(user); + if (client) { + await client.convert(rule.bucket, rule.prefix, { + toPlain: false, + skip: (key) => plainCovered(remaining, rule.bucket, key), + onProgress: (p) => broadcast('convert', { rule: rule.id, ...p }), + }); + } + return { ok: true }; + }); + + // The whole-drive "Access" dialog, answered with a public rule on "". + async function drivePolicy(bucket) { + const r = (await rules()).find((x) => x.type === 'public' && x.bucket === bucket && x.prefix === ''); + return { policy: null, public: r ? r.rights : 'private', hosted: true }; + } + async function setDrivePolicy(user, bucket, mode) { + const current = (await rules()).filter((x) => x.type === 'public' && x.bucket === bucket && x.prefix === ''); + const client = await s3For(user); + for (const r of current) await wrap(() => gw().call('DELETE', `/hosted/v1/rules?id=${encodeURIComponent(r.id)}`)); + dropRules(); + if (mode === 'private') { + const remaining = await rules(); + await client.convert(bucket, '', { toPlain: false, skip: (key) => plainCovered(remaining, bucket, key), onProgress: (p) => broadcast('convert', p) }); + return; + } + await wrap(() => gw().call('POST', '/hosted/v1/rules', { body: { type: 'public', bucket, prefix: '', rights: mode } })); + dropRules(); + await client.convert(bucket, '', { toPlain: true, onProgress: (p) => broadcast('convert', p) }); + } + + // "Share link" for one file: a link rule, and the file stored readable. + async function shareLink(user, bucket, key, seconds) { + const client = await s3For(user); + const expires = Date.now() + Math.min(Math.max(60, seconds | 0), 30 * 86400) * 1000; + const rule = await wrap(() => gw().call('POST', '/hosted/v1/rules', { body: { type: 'link', bucket, prefix: key, expires } })); + dropRules(); + await client.convert(bucket, key, { toPlain: true, skip: (k) => k !== key }); + const links = (prefs().hostedLinks || []).filter((l) => l.expires > Date.now()); + links.push({ user, bucket, key, expires }); + writePrefs(configFile, { hostedLinks: links }); + return `${hp().url}/l/${rule.token}`; + } + + // Files shared by a link that has run out go back to being encrypted. + async function sweepLinks() { + if (!isHosted()) return; + const all = prefs().hostedLinks || []; + const expired = all.filter((l) => l.expires <= Date.now()); + if (!expired.length) return; + dropRules(); + const remaining = await rules(); + for (const l of expired) { + if (plainCovered(remaining, l.bucket, l.key)) continue; + const client = await s3For(l.user).catch(() => null); + await client?.convert(l.bucket, l.key, { toPlain: false, skip: (k) => k !== l.key }).catch(() => {}); + } + writePrefs(configFile, { hostedLinks: all.filter((l) => l.expires > Date.now()) }); + } + const sweeper = setInterval(() => { sweepLinks().catch(() => {}); }, 10 * 60_000); + sweeper.unref?.(); + + // Key file export/import for hosts without a vault. + route('GET', '/api/hosted/secret', () => { + if (!secrets.exportSecret) throw new HttpError(400, 'your keys come from the Theseus vault; back up the vault instead'); + return secrets.exportSecret(); + }); + route('POST', '/api/hosted/secret', async (req) => { + if (!secrets.importSecret) throw new HttpError(400, 'your keys come from the Theseus vault'); + if (isHosted() || hp()?.token) throw new HttpError(409, 'sign out of Silent Mode before replacing your key'); + secrets.importSecret(await readJson(req)); + return { ok: true }; + }); + + return { + isHosted, intercept, s3For, hostedStatus, drivePolicy, setDrivePolicy, shareLink, + remoteDaemon: () => remote?.daemon || { state: 'stopped' }, + close() { clearInterval(poller); clearInterval(sweeper); }, + }; +} + +async function readJson(req) { + const parts = []; + for await (const c of req) parts.push(c); + const text = Buffer.concat(parts).toString('utf8'); + if (!text) return {}; + try { return JSON.parse(text); } catch { return {}; } +} diff --git a/core/hosted.js b/core/hosted.js new file mode 100644 index 0000000..404c448 --- /dev/null +++ b/core/hosted.js @@ -0,0 +1,369 @@ +// The client half of "On Silent Mode": signing in to pithos-hosted, the S3 +// client that encrypts on the way out and decrypts on the way in, converting +// folders between private (encrypted) and readable when they are shared, and +// moving the s3d data folder between this computer and the server. +// +// Nothing secret leaves this process: the identity key signs a nonce, the +// encryption key never goes anywhere, and the Sia recovery phrase is not +// involved at all (only the connected s3d data folder moves). + +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { S3Client, readBody, objectPath } from './s3.js'; +import { + driveKeys, encryptPath, decryptPath, encryptBody, decryptBody, cipherRange, plainSize, parseRange, HEADER_LEN, +} from './crypt.js'; + +// ---- secrets --------------------------------------------------------------- + +const PKCS8_ED25519 = Buffer.from('302e020100300506032b657004220420', 'hex'); + +// Hosts with a vault (Theseus) pass their own; everyone else gets a random +// secret in a file beside s3d.yml. Losing that file loses the files it +// encrypted, so the UI offers to export it. +export function fileSecrets(configFile) { + const file = path.join(path.dirname(configFile), 'pithos-hosted-secret.json'); + let root = null; + function load() { + if (root) return root; + try { root = Buffer.from(JSON.parse(fs.readFileSync(file, 'utf8')).secret, 'base64'); } catch {} + if (!root || root.length !== 32) { + root = crypto.randomBytes(32); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ v: 1, secret: root.toString('base64'), note: 'Pithos hosted identity and encryption key. Anyone with this file can read your encrypted files; without it they are lost.' }, null, 2), { mode: 0o600 }); + } + return root; + } + const derive = (label) => Buffer.from(crypto.hkdfSync('sha256', load(), Buffer.alloc(0), label, 32)); + return { + kind: 'file', + file, + identitySeed: async () => derive('pithos/hosted-identity/v1'), + encryptionKey: async () => derive('pithos/hosted-encryption/v1'), + exportSecret: () => JSON.parse(fs.readFileSync(file, 'utf8')), + importSecret(obj) { + const b = Buffer.from(String(obj?.secret || ''), 'base64'); + if (b.length !== 32) throw new Error('this is not a Pithos key file'); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ v: 1, secret: b.toString('base64'), note: 'Pithos hosted identity and encryption key.' }, null, 2), { mode: 0o600 }); + root = b; + }, + }; +} + +export function identityFromSeed(seed) { + const key = crypto.createPrivateKey({ key: Buffer.concat([PKCS8_ED25519, seed]), format: 'der', type: 'pkcs8' }); + const pub = crypto.createPublicKey(key).export({ format: 'der', type: 'spki' }).subarray(-32); + return { pubkey: pub.toString('hex'), sign: (msg) => crypto.sign(null, Buffer.from(msg, 'utf8'), key).toString('base64') }; +} + +// ---- gateway calls ------------------------------------------------------------- + +export class HostedError extends Error { + constructor(status, message, data) { super(message); this.status = status; this.data = data; } +} + +export function gateway(url, token) { + const base = url.replace(/\/$/, ''); + async function call(method, p, { body, headers = {}, raw, stream } = {}) { + const h = { ...headers }; + if (token) h.authorization = `Bearer ${token}`; + let payload = body; + if (body !== undefined && !stream && !Buffer.isBuffer(body)) { h['content-type'] = 'application/json'; payload = JSON.stringify(body); } + const res = await fetch(base + p, { method, headers: h, body: payload, duplex: stream ? 'half' : undefined }); + if (raw) return res; + const text = await res.text(); + let data = null; + try { data = text ? JSON.parse(text) : null; } catch { data = { error: text.slice(0, 300) }; } + if (!res.ok) throw new HostedError(res.status, data?.error || `HTTP ${res.status}`, data); + return data; + } + return { base, call, auth: token ? `Bearer ${token}` : null }; +} + +export async function signIn(url, secrets, signup) { + const g = gateway(url); + const id = identityFromSeed(await secrets.identitySeed()); + const { nonce, message } = await g.call('GET', '/hosted/v1/challenge'); + try { + return await g.call('POST', '/hosted/v1/session', { body: { pubkey: id.pubkey, nonce, signature: id.sign(message), signup } }); + } catch (e) { + if (e.status === 404 && e.data?.needsSignup) return { needsSignup: true, ...e.data }; + throw e; + } +} + +// ---- which files are readable ------------------------------------------------------ + +// Files anyone else can reach (public folders, shared keys, share links) must +// be stored readable; everything else is encrypted. +export function plainCovered(rules, bucket, key) { + const now = Date.now(); + return rules.some((r) => r.bucket === bucket && (!r.expires || r.expires > now) && ( + r.type === 'link' ? r.prefix === key : key.startsWith(r.prefix) + )); +} + +const MIME = { + html: 'text/html', htm: 'text/html', txt: 'text/plain', md: 'text/markdown', css: 'text/css', js: 'text/javascript', + json: 'application/json', xml: 'application/xml', csv: 'text/csv', pdf: 'application/pdf', zip: 'application/zip', + png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', svg: 'image/svg+xml', avif: 'image/avif', ico: 'image/x-icon', + mp3: 'audio/mpeg', ogg: 'audio/ogg', oga: 'audio/ogg', wav: 'audio/wav', flac: 'audio/flac', m4a: 'audio/mp4', opus: 'audio/opus', + mp4: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', mkv: 'video/x-matroska', +}; +export const guessType = (key) => MIME[String(key).split('.').pop().toLowerCase()] || 'application/octet-stream'; + +// ---- the encrypting S3 client ---------------------------------------------------------- + +// Same surface as S3Client for everything the control server's object routes +// use. Keys and sizes it returns are the plain ones. `encryptWrites` is false +// on this computer (local s3d already keeps files private) but reads still +// decrypt, so drives brought back from Silent Mode stay readable. +export class EncryptingS3Client { + constructor({ endpoint, accessKeyId, secretKey, master, rules = async () => [], encryptWrites = true }) { + this.inner = new S3Client({ endpoint, accessKeyId, secretKey }); + this.master = master; // async () => Buffer(32) + this.rules = rules; + this.encryptWrites = encryptWrites; + this.keys = new Map(); + } + + async dk(bucket) { + if (!this.keys.has(bucket)) this.keys.set(bucket, driveKeys(await this.master(), bucket)); + return this.keys.get(bucket); + } + + async storeAsPlain(bucket, key) { + return !this.encryptWrites || plainCovered(await this.rules(), bucket, key); + } + + listBuckets() { return this.inner.listBuckets(); } + createBucket(b) { return this.inner.createBucket(b); } + deleteBucket(b) { return this.inner.deleteBucket(b); } + presign(...a) { return this.inner.presign(...a); } + getPolicy(b) { return this.inner.getPolicy(b); } + putPolicy(b, p) { return this.inner.putPolicy(b, p); } + deletePolicy(b) { return this.inner.deletePolicy(b); } + + // A folder can hold both forms (files from before hosting are plain), so a + // listing below the root asks for both prefixes and merges them. + async listObjects(bucket, { prefix = '', delimiter = '/', token, max = 1000 } = {}) { + const dk = await this.dk(bucket); + const cut = prefix.lastIndexOf('/') + 1; + const folder = prefix.slice(0, cut); + const encFolder = folder ? encryptPath(dk, folder) : ''; + const state = token ? JSON.parse(Buffer.from(token, 'base64url').toString('utf8')) : {}; + const sources = folder ? [['e', encFolder], ['p', folder]] : [['p', '']]; + const folders = new Map(); + const objects = new Map(); + const next = {}; + for (const [name, pfx] of sources) { + if (state[name] === null) continue; + const page = await this.inner.listObjects(bucket, { prefix: pfx, delimiter, token: state[name] || undefined, max }); + next[name] = page.truncated ? page.nextToken : null; + for (const f of page.folders) { + const d = decryptPath(dk, f); + if (d.path.startsWith(prefix)) folders.set(d.path, { path: d.path, encrypted: d.encrypted }); + } + for (const o of page.objects) { + const d = decryptPath(dk, o.key); + if (!d.path.startsWith(prefix)) continue; + const size = d.encrypted && !d.path.endsWith('/') ? plainSize(o.size) : o.size; + // Same name in both forms (re-uploaded after hosting): the encrypted one is newer. + if (objects.has(d.path) && !d.encrypted) continue; + objects.set(d.path, { key: d.path, size: size ?? o.size, modified: o.modified, etag: o.etag, encrypted: d.encrypted, storageKey: o.key }); + } + } + const more = Object.values(next).some((v) => v); + const order = (a, b) => (a < b ? -1 : a > b ? 1 : 0); + return { + folders: [...folders.keys()].sort(order), + objects: [...objects.values()].sort((a, b) => order(a.key, b.key)), + truncated: more, + nextToken: more ? Buffer.from(JSON.stringify(next)).toString('base64url') : null, + }; + } + + async putObject(bucket, key, body, { contentType, contentLength } = {}) { + if (await this.storeAsPlain(bucket, key)) return this.inner.putObject(bucket, key, body, { contentType, contentLength }); + const dk = await this.dk(bucket); + const ek = encryptPath(dk, key); + if (key.endsWith('/')) return this.inner.putObject(bucket, ek, Buffer.alloc(0), { contentLength: 0 }); + const len = Number(contentLength ?? (Buffer.isBuffer(body) ? body.length : NaN)); + if (!Number.isFinite(len)) throw new Error('encrypted uploads need a Content-Length'); + const { stream, size } = encryptBody(dk, key, body, len); + const r = await this.inner.putObject(bucket, ek, stream, { contentType: 'application/octet-stream', contentLength: size }); + // An older readable copy under the same name would shadow nothing but + // would still be readable on the server: remove it. + if (ek !== key) await this.inner.deleteObject(bucket, key).catch(() => {}); + return r; + } + + // Resolves to a readable stream carrying statusCode and headers, like the + // http.IncomingMessage S3Client.getObject returns. + async getObject(bucket, key, { range } = {}) { + const dk = await this.dk(bucket); + const ek = encryptPath(dk, key); + const head = await this.inner.request('HEAD', objectPath(bucket, ek)); + head.resume(); + if (head.statusCode === 404 || head.statusCode === 403) return this.inner.getObject(bucket, key, { range }); + if (head.statusCode >= 300) return this.inner.getObject(bucket, ek, { range }); + const total = Number(head.headers['content-length']); + const r = cipherRange(total, range ? parseRange(range, plainSize(total) ?? 0) : null); + const base = { 'content-type': guessType(key), 'accept-ranges': 'bytes' }; + if (head.headers.etag) base.etag = head.headers.etag; + if (head.headers['last-modified']) base['last-modified'] = head.headers['last-modified']; + if (r.unsatisfiable) return withStatus(Readable.from([]), 416, { ...base, 'content-range': `bytes */${r.plain}` }); + const headerRes = await this.inner.getObject(bucket, ek, { range: `bytes=0-${r.first === 0 ? r.to : HEADER_LEN - 1}` }); + if (headerRes.statusCode >= 300) return headerRes; + // The header arrives first; when the wanted chunks follow it directly the + // same response carries them too. + const iter = headerRes[Symbol.asyncIterator](); + let first = Buffer.alloc(0); + while (first.length < HEADER_LEN) { + const { value, done } = await iter.next(); + if (done) throw new Error('encrypted file is shorter than its header'); + first = Buffer.concat([first, value]); + } + const header = first.subarray(0, HEADER_LEN); + let body; + if (r.first === 0) { + const rest = first.subarray(HEADER_LEN); + body = Readable.from((async function* () { if (rest.length) yield rest; for (;;) { const { value, done } = await iter.next(); if (done) return; yield value; } })()); + } else { + iter.return?.(); + headerRes.destroy(); + body = await this.inner.getObject(bucket, ek, { range: `bytes=${r.from}-${r.to}` }); + if (body.statusCode >= 300) return body; + } + const out = decryptBody(dk, key, header, r.plain ? body : Readable.from([]), r); + const headers = { ...base, 'content-length': String(r.plain ? r.end - r.start + 1 : 0) }; + let status = 200; + if (range) { status = 206; headers['content-range'] = `bytes ${r.start}-${r.end}/${r.plain}`; } + return withStatus(out, status, headers); + } + + async deleteObject(bucket, key) { + const dk = await this.dk(bucket); + const ek = encryptPath(dk, key); + await this.inner.deleteObject(bucket, ek); + if (ek !== key) await this.inner.deleteObject(bucket, key); + } + + async deletePrefix(bucket, prefix) { + let n = 0; + let token; + do { + const page = await this.listObjects(bucket, { prefix, delimiter: '', token }); + for (const o of page.objects) { await this.inner.deleteObject(bucket, o.storageKey); n++; } + token = page.truncated ? page.nextToken : null; + } while (token); + return n; + } + + // Re-stores every file under prefix in the wanted form: readable (shared) + // or encrypted (private). Rules must already say what the prefix should be. + async convert(bucket, prefix, { toPlain, onProgress = () => {}, skip = () => false }) { + const dk = await this.dk(bucket); + let done = 0; + let token; + const todo = []; + do { + const page = await this.listObjects(bucket, { prefix, delimiter: '', token }); + for (const o of page.objects) if (o.encrypted === toPlain && !skip(o.key)) todo.push(o); + token = page.truncated ? page.nextToken : null; + } while (token); + for (const o of todo) { + onProgress({ done, total: todo.length, key: o.key }); + const target = toPlain ? o.key : encryptPath(dk, o.key); + if (o.key.endsWith('/')) { + await this.inner.putObject(bucket, target, Buffer.alloc(0), { contentLength: 0 }); + } else if (toPlain) { + const src = await this.getObject(bucket, o.key); + if (src.statusCode >= 300) throw new Error(`could not read ${o.key}: HTTP ${src.statusCode}`); + await this.inner.putObject(bucket, target, src, { contentType: guessType(o.key), contentLength: o.size }); + } else { + const src = await this.inner.getObject(bucket, o.storageKey); + if (src.statusCode >= 300) throw new Error(`could not read ${o.key}: HTTP ${src.statusCode}`); + const { stream, size } = encryptBody(dk, o.key, src, o.size); + await this.inner.putObject(bucket, target, stream, { contentType: 'application/octet-stream', contentLength: size }); + } + await this.inner.deleteObject(bucket, o.storageKey); + done++; + } + onProgress({ done, total: todo.length }); + return { converted: done }; + } +} + +function withStatus(stream, statusCode, headers) { + stream.statusCode = statusCode; + stream.headers = headers; + return stream; +} + +// ---- moving the data folder ---------------------------------------------------------------- + +// What travels: the database and objects still waiting for Sia. Logs, temp +// files, the local config and earlier accounts stay on this computer. +const SKIP_TOP = new Set(['s3d.yml', 'pithos-prefs.json', 'pithos-hosted-secret.json', 'pithos-archive.json', 'tmp', 's3d.log']); +export function dataFiles(dataDir) { + const out = []; + const rec = (rel) => { + for (const e of fs.readdirSync(path.join(dataDir, rel), { withFileTypes: true })) { + const r = rel ? `${rel}/${e.name}` : e.name; + if (!rel && (SKIP_TOP.has(e.name) || e.name.startsWith('previous-account-') || e.name.endsWith('.log') || e.name.endsWith('.tmp'))) continue; + if (e.isDirectory()) rec(r); + else if (e.isFile()) { + const data = fs.readFileSync(path.join(dataDir, r)); + out.push({ path: r, size: data.length, sha256: crypto.createHash('sha256').update(data).digest('hex') }); + } + } + }; + rec(''); + return out; +} + +export async function uploadData(g, dataDir, files, onProgress = () => {}) { + let sent = 0; + const total = files.reduce((n, f) => n + f.size, 0); + for (const f of files) { + onProgress({ phase: 'upload', file: f.path, sent, total }); + const res = await fetch(`${g.base}/hosted/v1/data/file?path=${encodeURIComponent(f.path)}`, { + method: 'PUT', + headers: { authorization: g.auth, 'content-length': String(f.size) }, + body: fs.createReadStream(path.join(dataDir, f.path)), + duplex: 'half', + }); + const r = await res.json().catch(() => ({})); + if (!res.ok) throw new HostedError(res.status, r.error || `upload of ${f.path} failed`); + if (r.sha256 !== f.sha256) throw new Error(`${f.path} changed while it was being sent`); + sent += f.size; + } + onProgress({ phase: 'upload', sent, total }); +} + +export async function downloadData(g, dataDir, files, onProgress = () => {}) { + let got = 0; + const total = files.reduce((n, f) => n + f.size, 0); + for (const f of files) { + onProgress({ phase: 'download', file: f.path, got, total }); + const target = path.join(dataDir, ...f.path.split('/')); + if (!target.startsWith(path.resolve(dataDir))) throw new Error('bad path from server'); + fs.mkdirSync(path.dirname(target), { recursive: true }); + const res = await fetch(`${g.base}/hosted/v1/data/file?path=${encodeURIComponent(f.path)}`, { headers: { authorization: g.auth } }); + if (!res.ok) throw new HostedError(res.status, (await res.json().catch(() => ({}))).error || `download of ${f.path} failed`); + const hash = crypto.createHash('sha256'); + const src = Readable.fromWeb(res.body); + src.on('data', (c) => hash.update(c)); + await pipeline(src, fs.createWriteStream(target)); + if (hash.digest('hex') !== f.sha256) throw new Error(`${f.path} arrived damaged`); + got += f.size; + } + onProgress({ phase: 'download', got, total }); +} + +export { readBody }; diff --git a/core/s3.js b/core/s3.js index 2153ab0..31ce226 100644 --- a/core/s3.js +++ b/core/s3.js @@ -4,16 +4,17 @@ import crypto from 'node:crypto'; import http from 'node:http'; +import https from 'node:https'; import { Readable } from 'node:stream'; -const REGION = 'us-east-1'; -const SERVICE = 's3'; +export const REGION = 'us-east-1'; +export const SERVICE = 's3'; const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex'); const hmac = (k, s) => crypto.createHmac('sha256', k).update(s).digest(); // RFC 3986 encoding as SigV4 wants it; '/' kept for object keys in paths. -function uriEncode(str, keepSlash) { +export function uriEncode(str, keepSlash) { const out = encodeURIComponent(str) .replace(/[!'()*]/g, (c) => '%' + c.charCodeAt(0).toString(16).toUpperCase()); // (A "never matches" regex like /$^/ is not one: it matches the empty @@ -21,17 +22,17 @@ function uriEncode(str, keepSlash) { return keepSlash ? out.replace(/%2F/g, '/') : out; } -function canonicalQuery(query) { +export function canonicalQuery(query) { return Object.keys(query).sort() .map((k) => `${uriEncode(k)}=${uriEncode(String(query[k] ?? ''))}`) .join('&'); } -function amzDate(d = new Date()) { +export function amzDate(d = new Date()) { return d.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, ''); } -function signingKey(secret, date) { +export function signingKey(secret, date) { return hmac(hmac(hmac(hmac('AWS4' + secret, date), REGION), SERVICE), 'aws4_request'); } @@ -84,11 +85,13 @@ export class S3Client { // Low-level request. body: Buffer | string | Readable | undefined. // Resolves the raw http.IncomingMessage so callers can stream it. - request(method, path, { query = {}, headers = {}, body } = {}) { - const signedHeaders = this.sign(method, path, query, headers); + request(method, path, { query = {}, headers = {}, body, payloadHash } = {}) { + const signedHeaders = this.sign(method, path, query, headers, payloadHash ? { payloadHash } : undefined); const qs = canonicalQuery(query); + // The hosted gateway is https; a local s3d is plain http on loopback. + const transport = this.endpoint.protocol === 'https:' ? https : http; return new Promise((resolve, reject) => { - const req = http.request({ + const req = transport.request({ protocol: this.endpoint.protocol, hostname: this.endpoint.hostname, port: this.endpoint.port, diff --git a/core/server.js b/core/server.js index c890e2f..bfb486f 100644 --- a/core/server.js +++ b/core/server.js @@ -24,6 +24,7 @@ import { installS3d, S3D_VERSION, assetForPlatform } from './binary.js'; import * as accounts from './accounts.js'; import { readPrefs, writePrefs } from './prefs.js'; import { createAutoFlush, clampMinutes, DEFAULT_MINUTES } from './autoflush.js'; +import { installHosted } from './hosted-routes.js'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const UI_DIR = path.join(HERE, '..', 'ui'); @@ -49,6 +50,7 @@ export async function createPithos(opts = {}) { hostName = 'web', // 'web' | 'desktop' | 'theseus' — UI tweaks only openExternal, // (url) => void, for hosts that can open the system browser showPanel, // () => void, for hosts with a side panel to switch back to (Theseus) + secrets, // hosted identity + encryption keys; the Theseus vault, else a local key file } = opts; const configFile = resolveConfigPath(configOpt); @@ -67,6 +69,7 @@ export async function createPithos(opts = {}) { const subscribers = new Set(); let versionCache = null; + let hosted = null; // "On Silent Mode", installed below const broadcast = (event, data) => { const msg = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; for (const res of subscribers) res.write(msg); @@ -74,7 +77,8 @@ export async function createPithos(opts = {}) { daemon.on('status', (s) => { // A registration done outside Pithos (s3d login by hand) shows up on the next start. if (s.state === 'starting' && !regCache?.registered) regCache = null; - broadcast('status', s); + // While drives are hosted the UI follows the server's s3d instead. + if (!hosted?.isHosted()) broadcast('status', s); }); daemon.on('log', (l) => broadcast('log', l)); login.on('state', (s) => { @@ -119,8 +123,11 @@ export async function createPithos(opts = {}) { } async function s3For(user) { + if (hosted.isHosted()) return hosted.s3For(user); const keys = await cli.listKeys(user); if (!keys.length) throw new HttpError(409, `user "${user}" has no access keys yet - create one first`); + const decrypting = await hosted.s3For(user, keys); + if (decrypting) return decrypting; const c = cfg(); return new S3Client({ endpoint: `http://${c.apiAddress}`, accessKeyId: keys[0].accessKeyId, secretKey: keys[0].secretKey }); } @@ -143,7 +150,7 @@ export async function createPithos(opts = {}) { autostart('Pithos was opened'); } const v = await version(); - return { + const local = { host: hostName, daemon: daemon.status(), external: await probeExternal(c), @@ -152,6 +159,7 @@ export async function createPithos(opts = {}) { login: login.snapshot(), registration: await registration(), }; + return hosted.isHosted() ? hosted.hostedStatus(local) : local; }); // Start and Stop also record what the user wants, so s3d comes back by @@ -356,6 +364,8 @@ export async function createPithos(opts = {}) { // Objects are user content: never let them run script on this origin. headers['content-security-policy'] = "sandbox; default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'"; res.writeHead(up.statusCode, headers); + // A decryption failure mid-file must not look like a complete download. + up.on('error', () => res.destroy()); up.pipe(res); return STREAMED; }); @@ -404,15 +414,22 @@ export async function createPithos(opts = {}) { route('GET', '/api/s3/:user/buckets/:bucket/presign', async (req, url, p) => { const q = url.searchParams; + if (hosted.isHosted()) return { url: await hosted.shareLink(p.user, p.bucket, requireKey(url), Number(q.get('expires') || 3600)) }; const client = await s3For(p.user); return { url: client.presign(p.bucket, requireKey(url), Number(q.get('expires') || 3600), q.get('endpoint') || undefined) }; }); route('GET', '/api/s3/:user/buckets/:bucket/policy', async (req, url, p) => { + if (hosted.isHosted()) return hosted.drivePolicy(p.bucket); const policy = await (await s3For(p.user)).getPolicy(p.bucket); return { policy, public: describePolicy(policy) }; }); route('PUT', '/api/s3/:user/buckets/:bucket/policy', async (req, url, p) => { const { mode } = await jsonBody(req); // 'private' | 'read' | 'read-list' + if (hosted.isHosted()) { + if (!['private', 'read', 'read-list'].includes(mode)) throw new HttpError(400, 'mode must be private, read or read-list'); + await hosted.setDrivePolicy(p.user, p.bucket, mode); + return { ok: true }; + } const client = await s3For(p.user); if (mode === 'private') { await client.deletePolicy(p.bucket).catch((e) => { if (e.status !== 404) throw e; }); } else if (mode === 'read' || mode === 'read-list') await client.putPolicy(p.bucket, publicReadPolicy(p.bucket, { list: mode === 'read-list' })); @@ -420,6 +437,12 @@ export async function createPithos(opts = {}) { return { ok: true }; }); + hosted = installHosted({ + route, configFile, cfg, daemon, broadcast, registration, HttpError, secrets, + resetRegistration: () => { regCache = null; }, + hasSubscribers: () => subscribers.size > 0, + }); + // ---- server ------------------------------------------------------------ const STREAMED = Symbol('streamed'); @@ -493,9 +516,11 @@ export async function createPithos(opts = {}) { if (!viaTicket && !cookieSession(req)) throw new HttpError(401, 'not signed in'); if (req.method !== 'GET') requireCsrf(req); + if (await hosted.intercept(req, res, url, (out) => sendJson(res, 200, out))) return; + if (url.pathname === '/api/events') { res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-store', connection: 'keep-alive' }); - res.write(`event: status\ndata: ${JSON.stringify(daemon.status())}\n\n`); + res.write(`event: status\ndata: ${JSON.stringify(hosted.isHosted() ? hosted.remoteDaemon() : daemon.status())}\n\n`); res.write(`event: login\ndata: ${JSON.stringify(login.snapshot())}\n\n`); subscribers.add(res); const ping = setInterval(() => res.write(': ping\n\n'), 25_000); @@ -590,6 +615,7 @@ export async function createPithos(opts = {}) { }, async close() { clearInterval(autoFlushTimer); + hosted.close(); login.cancel(); for (const s of subscribers) s.end(); await daemon.stop(); diff --git a/hosted/auth.js b/hosted/auth.js new file mode 100644 index 0000000..14ccaf0 --- /dev/null +++ b/hosted/auth.js @@ -0,0 +1,65 @@ +// Sign-in: a person is an Ed25519 key (from their Theseus vault, or a key file +// on hosts without one). The gateway hands out a nonce, the client signs it, +// and gets a session token back. Only token hashes are stored. + +import crypto from 'node:crypto'; +import path from 'node:path'; +import { readJson, writeJson } from './store.js'; + +const SPKI_ED25519 = Buffer.from('302a300506032b6570032100', 'hex'); +const SESSION_DAYS = 90; + +export const accountId = (pubkeyHex) => crypto.createHash('sha256').update(Buffer.from(pubkeyHex, 'hex')).digest('hex').slice(0, 32); + +export function challengeText(publicUrl, nonce) { + return `pithos-hosted-v1\n${publicUrl}\n${nonce}`; +} + +export function verifyEd25519(pubkeyHex, message, signatureB64) { + if (!/^[0-9a-f]{64}$/i.test(pubkeyHex || '')) return false; + try { + const key = crypto.createPublicKey({ key: Buffer.concat([SPKI_ED25519, Buffer.from(pubkeyHex, 'hex')]), format: 'der', type: 'spki' }); + return crypto.verify(null, Buffer.from(message, 'utf8'), key, Buffer.from(String(signatureB64 || ''), 'base64')); + } catch { return false; } +} + +export function createAuth(root) { + const file = path.join(root, 'sessions.json'); + const nonces = new Map(); // nonce -> expiry + const sessions = readJson(file, {}); + + const hash = (t) => crypto.createHash('sha256').update(t).digest('hex'); + + return { + challenge() { + const now = Date.now(); + for (const [n, exp] of nonces) if (exp < now) nonces.delete(n); + if (nonces.size > 10_000) throw Object.assign(new Error('too many sign-ins in progress, try again shortly'), { status: 503 }); + const nonce = crypto.randomBytes(24).toString('base64url'); + nonces.set(nonce, now + 5 * 60_000); + return nonce; + }, + // Consumes the nonce whether or not the signature holds. + takeNonce(nonce) { + const exp = nonces.get(nonce); + nonces.delete(nonce); + return !!exp && exp > Date.now(); + }, + issue(id) { + const token = crypto.randomBytes(32).toString('base64url'); + const now = Date.now(); + for (const [h, s] of Object.entries(sessions)) if (s.exp < now) delete sessions[h]; + sessions[hash(token)] = { id, exp: now + SESSION_DAYS * 86_400_000 }; + writeJson(file, sessions); + return token; + }, + lookup(token) { + const s = token && sessions[hash(token)]; + return s && s.exp > Date.now() ? s.id : null; + }, + revoke(token) { + delete sessions[hash(token)]; + writeJson(file, sessions); + }, + }; +} diff --git a/hosted/deploy/nginx-s3.silentmode.st.conf b/hosted/deploy/nginx-s3.silentmode.st.conf new file mode 100644 index 0000000..5d85321 --- /dev/null +++ b/hosted/deploy/nginx-s3.silentmode.st.conf @@ -0,0 +1,31 @@ +# s3.silentmode.st -> pithos-hosted. Bodies stream straight through: uploads +# can be gigabytes, and SigV4 signs the exact bytes and headers. +server { + listen 443 ssl; + http2 on; + server_name s3.silentmode.st; + + ssl_certificate /etc/letsencrypt/live/s3.silentmode.st/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/s3.silentmode.st/privkey.pem; + + client_max_body_size 5g; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + + location / { + proxy_pass http://127.0.0.1:8720; + proxy_http_version 1.1; + # s3d checks the signature against the Host the client signed. + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Connection ""; + } +} + +server { + listen 80; + server_name s3.silentmode.st; + return 301 https://$host$request_uri; +} diff --git a/hosted/deploy/pithos-hosted.example.json b/hosted/deploy/pithos-hosted.example.json new file mode 100644 index 0000000..9500864 --- /dev/null +++ b/hosted/deploy/pithos-hosted.example.json @@ -0,0 +1,19 @@ +{ + "listen": "127.0.0.1:8720", + "publicUrl": "https://s3.silentmode.st", + "root": "/var/lib/pithos-hosted", + "s3dBinary": "/opt/pithos/bin/s3d", + "portBase": 21000, + "maxRunning": 30, + "idleMinutes": 30, + "bufferLimitBytes": 2147483648, + "maxDataBytes": 4294967296, + "downloadCapBytes": 53687091200, + "trustProxy": true, + "gate": { + "bns": { "api": "https://navigate.st" }, + "deposit": null, + "invites": [], + "perIpPerDay": 3 + } +} diff --git a/hosted/deploy/pithos-hosted.service b/hosted/deploy/pithos-hosted.service new file mode 100644 index 0000000..3ae6123 --- /dev/null +++ b/hosted/deploy/pithos-hosted.service @@ -0,0 +1,22 @@ +[Unit] +Description=Pithos on Silent Mode (hosted s3d gateway) +After=network-online.target +Wants=network-online.target + +[Service] +User=pithos +Group=pithos +WorkingDirectory=/opt/pithos +ExecStart=/usr/bin/node /opt/pithos/hosted/main.js --config /etc/pithos-hosted.json +Restart=on-failure +RestartSec=5 +# Stopping flushes every s3d's leftovers to Sia first. +TimeoutStopSec=300 +KillMode=mixed +NoNewPrivileges=true +ProtectSystem=strict +ReadWritePaths=/var/lib/pithos-hosted +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/hosted/electrum.js b/hosted/electrum.js new file mode 100644 index 0000000..ff286b6 --- /dev/null +++ b/hosted/electrum.js @@ -0,0 +1,30 @@ +// The one Electrum call the deposit gate needs: a verbose transaction. + +import tls from 'node:tls'; +import net from 'node:net'; + +export function electrumCall({ host, port, tls: useTls = true, timeoutMs = 15_000 }, method, params) { + return new Promise((resolve, reject) => { + const sock = useTls ? tls.connect({ host, port, servername: host }) : net.connect({ host, port }); + let buf = ''; + const timer = setTimeout(() => { sock.destroy(); reject(new Error('Electrum server did not answer')); }, timeoutMs); + const done = (err, val) => { clearTimeout(timer); sock.end(); if (err) reject(err); else resolve(val); }; + sock.setEncoding('utf8'); + sock.on('error', (e) => done(e)); + sock.write(JSON.stringify({ jsonrpc: '2.0', id: 0, method: 'server.version', params: ['pithos-hosted', '1.4'] }) + '\n'); + sock.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }) + '\n'); + sock.on('data', (d) => { + buf += d; + let i; + while ((i = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, i); + buf = buf.slice(i + 1); + let msg; + try { msg = JSON.parse(line); } catch { continue; } + if (msg.id !== 1) continue; + if (msg.error) return done(new Error(msg.error.message || 'Electrum error')); + return done(null, msg.result); + } + }); + }); +} diff --git a/hosted/gate.js b/hosted/gate.js new file mode 100644 index 0000000..ab373bf --- /dev/null +++ b/hosted/gate.js @@ -0,0 +1,126 @@ +// Who may open a hosted slot. A slot costs us memory, disk and traffic, so +// sign-up asks for one of: +// bns a signature from the wallet that owns a BNS name +// (one slot per name) +// deposit a BCH payment of at least `minSats` to the operator's deposit +// address carrying OP_RETURN "pithos:" (one slot per +// tx; refunds are an operator decision) +// invite a code the operator handed out (one use each) +// plus a per-IP limit on sign-ups per day. + +import path from 'node:path'; +import { + sha256, hash160, secp256k1, utf8ToBin, binToHex, base64ToBin, cashAddressToLockingBytecode, +} from '@bitauth/libauth'; +import { readJson, writeJson } from './store.js'; +import { electrumCall } from './electrum.js'; + +export const bnsMessage = (id) => `pithos-hosted signup ${id}`; +export const depositMemo = (id) => `pithos:${id}`; + +function varStr(bytes) { + const n = bytes.length; + const prefix = n < 0xfd ? [n] : [0xfd, n & 0xff, (n >> 8) & 0xff]; + return Uint8Array.from([...prefix, ...bytes]); +} + +// BIP-137: recover the signer, then compare its locking bytecode with the +// address we expected. Recovery alone proves nothing: any signature recovers +// to *some* key. +export function signedByAddress(message, signatureB64, expectedAddress) { + let sig; + try { sig = base64ToBin(String(signatureB64).trim()); } catch { return false; } + if (!(sig instanceof Uint8Array) || sig.length !== 65 || sig[0] < 27 || sig[0] > 34) return false; + const compressed = sig[0] >= 31; + const digest = sha256.hash(sha256.hash(Uint8Array.from([ + ...varStr(utf8ToBin('Bitcoin Signed Message:\n')), ...varStr(utf8ToBin(message)), + ]))); + const recover = compressed ? secp256k1.recoverPublicKeyCompressed : secp256k1.recoverPublicKeyUncompressed; + const pub = recover(sig.slice(1), (sig[0] - 27) & 3, digest); + if (typeof pub === 'string') return false; + const signer = binToHex(Uint8Array.from([0x76, 0xa9, 0x14, ...hash160(pub), 0x88, 0xac])); + const expected = cashAddressToLockingBytecode(expectedAddress); + return typeof expected !== 'string' && binToHex(expected.bytecode) === signer; +} + +export function createGate(root, cfg, { fetchImpl = fetch, electrum = electrumCall } = {}) { + const file = path.join(root, 'gate.json'); + const state = readJson(file, { names: {}, txids: {}, invites: {}, ip: {} }); + const save = () => writeJson(file, state); + const today = () => new Date().toISOString().slice(0, 10); + + async function bns({ name, signature }, id) { + if (!cfg.bns) throw gateError('BNS sign-up is not enabled here'); + const n = String(name || '').trim().toLowerCase(); + if (!/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(n)) throw gateError('enter a BNS name like yourname.bch'); + if (state.names[n] && state.names[n] !== id) throw gateError(`${n} already opened a hosted slot`); + const r = await fetchImpl(`${cfg.bns.api.replace(/\/$/, '')}/api/name/${encodeURIComponent(n)}`); + const info = await r.json().catch(() => ({})); + if (!r.ok || !info.registered || !info.owner) throw gateError(`${n} is not a registered BNS name`); + if (!signedByAddress(bnsMessage(id), signature, info.owner)) { + throw gateError(`the signature is not from the wallet that owns ${n} (${info.owner})`); + } + return { type: 'bns', name: n, owner: info.owner }; + } + + async function deposit({ txid }, id) { + const d = cfg.deposit; + if (!d) throw gateError('deposit sign-up is not enabled here'); + const t = String(txid || '').trim().toLowerCase(); + if (!/^[0-9a-f]{64}$/.test(t)) throw gateError('enter the 64-character transaction id'); + if (state.txids[t] && state.txids[t] !== id) throw gateError('this deposit already opened a hosted slot'); + const tx = await electrum(d.electrum, 'blockchain.transaction.get', [t, true]); + if ((tx.confirmations || 0) < (d.confirmations ?? 1)) throw gateError('the deposit has no confirmation yet; try again in about ten minutes'); + const want = cashAddressToLockingBytecode(d.address); + if (typeof want === 'string') throw new Error('deposit address in the gateway config is invalid'); + const wantHex = binToHex(want.bytecode); + const memoHex = Buffer.from(depositMemo(id), 'utf8').toString('hex'); + let paid = 0; + let memo = false; + for (const o of tx.vout || []) { + const hex = String(o.scriptPubKey?.hex || ''); + if (hex === wantHex) paid += Math.round(Number(o.value) * 1e8); + if (hex.startsWith('6a') && hex.includes(memoHex)) memo = true; + } + if (paid < d.minSats) throw gateError(`the deposit must send at least ${d.minSats} sats to ${d.address}`); + if (!memo) throw gateError(`the deposit must carry the memo "${depositMemo(id)}" (OP_RETURN)`); + return { type: 'deposit', txid: t, sats: paid }; + } + + function invite({ code }, id) { + const c = String(code || '').trim(); + if (!(cfg.invites || []).includes(c)) throw gateError('this invite code is not valid'); + if (state.invites[c] && state.invites[c] !== id) throw gateError('this invite code was already used'); + return { type: 'invite', code: c }; + } + + return { + describe() { + return { + bns: !!cfg.bns, + deposit: cfg.deposit ? { address: cfg.deposit.address, minSats: cfg.deposit.minSats } : null, + invite: !!(cfg.invites && cfg.invites.length), + }; + }, + async admit(proof, id, ip) { + const day = today(); + for (const key of Object.keys(state.ip)) if (!key.startsWith(day)) delete state.ip[key]; + const ipKey = `${day} ${ip}`; + if ((state.ip[ipKey] || 0) >= (cfg.perIpPerDay ?? 3)) throw gateError('too many sign-ups from this network today', 429); + const kind = proof?.type; + let r; + if (kind === 'bns') r = await bns(proof, id); + else if (kind === 'deposit') r = await deposit(proof, id); + else if (kind === 'invite') r = invite(proof, id); + else throw gateError('choose how to open your slot: BNS name, deposit or invite', 400); + if (r.name) state.names[r.name] = id; + if (r.txid) state.txids[r.txid] = id; + if (r.code) state.invites[r.code] = id; + state.ip[ipKey] = (state.ip[ipKey] || 0) + 1; + save(); + return r; + }, + }; +} + +function gateError(message, status = 403) { return Object.assign(new Error(message), { status }); } diff --git a/hosted/main.js b/hosted/main.js new file mode 100644 index 0000000..c34d266 --- /dev/null +++ b/hosted/main.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// pithos-hosted entry: node hosted/main.js --config /etc/pithos-hosted.json + +import fs from 'node:fs'; +import { createHosted } from './server.js'; + +const DEFAULTS = { + listen: '127.0.0.1:8720', + publicUrl: 'https://s3.silentmode.st', + root: '/var/lib/pithos-hosted', + s3dBinary: null, + portBase: 21000, + maxRunning: 30, + idleMinutes: 30, + bufferLimitBytes: 2 * 1024 ** 3, + maxDataBytes: 4 * 1024 ** 3, + downloadCapBytes: 50 * 1024 ** 3, + trustProxy: false, + gate: { bns: { api: 'https://navigate.st' }, deposit: null, invites: [], perIpPerDay: 3 }, +}; + +const i = process.argv.indexOf('--config'); +const file = i > 0 ? process.argv[i + 1] : process.env.PITHOS_HOSTED_CONFIG; +if (!file) { console.error('usage: pithos-hosted --config '); process.exit(2); } +const cfg = { ...DEFAULTS, ...JSON.parse(fs.readFileSync(file, 'utf8')) }; +if (!cfg.s3dBinary || !fs.existsSync(cfg.s3dBinary)) { console.error(`s3dBinary not found: ${cfg.s3dBinary}`); process.exit(2); } +cfg.publicUrl = cfg.publicUrl.replace(/\/$/, ''); + +const hosted = await createHosted(cfg); +let stopping = false; +for (const sig of ['SIGTERM', 'SIGINT']) { + process.on(sig, async () => { + if (stopping) return; + stopping = true; + console.log(`${sig}: flushing and stopping every s3d`); + await hosted.close().catch((e) => console.error(e)); + process.exit(0); + }); +} diff --git a/hosted/package.json b/hosted/package.json new file mode 100644 index 0000000..75896c6 --- /dev/null +++ b/hosted/package.json @@ -0,0 +1,18 @@ +{ + "name": "pithos-hosted", + "version": "0.1.0", + "private": true, + "description": "Pithos on Silent Mode: one s3d per person, on their own Sia account, behind a gateway that adds sign-in, per-folder access and client-side encryption.", + "license": "MPL-2.0", + "type": "module", + "main": "main.js", + "scripts": { + "start": "node main.js --config /etc/pithos-hosted.json" + }, + "engines": { + "node": ">=20" + }, + "dependencies": { + "@bitauth/libauth": "3.1.0-next.8" + } +} diff --git a/hosted/rules.js b/hosted/rules.js new file mode 100644 index 0000000..9927c6c --- /dev/null +++ b/hosted/rules.js @@ -0,0 +1,93 @@ +// Access rules for one hosted user: what s3d cannot express on its own. +// +// public anyone may read a drive or a folder (rights: read | read-list) +// link anyone holding the link may read one file until it expires +// grant whoever holds a gateway-minted S3 key may act on a drive or a +// folder with the rights given: read, add (new files only) or edit +// +// Prefixes are plain paths. Files under a rule's prefix are stored readable +// (the owner's Pithos converts them), because the people a rule is for have +// no decryption key. + +import crypto from 'node:crypto'; + +export const RIGHTS = { + public: ['read', 'read-list'], + link: ['read'], + grant: ['read', 'add', 'edit'], +}; + +const id = (n = 12) => crypto.randomBytes(n).toString('base64url'); + +export function validateRule(input, buckets) { + const type = input.type; + if (!RIGHTS[type]) throw new Error('rule type must be public, link or grant'); + const rights = input.rights || RIGHTS[type][0]; + if (!RIGHTS[type].includes(rights)) throw new Error(`a ${type} rule allows ${RIGHTS[type].join(', ')}`); + const bucket = String(input.bucket || ''); + if (buckets && !buckets.includes(bucket)) throw new Error(`no drive named "${bucket}"`); + const prefix = String(input.prefix || ''); + if (prefix.startsWith('/') || prefix.includes('//')) throw new Error('bad folder path'); + if (type === 'link' && (!prefix || prefix.endsWith('/'))) throw new Error('a link is for one file'); + if (type !== 'link' && prefix && !prefix.endsWith('/')) throw new Error('rules apply to a drive or a folder (end it with /)'); + let expires = null; + if (input.expires != null) { + expires = Number(input.expires); + if (!Number.isFinite(expires) || expires < Date.now()) throw new Error('the expiry must be in the future'); + } + if (type === 'link' && !expires) throw new Error('links need an expiry'); + const rule = { id: id(), type, bucket, prefix, rights, expires, label: String(input.label || '').slice(0, 120), createdAt: Date.now() }; + if (type === 'link') rule.token = id(24); + if (type === 'grant') { rule.accessKeyId = 'PG' + crypto.randomBytes(9).toString('hex').toUpperCase(); rule.secretKey = id(30); } + return rule; +} + +export const live = (r, now = Date.now()) => !r.expires || r.expires > now; + +// What an S3 request wants to do, from method, query and key. +export function classify(method, query, key, headers = {}) { + if (key == null || key === '') { + if (method === 'GET' && !hasAny(query, ['policy', 'acl', 'cors', 'lifecycle', 'versioning', 'tagging', 'uploads'])) return { op: 'list' }; + return { op: 'bucket-admin' }; + } + if (headers['x-amz-copy-source']) return { op: 'copy' }; + if (hasAny(query, ['acl', 'tagging', 'retention', 'legal-hold'])) return { op: 'object-admin' }; + if (method === 'GET' || method === 'HEAD') return { op: 'read' }; + if (method === 'PUT' || method === 'POST') return { op: 'write' }; + if (method === 'DELETE') return { op: hasAny(query, ['uploadId']) ? 'write' : 'delete' }; + return { op: 'object-admin' }; +} + +function hasAny(query, names) { return names.some((n) => Object.prototype.hasOwnProperty.call(query, n)); } + +// Decides a grant-key request. Returns { allow, needsAbsent } or { allow:false, reason }. +// needsAbsent: an "add" grant may only create keys that do not exist yet. +export function decideGrant(rule, bucket, op, key, listPrefix = '') { + if (!live(rule)) return { allow: false, reason: 'this key has expired' }; + if (bucket !== rule.bucket) return { allow: false, reason: 'this key is for another drive' }; + if (op === 'list') { + return listPrefix.startsWith(rule.prefix) ? { allow: true } : { allow: false, reason: `this key may only list ${rule.prefix || 'the drive'}` }; + } + if (!String(key).startsWith(rule.prefix)) return { allow: false, reason: `this key only reaches ${rule.prefix || 'the drive'}` }; + if (op === 'read') return { allow: true }; + if (op === 'write') { + if (rule.rights === 'edit') return { allow: true }; + if (rule.rights === 'add') return { allow: true, needsAbsent: true }; + return { allow: false, reason: 'this key is read-only' }; + } + if (op === 'delete') return rule.rights === 'edit' ? { allow: true } : { allow: false, reason: 'this key may not delete' }; + return { allow: false, reason: 'not allowed with a shared key' }; +} + +// Longest live public rule covering bucket/key, or null. +export function publicRuleFor(rules, bucket, key) { + let best = null; + for (const r of rules) { + if (r.type !== 'public' || r.bucket !== bucket || !live(r) || !key.startsWith(r.prefix)) continue; + if (!best || r.prefix.length > best.prefix.length) best = r; + } + return best; +} + +// Rules as the owner sees them; grant secrets stay in (the owner made them). +export function forOwner(rules) { return rules.filter((r) => live(r)); } diff --git a/hosted/server.js b/hosted/server.js new file mode 100644 index 0000000..df1020d --- /dev/null +++ b/hosted/server.js @@ -0,0 +1,531 @@ +// pithos-hosted: s3d on Silent Mode with each person's own Sia account. +// +// One HTTP listener (behind nginx/TLS) serves four kinds of traffic: +// +// /hosted/v1/... sign-in, sign-up gate, access rules, moving a data +// folder in and out (Bearer session token) +// /api/... the owner's own Pithos API, forwarded to their +// per-account control server (Bearer session token) +// /p///.. public files, /l/ share links (no auth) +// anything else S3, path-style. Owner keys pass straight through to +// their s3d (which checks the signature itself); keys +// minted for other people are verified and limited here, +// then re-signed with the owner's key. + +import http from 'node:http'; +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import { readConfig } from '../core/config.js'; +import { objectPath } from '../core/s3.js'; +import { createAuth, accountId, challengeText, verifyEd25519 } from './auth.js'; +import { createGate, bnsMessage, depositMemo } from './gate.js'; +import { createSupervisor, httpError } from './supervisor.js'; +import { parseAuth, verify, SigError } from './sigv4.js'; +import { validateRule, classify, decideGrant, publicRuleFor, live, forOwner } from './rules.js'; +import { readJson, writeJson } from './store.js'; + +// Owner API calls the gateway forwards. Everything that would act on the +// server itself (binary, s3d.yml, Sia login, account switching) is the +// gateway's business, and object routes are refused so nothing can bypass +// the client-side encryption. +const FORWARD = [ + ['GET', /^\/api\/status$/], ['GET', /^\/api\/logs$/], ['GET', /^\/api\/stats$/], + ['POST', /^\/api\/flush$/], ['GET', /^\/api\/autoflush$/], + ['POST', /^\/api\/daemon\/(start|restart)$/], + ['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/], + ['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/], + ['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/], +]; +const KEY_CHANGES = /^\/api\/(users|keys)/; + +export async function createHosted(cfg, log = console.log) { + fs.mkdirSync(cfg.root, { recursive: true }); + const auth = createAuth(cfg.root); + const gate = createGate(cfg.root, cfg.gate || {}); + const sup = createSupervisor(cfg, log); + const grantsFile = path.join(cfg.root, 'grants.json'); + const linksFile = path.join(cfg.root, 'links.json'); + const grants = readJson(grantsFile, {}); // accessKeyId -> { id, rule } + const links = readJson(linksFile, {}); // token -> { id, rule } + + const rulesFile = (id) => path.join(sup.dir(id), 'rules.json'); + const rulesOf = (id) => readJson(rulesFile(id), []); + function saveRules(id, rules) { + writeJson(rulesFile(id), rules); + for (const [k, v] of Object.entries(grants)) if (v.id === id) delete grants[k]; + for (const [k, v] of Object.entries(links)) if (v.id === id) delete links[k]; + for (const r of rules) { + if (r.type === 'grant') grants[r.accessKeyId] = { id, rule: r.id }; + if (r.type === 'link') links[r.token] = { id, rule: r.id }; + } + writeJson(grantsFile, grants); + writeJson(linksFile, links); + } + const ruleById = (id, ruleId) => rulesOf(id).find((r) => r.id === ruleId) || null; + + // ---- traffic meter ------------------------------------------------------- + const month = () => new Date().toISOString().slice(0, 7); + const usage = new Map(); // id -> { month, down, up } + function usageOf(id) { + let u = usage.get(id); + if (!u) { u = readJson(path.join(sup.dir(id), 'usage.json'), null) || { month: month(), down: 0, up: 0 }; usage.set(id, u); } + if (u.month !== month()) { u.month = month(); u.down = 0; u.up = 0; } + return u; + } + const meterTimer = setInterval(() => { + for (const [id, u] of usage) writeJson(path.join(sup.dir(id), 'usage.json'), u); + }, 30_000); + meterTimer.unref?.(); + function checkCap(id) { + if (cfg.downloadCapBytes && usageOf(id).down >= cfg.downloadCapBytes) { + throw httpError(429, `this account has used its ${fmtBytes(cfg.downloadCapBytes)} of downloads this month`); + } + } + + // ---- helpers --------------------------------------------------------------- + const clientIp = (req) => (cfg.trustProxy && String(req.headers['x-forwarded-for'] || '').split(',')[0].trim()) || req.socket.remoteAddress || ''; + function bearer(req) { + const m = /^Bearer\s+(\S+)$/.exec(req.headers.authorization || ''); + const id = m && auth.lookup(m[1]); + if (!id) throw httpError(401, 'sign in first'); + return { id, token: m[1] }; + } + + async function forwardApi(req, res, url, id) { + const a = sup.account(id); + if (url.pathname === '/api/status' && a.state !== 'active') { + return json(res, 200, hostedStatus(a, { daemon: { state: 'stopped' }, registration: { registered: false } })); + } + const inst = await sup.ensure(id); + const release = sup.hold(inst); + try { + const target = new URL(url.pathname + url.search, inst.pithos.url); + const up = await new Promise((resolve, reject) => { + const r = http.request(target, { + method: req.method, + headers: { cookie: inst.cookie, 'x-pithos': '1', 'content-type': req.headers['content-type'] || 'application/json' }, + }, resolve); + r.on('error', reject); + req.pipe(r); + }); + const text = await readAll(up); + if (req.method !== 'GET' && KEY_CHANGES.test(url.pathname)) await sup.indexKeys(inst); + let body = text; + if (up.statusCode === 200 && url.pathname === '/api/status') body = JSON.stringify(hostedStatus(a, JSON.parse(text))); + if (up.statusCode === 200 && url.pathname === '/api/account') { + const acc = JSON.parse(text); + body = JSON.stringify({ indexerUrl: acc.indexerUrl, registered: acc.registered, label: acc.label, archives: [], hosted: true }); + } + res.writeHead(up.statusCode, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' }); + res.end(body); + } finally { release(); } + } + + // Server paths stay on the server. + function hostedStatus(a, st) { + return { + ...st, + host: 'hosted', + config: st.config ? { apiAddress: null, adminAddress: null, file: null, directory: null, https: null, exists: true } : undefined, + hosted: { id: a.id, state: a.state, publicUrl: cfg.publicUrl, usage: usageOf(a.id), downloadCapBytes: cfg.downloadCapBytes || null }, + }; + } + + // ---- data folder in and out ------------------------------------------------ + const incomingDir = (id) => path.join(sup.dir(id), 'incoming'); + const incomingManifest = (id) => path.join(sup.dir(id), 'incoming.json'); + + function safeRel(rel) { + const r = String(rel || '').replace(/\\/g, '/'); + if (!r || r.startsWith('/') || r.split('/').some((s) => s === '..' || s === '' || s === '.') || /^[A-Za-z]:/.test(r)) throw httpError(400, 'bad file path'); + return r; + } + + async function receiveFile(req, id, rel) { + const a = sup.account(id); + if (a.state !== 'empty') throw httpError(409, 'drives are already hosted here; move them back to your computer first'); + const file = path.join(incomingDir(id), safeRel(rel)); + fs.mkdirSync(path.dirname(file), { recursive: true }); + const hash = crypto.createHash('sha256'); + let size = 0; + const man = readJson(incomingManifest(id), {}); + const total = Object.values(man).reduce((n, f) => n + f.size, 0); + const limit = cfg.maxDataBytes || 4 * 1024 ** 3; + req.on('data', (c) => { hash.update(c); size += c.length; if (total + size > limit) req.destroy(httpError(413, 'the data folder is larger than this server accepts')); }); + await pipeline(req, fs.createWriteStream(file)); + man[safeRel(rel)] = { size, sha256: hash.digest('hex') }; + writeJson(incomingManifest(id), man); + return man[safeRel(rel)]; + } + + async function commit(id, files) { + const a = sup.account(id); + if (a.state !== 'empty') throw httpError(409, 'drives are already hosted here'); + const man = readJson(incomingManifest(id), {}); + if (!Array.isArray(files) || !files.length) throw httpError(400, 'no files listed'); + for (const f of files) { + const got = man[safeRel(f.path)]; + if (!got || got.sha256 !== f.sha256 || got.size !== f.size) throw httpError(400, `${f.path} did not arrive intact; send it again`); + } + if (Object.keys(man).length !== files.length) throw httpError(400, 'the server has files the list does not mention; start the move again'); + if (!fs.existsSync(path.join(incomingDir(id), 's3d.db'))) throw httpError(400, 'this is not an s3d data folder (no s3d.db)'); + await sup.close(id, 'new data folder'); + fs.rmSync(sup.dataDir(id), { recursive: true, force: true }); + fs.renameSync(incomingDir(id), sup.dataDir(id)); + fs.rmSync(incomingManifest(id), { force: true }); + sup.setState(id, 'active'); + let inst; + try { + inst = await sup.ensure(id); + } catch (e) { + sup.setState(id, 'empty'); + throw httpError(502, `s3d could not start on the moved folder: ${e.message}`); + } + const reg = await inst.cli.registration().catch(() => ({ registered: false })); + const keys = await sup.indexKeys(inst); + return { registered: reg.registered, indexerUrl: reg.indexerUrl || null, users: [...new Set(keys.map((k) => k.user))] }; + } + + function walk(base) { + const out = []; + const rec = (d) => { + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) rec(p); + else if (e.isFile()) out.push(p); + } + }; + rec(base); + return out.map((p) => { + const data = fs.readFileSync(p); + return { path: path.relative(base, p).split(path.sep).join('/'), size: data.length, sha256: crypto.createHash('sha256').update(data).digest('hex') }; + }); + } + + async function checkout(id, force) { + const a = sup.account(id); + if (a.state === 'moving-out') return { files: walk(sup.dataDir(id)) }; + if (a.state !== 'active') throw httpError(409, 'nothing is hosted here'); + const inst = await sup.ensure(id); + const flushed = await sup.flushQuietly(inst, 10 * 60_000); + if (!flushed && !force) throw httpError(409, 'some uploads could not be sent to Sia yet; try again later, or move anyway and they will finish on your computer'); + await sup.close(id, 'moving to the owner\'s computer'); + sup.setState(id, 'moving-out'); + return { files: walk(sup.dataDir(id)) }; + } + + function release(id) { + const a = sup.account(id); + if (a.state !== 'moving-out') throw httpError(409, 'start the move first'); + fs.rmSync(sup.dataDir(id), { recursive: true, force: true }); + fs.mkdirSync(sup.dataDir(id), { recursive: true }); + saveRules(id, []); + sup.setState(id, 'empty'); + return { ok: true }; + } + + // ---- S3 ---------------------------------------------------------------------- + function splitS3Path(rawPath) { + const parts = rawPath.split('/').slice(1); + const dec = (s) => { try { return decodeURIComponent(s); } catch { return s; } }; + const bucket = dec(parts[0] || ''); + const key = parts.length > 1 ? parts.slice(1).map(dec).join('/') : null; + return { bucket, key }; + } + + async function s3(req, res, url) { + const a = parseAuth(req, url); + if (!a) return s3Error(res, 403, 'AccessDenied', 'sign requests with a key; public files are at /p///'); + const ownerId = sup.ownerOfKey(a.akid); + if (ownerId) { + checkReadCap(req, ownerId); + const inst = await sup.ensure(ownerId); + return rawProxy(req, res, inst); + } + const g = grants[a.akid]; + const rule = g && ruleById(g.id, g.rule); + if (!rule) return s3Error(res, 403, 'InvalidAccessKeyId', 'The AWS Access Key Id you provided does not exist in our records.'); + verify(req, url, a, rule.secretKey); + const raw = req.url.split('?')[0]; + const { bucket, key } = splitS3Path(raw); + const query = Object.fromEntries(url.searchParams); + const { op } = classify(req.method, query, key, req.headers); + const d = decideGrant(rule, bucket, op, key, query.prefix || ''); + if (!d.allow) return s3Error(res, 403, 'AccessDenied', d.reason); + const payload = req.headers['x-amz-content-sha256'] || 'UNSIGNED-PAYLOAD'; + if (payload === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { + return s3Error(res, 501, 'NotImplemented', 'signed streaming uploads are not supported with shared keys; turn off payload signing in your S3 client'); + } + checkReadCap(req, g.id); + const inst = await sup.ensure(g.id); + const client = await sup.bucketClient(inst, bucket); + if (!client) return s3Error(res, 404, 'NoSuchBucket', 'The specified bucket does not exist'); + if (d.needsAbsent) { + const head = await client.request('HEAD', objectPath(bucket, key)); + head.resume(); + if (head.statusCode === 200) return s3Error(res, 403, 'AccessDenied', 'this key may add files but not replace them'); + } + const pass = {}; + for (const [k, v] of Object.entries(req.headers)) { + if (/^(content-type|content-length|content-md5|content-encoding|content-disposition|cache-control|range|if-(match|none-match|modified-since|unmodified-since))$/.test(k) || (k.startsWith('x-amz-') && !/^x-amz-(date|content-sha256|security-token)$/.test(k))) pass[k] = v; + } + const release = sup.hold(inst); + try { + const up = await client.request(req.method, key == null ? objectPath(bucket) : objectPath(bucket, key), { query, headers: pass, body: req, payloadHash: payload }); + relay(up, res, g.id, req.method); + await done(res); + } finally { release(); } + } + + function checkReadCap(req, id) { + if (req.method === 'GET') checkCap(id); + } + + async function rawProxy(req, res, inst) { + const c = readConfig(sup.configFile(inst.id)); + const [host, port] = c.apiAddress.split(':'); + const release = sup.hold(inst); + const u = usageOf(inst.id); + try { + const up = await new Promise((resolve, reject) => { + const headers = { ...req.headers }; + delete headers['x-forwarded-for']; delete headers['x-real-ip']; delete headers.connection; + const r = http.request({ host, port: Number(port), method: req.method, path: req.url, headers }, resolve); + r.on('error', reject); + req.on('data', (chunk) => { u.up += chunk.length; }); + req.pipe(r); + }); + relay(up, res, inst.id, req.method); + await done(res); + } finally { release(); } + } + + function relay(up, res, id, method) { + const u = usageOf(id); + res.writeHead(up.statusCode, up.headers); + if (method === 'GET') up.on('data', (c) => { u.down += c.length; }); + up.pipe(res); + } + + // ---- public files and links ---------------------------------------------------- + async function serveObject(req, res, id, bucket, key, { inline = true } = {}) { + checkCap(id); + const inst = await sup.ensure(id); + const client = await sup.bucketClient(inst, bucket); + if (!client) throw httpError(404, 'not found'); + const release = sup.hold(inst); + try { + const up = await client.getObject(bucket, key, { range: req.headers.range }); + if (up.statusCode >= 300 && up.statusCode !== 304) { up.resume(); throw httpError(up.statusCode === 403 ? 404 : up.statusCode, 'not found'); } + const headers = {}; + for (const h of ['content-type', 'content-length', 'content-range', 'accept-ranges', 'etag', 'last-modified']) if (up.headers[h]) headers[h] = up.headers[h]; + const name = key.split('/').pop() || 'download'; + headers['content-disposition'] = `${inline ? 'inline' : 'attachment'}; filename*=UTF-8''${encodeURIComponent(name)}`; + headers['x-content-type-options'] = 'nosniff'; + headers['content-security-policy'] = "sandbox; default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'"; + headers['cache-control'] = 'public, max-age=60'; + const u = usageOf(id); + res.writeHead(up.statusCode, headers); + if (req.method === 'HEAD') { up.resume(); return res.end(); } + up.on('data', (c) => { u.down += c.length; }); + up.pipe(res); + await done(res); + } finally { release(); } + } + + async function servePublic(req, res, url) { + const m = /^\/p\/([0-9a-f]{32})\/([^/]+)\/?(.*)$/.exec(url.pathname); + if (!m) throw httpError(404, 'not found'); + const [, id, bucket] = m; + const key = decodeURIComponent(m[3]); + const rule = publicRuleFor(rulesOf(id), bucket, key); + if (!rule) throw httpError(404, 'not found'); + if (key === '' || key.endsWith('/')) { + if (rule.rights !== 'read-list') throw httpError(404, 'not found'); + const inst = await sup.ensure(id); + const client = await sup.bucketClient(inst, bucket); + if (!client) throw httpError(404, 'not found'); + const page = await client.listObjects(bucket, { prefix: key, token: url.searchParams.get('token') || undefined }); + return json(res, 200, { + folders: page.folders.map((f) => f.slice(key.length)), + files: page.objects.filter((o) => o.key !== key).map((o) => ({ name: o.key.slice(key.length), size: o.size, modified: o.modified })), + next: page.truncated ? page.nextToken : null, + }); + } + return serveObject(req, res, id, bucket, key); + } + + async function serveLink(req, res, url) { + const token = url.pathname.slice(3); + const l = links[token]; + const rule = l && ruleById(l.id, l.rule); + if (!rule || !live(rule)) throw httpError(410, 'this link has expired'); + return serveObject(req, res, l.id, rule.bucket, rule.prefix, { inline: !url.searchParams.has('download') }); + } + + // ---- routing ------------------------------------------------------------------- + async function handle(req, res) { + const url = new URL(req.url, cfg.publicUrl); + const isS3 = /^AWS4-HMAC-SHA256 /.test(req.headers.authorization || '') || url.searchParams.has('X-Amz-Algorithm'); + if (isS3) return s3(req, res, url); + const p = url.pathname; + + if (p.startsWith('/p/') && (req.method === 'GET' || req.method === 'HEAD')) return servePublic(req, res, url); + if (p.startsWith('/l/') && (req.method === 'GET' || req.method === 'HEAD')) return serveLink(req, res, url); + + if (p === '/hosted/v1/info' && req.method === 'GET') { + return json(res, 200, { publicUrl: cfg.publicUrl, gate: gate.describe(), downloadCapBytes: cfg.downloadCapBytes || null, s3Endpoint: cfg.publicUrl }); + } + if (p === '/hosted/v1/challenge' && req.method === 'GET') { + const nonce = auth.challenge(); + return json(res, 200, { nonce, message: challengeText(cfg.publicUrl, nonce) }); + } + if (p === '/hosted/v1/session' && req.method === 'POST') { + const body = await jsonBody(req); + if (!auth.takeNonce(body.nonce)) throw httpError(401, 'the sign-in request expired; try again'); + if (!verifyEd25519(body.pubkey, challengeText(cfg.publicUrl, body.nonce), body.signature)) throw httpError(401, 'the signature does not match'); + const id = accountId(body.pubkey); + let a = sup.account(id); + if (!a) { + if (!body.signup) { + return json(res, 404, { error: 'no hosted account for this key yet', needsSignup: true, id, gate: gate.describe(), bnsMessage: bnsMessage(id), depositMemo: depositMemo(id) }); + } + const g = await gate.admit(body.signup, id, clientIp(req)); + a = sup.createAccount({ id, pubkey: body.pubkey.toLowerCase(), gate: g }); + log(`[${id.slice(0, 8)}] new account via ${g.type}`); + } + return json(res, 200, { token: auth.issue(id), id, state: a.state }); + } + + if (p.startsWith('/hosted/v1/') || p.startsWith('/api/')) { + const { id, token } = bearer(req); + if (p.startsWith('/api/')) { + if (!FORWARD.some(([m, re]) => m === req.method && re.test(p))) throw httpError(403, 'not available on a hosted account'); + return forwardApi(req, res, url, id); + } + const a = sup.account(id); + const q = url.searchParams; + switch (`${req.method} ${p}`) { + case 'GET /hosted/v1/me': + return json(res, 200, { id, state: a.state, gate: { type: a.gate?.type, name: a.gate?.name || null }, createdAt: a.createdAt, usage: usageOf(id), downloadCapBytes: cfg.downloadCapBytes || null, publicUrl: cfg.publicUrl }); + case 'DELETE /hosted/v1/session': + auth.revoke(token); + return json(res, 200, { ok: true }); + case 'GET /hosted/v1/rules': + return json(res, 200, forOwner(rulesOf(id))); + case 'POST /hosted/v1/rules': { + const inst = await sup.ensure(id); + const rule = validateRule(await jsonBody(req), await sup.mapBuckets(inst)); + const rules = rulesOf(id).filter((r) => live(r)); + if (rules.length >= 500) throw httpError(400, 'too many rules on one account'); + rules.push(rule); + saveRules(id, rules); + return json(res, 200, rule); + } + case 'DELETE /hosted/v1/rules': { + const rules = rulesOf(id); + const next = rules.filter((r) => r.id !== q.get('id')); + if (next.length === rules.length) throw httpError(404, 'no such rule'); + saveRules(id, next); + return json(res, 200, { ok: true }); + } + case 'PUT /hosted/v1/data/file': + return json(res, 200, await receiveFile(req, id, q.get('path'))); + case 'POST /hosted/v1/data/commit': + return json(res, 200, await commit(id, (await jsonBody(req)).files)); + case 'POST /hosted/v1/data/abort': + if (a.state !== 'empty') throw httpError(409, 'nothing to abort'); + fs.rmSync(incomingDir(id), { recursive: true, force: true }); + fs.rmSync(incomingManifest(id), { force: true }); + return json(res, 200, { ok: true }); + case 'POST /hosted/v1/data/checkout': + return json(res, 200, await checkout(id, q.get('force') === '1')); + case 'GET /hosted/v1/data/file': { + if (a.state !== 'moving-out') throw httpError(409, 'start the move first'); + const file = path.join(sup.dataDir(id), safeRel(q.get('path'))); + if (!fs.existsSync(file)) throw httpError(404, 'no such file'); + res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-length': fs.statSync(file).size }); + await pipeline(fs.createReadStream(file), res); + return; + } + case 'POST /hosted/v1/data/cancel-checkout': + if (a.state !== 'moving-out') throw httpError(409, 'no move in progress'); + sup.setState(id, 'active'); + return json(res, 200, { ok: true }); + case 'POST /hosted/v1/data/release': + return json(res, 200, release(id)); + default: + throw httpError(404, 'no such endpoint'); + } + } + throw httpError(404, 'not found'); + } + + const server = http.createServer((req, res) => { + handle(req, res).catch((err) => { + const status = err instanceof SigError ? err.status : err.status || 500; + if (status >= 500) log('[hosted]', err.stack || err.message); + if (res.headersSent) return res.destroy(); + const isS3 = /^AWS4-HMAC-SHA256 /.test(req.headers.authorization || ''); + if (isS3) return s3Error(res, status, err.code || (status === 404 ? 'NoSuchKey' : 'InternalError'), err.message); + json(res, status, { error: status >= 500 && !err.status ? 'internal error' : err.message }); + }); + }); + server.requestTimeout = 0; // large uploads and downloads stream for a long time + + const [host, port] = cfg.listen.split(':'); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(Number(port), host, resolve); }); + log(`pithos-hosted on ${cfg.listen} for ${cfg.publicUrl}`); + + return { + server, + supervisor: sup, + async close() { + clearInterval(meterTimer); + for (const [id, u] of usage) writeJson(path.join(sup.dir(id), 'usage.json'), u); + await new Promise((r) => server.close(r)); + await sup.closeAll(); + }, + }; +} + +function json(res, status, body) { + const data = JSON.stringify(body); + res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' }); + res.end(data); +} + +function s3Error(res, status, code, message) { + const esc = (s) => String(s).replace(/[<>&'"]/g, (c) => `&#${c.charCodeAt(0)};`); + res.writeHead(status, { 'content-type': 'application/xml' }); + res.end(`${esc(code)}${esc(message)}`); +} + +function readAll(stream) { + return new Promise((resolve, reject) => { + const parts = []; + stream.on('data', (c) => parts.push(c)); + stream.on('end', () => resolve(Buffer.concat(parts).toString('utf8'))); + stream.on('error', reject); + }); +} + +async function jsonBody(req) { + let size = 0; + const parts = []; + for await (const c of req) { size += c.length; if (size > 1024 * 1024) throw httpError(413, 'request too large'); parts.push(c); } + try { return JSON.parse(Buffer.concat(parts).toString('utf8') || '{}'); } catch { throw httpError(400, 'invalid JSON'); } +} + +function done(res) { + return new Promise((resolve) => { if (res.writableFinished) return resolve(); res.on('finish', resolve); res.on('close', resolve); }); +} + +function fmtBytes(n) { + const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + let i = 0; + while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; } + return `${n.toFixed(n < 10 && i ? 1 : 0)} ${u[i]}`; +} diff --git a/hosted/sigv4.js b/hosted/sigv4.js new file mode 100644 index 0000000..ac2efbd --- /dev/null +++ b/hosted/sigv4.js @@ -0,0 +1,96 @@ +// SigV4 on the receiving side. The gateway reads the access key out of every +// S3 request to know whose s3d it is for. Keys it minted itself (grants to +// other people) are fully verified here, because s3d never sees them; an +// owner's own s3d keys are verified again by s3d, so for those the gateway +// only needs the key id. + +import crypto from 'node:crypto'; +import { uriEncode, canonicalQuery, signingKey, REGION, SERVICE } from '../core/s3.js'; + +const MAX_SKEW_MS = 15 * 60_000; + +// { akid, presigned, signedHeaders, signature, date, scope } or null. +export function parseAuth(req, url) { + const h = req.headers.authorization; + if (h) { + const m = /^AWS4-HMAC-SHA256\s+Credential=([^/,\s]+)\/(\d{8})\/([^/]+)\/([^/]+)\/aws4_request,\s*SignedHeaders=([^,\s]+),\s*Signature=([0-9a-f]{64})\s*$/i.exec(h); + if (!m) return null; + return { akid: m[1], day: m[2], region: m[3], service: m[4], signedHeaders: m[5].toLowerCase(), signature: m[6].toLowerCase(), date: req.headers['x-amz-date'], presigned: false }; + } + const q = url.searchParams; + if (q.get('X-Amz-Algorithm') === 'AWS4-HMAC-SHA256') { + const cred = /^([^/]+)\/(\d{8})\/([^/]+)\/([^/]+)\/aws4_request$/.exec(q.get('X-Amz-Credential') || ''); + if (!cred) return null; + return { + akid: cred[1], day: cred[2], region: cred[3], service: cred[4], + signedHeaders: (q.get('X-Amz-SignedHeaders') || '').toLowerCase(), + signature: (q.get('X-Amz-Signature') || '').toLowerCase(), + date: q.get('X-Amz-Date'), expires: Number(q.get('X-Amz-Expires') || 0), presigned: true, + }; + } + return null; +} + +function parseAmzDate(s) { + const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(s || ''); + return m ? Date.UTC(+m[1], m[2] - 1, +m[3], +m[4], +m[5], +m[6]) : NaN; +} + +// The raw path as the client sent it, decoded and re-encoded the one way +// SigV4 for S3 defines, so clients that over- or under-escape still match. +export function canonicalPath(rawPath) { + const decoded = rawPath.split('/').map((s) => { try { return decodeURIComponent(s); } catch { return s; } }); + return decoded.map((s) => uriEncode(s)).join('/'); +} + +function rawQuery(req) { + const i = req.url.indexOf('?'); + const out = {}; + if (i < 0) return out; + for (const part of req.url.slice(i + 1).split('&')) { + if (!part) continue; + const eq = part.indexOf('='); + const dec = (s) => { try { return decodeURIComponent(s.replace(/\+/g, '%20')); } catch { return s; } }; + const k = dec(eq < 0 ? part : part.slice(0, eq)); + out[k] = eq < 0 ? '' : dec(part.slice(eq + 1)); + } + return out; +} + +// Throws with an S3-style code when the signature does not hold. +export function verify(req, url, auth, secretKey, now = Date.now()) { + if (auth.region !== REGION || auth.service !== SERVICE) throw sigError('AuthorizationHeaderMalformed', `use region ${REGION}`); + const t = parseAmzDate(auth.date); + if (!Number.isFinite(t) || auth.date.slice(0, 8) !== auth.day) throw sigError('AccessDenied', 'missing or inconsistent X-Amz-Date'); + if (auth.presigned) { + if (!(auth.expires > 0 && auth.expires <= 604800)) throw sigError('AuthorizationQueryParametersError', 'bad X-Amz-Expires'); + if (now > t + auth.expires * 1000) throw sigError('AccessDenied', 'Request has expired'); + if (t - now > MAX_SKEW_MS) throw sigError('RequestTimeTooSkewed', 'request date is in the future'); + } else if (Math.abs(now - t) > MAX_SKEW_MS) { + throw sigError('RequestTimeTooSkewed', 'the difference between the request time and the server time is too large'); + } + const query = rawQuery(req); + if (auth.presigned) delete query['X-Amz-Signature']; + const names = auth.signedHeaders.split(';').filter(Boolean); + if (!names.includes('host')) throw sigError('AccessDenied', 'host must be signed'); + const headerLines = names.map((n) => { + const v = req.headers[n]; + if (v == null) throw sigError('AccessDenied', `signed header ${n} is missing`); + return `${n}:${String(Array.isArray(v) ? v.join(',') : v).trim().replace(/\s+/g, ' ')}\n`; + }).join(''); + const payload = auth.presigned ? 'UNSIGNED-PAYLOAD' : (req.headers['x-amz-content-sha256'] || 'UNSIGNED-PAYLOAD'); + const canonical = [req.method, canonicalPath(url.pathname === '' ? '/' : req.url.split('?')[0]), canonicalQuery(query), headerLines, auth.signedHeaders, payload].join('\n'); + const scope = `${auth.day}/${REGION}/${SERVICE}/aws4_request`; + const toSign = ['AWS4-HMAC-SHA256', auth.date, scope, crypto.createHash('sha256').update(canonical).digest('hex')].join('\n'); + const want = crypto.createHmac('sha256', signingKey(secretKey, auth.day)).update(toSign).digest(); + const got = Buffer.from(auth.signature, 'hex'); + if (got.length !== want.length || !crypto.timingSafeEqual(got, want)) { + throw sigError('SignatureDoesNotMatch', 'The request signature we calculated does not match the signature you provided'); + } +} + +export class SigError extends Error { + constructor(code, message, status = 403) { super(message); this.code = code; this.status = status; } +} + +function sigError(code, message) { return new SigError(code, message, code === 'AuthorizationHeaderMalformed' || code === 'AuthorizationQueryParametersError' ? 400 : 403); } diff --git a/hosted/store.js b/hosted/store.js new file mode 100644 index 0000000..883ef29 --- /dev/null +++ b/hosted/store.js @@ -0,0 +1,16 @@ +// Small JSON files written atomically: the gateway's whole state is files +// under its root, so a backup is a copy of one folder. + +import fs from 'node:fs'; +import path from 'node:path'; + +export function readJson(file, fallback) { + try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return fallback; } +} + +export function writeJson(file, value) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.${process.pid}.tmp`; + fs.writeFileSync(tmp, JSON.stringify(value, null, 2), { mode: 0o600 }); + fs.renameSync(tmp, file); +} diff --git a/hosted/supervisor.js b/hosted/supervisor.js new file mode 100644 index 0000000..c03c038 --- /dev/null +++ b/hosted/supervisor.js @@ -0,0 +1,206 @@ +// One s3d per hosted account, each wrapped in the same createPithos() the +// desktop runs, so the server speaks exactly the /api the Pithos UI already +// knows. Accounts live under /users//: +// +// account.json who (public key), how they got in (gate), port slot, state +// s3d.yml loopback-only addresses from the slot, random admin password +// data/ the s3d data folder moved here from the user's PC +// rules.json public folders, links and shared keys +// usage.json bytes served this month +// +// An s3d starts on the account's first request and is stopped after +// `idleMinutes` without one, flushing leftovers to Sia first. At most +// `maxRunning` run at once; the least recently used idle one makes room. + +import fs from 'node:fs'; +import path from 'node:path'; +import { createPithos } from '../core/server.js'; +import { makeCli } from '../core/cli.js'; +import { readConfig, writeConfig, randomPassword } from '../core/config.js'; +import { writePrefs } from '../core/prefs.js'; +import { S3Client } from '../core/s3.js'; +import * as admin from '../core/admin.js'; +import { readJson, writeJson } from './store.js'; + +export function createSupervisor(cfg, log = console.log) { + const usersDir = path.join(cfg.root, 'users'); + const slotsFile = path.join(cfg.root, 'slots.json'); + const keysFile = path.join(cfg.root, 'keys.json'); + const running = new Map(); // id -> instance + const loading = new Map(); // id -> Promise + const keyIndex = readJson(keysFile, {}); // accessKeyId -> account id (owners' s3d keys) + + const dir = (id) => path.join(usersDir, id); + const accountFile = (id) => path.join(dir(id), 'account.json'); + const configFile = (id) => path.join(dir(id), 's3d.yml'); + const dataDir = (id) => path.join(dir(id), 'data'); + + function account(id) { + if (!/^[0-9a-f]{32}$/.test(id)) return null; + return readJson(accountFile(id), null); + } + + function saveAccount(a) { writeJson(accountFile(a.id), a); } + + function createAccount({ id, pubkey, gate }) { + const slots = readJson(slotsFile, { next: 0 }); + const slot = slots.next++; + writeJson(slotsFile, slots); + const a = { id, pubkey, gate, slot, state: 'empty', createdAt: Date.now() }; + fs.mkdirSync(dataDir(id), { recursive: true }); + const port = cfg.portBase + slot * 2; + writeConfig(configFile(id), { + apiAddress: `127.0.0.1:${port}`, + adminAddress: `127.0.0.1:${port + 1}`, + adminPassword: randomPassword(), + directory: dataDir(id), + 'sia.diskUsageLimit': cfg.bufferLimitBytes, + }); + // Leftovers always go to Sia in hosted mode: an idle s3d gets stopped. + writePrefs(configFile(id), { autostart: false, autoFlush: true, autoFlushMinutes: 10 }); + saveAccount(a); + return a; + } + + function setState(id, state) { + const a = account(id); + a.state = state; + saveAccount(a); + writePrefs(configFile(id), { autostart: state === 'active' }); + return a; + } + + async function load(id) { + if (running.has(id)) return running.get(id); + if (loading.has(id)) return loading.get(id); + const p = (async () => { + await makeRoom(); + const pithos = await createPithos({ + host: '127.0.0.1', port: 0, hostName: 'hosted', + configFile: configFile(id), binary: cfg.s3dBinary, + }); + const inst = { id, pithos, cli: makeCli(pithos.daemon), cookie: pithos.internalSession(), lastUsed: Date.now(), busy: 0, buckets: new Map() }; + running.set(id, inst); + pithos.daemon.on('log', (l) => { if (l.stream !== 'out') log(`[${id.slice(0, 8)}] ${l.line}`); }); + return inst; + })().finally(() => loading.delete(id)); + loading.set(id, p); + return p; + } + + // The instance with s3d up, for anything that talks S3. + async function ensure(id) { + const a = account(id); + if (!a) throw httpError(404, 'no such account'); + if (a.state === 'empty') throw httpError(409, 'nothing is hosted yet: move your drives here from Pithos first'); + if (a.state === 'moving-out') throw httpError(409, 'this account is being moved to its owner\'s computer'); + const inst = await load(id); + inst.lastUsed = Date.now(); + const d = inst.pithos.daemon; + if (d.state !== 'running') { + if (d.state !== 'starting') d.start(); + const until = Date.now() + 30_000; + while (d.state === 'starting' && Date.now() < until) await sleep(200); + if (d.state !== 'running') throw httpError(503, 'your s3d did not start; try again in a minute'); + await indexKeys(inst); + } + return inst; + } + + async function indexKeys(inst) { + const keys = await inst.cli.listKeys().catch(() => []); + let changed = false; + for (const [k, v] of Object.entries(keyIndex)) if (v === inst.id && !keys.some((x) => x.accessKeyId === k)) { delete keyIndex[k]; changed = true; } + for (const k of keys) if (keyIndex[k.accessKeyId] !== inst.id) { keyIndex[k.accessKeyId] = inst.id; changed = true; } + if (changed) writeJson(keysFile, keyIndex); + inst.keys = keys; + inst.buckets.clear(); + return keys; + } + + // Every drive on the account, mapped to a key of the s3d user owning it. + async function mapBuckets(inst) { + inst.buckets.clear(); + for (const k of inst.keys || await indexKeys(inst)) { + const list = await clientFor(inst, k).listBuckets().catch(() => []); + for (const b of list) if (!inst.buckets.has(b.name)) inst.buckets.set(b.name, k); + } + return [...inst.buckets.keys()]; + } + + // An owner key that can act on `bucket`. + async function bucketClient(inst, bucket) { + if (!inst.buckets.has(bucket)) await mapBuckets(inst); + const k = inst.buckets.get(bucket); + return k ? clientFor(inst, k) : null; + } + + function clientFor(inst, k) { + return new S3Client({ endpoint: `http://${readConfig(configFile(inst.id)).apiAddress}`, accessKeyId: k.accessKeyId, secretKey: k.secretKey }); + } + + async function close(id, reason) { + const inst = running.get(id); + if (!inst) return; + running.delete(id); + log(`[${id.slice(0, 8)}] stopping s3d (${reason})`); + await inst.pithos.close().catch((e) => log(`[${id.slice(0, 8)}] close failed: ${e.message}`)); + } + + async function flushQuietly(inst, timeoutMs = 120_000) { + const c = readConfig(configFile(inst.id)); + try { + const st = await admin.uploadStats(c); + if (!st.pendingObjects) return true; + await Promise.race([admin.flush(c), sleep(timeoutMs).then(() => { throw new Error('timed out'); })]); + return true; + } catch (e) { + log(`[${inst.id.slice(0, 8)}] flush before stop failed: ${e.message}`); + return false; + } + } + + async function makeRoom() { + if (running.size < cfg.maxRunning) return; + const idle = [...running.values()].filter((i) => !i.busy).sort((a, b) => a.lastUsed - b.lastUsed); + if (!idle.length) throw httpError(503, 'the server is busy; try again in a minute'); + await flushQuietly(idle[0]); + await close(idle[0].id, 'making room'); + } + + async function sweep() { + const cutoff = Date.now() - cfg.idleMinutes * 60_000; + for (const inst of [...running.values()]) { + if (inst.busy || inst.lastUsed > cutoff) continue; + await flushQuietly(inst); + if (!inst.busy && inst.lastUsed <= cutoff) await close(inst.id, 'idle'); + } + } + const sweeper = setInterval(() => { sweep().catch((e) => log('sweep failed:', e.message)); }, 60_000); + sweeper.unref?.(); + + // Holds an instance "in use" for the duration of a streamed request, so the + // sweeper does not stop s3d under a long download. + function hold(inst) { + inst.busy++; + let done = false; + return () => { if (!done) { done = true; inst.busy--; inst.lastUsed = Date.now(); } }; + } + + return { + dir, dataDir, configFile, account, createAccount, saveAccount, setState, + load, ensure, close, flushQuietly, bucketClient, mapBuckets, indexKeys, hold, + ownerOfKey: (akid) => keyIndex[akid] || null, + async closeAll() { + clearInterval(sweeper); + for (const id of [...running.keys()]) { + await flushQuietly(running.get(id), 30_000); + await close(id, 'shutdown'); + } + }, + runningCount: () => running.size, + }; +} + +export function httpError(status, message) { return Object.assign(new Error(message), { status }); } +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); diff --git a/theseus/index.js b/theseus/index.js index 0531eaa..e34b16b 100644 --- a/theseus/index.js +++ b/theseus/index.js @@ -36,6 +36,15 @@ module.exports = { openExternal: (url) => api.openTab(url), // "Back to sidebar" from the full-tab view. showPanel: () => api.revealSidebar("main"), + // "On Silent Mode": sign-in identity and file encryption keys come + // from the vault, so another computer with the same vault reaches + // and reads the same drives. NEVER CHANGE THESE PATHS: encrypted + // files would become unreadable. + secrets: { + kind: "vault", + identitySeed: () => vaultKey("pithos/hosted-identity/v1", "Sign in to your drives on Silent Mode."), + encryptionKey: () => vaultKey("pithos/hosted-encryption/v1", "Open your encrypted files on Silent Mode."), + }, }); cookie = p.internalSession(); pithos = p; @@ -138,6 +147,20 @@ module.exports = { api.onMessage("vault-status", () => vaultStatus()); + async function vaultKey(purpose, reason) { + const st = await vaultStatus(); + if (!st.setup) throw new Error("Set up the password vault in Settings › Passwords first."); + if (!st.unlocked) { + if (!canPrompt) throw new Error("Unlock the password vault in Settings › Passwords first."); + const r = await vault.requestUnlock({ reason }); + if (!r.ok) throw new Error("The vault stayed locked."); + } + const bytes = await vault.derive(purpose); + const key = Buffer.from(bytes.subarray(0, 32)); + bytes.fill(0); + return key; + } + // Watch for the vault locking (Settings › Lock now, or a lock from another // extension) and tell the panel at once. This runs in the main process: // a hidden panel's own timers are throttled to about once a minute.