92260d177e
In Theseus, Pithos now links the person's Theseus ID to their hosted account by itself, once, right after signing in to Silent Mode (or on the first look at an account already signed in): one consent prompt, then Pithos is listed under Settings › Theseus ID. On a Theseus without the add-on API it does nothing, so the add-on can ship before the Theseus release that carries it. The gateway serves /console on its own origin: sign in with the Theseus ID and see the account's state, sign-up, usage and Theseus ID, with a console session that cannot reach files. Three fixed files under a CSP that allows only this origin. The bundled copy in TheseusNavigator/bundled-addons/pithos is rebuilt from Pithos/ with no version bump; the release that ships it bumps the version.
753 lines
40 KiB
JavaScript
753 lines
40 KiB
JavaScript
// pithos-hosted: s3d on Silent Mode with each person's own Sia account.
|
|
//
|
|
// One HTTP listener (behind nginx/TLS) serves four kinds of traffic:
|
|
//
|
|
// /hosted/v1/... sign-in, sign-up gate, access rules, moving a data
|
|
// folder in and out (Bearer session token); Theseus ID
|
|
// linking and the read-only account console
|
|
// /api/... the owner's own Pithos API, forwarded to their
|
|
// per-account control server (Bearer session token)
|
|
// /p/<id>/<drive>/.. public files, /l/<token> share links (no auth)
|
|
// anything else S3, path-style. Owner keys pass straight through to
|
|
// their s3d (which checks the signature itself); keys
|
|
// minted for other people are verified and limited here,
|
|
// then re-signed with the owner's key.
|
|
|
|
import http from 'node:http';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import crypto from 'node:crypto';
|
|
import { pipeline } from 'node:stream/promises';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { readConfig } from '../core/config.js';
|
|
import { objectPath } from '../core/s3.js';
|
|
import { createAuth, accountId, challengeText, verifyEd25519 } from './auth.js';
|
|
import { createGate, bnsMessage, depositMemo } from './gate.js';
|
|
import { createSupervisor, httpError } from './supervisor.js';
|
|
import { parseAuth, verify, SigError } from './sigv4.js';
|
|
import { validateRule, classify, decideGrant, publicRuleFor, live, forOwner } from './rules.js';
|
|
import { readJson, writeJson } from './store.js';
|
|
import { isTransferPath, transferTarget, hashFile } from '../core/hosted.js';
|
|
import { createPithosVerifier, PROJECT as TID_PROJECT } from './theseus-id/index.js';
|
|
|
|
// Owner API calls the gateway forwards. Everything that would act on the
|
|
// server itself (binary, s3d.yml, Sia login, account switching) is the
|
|
// gateway's business, and object routes are refused so nothing can bypass
|
|
// the client-side encryption.
|
|
const FORWARD = [
|
|
['GET', /^\/api\/status$/], ['GET', /^\/api\/logs$/], ['GET', /^\/api\/stats$/],
|
|
['POST', /^\/api\/flush$/], ['GET', /^\/api\/autoflush$/],
|
|
['POST', /^\/api\/daemon\/(start|restart)$/],
|
|
['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/],
|
|
['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/],
|
|
['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/],
|
|
['GET', /^\/api\/space$/],
|
|
];
|
|
const KEY_CHANGES = /^\/api\/(users|keys)/;
|
|
|
|
export async function createHosted(cfg, log = console.log) {
|
|
fs.mkdirSync(cfg.root, { recursive: true });
|
|
const auth = createAuth(cfg.root);
|
|
const grantsFile = path.join(cfg.root, 'grants.json');
|
|
const linksFile = path.join(cfg.root, 'links.json');
|
|
const grants = readJson(grantsFile, {}); // accessKeyId -> { id, rule }
|
|
const links = readJson(linksFile, {}); // token -> { id, rule }
|
|
const gate = createGate(cfg.root, cfg.gate || {});
|
|
// Theseus ID -> account id, for the console sign-in (DESIGN-theseus-id.md §11.3).
|
|
const tidFile = path.join(cfg.root, 'tid-links.json');
|
|
const tidLinks = readJson(tidFile, {});
|
|
const tidVerifier = createPithosVerifier((nonce) => auth.takeNonce(nonce));
|
|
// Grant key ids are the gateway's: no account's s3d may route the same id.
|
|
const sup = createSupervisor(cfg, log, { reservedKey: (akid) => /^PG/.test(akid) || Object.hasOwn(grants, akid) });
|
|
|
|
const rulesFile = (id) => path.join(sup.dir(id), 'rules.json');
|
|
const rulesOf = (id) => readJson(rulesFile(id), []);
|
|
function saveRules(id, rules) {
|
|
writeJson(rulesFile(id), rules);
|
|
for (const [k, v] of Object.entries(grants)) if (v.id === id) delete grants[k];
|
|
for (const [k, v] of Object.entries(links)) if (v.id === id) delete links[k];
|
|
for (const r of rules) {
|
|
if (r.type === 'grant') grants[r.accessKeyId] = { id, rule: r.id };
|
|
if (r.type === 'link') links[r.token] = { id, rule: r.id };
|
|
}
|
|
writeJson(grantsFile, grants);
|
|
writeJson(linksFile, links);
|
|
}
|
|
const ruleById = (id, ruleId) => rulesOf(id).find((r) => r.id === ruleId) || null;
|
|
|
|
// ---- traffic meter -------------------------------------------------------
|
|
const month = () => new Date().toISOString().slice(0, 7);
|
|
const usage = new Map(); // id -> { month, down, up }
|
|
function usageOf(id) {
|
|
let u = usage.get(id);
|
|
if (!u) { u = readJson(path.join(sup.dir(id), 'usage.json'), null) || { month: month(), down: 0, up: 0 }; usage.set(id, u); }
|
|
if (u.month !== month()) { u.month = month(); u.down = 0; u.up = 0; }
|
|
return u;
|
|
}
|
|
const meterTimer = setInterval(() => {
|
|
for (const [id, u] of usage) writeJson(path.join(sup.dir(id), 'usage.json'), u);
|
|
}, 30_000);
|
|
meterTimer.unref?.();
|
|
function checkCap(id) {
|
|
if (cfg.downloadCapBytes && usageOf(id).down >= cfg.downloadCapBytes) {
|
|
throw httpError(429, `this account has used its ${fmtBytes(cfg.downloadCapBytes)} of downloads this month`);
|
|
}
|
|
}
|
|
|
|
// ---- helpers ---------------------------------------------------------------
|
|
const clientIp = (req) => (cfg.trustProxy && String(req.headers['x-forwarded-for'] || '').split(',')[0].trim()) || req.socket.remoteAddress || '';
|
|
function bearer(req) {
|
|
const m = /^Bearer\s+(\S+)$/.exec(req.headers.authorization || '');
|
|
const s = m && auth.lookup(m[1]);
|
|
if (!s) throw httpError(401, 'sign in first');
|
|
return { id: s.id, scope: s.scope, token: m[1] };
|
|
}
|
|
|
|
// A Theseus ID sign-in for project "pithos", checked like every other
|
|
// Theseus ID verifier (§4.3) against nonces this gateway handed out.
|
|
async function theseusIdSignIn(body) {
|
|
try {
|
|
return await tidVerifier.verifySignIn({ message: String(body.message || ''), signature: String(body.signature || ''), move: body.move });
|
|
} catch (e) {
|
|
throw httpError(401, e.code === 'bad-nonce' ? 'the sign-in request expired; try again' : `Theseus ID: ${e.code || e.message}`);
|
|
}
|
|
}
|
|
function linkTheseusId(id, account) {
|
|
const a = sup.account(id);
|
|
if (a.theseusId && a.theseusId.account !== account) delete tidLinks[a.theseusId.account];
|
|
tidLinks[account] = id;
|
|
writeJson(tidFile, tidLinks);
|
|
sup.saveAccount({ ...a, theseusId: { account, linkedAt: new Date().toISOString() } });
|
|
}
|
|
|
|
async function forwardApi(req, res, url, id) {
|
|
const a = sup.account(id);
|
|
if (url.pathname === '/api/status' && a.state !== 'active') {
|
|
return json(res, 200, hostedStatus(a, { daemon: { state: 'stopped' }, registration: { registered: false } }));
|
|
}
|
|
const reqBody = req.method === 'GET' ? null : JSON.stringify(await jsonBody(req));
|
|
// A chosen access key id must not be one another account (or a grant)
|
|
// routes: S3 traffic is sent to whichever s3d owns the id.
|
|
if (req.method === 'POST' && url.pathname === '/api/keys') {
|
|
const want = JSON.parse(reqBody).accessKey;
|
|
if (want && sup.keyTaken(String(want), id)) throw httpError(409, 'that access key id is already in use; leave it empty to get a random one');
|
|
}
|
|
const inst = await sup.ensure(id);
|
|
const release = sup.hold(inst);
|
|
try {
|
|
const target = new URL(url.pathname + url.search, inst.pithos.url);
|
|
const up = await new Promise((resolve, reject) => {
|
|
const r = http.request(target, {
|
|
method: req.method,
|
|
headers: { cookie: inst.cookie, 'x-pithos': '1', 'content-type': 'application/json' },
|
|
}, resolve);
|
|
r.on('error', reject);
|
|
r.end(reqBody ?? undefined);
|
|
});
|
|
const text = await readAll(up);
|
|
if (req.method !== 'GET' && KEY_CHANGES.test(url.pathname)) await sup.indexKeys(inst);
|
|
let body = text;
|
|
if (up.statusCode === 200 && url.pathname === '/api/status') body = JSON.stringify(hostedStatus(a, JSON.parse(text)));
|
|
if (up.statusCode === 200 && url.pathname === '/api/account') {
|
|
const acc = JSON.parse(text);
|
|
body = JSON.stringify({ indexerUrl: acc.indexerUrl, registered: acc.registered, label: acc.label, sia: acc.sia || null, archives: [], hosted: true });
|
|
}
|
|
res.writeHead(up.statusCode, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' });
|
|
res.end(body);
|
|
} finally { release(); }
|
|
}
|
|
|
|
// Server paths stay on the server.
|
|
function hostedStatus(a, st) {
|
|
return {
|
|
...st,
|
|
host: 'hosted',
|
|
config: st.config ? { apiAddress: null, adminAddress: null, file: null, directory: null, https: null, exists: true } : undefined,
|
|
hosted: { id: a.id, name: a.gate?.type === 'bns' ? a.gate.name : null, state: a.state, publicUrl: cfg.publicUrl, usage: usageOf(a.id), downloadCapBytes: cfg.downloadCapBytes || null },
|
|
};
|
|
}
|
|
|
|
// ---- data folder in and out ------------------------------------------------
|
|
const incomingDir = (id) => path.join(sup.dir(id), 'incoming');
|
|
const incomingManifest = (id) => path.join(sup.dir(id), 'incoming.json');
|
|
|
|
function safeRel(rel) {
|
|
if (!isTransferPath(rel)) throw httpError(400, 'bad file path');
|
|
return rel;
|
|
}
|
|
|
|
// Bytes actually on disk under a folder: aborted uploads count too.
|
|
function diskBytes(dir) {
|
|
let n = 0;
|
|
const rec = (d) => {
|
|
let entries = [];
|
|
try { entries = fs.readdirSync(d, { withFileTypes: true }); } catch { return; }
|
|
for (const e of entries) {
|
|
const p = path.join(d, e.name);
|
|
if (e.isDirectory()) rec(p);
|
|
else if (e.isFile()) { try { n += fs.statSync(p).size; } catch {} }
|
|
}
|
|
};
|
|
rec(dir);
|
|
return n;
|
|
}
|
|
|
|
// One upload at a time per account, so the size limit cannot be raced.
|
|
const receiving = new Set();
|
|
async function receiveFile(req, id, rel) {
|
|
const a = sup.account(id);
|
|
if (a.state !== 'empty') throw httpError(409, 'drives are already hosted here; move them back to your computer first');
|
|
const name = safeRel(rel);
|
|
if (receiving.has(id)) throw httpError(409, 'send one file at a time');
|
|
receiving.add(id);
|
|
const file = transferTarget(incomingDir(id), name);
|
|
try {
|
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
fs.rmSync(file, { force: true });
|
|
const limit = cfg.maxDataBytes || 4 * 1024 ** 3;
|
|
const room = limit - diskBytes(incomingDir(id));
|
|
const declared = Number(req.headers['content-length']);
|
|
if (Number.isFinite(declared) && declared > room) throw httpError(413, 'the data folder is larger than this server accepts');
|
|
const hash = crypto.createHash('sha256');
|
|
let size = 0;
|
|
req.on('data', (c) => { hash.update(c); size += c.length; if (size > room) req.destroy(httpError(413, 'the data folder is larger than this server accepts')); });
|
|
await pipeline(req, fs.createWriteStream(file));
|
|
const man = readJson(incomingManifest(id), {});
|
|
man[name] = { size, sha256: hash.digest('hex') };
|
|
writeJson(incomingManifest(id), man);
|
|
return man[name];
|
|
} catch (e) {
|
|
fs.rmSync(file, { force: true });
|
|
throw e;
|
|
} finally {
|
|
receiving.delete(id);
|
|
}
|
|
}
|
|
|
|
async function commit(id, files) {
|
|
const a = sup.account(id);
|
|
if (a.state !== 'empty') throw httpError(409, 'drives are already hosted here');
|
|
const man = readJson(incomingManifest(id), {});
|
|
if (!Array.isArray(files) || !files.length) throw httpError(400, 'no files listed');
|
|
for (const f of files) {
|
|
const got = Object.hasOwn(man, safeRel(f.path)) ? man[f.path] : null;
|
|
if (!got || got.sha256 !== f.sha256 || got.size !== f.size) throw httpError(400, `${f.path} did not arrive intact; send it again`);
|
|
}
|
|
if (Object.keys(man).length !== files.length) throw httpError(400, 'the server has files the list does not mention; start the move again');
|
|
if (!fs.existsSync(path.join(incomingDir(id), 's3d.db'))) throw httpError(400, 'this is not an s3d data folder (no s3d.db)');
|
|
await sup.close(id, 'new data folder');
|
|
fs.rmSync(sup.dataDir(id), { recursive: true, force: true });
|
|
fs.renameSync(incomingDir(id), sup.dataDir(id));
|
|
fs.rmSync(incomingManifest(id), { force: true });
|
|
sup.setState(id, 'active');
|
|
// Anything wrong with the folder puts the account back to empty, so the
|
|
// owner's computer (which asks the server's state) keeps its copy.
|
|
const refuse = async (status, message) => {
|
|
await sup.close(id, 'refused data folder');
|
|
fs.rmSync(sup.dataDir(id), { recursive: true, force: true });
|
|
fs.mkdirSync(sup.dataDir(id), { recursive: true });
|
|
sup.setState(id, 'empty');
|
|
throw httpError(status, message);
|
|
};
|
|
let inst;
|
|
try { inst = await sup.ensure(id); } catch (e) { return refuse(502, `s3d could not start on the moved folder: ${e.message}`); }
|
|
const reg = await inst.cli.registration().catch(() => ({ registered: false }));
|
|
if (!reg.registered) return refuse(422, 'the moved folder has no Sia connection');
|
|
const keys = await sup.indexKeys(inst);
|
|
if (inst.conflicts?.length) return refuse(409, `these access keys are already used on this server: ${inst.conflicts.join(', ')}; create new keys on your computer and try again`);
|
|
return { registered: true, indexerUrl: reg.indexerUrl || null, users: [...new Set(keys.map((k) => k.user))] };
|
|
}
|
|
|
|
// Hashes stream from disk (buffered objects can be gigabytes) and only
|
|
// paths the owner's Pithos will accept are listed.
|
|
async function walk(base) {
|
|
const out = [];
|
|
const rec = async (d) => {
|
|
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
|
|
const p = path.join(d, e.name);
|
|
const rel = path.relative(base, p).split(path.sep).join('/');
|
|
if (e.isDirectory()) await rec(p);
|
|
else if (e.isFile() && isTransferPath(rel)) out.push({ path: rel, ...(await hashFile(p)) });
|
|
}
|
|
};
|
|
await rec(base);
|
|
return out;
|
|
}
|
|
|
|
async function checkout(id, force) {
|
|
const a = sup.account(id);
|
|
if (a.state === 'moving-out') return { files: await walk(sup.dataDir(id)) };
|
|
if (a.state !== 'active') throw httpError(409, 'nothing is hosted here');
|
|
const inst = await sup.ensure(id);
|
|
const flushed = await sup.flushQuietly(inst, 10 * 60_000);
|
|
if (!flushed && !force) throw httpError(409, 'some uploads could not be sent to Sia yet; try again later, or move anyway and they will finish on your computer');
|
|
// State first: from now on no request can start s3d on this folder again.
|
|
sup.setState(id, 'moving-out');
|
|
await sup.close(id, 'moving to the owner\'s computer');
|
|
return { files: await walk(sup.dataDir(id)) };
|
|
}
|
|
|
|
function release(id) {
|
|
const a = sup.account(id);
|
|
if (a.state !== 'moving-out') throw httpError(409, 'start the move first');
|
|
fs.rmSync(sup.dataDir(id), { recursive: true, force: true });
|
|
fs.mkdirSync(sup.dataDir(id), { recursive: true });
|
|
saveRules(id, []);
|
|
sup.setState(id, 'empty');
|
|
return { ok: true };
|
|
}
|
|
|
|
// ---- S3 ----------------------------------------------------------------------
|
|
// For shared keys: "." and ".." segments are refused outright, so a key can
|
|
// never be made to look inside its folder while naming something outside.
|
|
function splitS3Path(rawPath) {
|
|
const parts = rawPath.split('/').slice(1);
|
|
const dec = (s) => { try { return decodeURIComponent(s); } catch { throw httpError(400, 'malformed path'); } };
|
|
const bucket = dec(parts[0] || '');
|
|
const keyParts = parts.slice(1).map(dec);
|
|
const key = parts.length > 1 ? keyParts.join('/') : null;
|
|
if (key != null && key.split('/').some((x) => x === '.' || x === '..')) throw httpError(400, 'paths with . or .. segments are not allowed');
|
|
return { bucket, key };
|
|
}
|
|
|
|
async function s3(req, res, url) {
|
|
const a = parseAuth(req, url);
|
|
if (!a) return s3Error(res, 403, 'AccessDenied', 'sign requests with a key; public files are at /p/<account>/<drive>/<path>');
|
|
const g = Object.hasOwn(grants, a.akid) ? grants[a.akid] : null;
|
|
const ownerId = g ? null : sup.ownerOfKey(a.akid);
|
|
if (ownerId) {
|
|
checkReadCap(req, ownerId);
|
|
const inst = await sup.ensure(ownerId);
|
|
return rawProxy(req, res, inst);
|
|
}
|
|
const rule = g && ruleById(g.id, g.rule);
|
|
if (!rule) return s3Error(res, 403, 'InvalidAccessKeyId', 'The AWS Access Key Id you provided does not exist in our records.');
|
|
verify(req, url, a, rule.secretKey);
|
|
const raw = req.url.split('?')[0];
|
|
const { bucket, key } = splitS3Path(raw);
|
|
// The client's own signature parameters (presigned URLs) must not reach
|
|
// s3d next to the owner signature the gateway adds.
|
|
const query = Object.fromEntries([...url.searchParams].filter(([k]) => !/^x-amz-/i.test(k)));
|
|
const { op } = classify(req.method, query, key, req.headers);
|
|
const d = decideGrant(rule, bucket, op, key, query.prefix || '');
|
|
if (!d.allow) return s3Error(res, 403, 'AccessDenied', d.reason);
|
|
const payload = req.headers['x-amz-content-sha256'] || 'UNSIGNED-PAYLOAD';
|
|
if (payload === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
|
|
return s3Error(res, 501, 'NotImplemented', 'signed streaming uploads are not supported with shared keys; turn off payload signing in your S3 client');
|
|
}
|
|
checkReadCap(req, g.id);
|
|
const inst = await sup.ensure(g.id);
|
|
const client = await sup.bucketClient(inst, bucket);
|
|
if (!client) return s3Error(res, 404, 'NoSuchBucket', 'The specified bucket does not exist');
|
|
if (d.needsAbsent) {
|
|
const head = await client.request('HEAD', objectPath(bucket, key));
|
|
head.resume();
|
|
if (head.statusCode === 200) return s3Error(res, 403, 'AccessDenied', 'this key may add files but not replace them');
|
|
}
|
|
const pass = {};
|
|
for (const [k, v] of Object.entries(req.headers)) {
|
|
if (/^(content-type|content-length|content-md5|content-encoding|content-disposition|cache-control|range|if-(match|none-match|modified-since|unmodified-since))$/.test(k) || (k.startsWith('x-amz-') && !/^x-amz-(date|content-sha256|security-token)$/.test(k))) pass[k] = v;
|
|
}
|
|
const release = sup.hold(inst);
|
|
try {
|
|
const up = await client.request(req.method, key == null ? objectPath(bucket) : objectPath(bucket, key), { query, headers: pass, body: req, payloadHash: payload });
|
|
relay(up, res, g.id, req.method);
|
|
await done(res);
|
|
} finally { release(); }
|
|
}
|
|
|
|
function checkReadCap(req, id) {
|
|
if (req.method === 'GET') checkCap(id);
|
|
}
|
|
|
|
async function rawProxy(req, res, inst) {
|
|
const c = readConfig(sup.configFile(inst.id));
|
|
const [host, port] = c.apiAddress.split(':');
|
|
const release = sup.hold(inst);
|
|
const u = usageOf(inst.id);
|
|
try {
|
|
const up = await new Promise((resolve, reject) => {
|
|
const headers = { ...req.headers };
|
|
delete headers['x-forwarded-for']; delete headers['x-real-ip']; delete headers.connection;
|
|
const r = http.request({ host, port: Number(port), method: req.method, path: req.url, headers }, resolve);
|
|
r.on('error', reject);
|
|
req.on('data', (chunk) => { u.up += chunk.length; });
|
|
req.pipe(r);
|
|
});
|
|
relay(up, res, inst.id, req.method);
|
|
await done(res);
|
|
} finally { release(); }
|
|
}
|
|
|
|
function relay(up, res, id, method) {
|
|
const u = usageOf(id);
|
|
res.writeHead(up.statusCode, up.headers);
|
|
if (method === 'GET') up.on('data', (c) => { u.down += c.length; });
|
|
up.pipe(res);
|
|
}
|
|
|
|
// ---- public files and links ----------------------------------------------------
|
|
async function serveObject(req, res, id, bucket, key, { inline = true } = {}) {
|
|
checkCap(id);
|
|
const inst = await sup.ensure(id);
|
|
const client = await sup.bucketClient(inst, bucket);
|
|
if (!client) throw httpError(404, 'not found');
|
|
const release = sup.hold(inst);
|
|
try {
|
|
const up = await client.getObject(bucket, key, { range: req.headers.range });
|
|
if (up.statusCode >= 300 && up.statusCode !== 304) { up.resume(); throw httpError(up.statusCode === 403 ? 404 : up.statusCode, 'not found'); }
|
|
const headers = {};
|
|
for (const h of ['content-type', 'content-length', 'content-range', 'accept-ranges', 'etag', 'last-modified']) if (up.headers[h]) headers[h] = up.headers[h];
|
|
const name = key.split('/').pop() || 'download';
|
|
headers['content-disposition'] = `${inline ? 'inline' : 'attachment'}; filename*=UTF-8''${encodeURIComponent(name)}`;
|
|
headers['x-content-type-options'] = 'nosniff';
|
|
headers['content-security-policy'] = "sandbox; default-src 'none'; img-src 'self'; media-src 'self'; style-src 'unsafe-inline'";
|
|
headers['cache-control'] = 'public, max-age=60';
|
|
const u = usageOf(id);
|
|
res.writeHead(up.statusCode, headers);
|
|
if (req.method === 'HEAD') { up.resume(); return res.end(); }
|
|
up.on('data', (c) => { u.down += c.length; });
|
|
up.pipe(res);
|
|
await done(res);
|
|
} finally { release(); }
|
|
}
|
|
|
|
// /p/<account id or BNS name>/<drive>/<path>. Also reached as
|
|
// navigate.st/s3/…, so every link and redirect here is relative.
|
|
async function servePublic(req, res, url) {
|
|
const m = /^\/p\/([^/]+)\/([^/]+)(\/?)(.*)$/.exec(url.pathname);
|
|
if (!m) throw httpError(404, 'not found');
|
|
const [, who, bucket, slash] = m;
|
|
let whoName = who;
|
|
try { whoName = decodeURIComponent(who); } catch {}
|
|
const id = /^[0-9a-f]{32}$/.test(who) ? who : sup.accountByName(whoName);
|
|
if (!id) throw httpError(404, 'not found');
|
|
// The drive itself without its slash: add it, so relative links resolve.
|
|
if (!slash && !m[4]) { res.writeHead(301, { location: `${encodeURIComponent(bucket)}/${url.search}` }); return res.end(); }
|
|
let key;
|
|
try { key = decodeURIComponent(m[3]); } catch { throw httpError(404, 'not found'); }
|
|
if (key.split('/').some((x) => x === '.' || x === '..')) throw httpError(404, 'not found');
|
|
const rule = publicRuleFor(rulesOf(id), bucket, key);
|
|
if (!rule) throw httpError(404, 'not found');
|
|
if (key === '' || key.endsWith('/')) {
|
|
if (rule.rights !== 'read-list') throw httpError(404, 'not found');
|
|
const inst = await sup.ensure(id);
|
|
const client = await sup.bucketClient(inst, bucket);
|
|
if (!client) throw httpError(404, 'not found');
|
|
const page = await client.listObjects(bucket, { prefix: key, token: url.searchParams.get('token') || undefined });
|
|
const listing = {
|
|
folders: page.folders.map((f) => f.slice(key.length)),
|
|
files: page.objects.filter((o) => o.key !== key).map((o) => ({ name: o.key.slice(key.length), size: o.size, modified: o.modified })),
|
|
next: page.truncated ? page.nextToken : null,
|
|
};
|
|
// Browsers get a page with links and a player; tools get the JSON.
|
|
if (/text\/html/.test(req.headers.accept || '') && !url.searchParams.has('json')) {
|
|
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store', 'content-security-policy': "default-src 'self'; style-src 'unsafe-inline'; img-src 'self'; media-src 'self'; script-src 'unsafe-inline'; frame-ancestors 'none'", 'x-content-type-options': 'nosniff' });
|
|
return res.end(folderPage({ owner: whoName, bucket, key, listing, token: url.searchParams.get('token') }));
|
|
}
|
|
return json(res, 200, listing);
|
|
}
|
|
return serveObject(req, res, id, bucket, key);
|
|
}
|
|
|
|
async function serveLink(req, res, url) {
|
|
const token = url.pathname.slice(3);
|
|
const l = Object.hasOwn(links, token) ? links[token] : null;
|
|
const rule = l && ruleById(l.id, l.rule);
|
|
if (!rule || !live(rule)) throw httpError(410, 'this link has expired');
|
|
return serveObject(req, res, l.id, rule.bucket, rule.prefix, { inline: !url.searchParams.has('download') });
|
|
}
|
|
|
|
// ---- the browser app ---------------------------------------------------------------
|
|
// cfg.app = { dir, hosts: ['pithos.silentmode.st'] }. Its own origin keeps the
|
|
// keys a signed-in page holds away from every other site, and the strict CSP
|
|
// keeps the page from loading or talking to anything but this origin.
|
|
const APP_TYPES = { '.html': 'text/html; charset=utf-8', '.js': 'text/javascript; charset=utf-8', '.css': 'text/css; charset=utf-8', '.svg': 'image/svg+xml', '.png': 'image/png', '.woff2': 'font/woff2', '.json': 'application/json', '.webmanifest': 'application/manifest+json' };
|
|
const APP_CSP = "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; connect-src 'self'; worker-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'; object-src 'none'";
|
|
function appHost(req) {
|
|
if (!cfg.app?.dir) return false;
|
|
const h = String(req.headers.host || '').toLowerCase().replace(/:\d+$/, '');
|
|
return (cfg.app.hosts || []).includes(h);
|
|
}
|
|
function serveApp(req, res, p) {
|
|
const root = path.resolve(cfg.app.dir);
|
|
const rel = p === '/' ? 'index.html' : decodeURIComponent(p.slice('/_app/'.length));
|
|
const file = path.resolve(root, rel);
|
|
const type = APP_TYPES[path.extname(file).toLowerCase()];
|
|
if (!file.startsWith(root + path.sep) || !type || rel.split(/[\\/]/).some((s) => s.startsWith('.')) || !fs.existsSync(file) || !fs.statSync(file).isFile()) throw httpError(404, 'not found');
|
|
const body = fs.readFileSync(file);
|
|
res.writeHead(200, {
|
|
'content-type': type,
|
|
'content-length': body.length,
|
|
'cache-control': 'no-cache',
|
|
'content-security-policy': APP_CSP,
|
|
'x-content-type-options': 'nosniff',
|
|
'referrer-policy': 'no-referrer',
|
|
'cross-origin-opener-policy': 'same-origin',
|
|
'permissions-policy': 'camera=(), microphone=(), geolocation=()',
|
|
});
|
|
res.end(req.method === 'HEAD' ? undefined : body);
|
|
}
|
|
|
|
// The account console (hosted/console/, DESIGN-theseus-id.md §11.3): three
|
|
// fixed files on the gateway's own origin, which is on the "pithos" origin
|
|
// list, so Theseus will sign a Theseus ID sign-in for it here.
|
|
const CONSOLE_DIR = fileURLToPath(new URL('./console/', import.meta.url));
|
|
const CONSOLE_FILES = {
|
|
'/console': ['index.html', 'text/html; charset=utf-8'],
|
|
'/console/console.js': ['console.js', 'text/javascript; charset=utf-8'],
|
|
'/console/console.css': ['console.css', 'text/css; charset=utf-8'],
|
|
};
|
|
const CONSOLE_CSP = "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'";
|
|
function serveConsole(req, res, p) {
|
|
const [file, type] = CONSOLE_FILES[p];
|
|
const body = fs.readFileSync(path.join(CONSOLE_DIR, file));
|
|
res.writeHead(200, {
|
|
'content-type': type, 'content-length': body.length, 'cache-control': 'no-cache',
|
|
'content-security-policy': CONSOLE_CSP, 'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer',
|
|
});
|
|
res.end(req.method === 'HEAD' ? undefined : body);
|
|
}
|
|
|
|
// ---- routing -------------------------------------------------------------------
|
|
async function handle(req, res) {
|
|
const url = new URL(req.url, cfg.publicUrl);
|
|
const isS3 = /^AWS4-HMAC-SHA256 /.test(req.headers.authorization || '') || url.searchParams.has('X-Amz-Algorithm');
|
|
if (isS3) return s3(req, res, url);
|
|
const p = url.pathname;
|
|
|
|
// The browser app (web/) on its own hostname: "/" and "/_app/*". Signed S3
|
|
// requests were routed above, so a drive named "_app" is unaffected.
|
|
if (appHost(req) && (req.method === 'GET' || req.method === 'HEAD') && (p === '/' || p.startsWith('/_app/'))) return serveApp(req, res, p);
|
|
|
|
if (Object.hasOwn(CONSOLE_FILES, p) && (req.method === 'GET' || req.method === 'HEAD')) return serveConsole(req, res, p);
|
|
if (p.startsWith('/p/') && (req.method === 'GET' || req.method === 'HEAD')) return servePublic(req, res, url);
|
|
if (p.startsWith('/l/') && (req.method === 'GET' || req.method === 'HEAD')) return serveLink(req, res, url);
|
|
|
|
if (p === '/hosted/v1/info' && req.method === 'GET') {
|
|
return json(res, 200, { publicUrl: cfg.publicUrl, gate: gate.describe(), downloadCapBytes: cfg.downloadCapBytes || null, s3Endpoint: cfg.publicUrl });
|
|
}
|
|
if (p === '/hosted/v1/challenge' && req.method === 'GET') {
|
|
const nonce = auth.challenge();
|
|
return json(res, 200, { nonce, message: challengeText(cfg.publicUrl, nonce) });
|
|
}
|
|
if (p === '/hosted/v1/session' && req.method === 'POST') {
|
|
const body = await jsonBody(req);
|
|
if (!auth.takeNonce(body.nonce)) throw httpError(401, 'the sign-in request expired; try again');
|
|
if (!verifyEd25519(body.pubkey, challengeText(cfg.publicUrl, body.nonce), body.signature)) throw httpError(401, 'the signature does not match');
|
|
const id = accountId(body.pubkey);
|
|
let a = sup.account(id);
|
|
if (!a) {
|
|
if (!body.signup) {
|
|
return json(res, 404, { error: 'no hosted account for this key yet', needsSignup: true, id, gate: gate.describe(), bnsMessage: bnsMessage(id), depositMemo: depositMemo(id) });
|
|
}
|
|
const g = await gate.admit(body.signup, id, clientIp(req));
|
|
a = sup.createAccount({ id, pubkey: body.pubkey.toLowerCase(), gate: g });
|
|
log(`[${id.slice(0, 8)}] new account via ${g.type}`);
|
|
}
|
|
return json(res, 200, { token: auth.issue(id), id, state: a.state });
|
|
}
|
|
|
|
if (p === '/hosted/v1/theseus-id/challenge' && req.method === 'GET') {
|
|
return json(res, 200, { projectId: TID_PROJECT, nonce: auth.challenge() });
|
|
}
|
|
// The console: a linked Theseus ID signs in to see the account, never its files.
|
|
if (p === '/hosted/v1/console/session' && req.method === 'POST') {
|
|
const v = await theseusIdSignIn(await jsonBody(req));
|
|
let id = tidLinks[v.account];
|
|
if (!id && v.movedFrom && tidLinks[v.movedFrom]) {
|
|
// The person moved to a new ID in Theseus (§6.3); the account follows.
|
|
id = tidLinks[v.movedFrom];
|
|
delete tidLinks[v.movedFrom];
|
|
linkTheseusId(id, v.account);
|
|
}
|
|
if (!id || !sup.account(id)) throw httpError(404, 'no hosted account is linked to this Theseus ID; sign in to Pithos in Theseus once to link it');
|
|
return json(res, 200, { token: auth.issue(id, 'console'), id, scope: 'console', moved: !!v.movedFrom });
|
|
}
|
|
|
|
if (p.startsWith('/hosted/v1/') || p.startsWith('/api/')) {
|
|
const { id, scope, token } = bearer(req);
|
|
// Console sessions see the account and nothing that touches its data.
|
|
const CONSOLE_OK = ['GET /hosted/v1/me', 'DELETE /hosted/v1/session', 'GET /hosted/v1/theseus-id'];
|
|
if (scope !== 'owner' && !CONSOLE_OK.includes(`${req.method} ${p}`)) throw httpError(403, 'sign in with your Pithos key for that; a Theseus ID console session cannot reach files or settings');
|
|
if (p.startsWith('/api/')) {
|
|
if (!FORWARD.some(([m, re]) => m === req.method && re.test(p))) throw httpError(403, 'not available on a hosted account');
|
|
return forwardApi(req, res, url, id);
|
|
}
|
|
const a = sup.account(id);
|
|
const q = url.searchParams;
|
|
switch (`${req.method} ${p}`) {
|
|
case 'GET /hosted/v1/theseus-id':
|
|
return json(res, 200, { account: a.theseusId?.account || null, linkedAt: a.theseusId?.linkedAt || null });
|
|
case 'POST /hosted/v1/theseus-id': {
|
|
// Link: the owner session proves the Ed25519 key, the sign-in proves the ID.
|
|
const v = await theseusIdSignIn(await jsonBody(req));
|
|
if (tidLinks[v.account] && tidLinks[v.account] !== id) throw httpError(409, 'this Theseus ID is already linked to another hosted account');
|
|
linkTheseusId(id, v.account);
|
|
return json(res, 200, { account: v.account });
|
|
}
|
|
case 'DELETE /hosted/v1/theseus-id':
|
|
if (a.theseusId) {
|
|
delete tidLinks[a.theseusId.account];
|
|
writeJson(tidFile, tidLinks);
|
|
const { theseusId: _gone, ...rest } = a;
|
|
sup.saveAccount(rest);
|
|
}
|
|
return json(res, 200, { ok: true });
|
|
case 'GET /hosted/v1/me':
|
|
return json(res, 200, { id, scope, theseusId: a.theseusId?.account || null, state: a.state, gate: { type: a.gate?.type, name: a.gate?.name || null }, createdAt: a.createdAt, usage: usageOf(id), downloadCapBytes: cfg.downloadCapBytes || null, publicUrl: cfg.publicUrl });
|
|
case 'DELETE /hosted/v1/session':
|
|
auth.revoke(token);
|
|
return json(res, 200, { ok: true });
|
|
case 'GET /hosted/v1/rules':
|
|
return json(res, 200, forOwner(rulesOf(id)));
|
|
case 'POST /hosted/v1/rules': {
|
|
const inst = await sup.ensure(id);
|
|
const rule = validateRule(await jsonBody(req), await sup.mapBuckets(inst));
|
|
const rules = rulesOf(id).filter((r) => live(r));
|
|
if (rules.length >= 500) throw httpError(400, 'too many rules on one account');
|
|
rules.push(rule);
|
|
saveRules(id, rules);
|
|
return json(res, 200, rule);
|
|
}
|
|
case 'DELETE /hosted/v1/rules': {
|
|
const rules = rulesOf(id);
|
|
const next = rules.filter((r) => r.id !== q.get('id'));
|
|
if (next.length === rules.length) throw httpError(404, 'no such rule');
|
|
saveRules(id, next);
|
|
return json(res, 200, { ok: true });
|
|
}
|
|
case 'PUT /hosted/v1/data/file':
|
|
return json(res, 200, await receiveFile(req, id, q.get('path')));
|
|
case 'POST /hosted/v1/data/commit':
|
|
return json(res, 200, await commit(id, (await jsonBody(req)).files));
|
|
case 'POST /hosted/v1/data/abort':
|
|
if (a.state !== 'empty') throw httpError(409, 'nothing to abort');
|
|
fs.rmSync(incomingDir(id), { recursive: true, force: true });
|
|
fs.rmSync(incomingManifest(id), { force: true });
|
|
return json(res, 200, { ok: true });
|
|
case 'POST /hosted/v1/data/checkout':
|
|
return json(res, 200, await checkout(id, q.get('force') === '1'));
|
|
case 'GET /hosted/v1/data/file': {
|
|
if (a.state !== 'moving-out') throw httpError(409, 'start the move first');
|
|
const file = path.join(sup.dataDir(id), safeRel(q.get('path')));
|
|
if (!fs.existsSync(file)) throw httpError(404, 'no such file');
|
|
res.writeHead(200, { 'content-type': 'application/octet-stream', 'content-length': fs.statSync(file).size });
|
|
await pipeline(fs.createReadStream(file), res);
|
|
return;
|
|
}
|
|
case 'POST /hosted/v1/data/cancel-checkout':
|
|
if (a.state !== 'moving-out') throw httpError(409, 'no move in progress');
|
|
sup.setState(id, 'active');
|
|
return json(res, 200, { ok: true });
|
|
case 'POST /hosted/v1/data/release':
|
|
return json(res, 200, release(id));
|
|
default:
|
|
throw httpError(404, 'no such endpoint');
|
|
}
|
|
}
|
|
throw httpError(404, 'not found');
|
|
}
|
|
|
|
const server = http.createServer((req, res) => {
|
|
handle(req, res).catch((err) => {
|
|
const status = err instanceof SigError ? err.status : err.status || 500;
|
|
if (status >= 500) log('[hosted]', err.stack || err.message);
|
|
if (res.headersSent) return res.destroy();
|
|
const isS3 = /^AWS4-HMAC-SHA256 /.test(req.headers.authorization || '');
|
|
if (isS3) return s3Error(res, status, err.code || (status === 404 ? 'NoSuchKey' : 'InternalError'), err.message);
|
|
json(res, status, { error: status >= 500 && !err.status ? 'internal error' : err.message });
|
|
});
|
|
});
|
|
server.requestTimeout = 0; // large uploads and downloads stream for a long time
|
|
|
|
const [host, port] = cfg.listen.split(':');
|
|
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(Number(port), host, resolve); });
|
|
log(`pithos-hosted on ${cfg.listen} for ${cfg.publicUrl}`);
|
|
|
|
return {
|
|
server,
|
|
supervisor: sup,
|
|
async close() {
|
|
clearInterval(meterTimer);
|
|
for (const [id, u] of usage) writeJson(path.join(sup.dir(id), 'usage.json'), u);
|
|
await new Promise((r) => server.close(r));
|
|
await sup.closeAll();
|
|
},
|
|
};
|
|
}
|
|
|
|
// A public folder as a page: breadcrumbs, folders, files (links are relative
|
|
// to the folder, so the page works under /p/ and under navigate.st/s3/),
|
|
// pictures shown small, music and video playable in place.
|
|
const PAGE_ESC = (s) => String(s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
|
|
const fmtSize = (n) => { const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; let i = 0; n = Number(n) || 0; while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; } return `${i ? n.toFixed(1) : n} ${u[i]}`; };
|
|
export function folderPage({ owner, bucket, key, listing, token }) {
|
|
const parts = key.split('/').filter(Boolean);
|
|
const up = (n) => '../'.repeat(n) || './';
|
|
const crumbs = [`<a href="${up(parts.length)}">${PAGE_ESC(bucket)}</a>`, ...parts.map((p, i) => i === parts.length - 1 ? `<span>${PAGE_ESC(p)}</span>` : `<a href="${up(parts.length - 1 - i)}">${PAGE_ESC(p)}</a>`)].join(' / ');
|
|
const enc = (name) => name.split('/').map(encodeURIComponent).join('/');
|
|
const kind = (n) => (/\.(png|jpe?g|gif|webp|avif|svg)$/i.test(n) ? 'img' : /\.(mp3|m4a|aac|ogg|oga|opus|flac|wav)$/i.test(n) ? 'audio' : /\.(mp4|webm|mov|m4v)$/i.test(n) ? 'video' : '');
|
|
const rows = [
|
|
...listing.folders.map((f) => `<li class="dir"><a href="${enc(f)}">📁 ${PAGE_ESC(f.replace(/\/$/, ''))}</a></li>`),
|
|
...listing.files.map((f) => {
|
|
const k = kind(f.name);
|
|
const media = k === 'img' ? `<img loading="lazy" src="${enc(f.name)}" alt="">` : k ? `<button type="button" class="play" data-src="${enc(f.name)}" data-kind="${k}">▶</button>` : '';
|
|
return `<li>${media}<a href="${enc(f.name)}">${PAGE_ESC(f.name)}</a><span class="sz">${fmtSize(f.size)}</span></li>`;
|
|
}),
|
|
].join('');
|
|
const more = listing.next ? `<p><a href="?token=${encodeURIComponent(listing.next)}">More…</a></p>` : '';
|
|
return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>${PAGE_ESC(parts.at(-1) || bucket)} · ${PAGE_ESC(owner)}</title>
|
|
<style>body{margin:0;background:#0b100e;color:#e6ece9;font:15px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif}main{max-width:860px;margin:0 auto;padding:28px 18px 120px}
|
|
a{color:#3ddc97;text-decoration:none}a:hover{text-decoration:underline}.owner{color:#8e9c96;font-size:13px}h1{font-size:20px;margin:4px 0 18px;font-weight:600}h1 span{color:#e6ece9}
|
|
ul{list-style:none;padding:0;margin:0;border:1px solid rgba(255,255,255,.08);border-radius:10px;overflow:hidden}li{display:flex;align-items:center;gap:10px;padding:9px 14px;border-top:1px solid rgba(255,255,255,.06);min-width:0}li:first-child{border-top:none}
|
|
li a{flex:1;min-width:0;overflow-wrap:anywhere}li img{width:44px;height:44px;object-fit:cover;border-radius:6px;flex:none}.sz{color:#8e9c96;font-size:13px;white-space:nowrap}
|
|
.play{flex:none;width:32px;height:32px;border-radius:50%;border:0;background:#3ddc97;color:#06231a;cursor:pointer;font-size:13px}
|
|
#player{position:fixed;left:0;right:0;bottom:0;background:#121a17;border-top:1px solid rgba(255,255,255,.08);padding:10px 16px;display:none}#player.on{display:block}#player audio,#player video{width:100%;max-height:50vh}.empty{color:#8e9c96;padding:18px}footer{color:#5f6b66;font-size:12.5px;margin-top:18px}</style></head>
|
|
<body><main><div class="owner">${PAGE_ESC(owner)} · public folder</div><h1>${crumbs}</h1>
|
|
${rows ? `<ul>${rows}</ul>` : '<p class="empty">This folder is empty.</p>'}${more}
|
|
<footer>Shared with Pithos on Silent Mode. <a href="?json">JSON</a></footer></main>
|
|
<div id="player"></div>
|
|
<script>document.addEventListener('click',function(e){var b=e.target.closest('.play');if(!b)return;var p=document.getElementById('player');var m=document.createElement(b.dataset.kind);m.src=b.dataset.src;m.controls=true;m.autoplay=true;p.innerHTML='';p.appendChild(m);p.className='on';});</script>
|
|
</body></html>`;
|
|
}
|
|
|
|
function json(res, status, body) {
|
|
const data = JSON.stringify(body);
|
|
res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' });
|
|
res.end(data);
|
|
}
|
|
|
|
function s3Error(res, status, code, message) {
|
|
const esc = (s) => String(s).replace(/[<>&'"]/g, (c) => `&#${c.charCodeAt(0)};`);
|
|
res.writeHead(status, { 'content-type': 'application/xml' });
|
|
res.end(`<?xml version="1.0" encoding="UTF-8"?><Error><Code>${esc(code)}</Code><Message>${esc(message)}</Message></Error>`);
|
|
}
|
|
|
|
function readAll(stream) {
|
|
return new Promise((resolve, reject) => {
|
|
const parts = [];
|
|
stream.on('data', (c) => parts.push(c));
|
|
stream.on('end', () => resolve(Buffer.concat(parts).toString('utf8')));
|
|
stream.on('error', reject);
|
|
});
|
|
}
|
|
|
|
async function jsonBody(req) {
|
|
let size = 0;
|
|
const parts = [];
|
|
for await (const c of req) { size += c.length; if (size > 1024 * 1024) throw httpError(413, 'request too large'); parts.push(c); }
|
|
try { return JSON.parse(Buffer.concat(parts).toString('utf8') || '{}'); } catch { throw httpError(400, 'invalid JSON'); }
|
|
}
|
|
|
|
function done(res) {
|
|
return new Promise((resolve) => { if (res.writableFinished) return resolve(); res.on('finish', resolve); res.on('close', resolve); });
|
|
}
|
|
|
|
function fmtBytes(n) {
|
|
const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
|
let i = 0;
|
|
while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; }
|
|
return `${n.toFixed(n < 10 && i ? 1 : 0)} ${u[i]}`;
|
|
}
|