theseus/bundled-addons/pithos/core/hosted.js

// The client half of "On Silent Mode": signing in to pithos-hosted, the S3
// client that encrypts on the way out and decrypts on the way in, converting
// folders between private (encrypted) and readable when they are shared, and
// moving the s3d data folder between this computer and the server.
//
// Nothing secret leaves this process: the identity key signs a nonce, the
// encryption key never goes anywhere, and the Sia recovery phrase is not
// involved at all (only the connected s3d data folder moves).

import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { S3Client, S3Error, readBody, objectPath } from './s3.js';
import {
  driveKeys, encryptPath, decryptPath, encryptBody, decryptBody, cipherRange, plainSize, parseRange, HEADER_LEN,
} from './crypt.js';

// ---- secrets ---------------------------------------------------------------

const PKCS8_ED25519 = Buffer.from('302e020100300506032b657004220420', 'hex');

// Hosts with a vault (Theseus) pass their own; everyone else gets a random
// secret in a file beside s3d.yml. Losing that file loses the files it
// encrypted, so the UI offers to export it.
export function fileSecrets(configFile) {
  const file = path.join(path.dirname(configFile), 'pithos-hosted-secret.json');
  let root = null;
  const NOTE = 'Pithos hosted identity and encryption key. Anyone with this file can read your encrypted files; without it they are lost.';
  function save(secret) {
    fs.mkdirSync(path.dirname(file), { recursive: true });
    const tmp = `${file}.${process.pid}.tmp`;
    fs.writeFileSync(tmp, JSON.stringify({ v: 1, secret: secret.toString('base64'), note: NOTE }, null, 2), { mode: 0o600 });
    fs.renameSync(tmp, file);
  }
  // A key is only ever created when there is no file at all. A file that
  // cannot be read or parsed is an error, never a reason to make a new key:
  // that would silently orphan everything the old one encrypted.
  function load() {
    if (root) return root;
    let text;
    try { text = fs.readFileSync(file, 'utf8'); } catch (e) {
      if (e.code !== 'ENOENT') throw new Error(`cannot read your Pithos key file (${file}): ${e.message}`);
      root = crypto.randomBytes(32);
      save(root);
      return root;
    }
    let b = null;
    try { b = Buffer.from(JSON.parse(text).secret, 'base64'); } catch {}
    if (!b || b.length !== 32) throw new Error(`your Pithos key file is damaged (${file}); restore it from your backup`);
    root = b;
    return root;
  }
  const derive = (label) => Buffer.from(crypto.hkdfSync('sha256', load(), Buffer.alloc(0), label, 32));
  return {
    kind: 'file',
    file,
    identitySeed: async () => derive('pithos/hosted-identity/v1'),
    encryptionKey: async () => derive('pithos/hosted-encryption/v1'),
    exportSecret: () => JSON.parse(fs.readFileSync(file, 'utf8')),
    importSecret(obj) {
      const b = Buffer.from(String(obj?.secret || ''), 'base64');
      if (b.length !== 32) throw new Error('this is not a Pithos key file');
      // The key being replaced may still be the only way to read some files.
      if (fs.existsSync(file)) fs.copyFileSync(file, file.replace(/\.json$/, `.replaced-${Date.now()}.json`));
      save(b);
      root = b;
    },
  };
}

export function identityFromSeed(seed) {
  const key = crypto.createPrivateKey({ key: Buffer.concat([PKCS8_ED25519, seed]), format: 'der', type: 'pkcs8' });
  const pub = crypto.createPublicKey(key).export({ format: 'der', type: 'spki' }).subarray(-32);
  return { pubkey: pub.toString('hex'), sign: (msg) => crypto.sign(null, Buffer.from(msg, 'utf8'), key).toString('base64') };
}

// ---- gateway calls -------------------------------------------------------------

export class HostedError extends Error {
  constructor(status, message, data) { super(message); this.status = status; this.data = data; }
}

export function gateway(url, token) {
  const base = url.replace(/\/$/, '');
  async function call(method, p, { body, headers = {}, raw, stream } = {}) {
    const h = { ...headers };
    if (token) h.authorization = `Bearer ${token}`;
    let payload = body;
    if (body !== undefined && !stream && !Buffer.isBuffer(body)) { h['content-type'] = 'application/json'; payload = JSON.stringify(body); }
    const res = await fetch(base + p, { method, headers: h, body: payload, duplex: stream ? 'half' : undefined });
    if (raw) return res;
    const text = await res.text();
    let data = null;
    try { data = text ? JSON.parse(text) : null; } catch { data = { error: text.slice(0, 300) }; }
    if (!res.ok) throw new HostedError(res.status, data?.error || `HTTP ${res.status}`, data);
    return data;
  }
  return { base, call, auth: token ? `Bearer ${token}` : null };
}

export async function signIn(url, secrets, signup) {
  const g = gateway(url);
  const id = identityFromSeed(await secrets.identitySeed());
  const { nonce, message } = await g.call('GET', '/hosted/v1/challenge');
  try {
    return await g.call('POST', '/hosted/v1/session', { body: { pubkey: id.pubkey, nonce, signature: id.sign(message), signup } });
  } catch (e) {
    if (e.status === 404 && e.data?.needsSignup) return { needsSignup: true, ...e.data };
    throw e;
  }
}

// ---- which files are readable ------------------------------------------------------

// Files anyone else can reach (public folders, shared keys, share links) must
// be stored readable; everything else is encrypted.
export function plainCovered(rules, bucket, key) {
  const now = Date.now();
  return rules.some((r) => r.bucket === bucket && (!r.expires || r.expires > now) && (
    r.type === 'link' ? r.prefix === key : key.startsWith(r.prefix)
  ));
}

const MIME = {
  html: 'text/html', htm: 'text/html', txt: 'text/plain', md: 'text/markdown', css: 'text/css', js: 'text/javascript', mjs: 'text/javascript',
  wasm: 'application/wasm', woff: 'font/woff', woff2: 'font/woff2', ttf: 'font/ttf', otf: 'font/otf', webmanifest: 'application/manifest+json',
  json: 'application/json', xml: 'application/xml', csv: 'text/csv', pdf: 'application/pdf', zip: 'application/zip',
  png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', svg: 'image/svg+xml', avif: 'image/avif', ico: 'image/x-icon',
  mp3: 'audio/mpeg', ogg: 'audio/ogg', oga: 'audio/ogg', wav: 'audio/wav', flac: 'audio/flac', m4a: 'audio/mp4', opus: 'audio/opus',
  mp4: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', mkv: 'video/x-matroska',
};
export const guessType = (key) => MIME[String(key).split('.').pop().toLowerCase()] || 'application/octet-stream';

// ---- the encrypting S3 client ----------------------------------------------------------

// Same surface as S3Client for everything the control server's object routes
// use. Keys and sizes it returns are the plain ones. `encryptWrites` is false
// on this computer (local s3d already keeps files private) but reads still
// decrypt, so drives brought back from Silent Mode stay readable.
export class EncryptingS3Client {
  constructor({ endpoint, accessKeyId, secretKey, master, rules = async () => [], encryptWrites = true }) {
    this.inner = new S3Client({ endpoint, accessKeyId, secretKey });
    this.master = master; // async () => Buffer(32)
    this.rules = rules;
    this.encryptWrites = encryptWrites;
    this.keys = new Map();
  }

  async dk(bucket) {
    if (!this.keys.has(bucket)) this.keys.set(bucket, driveKeys(await this.master(), bucket));
    return this.keys.get(bucket);
  }

  async storeAsPlain(bucket, key) {
    return !this.encryptWrites || plainCovered(await this.rules(), bucket, key);
  }

  listBuckets() { return this.inner.listBuckets(); }
  createBucket(b) { return this.inner.createBucket(b); }
  deleteBucket(b) { return this.inner.deleteBucket(b); }
  presign(...a) { return this.inner.presign(...a); }
  getPolicy(b) { return this.inner.getPolicy(b); }
  putPolicy(b, p) { return this.inner.putPolicy(b, p); }
  deletePolicy(b) { return this.inner.deletePolicy(b); }

  // A folder can hold both forms (files from before hosting are plain), so a
  // listing below the root asks for both prefixes and merges them.
  async listObjects(bucket, { prefix = '', delimiter = '/', token, max = 1000 } = {}) {
    const dk = await this.dk(bucket);
    const cut = prefix.lastIndexOf('/') + 1;
    const folder = prefix.slice(0, cut);
    const encFolder = folder ? encryptPath(dk, folder) : '';
    const state = token ? JSON.parse(Buffer.from(token, 'base64url').toString('utf8')) : {};
    const sources = folder ? [['e', encFolder], ['p', folder]] : [['p', '']];
    const folders = new Map();
    const objects = new Map();
    const next = {};
    for (const [name, pfx] of sources) {
      // A source that has run out stays out on later pages.
      if (state[name] === null) { next[name] = null; continue; }
      const page = await this.inner.listObjects(bucket, { prefix: pfx, delimiter, token: state[name] || undefined, max });
      next[name] = page.truncated ? page.nextToken : null;
      for (const f of page.folders) {
        const d = decryptPath(dk, f);
        if (d.path.startsWith(prefix)) folders.set(d.path, { path: d.path, encrypted: d.encrypted });
      }
      for (const o of page.objects) {
        const d = decryptPath(dk, o.key);
        if (!d.path.startsWith(prefix)) continue;
        const size = d.encrypted && !d.path.endsWith('/') ? plainSize(o.size) : o.size;
        // Same name in both forms (re-uploaded after hosting): the encrypted one is newer.
        if (objects.has(d.path) && !d.encrypted) continue;
        objects.set(d.path, { key: d.path, size: size ?? o.size, modified: o.modified, etag: o.etag, encrypted: d.encrypted, storageKey: o.key });
      }
    }
    const more = Object.values(next).some((v) => v);
    const order = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
    return {
      folders: [...folders.keys()].sort(order),
      objects: [...objects.values()].sort((a, b) => order(a.key, b.key)),
      truncated: more,
      nextToken: more ? Buffer.from(JSON.stringify(next)).toString('base64url') : null,
    };
  }

  async putObject(bucket, key, body, { contentType, contentLength } = {}) {
    const dk = await this.dk(bucket);
    if (await this.storeAsPlain(bucket, key)) {
      const r = await this.inner.putObject(bucket, key, body, { contentType, contentLength });
      // Reads prefer the encrypted form, so an older encrypted copy under the
      // same name would hide this new version.
      const ek = encryptPath(dk, key);
      if (ek !== key) await this.inner.deleteObject(bucket, ek).catch(() => {});
      return r;
    }
    const ek = encryptPath(dk, key);
    if (key.endsWith('/')) return this.inner.putObject(bucket, ek, Buffer.alloc(0), { contentLength: 0 });
    const len = Number(contentLength ?? (Buffer.isBuffer(body) ? body.length : NaN));
    if (!Number.isFinite(len)) throw new Error('encrypted uploads need a Content-Length');
    const { stream, size } = encryptBody(dk, key, body, len);
    const r = await this.inner.putObject(bucket, ek, stream, { contentType: 'application/octet-stream', contentLength: size });
    // An older readable copy under the same name would shadow nothing but
    // would still be readable on the server: remove it.
    if (ek !== key) await this.inner.deleteObject(bucket, key).catch(() => {});
    return r;
  }

  // Resolves to a readable stream carrying statusCode and headers, like the
  // http.IncomingMessage S3Client.getObject returns.
  async getObject(bucket, key, { range } = {}) {
    const dk = await this.dk(bucket);
    const ek = encryptPath(dk, key);
    const head = await this.inner.request('HEAD', objectPath(bucket, ek));
    head.resume();
    if (head.statusCode === 404 || head.statusCode === 403) return this.inner.getObject(bucket, key, { range });
    if (head.statusCode >= 300) return this.inner.getObject(bucket, ek, { range });
    const total = Number(head.headers['content-length']);
    const r = cipherRange(total, range ? parseRange(range, plainSize(total) ?? 0) : null);
    const base = { 'content-type': guessType(key), 'accept-ranges': 'bytes' };
    if (head.headers.etag) base.etag = head.headers.etag;
    if (head.headers['last-modified']) base['last-modified'] = head.headers['last-modified'];
    if (r.unsatisfiable) return withStatus(Readable.from([]), 416, { ...base, 'content-range': `bytes */${r.plain}` });
    const headerRes = await this.inner.getObject(bucket, ek, { range: `bytes=0-${r.first === 0 ? r.to : HEADER_LEN - 1}` });
    if (headerRes.statusCode >= 300) return headerRes;
    // The header arrives first; when the wanted chunks follow it directly the
    // same response carries them too.
    const iter = headerRes[Symbol.asyncIterator]();
    let first = Buffer.alloc(0);
    while (first.length < HEADER_LEN) {
      const { value, done } = await iter.next();
      if (done) throw new Error('encrypted file is shorter than its header');
      first = Buffer.concat([first, value]);
    }
    const header = first.subarray(0, HEADER_LEN);
    let body;
    if (r.first === 0) {
      const rest = first.subarray(HEADER_LEN);
      body = Readable.from((async function* () { if (rest.length) yield rest; for (;;) { const { value, done } = await iter.next(); if (done) return; yield value; } })());
    } else {
      iter.return?.();
      headerRes.destroy();
      body = await this.inner.getObject(bucket, ek, { range: `bytes=${r.from}-${r.to}` });
      if (body.statusCode >= 300) return body;
    }
    const out = decryptBody(dk, key, header, r.plain ? body : Readable.from([]), r);
    const headers = { ...base, 'content-length': String(r.plain ? r.end - r.start + 1 : 0) };
    let status = 200;
    if (range) { status = 206; headers['content-range'] = `bytes ${r.start}-${r.end}/${r.plain}`; }
    return withStatus(out, status, headers);
  }

  // Renames (the routes' renameObject / renamePrefix). Encrypted names and
  // contents are bound to their path and drive, so only a readable file
  // staying readable can use s3d's server-side copy; everything else streams
  // through here, decrypted and encrypted again for its new place.
  async renameObject(bucket, from, to, dstBucket = bucket) {
    if (from === to && dstBucket === bucket) return;
    const dk = await this.dk(bucket);
    const ek = encryptPath(dk, from);
    const head = await this.inner.request('HEAD', objectPath(bucket, ek));
    head.resume();
    const srcPlain = head.statusCode === 404 || head.statusCode === 403;
    if (srcPlain && await this.storeAsPlain(dstBucket, to)) {
      await this.inner.renameObject(bucket, from, to, dstBucket);
      // An older encrypted copy under the new name would shadow this one.
      await this.inner.deleteObject(dstBucket, encryptPath(await this.dk(dstBucket), to)).catch(() => {});
      return;
    }
    if (from.endsWith('/')) {
      await this.putObject(dstBucket, to, Buffer.alloc(0), { contentLength: 0 });
    } else {
      const src = await this.getObject(bucket, from);
      if (src.statusCode >= 300) { src.resume?.(); throw new Error(`could not read ${from}: HTTP ${src.statusCode}`); }
      await this.putObject(dstBucket, to, src, { contentType: guessType(to), contentLength: Number(src.headers['content-length']) });
    }
    await this.deleteObject(bucket, from);
  }

  async renamePrefix(bucket, from, to, dstBucket = bucket) {
    // Sharing rules name their drive; a renamed drive would silently lose them.
    if (dstBucket !== bucket && this.encryptWrites && (await this.rules()).some((r) => r.bucket === bucket)) {
      throw new S3Error(409, 'Conflict', 'Stop sharing in this drive (Access) before renaming it; its shared folders and links belong to the old name.');
    }
    // Listed in full first, so the loop never walks into its own copies.
    const keys = [];
    let token;
    do {
      const page = await this.listObjects(bucket, { prefix: from, delimiter: '', token });
      for (const o of page.objects) keys.push(o.key);
      token = page.truncated ? page.nextToken : null;
    } while (token);
    for (const k of keys) await this.renameObject(bucket, k, to + k.slice(from.length), dstBucket);
    return keys.length;
  }

  async deleteObject(bucket, key) {
    const dk = await this.dk(bucket);
    const ek = encryptPath(dk, key);
    await this.inner.deleteObject(bucket, ek);
    if (ek !== key) await this.inner.deleteObject(bucket, key);
  }

  async deletePrefix(bucket, prefix) {
    let n = 0;
    let token;
    do {
      const page = await this.listObjects(bucket, { prefix, delimiter: '', token });
      for (const o of page.objects) { await this.inner.deleteObject(bucket, o.storageKey); n++; }
      token = page.truncated ? page.nextToken : null;
    } while (token);
    return n;
  }

  // Re-stores every file under prefix in the wanted form: readable (shared)
  // or encrypted (private). Rules must already say what the prefix should be.
  async convert(bucket, prefix, { toPlain, onProgress = () => {}, skip = () => false }) {
    const dk = await this.dk(bucket);
    let done = 0;
    let token;
    const todo = [];
    do {
      const page = await this.listObjects(bucket, { prefix, delimiter: '', token });
      for (const o of page.objects) if (o.encrypted === toPlain && !skip(o.key)) todo.push(o);
      token = page.truncated ? page.nextToken : null;
    } while (token);
    for (const o of todo) {
      onProgress({ done, total: todo.length, key: o.key });
      const target = toPlain ? o.key : encryptPath(dk, o.key);
      if (o.key.endsWith('/')) {
        await this.inner.putObject(bucket, target, Buffer.alloc(0), { contentLength: 0 });
      } else if (toPlain) {
        const src = await this.getObject(bucket, o.key);
        if (src.statusCode >= 300) throw new Error(`could not read ${o.key}: HTTP ${src.statusCode}`);
        await this.inner.putObject(bucket, target, src, { contentType: guessType(o.key), contentLength: o.size });
      } else {
        const src = await this.inner.getObject(bucket, o.storageKey);
        if (src.statusCode >= 300) throw new Error(`could not read ${o.key}: HTTP ${src.statusCode}`);
        const { stream, size } = encryptBody(dk, o.key, src, o.size);
        await this.inner.putObject(bucket, target, stream, { contentType: 'application/octet-stream', contentLength: size });
      }
      await this.inner.deleteObject(bucket, o.storageKey);
      done++;
    }
    onProgress({ done, total: todo.length });
    return { converted: done };
  }
}

function withStatus(stream, statusCode, headers) {
  stream.statusCode = statusCode;
  stream.headers = headers;
  return stream;
}

// ---- moving the data folder ----------------------------------------------------------------

// What travels: the database and objects still waiting for Sia. Logs, temp
// files, the local config, the key file and earlier accounts stay on this
// computer, and a path naming any of them is refused in either direction.
const SKIP_TOP = new Set(['s3d.yml', 'pithos-prefs.json', 'pithos-hosted-secret.json', 'pithos-archive.json', 'pithos.json', 'tmp', 's3d.log']);
const skipTop = (name) => SKIP_TOP.has(name) || name.startsWith('previous-account-') || name.startsWith('pithos-hosted-secret') || name.endsWith('.log') || name.endsWith('.tmp');

// "uploads/ab.obj" style paths only: forward slashes, no empty, "." or ".."
// segments, no drive letters or backslashes, and nothing that is ours.
export function isTransferPath(rel) {
  if (typeof rel !== 'string' || !rel || rel.length > 512) return false;
  if (/[\\:\x00]/.test(rel) || rel.startsWith('/')) return false;
  const segs = rel.split('/');
  if (segs.some((x) => !x || x === '.' || x === '..')) return false;
  return !skipTop(segs[0]);
}

// The file a transfer path names, proven to sit inside dir.
export function transferTarget(dir, rel) {
  if (!isTransferPath(rel)) throw new Error(`refusing an unsafe file path: ${String(rel).slice(0, 80)}`);
  const base = path.resolve(dir);
  const target = path.resolve(base, ...rel.split('/'));
  const r = path.relative(base, target);
  if (!r || r.startsWith('..') || path.isAbsolute(r)) throw new Error(`refusing an unsafe file path: ${rel}`);
  return target;
}

export function hashFile(file) {
  return new Promise((resolve, reject) => {
    const h = crypto.createHash('sha256');
    let size = 0;
    fs.createReadStream(file)
      .on('data', (c) => { h.update(c); size += c.length; })
      .on('error', reject)
      .on('end', () => resolve({ size, sha256: h.digest('hex') }));
  });
}

// Hashes stream from disk: buffered objects can be gigabytes.
export async function dataFiles(dataDir) {
  const out = [];
  const rec = async (rel) => {
    for (const e of fs.readdirSync(path.join(dataDir, rel), { withFileTypes: true })) {
      const r = rel ? `${rel}/${e.name}` : e.name;
      if (!rel && skipTop(e.name)) continue;
      if (e.isDirectory()) await rec(r);
      else if (e.isFile() && isTransferPath(r)) out.push({ path: r, ...(await hashFile(path.join(dataDir, r))) });
    }
  };
  await rec('');
  return out;
}

export async function uploadData(g, dataDir, files, onProgress = () => {}) {
  let sent = 0;
  const total = files.reduce((n, f) => n + f.size, 0);
  for (const f of files) {
    onProgress({ phase: 'upload', file: f.path, sent, total });
    const res = await fetch(`${g.base}/hosted/v1/data/file?path=${encodeURIComponent(f.path)}`, {
      method: 'PUT',
      headers: { authorization: g.auth, 'content-length': String(f.size) },
      body: fs.createReadStream(transferTarget(dataDir, f.path)),
      duplex: 'half',
    });
    const r = await res.json().catch(() => ({}));
    if (!res.ok) throw new HostedError(res.status, r.error || `upload of ${f.path} failed`);
    if (r.sha256 !== f.sha256) throw new Error(`${f.path} changed while it was being sent`);
    sent += f.size;
  }
  onProgress({ phase: 'upload', sent, total });
}

// The file list comes from the server, so every path is checked before
// anything is written, nothing existing is overwritten, and `written` lists
// exactly what this run created (the only files a failed move may remove).
export async function downloadData(g, dataDir, files, onProgress = () => {}, written = []) {
  if (!Array.isArray(files) || !files.length) throw new Error('the server sent no files');
  const targets = files.map((f) => {
    const t = transferTarget(dataDir, f.path);
    if (!/^[0-9a-f]{64}$/.test(String(f.sha256))) throw new Error(`no checksum for ${f.path}`);
    if (fs.existsSync(t)) throw new Error(`${f.path} already exists on this computer`);
    return t;
  });
  if (!files.some((f) => f.path === 's3d.db')) throw new Error('the server sent no s3d database');
  let got = 0;
  const total = files.reduce((n, f) => n + (Number(f.size) || 0), 0);
  for (let i = 0; i < files.length; i++) {
    const f = files[i];
    const target = targets[i];
    onProgress({ phase: 'download', file: f.path, got, total });
    fs.mkdirSync(path.dirname(target), { recursive: true });
    const res = await fetch(`${g.base}/hosted/v1/data/file?path=${encodeURIComponent(f.path)}`, { headers: { authorization: g.auth } });
    if (!res.ok) throw new HostedError(res.status, (await res.json().catch(() => ({}))).error || `download of ${f.path} failed`);
    const hash = crypto.createHash('sha256');
    const src = Readable.fromWeb(res.body);
    src.on('data', (c) => hash.update(c));
    written.push(target);
    await pipeline(src, fs.createWriteStream(target, { flags: 'wx' }));
    if (hash.digest('hex') !== f.sha256) throw new Error(`${f.path} arrived damaged`);
    got += Number(f.size) || 0;
  }
  onProgress({ phase: 'download', got, total });
}

export { readBody };