pithos/core/hosted-routes.js

// "On Silent Mode" inside the control server. In hosted mode the UI talks to
// this server exactly as before; this module answers by:
//   - forwarding daemon, users, keys, stats and status calls to the gateway,
//   - giving the object routes an S3 client that encrypts and decrypts here,
//   - turning "Access" and "Share link" into gateway rules (and converting the
//     files they cover to readable copies, since visitors have no key),
//   - moving the s3d data folder to the server and back.

import fs from 'node:fs';
import path from 'node:path';
import * as admin from './admin.js';
import * as accounts from './accounts.js';
import { readPrefs, writePrefs } from './prefs.js';
import { ensureConfig } from './config.js';
import {
  fileSecrets, signIn, gateway, HostedError, EncryptingS3Client, plainCovered,
  dataFiles, uploadData, downloadData,
} from './hosted.js';

export const DEFAULT_HOSTED_URL = 'https://s3.silentmode.st';

// Calls forwarded as they are. The gateway allows the same set.
const FORWARD = [
  ['GET', /^\/api\/status$/], ['GET', /^\/api\/logs$/], ['GET', /^\/api\/stats$/],
  ['POST', /^\/api\/flush$/], ['GET', /^\/api\/autoflush$/],
  ['POST', /^\/api\/daemon\/(start|restart)$/],
  ['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/],
  ['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/],
  ['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/],
  ['GET', /^\/api\/space$/],
];
// Things that act on this computer's s3d and make no sense while drives live
// on the server.
const LOCAL_ONLY = [
  [/^\/api\/binary\/install$/, 'There is nothing to install while your drives are on Silent Mode.'],
  [/^\/api\/login/, 'Your drives are on Silent Mode, already connected to your Sia account.'],
  [/^\/api\/account\/(switch|restore)$/, 'Move your drives back to this computer before switching Sia accounts.'],
  [/^\/api\/daemon\/stop$/, 'On Silent Mode, s3d stops by itself when nobody uses it.'],
];

export function installHosted(ctx) {
  const { route, configFile, cfg, daemon, broadcast, registration, resetRegistration, HttpError } = ctx;
  const secrets = ctx.secrets || fileSecrets(configFile);
  const prefs = () => readPrefs(configFile);
  const hp = () => prefs().hosted || null;
  const isHosted = () => prefs().mode === 'hosted' && !!hp()?.token;
  const gw = () => {
    const h = hp();
    if (!h?.token) throw new HttpError(401, 'sign in to Silent Mode first');
    return gateway(h.url, h.token);
  };
  const wrap = async (fn) => {
    try { return await fn(); } catch (e) {
      if (e instanceof HostedError) {
        if (e.status === 401) { writePrefs(configFile, { hosted: { ...hp(), token: null } }); }
        throw new HttpError(e.status, e.message);
      }
      throw e;
    }
  };

  // ---- rules, cached briefly ----
  let rulesCache = null;
  async function rules() {
    if (!isHosted()) return [];
    if (rulesCache && Date.now() - rulesCache.at < 30_000) return rulesCache.list;
    const list = await wrap(() => gw().call('GET', '/hosted/v1/rules'));
    rulesCache = { at: Date.now(), list };
    return list;
  }
  const dropRules = () => { rulesCache = null; };

  // ---- the S3 client for object routes ----
  const keyCache = new Map();
  async function hostedKeys(user) {
    const c = keyCache.get(user);
    if (c && Date.now() - c.at < 60_000) return c.keys;
    const keys = await wrap(() => gw().call('GET', `/api/keys?user=${encodeURIComponent(user)}`));
    keyCache.set(user, { at: Date.now(), keys });
    return keys;
  }

  async function s3For(user, localKeys) {
    if (isHosted()) {
      const keys = await hostedKeys(user);
      if (!keys.length) throw new HttpError(409, `user "${user}" has no access keys yet - create one first`);
      return new EncryptingS3Client({
        endpoint: hp().url, accessKeyId: keys[0].accessKeyId, secretKey: keys[0].secretKey,
        master: () => secrets.encryptionKey(), rules,
      });
    }
    // Back on this computer, files encrypted while hosted still decrypt.
    if (prefs().encryptedData && localKeys?.length) {
      return new EncryptingS3Client({
        endpoint: `http://${cfg().apiAddress}`, accessKeyId: localKeys[0].accessKeyId, secretKey: localKeys[0].secretKey,
        master: () => secrets.encryptionKey(), encryptWrites: false,
      });
    }
    return null;
  }

  // ---- status, logs and events from the server ----
  let remote = null; // last /api/status from the gateway
  async function hostedStatus(local) {
    const h = hp();
    try {
      remote = await wrap(() => gw().call('GET', '/api/status'));
      return { ...remote, host: local.host, mode: 'hosted', login: local.login, hosted: { ...remote.hosted, url: h.url } };
    } catch (e) {
      return { ...local, mode: 'hosted', daemon: { state: 'stopped', error: e.message }, hosted: { url: h.url, error: e.message }, registration: { registered: true } };
    }
  }
  let lastSeq = 0;
  let lastDaemon = '';
  async function pollRemote() {
    if (!isHosted() || !ctx.hasSubscribers()) return;
    try {
      const st = await gw().call('GET', '/api/status');
      remote = st;
      const d = JSON.stringify(st.daemon);
      if (d !== lastDaemon) { lastDaemon = d; broadcast('status', st.daemon); }
      const logs = await gw().call('GET', `/api/logs?since=${lastSeq}`);
      for (const l of logs || []) { lastSeq = Math.max(lastSeq, l.seq); broadcast('log', l); }
    } catch { /* the next poll tries again */ }
  }
  const poller = setInterval(() => { pollRemote(); }, 5000);
  poller.unref?.();

  // ---- intercept: runs before the local routes in hosted mode ----
  async function intercept(req, res, url, send) {
    if (!isHosted()) return false;
    const p = url.pathname;
    for (const [re, msg] of LOCAL_ONLY) if (re.test(p)) throw new HttpError(409, msg);
    if (p === '/api/config' && req.method === 'PUT') throw new HttpError(409, 'Server settings are managed by Silent Mode while your drives are there.');
    if (p === '/api/status') return false; // answered by the local route, which asks us
    if (!FORWARD.some(([m, re]) => m === req.method && re.test(p))) return false;
    const body = req.method === 'GET' ? undefined : await readJson(req);
    const out = await wrap(() => gw().call(req.method, p + url.search, { body }));
    if (/^\/api\/(users|keys)/.test(p) && req.method !== 'GET') keyCache.clear();
    send(out);
    return true;
  }

  // ---- routes ----
  route('GET', '/api/hosted', async () => {
    const h = hp();
    const out = { mode: isHosted() ? 'hosted' : 'local', url: h?.url || DEFAULT_HOSTED_URL, id: h?.id || null, signedIn: !!h?.token, secrets: secrets.kind || 'vault' };
    if (h?.token) {
      try { const warn = await settleMoves(); if (warn) out.warning = warn; } catch {}
      out.mode = isHosted() ? 'hosted' : 'local';
      try { out.me = await gw().call('GET', '/hosted/v1/me'); } catch (e) { out.error = e.message; if (e.status === 401) out.signedIn = false; }
    }
    if (!out.signedIn) {
      try { out.info = await gateway(out.url).call('GET', '/hosted/v1/info'); } catch (e) { out.error = `Silent Mode is not reachable: ${e.message}`; }
    }
    return out;
  });

  route('POST', '/api/hosted/signin', async (req) => {
    const { url = DEFAULT_HOSTED_URL, signup } = await readJson(req);
    if (!/^https:\/\/[^\s/]+$|^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/.test(url.replace(/\/$/, ''))) throw new HttpError(400, 'the server address must be https');
    const r = await wrap(() => signIn(url.replace(/\/$/, ''), secrets, signup));
    if (r.needsSignup) return r;
    writePrefs(configFile, { hosted: { url: url.replace(/\/$/, ''), id: r.id, token: r.token } });
    return { ok: true, id: r.id, state: r.state };
  });

  route('POST', '/api/hosted/signout', async () => {
    const h = hp();
    if (h?.token) await gw().call('DELETE', '/hosted/v1/session').catch(() => {});
    writePrefs(configFile, { hosted: h ? { ...h, token: null } : null, mode: 'local' });
    dropRules();
    keyCache.clear();
    return { ok: true };
  });

  // Use drives already on Silent Mode from this computer (a second device).
  route('POST', '/api/hosted/mode', async (req) => {
    const { mode } = await readJson(req);
    if (mode === 'local') { writePrefs(configFile, { mode: 'local' }); return { mode: 'local' }; }
    if (mode !== 'hosted') throw new HttpError(400, 'mode is local or hosted');
    const me = await wrap(() => gw().call('GET', '/hosted/v1/me'));
    if (me.state !== 'active') throw new HttpError(409, 'Silent Mode has no drives for you yet. Move them there first.');
    writePrefs(configFile, { mode: 'hosted', encryptedData: true });
    return { mode: 'hosted' };
  });

  // Asks the server who holds the drives, a few times. null if it cannot tell.
  async function serverState(g) {
    for (let i = 0; i < 3; i++) {
      try { return (await g.call('GET', '/hosted/v1/me')).state; } catch { await new Promise((r) => setTimeout(r, 1000 * (i + 1))); }
    }
    return null;
  }

  // The server has the drives: set the local copy aside so it can never run
  // next to the server's (that would fork the account).
  function adoptHosted(reg, users) {
    const c = cfg();
    if (fs.existsSync(path.join(c.directory, 's3d.db'))) {
      accounts.archiveCurrent(c.directory, { indexerUrl: reg?.indexerUrl || null, users: users || [], movedTo: hp().url });
    }
    resetRegistration();
    writePrefs(configFile, { mode: 'hosted', encryptedData: true, autostart: false, moveUncertain: null });
    keyCache.clear();
    dropRules();
  }

  // Finishes a move whose last answer was lost (see move-up and move-down).
  async function settleMoves() {
    const p = prefs();
    if (!p.moveUncertain && !p.pendingRelease) return null;
    const g = gw();
    const state = await serverState(g);
    if (state == null) return 'Silent Mode is not reachable, so an earlier move is not finished yet.';
    if (p.moveUncertain) {
      if (state === 'active') adoptHosted(p.moveUncertain.reg, p.moveUncertain.users);
      else writePrefs(configFile, { moveUncertain: null }); // it never arrived; the local copy is the account
    }
    if (p.pendingRelease) {
      if (state === 'moving-out') await g.call('POST', '/hosted/v1/data/release').catch(() => {});
      if ((await serverState(g)) !== 'moving-out') writePrefs(configFile, { pendingRelease: null });
    }
    return null;
  }

  // Local -> Silent Mode. Nothing is re-uploaded to Sia: s3d's database and
  // any objects still waiting for Sia move, then s3d runs on the server.
  route('POST', '/api/hosted/move-up', async (req) => {
    const { force } = await readJson(req);
    const g = gw();
    const progress = (p) => broadcast('move', { direction: 'up', ...p });
    if (prefs().moveUncertain) throw new HttpError(409, 'An earlier move is not settled yet; open Settings again in a minute.');
    const me = await wrap(() => g.call('GET', '/hosted/v1/me'));
    if (me.state !== 'empty') throw new HttpError(409, 'Silent Mode already holds drives for you. Use them from here, or move them back first.');
    const c = cfg();
    const reg = await registration();
    if (!reg?.registered) throw new HttpError(409, 'Connect this computer to your Sia account first; then move the connection to Silent Mode.');
    const wasRunning = !!daemon.child;
    if (wasRunning) {
      progress({ phase: 'flush' });
      try { await admin.flush(c); } catch (e) {
        if (!force) throw new HttpError(409, `Some uploads could not reach Sia yet (${e.message}). Try again later, or move anyway: they will finish on the server.`);
      }
    }
    progress({ phase: 'stop' });
    await daemon.stop();
    const restartLocal = () => { if (wasRunning) { try { ensureConfig(configFile); daemon.start(); } catch {} } };
    let files;
    try {
      files = await dataFiles(c.directory);
      if (!files.some((f) => f.path === 's3d.db')) throw new HttpError(409, 'there is no s3d database to move');
      await uploadData(g, c.directory, files, progress);
    } catch (e) {
      // Nothing was committed: the server discards what arrived, s3d runs here again.
      await g.call('POST', '/hosted/v1/data/abort').catch(() => {});
      restartLocal();
      progress({ phase: 'failed', error: e.message });
      throw e;
    }
    progress({ phase: 'start' });
    let r = null;
    let commitError = null;
    try { r = await wrap(() => g.call('POST', '/hosted/v1/data/commit', { body: { files } })); } catch (e) { commitError = e; }
    // From here an answer can be lost in either direction, so the server's
    // state decides which copy is the account.
    const state = r ? 'active' : await serverState(g);
    if (state === 'active') {
      adoptHosted(reg, r?.users);
      progress({ phase: 'done' });
      return { ok: true, registered: true, users: r?.users || [] };
    }
    if (state === 'empty') {
      restartLocal();
      progress({ phase: 'failed', error: commitError?.message });
      throw commitError || new HttpError(502, 'the server did not take the drives');
    }
    // Unknown: keep s3d stopped here until the server can be asked again.
    writePrefs(configFile, { moveUncertain: { at: Date.now(), reg: { indexerUrl: reg.indexerUrl }, users: [] } });
    progress({ phase: 'failed', error: 'no answer from Silent Mode' });
    throw new HttpError(502, 'Silent Mode stopped answering during the move. s3d stays stopped here until Pithos can check which side has your drives; open Settings again in a minute.');
  });

  // Silent Mode -> local. The server flushes and stops its s3d, the folder
  // comes down, the local s3d starts on it, and only then is the server copy
  // released. Once release is sent the local copy is never removed.
  route('POST', '/api/hosted/move-down', async (req) => {
    const { force } = await readJson(req);
    const g = gw();
    const progress = (p) => broadcast('move', { direction: 'down', ...p });
    const c = cfg();
    if (daemon.child) await daemon.stop();
    let archived = null;
    if (fs.existsSync(path.join(c.directory, 's3d.db'))) {
      archived = accounts.archiveCurrent(c.directory, { note: 'set aside to bring drives back from Silent Mode' }).name;
    }
    progress({ phase: 'flush' });
    let man;
    try { man = await wrap(() => g.call('POST', `/hosted/v1/data/checkout${force ? '?force=1' : ''}`)); } catch (e) {
      if (archived) { try { accounts.restoreArchive(c.directory, archived, {}); } catch {} }
      throw e;
    }
    const written = [];
    try {
      await downloadData(g, c.directory, man.files, progress, written);
      progress({ phase: 'start' });
      ensureConfig(configFile);
      resetRegistration();
      daemon.start();
      const until = Date.now() + 30_000;
      while (daemon.state === 'starting' && Date.now() < until) await new Promise((r) => setTimeout(r, 200));
      if (daemon.state !== 'running') throw new Error('s3d did not start on the moved folder');
      const reg = await registration();
      if (!reg?.registered) throw new Error('the moved folder has no Sia connection');
    } catch (e) {
      // Only what this move wrote is removed; the server keeps its copy.
      await daemon.stop().catch(() => {});
      for (const f of written) fs.rmSync(f, { force: true });
      if (archived) { try { accounts.restoreArchive(c.directory, archived, {}); } catch {} }
      writePrefs(configFile, { mode: 'hosted' });
      await g.call('POST', '/hosted/v1/data/cancel-checkout').catch(() => {});
      progress({ phase: 'failed', error: e.message });
      throw new HttpError(502, `Moving back failed and nothing changed: ${e.message}`);
    }
    // The drives now run here. Whatever happens to the release, this copy stays.
    writePrefs(configFile, { mode: 'local', autostart: true, encryptedData: true, pendingRelease: true });
    keyCache.clear();
    dropRules();
    try {
      await g.call('POST', '/hosted/v1/data/release');
      writePrefs(configFile, { pendingRelease: null });
    } catch { /* settleMoves() retries; the server copy stays parked meanwhile */ }
    progress({ phase: 'done' });
    return { ok: true };
  });

  // Rules: public folders, shared keys, links. Files they cover become
  // readable copies; removing a rule encrypts them again.
  route('GET', '/api/hosted/rules', async () => rules());

  route('POST', '/api/hosted/rules', async (req) => {
    const body = await readJson(req);
    const client = await s3For(body.user);
    if (!client) throw new HttpError(409, 'not on Silent Mode');
    const rule = await wrap(() => gw().call('POST', '/hosted/v1/rules', { body }));
    dropRules();
    await client.convert(rule.bucket, rule.prefix, { toPlain: true, onProgress: (p) => broadcast('convert', { rule: rule.id, ...p }) });
    return rule;
  });

  route('DELETE', '/api/hosted/rules', async (req, url) => {
    const id = url.searchParams.get('id');
    const user = url.searchParams.get('user');
    const rule = (await rules()).find((r) => r.id === id);
    if (!rule) throw new HttpError(404, 'no such rule');
    await wrap(() => gw().call('DELETE', `/hosted/v1/rules?id=${encodeURIComponent(id)}`));
    dropRules();
    const remaining = await rules();
    const client = await s3For(user);
    if (client) {
      await client.convert(rule.bucket, rule.prefix, {
        toPlain: false,
        skip: (key) => plainCovered(remaining, rule.bucket, key),
        onProgress: (p) => broadcast('convert', { rule: rule.id, ...p }),
      });
    }
    return { ok: true };
  });

  // Files from before hosting (or from another S3 client) are stored as they
  // came. This encrypts every one that nothing shares.
  route('POST', '/api/hosted/encrypt', async (req) => {
    const { user, bucket } = await readJson(req);
    const client = await s3For(user);
    if (!isHosted() || !client) throw new HttpError(409, 'not on Silent Mode');
    const current = await rules();
    return client.convert(bucket, '', {
      toPlain: false,
      skip: (key) => plainCovered(current, bucket, key),
      onProgress: (p) => broadcast('convert', p),
    });
  });

  // The whole-drive "Access" dialog, answered with a public rule on "".
  async function drivePolicy(bucket) {
    const r = (await rules()).find((x) => x.type === 'public' && x.bucket === bucket && x.prefix === '');
    return { policy: null, public: r ? r.rights : 'private', hosted: true };
  }
  async function setDrivePolicy(user, bucket, mode) {
    const current = (await rules()).filter((x) => x.type === 'public' && x.bucket === bucket && x.prefix === '');
    const client = await s3For(user);
    for (const r of current) await wrap(() => gw().call('DELETE', `/hosted/v1/rules?id=${encodeURIComponent(r.id)}`));
    dropRules();
    if (mode === 'private') {
      const remaining = await rules();
      await client.convert(bucket, '', { toPlain: false, skip: (key) => plainCovered(remaining, bucket, key), onProgress: (p) => broadcast('convert', p) });
      return;
    }
    await wrap(() => gw().call('POST', '/hosted/v1/rules', { body: { type: 'public', bucket, prefix: '', rights: mode } }));
    dropRules();
    await client.convert(bucket, '', { toPlain: true, onProgress: (p) => broadcast('convert', p) });
  }

  // "Share link" for one file: a link rule, and the file stored readable.
  async function shareLink(user, bucket, key, seconds) {
    const client = await s3For(user);
    const expires = Date.now() + Math.min(Math.max(60, seconds | 0), 30 * 86400) * 1000;
    const rule = await wrap(() => gw().call('POST', '/hosted/v1/rules', { body: { type: 'link', bucket, prefix: key, expires } }));
    dropRules();
    await client.convert(bucket, key, { toPlain: true, skip: (k) => k !== key });
    const links = (prefs().hostedLinks || []).filter((l) => l.expires > Date.now());
    links.push({ user, bucket, key, expires });
    writePrefs(configFile, { hostedLinks: links });
    return `${hp().url}/l/${rule.token}`;
  }

  // Files shared by a link that has run out go back to being encrypted.
  async function sweepLinks() {
    if (!isHosted()) return;
    const all = prefs().hostedLinks || [];
    const expired = all.filter((l) => l.expires <= Date.now());
    if (!expired.length) return;
    dropRules();
    const remaining = await rules();
    for (const l of expired) {
      if (plainCovered(remaining, l.bucket, l.key)) continue;
      const client = await s3For(l.user).catch(() => null);
      await client?.convert(l.bucket, l.key, { toPlain: false, skip: (k) => k !== l.key }).catch(() => {});
    }
    writePrefs(configFile, { hostedLinks: all.filter((l) => l.expires > Date.now()) });
  }
  const sweeper = setInterval(() => { sweepLinks().catch(() => {}); }, 10 * 60_000);
  sweeper.unref?.();

  // Key file export/import for hosts without a vault.
  route('GET', '/api/hosted/secret', () => {
    if (!secrets.exportSecret) throw new HttpError(400, 'your keys come from the Theseus vault; back up the vault instead');
    return secrets.exportSecret();
  });
  route('POST', '/api/hosted/secret', async (req) => {
    if (!secrets.importSecret) throw new HttpError(400, 'your keys come from the Theseus vault');
    if (isHosted() || hp()?.token) throw new HttpError(409, 'sign out of Silent Mode before replacing your key');
    secrets.importSecret(await readJson(req));
    return { ok: true };
  });

  return {
    isHosted, intercept, s3For, hostedStatus, drivePolicy, setDrivePolicy, shareLink,
    userNames: async () => (await wrap(() => gw().call('GET', '/api/users'))).map((u) => u.name),
    remoteDaemon: () => remote?.daemon || { state: 'stopped' },
    close() { clearInterval(poller); clearInterval(sweeper); },
  };
}

async function readJson(req) {
  const parts = [];
  for await (const c of req) parts.push(c);
  const text = Buffer.concat(parts).toString('utf8');
  if (!text) return {};
  try { return JSON.parse(text); } catch { return {}; }
}