Pithos 0.3.16: public links at navigate.st/sia, and the Pithos name opens Overview

Files in your own Sia account can now be shared with anyone through a
navigate.st/sia link that keeps working with this computer off; the
navigate.st service is live, so the buttons can ship.

bundled-addons/pithos/addon.json

@@ -1,7 +1,7 @@
 {
   "id": "pithos",
   "name": "Pithos",
-  "version": "0.3.15",
+  "version": "0.3.16",
   "description": "Run s3d, the Sia S3 gateway, from the Theseus sidebar: connect it to a Sia indexer, create S3 users and access keys, browse and share buckets, and watch uploads reach Sia.",
   "author": "Silent Mode",
   "icon": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+PGRlZnM+PGxpbmVhckdyYWRpZW50IGlkPSJnIiB4MT0iMCIgeTE9IjAiIHgyPSIwIiB5Mj0iMSI+PHN0b3Agb2Zmc2V0PSIwIiBzdG9wLWNvbG9yPSIjM2RkYzk3Ii8+PHN0b3Agb2Zmc2V0PSIxIiBzdG9wLWNvbG9yPSIjMWU4ZjZhIi8+PC9saW5lYXJHcmFkaWVudD48L2RlZnM+PHBhdGggZD0iTTIyIDZoMjB2NWMwIDItMiAzLTIgNSA4IDMgMTQgMTEgMTQgMjEgMCAxMi05IDIxLTIyIDIxUzEwIDQ5IDEwIDM3YzAtMTAgNi0xOCAxNC0yMSAwLTItMi0zLTItNXoiIGZpbGw9InVybCgjZykiLz48cGF0aCBkPSJNMTQgMzBoMzZNMTMgNDBoMzgiIHN0cm9rZT0iIzBiMWYxOCIgc3Ryb2tlLXdpZHRoPSIzIiBzdHJva2UtbGluZWNhcD0icm91bmQiIG9wYWNpdHk9Ii41NSIvPjwvc3ZnPgo=",

bundled-addons/pithos/core/server.js

@@ -28,6 +28,7 @@ import { installHosted } from './hosted-routes.js';
 import { accountInfo, readConnection, fingerprint } from './sia-account.js';
 import { guessType } from './hosted.js';
 import { accountSpace } from './space.js';
+import { createShareUrl } from './sia-share.js';
 
 const HERE = path.dirname(fileURLToPath(import.meta.url));
 const UI_DIR = path.join(HERE, '..', 'ui');
@@ -466,6 +467,67 @@ export async function createPithos(opts = {}) {
     return { name, moved };
   });
 
+  // ---- public links: navigate.st/sia/<id>/<name> -----------------------------------
+  // A file in your own Sia account gets a Sia shared-object URL (it reads that
+  // one file and nothing else), registered at navigate.st, which streams it
+  // from Sia for anyone. Files still waiting on this computer go up first.
+  // The delete tokens stay here, beside s3d.yml, so links can be switched off.
+  const SIA_LINKS_API = process.env.PITHOS_SIA_LINKS_API || 'https://navigate.st/api/sia/links';
+  const siaLinksFile = () => path.join(path.dirname(configFile), 'pithos-sia-links.json');
+  const readSiaLinks = () => { try { return JSON.parse(fs.readFileSync(siaLinksFile(), 'utf8')); } catch { return []; } };
+  const writeSiaLinks = (list) => fs.writeFileSync(siaLinksFile(), JSON.stringify(list, null, 2), { mode: 0o600 });
+  async function shareUrlFor(bucket, key, until) {
+    const dir = cfg().directory;
+    try { return await createShareUrl(dir, bucket, key, until); }
+    catch (e) {
+      if (e.code !== 'pending') throw new HttpError(e.status || 502, e.message);
+      await admin.flush(cfg()); // send what is waiting, then try once more
+      try { return await createShareUrl(dir, bucket, key, until); }
+      catch (e2) { throw new HttpError(e2.status || 502, e2.code === 'pending' ? 'This file is still uploading to Sia. Try again in a minute.' : e2.message); }
+    }
+  }
+  route('POST', '/api/s3/:user/buckets/:bucket/sia-link', async (req, url, p) => {
+    if (hosted.isHosted()) throw new HttpError(409, 'Drives on Silent Mode share through Share… (navigate.st/s3).');
+    const { key, prefix, days = 365 } = await jsonBody(req);
+    const until = Math.floor(Date.now() / 1000) + Math.min(3650, Math.max(1, Number(days) || 365)) * 86400;
+    let body;
+    if (prefix) {
+      if (!prefix.endsWith('/')) throw new HttpError(400, 'a folder ends with /');
+      const s3 = await s3For(p.user);
+      const files = [];
+      let token;
+      do {
+        const page = await s3.listObjects(p.bucket, { prefix, token });
+        for (const o of page.objects) if (o.key !== prefix && o.size > 0) files.push(o.key);
+        token = page.truncated ? page.nextToken : null;
+      } while (token && files.length < 1000);
+      if (!files.length) throw new HttpError(400, 'This folder has no files to share (files in sub-folders are not included).');
+      const out = [];
+      for (const k of files) out.push({ name: k.slice(prefix.length), url: (await shareUrlFor(p.bucket, k, until)).url });
+      body = { name: prefix.slice(0, -1).split('/').pop(), folder: true, files: out };
+    } else {
+      if (!key || key.endsWith('/')) throw new HttpError(400, 'a file is required');
+      body = { name: key.split('/').pop(), url: (await shareUrlFor(p.bucket, key, until)).url };
+    }
+    const res = await fetch(SIA_LINKS_API, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body), signal: AbortSignal.timeout(180_000) });
+    const out = await res.json().catch(() => ({}));
+    if (!res.ok) throw new HttpError(res.status === 429 ? 429 : 502, out.error || `navigate.st answered ${res.status}`);
+    const list = readSiaLinks();
+    list.unshift({ id: out.id, url: out.url, deleteToken: out.deleteToken, user: p.user, bucket: p.bucket, key: key || null, prefix: prefix || null, until, created: Date.now(), files: body.files ? body.files.length : 1 });
+    writeSiaLinks(list);
+    return { id: out.id, url: out.url, until };
+  });
+  route('GET', '/api/sia-links', () => readSiaLinks().map(({ deleteToken, ...l }) => l));
+  route('DELETE', '/api/sia-links/:id', async (req, url, p) => {
+    const list = readSiaLinks();
+    const l = list.find((x) => x.id === p.id);
+    if (!l) throw new HttpError(404, 'no such link');
+    const res = await fetch(`${SIA_LINKS_API}/${encodeURIComponent(l.id)}`, { method: 'DELETE', headers: { 'x-delete-token': l.deleteToken }, signal: AbortSignal.timeout(30_000) });
+    if (!res.ok && res.status !== 404) throw new HttpError(502, `navigate.st answered ${res.status}`);
+    writeSiaLinks(list.filter((x) => x.id !== p.id));
+    return { ok: true };
+  });
+
   // ---- free space on the Sia account ----------------------------------------
   // From the indexer (signed with s3d's app key). Cached for a minute.
   let spaceCache = { at: 0, value: null };

bundled-addons/pithos/core/sia-share.js

@@ -0,0 +1,107 @@
+// Sia share links for files in your own Sia account. A shared object URL lets
+// anyone read ONE object (its data key travels in the URL fragment) and
+// nothing else in the account. It is made the way the Sia SDK makes it
+// (siastorage CreateSharedObjectURL, indexd api/app/client.go):
+//   1. s3d's database maps <drive>/<name> to the object's Sia id;
+//   2. the indexer returns the sealed object (signed GET /objects/<id>);
+//   3. its data key is opened with a key derived from s3d's app key:
+//      XChaCha20-Poly1305 under HKDF-BLAKE2b-256(appKey, salt = id, "dataKey");
+//   4. the link is a signed GET /objects/<id>/shared, valid until a date,
+//      with #encryption_key=<data key> (padded URL-safe base64).
+// The app key is read from s3d's database for this and never leaves.
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { blake2b256 } from './blake2b.js';
+import { readConnection, signRequest } from './sia-account.js';
+
+// ---- HMAC / HKDF over BLAKE2b-256 (block size 128) --------------------------------
+export function hmacBlake2b256(key, data) {
+  let k = Buffer.from(key);
+  if (k.length > 128) k = blake2b256(k);
+  const block = Buffer.alloc(128);
+  k.copy(block);
+  const ipad = Buffer.from(block.map((b) => b ^ 0x36));
+  const opad = Buffer.from(block.map((b) => b ^ 0x5c));
+  return blake2b256(Buffer.concat([opad, blake2b256(Buffer.concat([ipad, Buffer.from(data)]))]));
+}
+
+export function hkdfBlake2b256(ikm, salt, info, length) {
+  const prk = hmacBlake2b256(salt && salt.length ? salt : Buffer.alloc(32), ikm);
+  const out = [];
+  let t = Buffer.alloc(0);
+  for (let i = 1; Buffer.concat(out).length < length; i++) {
+    t = hmacBlake2b256(prk, Buffer.concat([t, Buffer.from(info || []), Buffer.from([i])]));
+    out.push(t);
+  }
+  return Buffer.concat(out).subarray(0, length);
+}
+
+// ---- XChaCha20-Poly1305 (open only) ------------------------------------------------
+const rotl = (v, n) => ((v << n) | (v >>> (32 - n))) >>> 0;
+function quarter(s, a, b, c, d) {
+  s[a] = (s[a] + s[b]) >>> 0; s[d] = rotl(s[d] ^ s[a], 16);
+  s[c] = (s[c] + s[d]) >>> 0; s[b] = rotl(s[b] ^ s[c], 12);
+  s[a] = (s[a] + s[b]) >>> 0; s[d] = rotl(s[d] ^ s[a], 8);
+  s[c] = (s[c] + s[d]) >>> 0; s[b] = rotl(s[b] ^ s[c], 7);
+}
+export function hchacha20(key, nonce16) {
+  const s = new Uint32Array(16);
+  s.set([0x61707865, 0x3320646e, 0x79622d32, 0x6b206574]);
+  for (let i = 0; i < 8; i++) s[4 + i] = key.readUInt32LE(i * 4);
+  for (let i = 0; i < 4; i++) s[12 + i] = nonce16.readUInt32LE(i * 4);
+  for (let r = 0; r < 10; r++) {
+    quarter(s, 0, 4, 8, 12); quarter(s, 1, 5, 9, 13); quarter(s, 2, 6, 10, 14); quarter(s, 3, 7, 11, 15);
+    quarter(s, 0, 5, 10, 15); quarter(s, 1, 6, 11, 12); quarter(s, 2, 7, 8, 13); quarter(s, 3, 4, 9, 14);
+  }
+  const out = Buffer.alloc(32);
+  [0, 1, 2, 3, 12, 13, 14, 15].forEach((w, i) => out.writeUInt32LE(s[w], i * 4));
+  return out;
+}
+export function xchachaOpen(key, nonce24, sealed) {
+  const sub = hchacha20(Buffer.from(key), Buffer.from(nonce24).subarray(0, 16));
+  const nonce12 = Buffer.concat([Buffer.alloc(4), Buffer.from(nonce24).subarray(16, 24)]);
+  const ct = Buffer.from(sealed);
+  const d = crypto.createDecipheriv('chacha20-poly1305', sub, nonce12, { authTagLength: 16 });
+  d.setAuthTag(ct.subarray(ct.length - 16));
+  return Buffer.concat([d.update(ct.subarray(0, ct.length - 16)), d.final()]);
+}
+
+// ---- s3d's database: <drive>/<name> -> Sia object id -------------------------------
+export async function siaObjectId(dataDir, bucket, key) {
+  const file = path.join(dataDir, 's3d.db');
+  if (!fs.existsSync(file)) throw new Error('s3d has no database yet');
+  const { DatabaseSync } = await import('node:sqlite');
+  const db = new DatabaseSync(file, { readOnly: true });
+  try {
+    const row = db.prepare(`SELECT o.sia_object_id AS id, o.size AS size FROM objects o JOIN buckets b ON b.id = o.bucket_id
+      WHERE b.name = ? AND o.name = ? AND o.is_latest = 1 AND o.is_delete_marker = 0`).get(bucket, key);
+    if (!row) return null;
+    return { id: row.id ? Buffer.from(row.id) : null, size: Number(row.size) };
+  } finally { db.close(); }
+}
+
+const b64urlPadded = (b) => Buffer.from(b).toString('base64').replace(/\+/g, '-').replace(/\//g, '_');
+
+// The shared-object URL for one file, valid until validUntil (unix seconds).
+export async function createShareUrl(dataDir, bucket, key, validUntil, { fetchImpl = fetch } = {}) {
+  const conn = await readConnection(dataDir);
+  if (!conn?.appKey || !conn.indexerUrl) throw Object.assign(new Error('s3d is not connected to a Sia account'), { status: 409 });
+  const obj = await siaObjectId(dataDir, bucket, key);
+  if (!obj) throw Object.assign(new Error('no such file'), { status: 404 });
+  if (!obj.id) throw Object.assign(new Error('This file has not reached Sia yet. Upload it now (Overview › Upload now) and try again.'), { status: 409, code: 'pending' });
+  const base = conn.indexerUrl.replace(/\/+$/, '');
+  const idHex = obj.id.toString('hex');
+  const res = await fetchImpl(signRequest(conn.appKey, 'GET', `${base}/objects/${idHex}`, Math.floor(Date.now() / 1000) + 120), { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(20_000) });
+  if (!res.ok) throw new Error(`indexer answered ${res.status}: ${(await res.text()).slice(0, 160)}`);
+  const sealed = await res.json();
+  const enc = Buffer.from(sealed.encryptedDataKey || '', 'base64');
+  if (enc.length < 24 + 16) throw new Error('the indexer returned no data key');
+  const kek = hkdfBlake2b256(conn.appKey, obj.id, Buffer.from('dataKey'), 32);
+  const dataKey = xchachaOpen(kek, enc.subarray(0, 24), enc.subarray(24));
+  kek.fill(0);
+  const u = signRequest(conn.appKey, 'GET', `${base}/objects/${idHex}/shared`, validUntil);
+  u.hash = `encryption_key=${b64urlPadded(dataKey)}`;
+  dataKey.fill(0);
+  return { url: u.toString(), size: obj.size, objectId: idHex };
+}

bundled-addons/pithos/ui/app.css

@@ -453,3 +453,7 @@ nav a.nav-account.active .na-label { color: var(--accent); }
 .acct-tile.add { appearance: none; font: inherit; color: var(--text); cursor: pointer; border-style: dashed; background: transparent; text-align: left; justify-content: center; }
 .acct-tile.add:hover { border-color: var(--accent); }
 .acct-plus { font-size: 20px; line-height: 1; color: var(--accent); }
+
+/* The Pithos name is a link to Overview */
+a.brand { color: inherit; text-decoration: none; }
+a.brand:hover > span:first-of-type { color: var(--accent); }

bundled-addons/pithos/ui/app.js

@@ -1563,7 +1563,7 @@ function objectBrowser(head, body, user, bucket, prefix, userSel) {
     }
     const folderName = (f) => f.slice(prefix.length).replace(/\/$/, '');
     const fileName = (o) => o.key.slice(prefix.length);
-    const folderMenu = (f) => itemMenu([['Open', () => go(f)], isHosted() && ['Share…', () => folderShareDialog(user, bucket, f)], ['Rename…', () => renameItem(f)], ['Delete', () => deleteFolder(f), 'danger']]);
+    const folderMenu = (f) => itemMenu([['Open', () => go(f)], isHosted() ? ['Share…', () => folderShareDialog(user, bucket, f)] : ['Public link…', () => siaFolderLink(f)], ['Rename…', () => renameItem(f)], ['Delete', () => deleteFolder(f), 'danger']]);
     const fileMenu = (o) => itemMenu([[fileKind(o.key) === 'audio' ? 'Play' : isHtml(o.key) ? 'Show as text' : 'Open', () => openViewer(o)], isHtml(o.key) && ['Open as page ↗', () => openPage(o.key)], ['Download', () => downloadObject(o.key)], ['Share link…', () => share(o.key)], ['Rename…', () => renameItem(o.key)], ['Delete', () => deleteObject(o.key), 'danger']]);
     const more = last?.truncated && h('div', { class: 'row', style: 'justify-content:center;margin-top:12px' }, h('button', { class: 'btn', onclick: () => load(last.nextToken) }, 'Load more'));
     const stop = (e) => e.stopPropagation();
@@ -1806,15 +1806,44 @@ function objectBrowser(head, body, user, bucket, prefix, userSel) {
     expires.addEventListener('change', gen);
     endpoint.addEventListener('change', gen);
     gen();
-    await modal('Share link', h('div', { class: 'stack' },
+    await modal('Share', h('div', { class: 'stack' },
-      h('p', { class: 'small muted', style: 'margin:0' }, 'A signed link that works without a key until it expires.'),
+      siaLinkBlock({ key }),
+      h('details', { class: 'more-opts', open: !!savedEndpoint },
+        h('summary', {}, 'Link for this computer, or your own server'),
+        h('div', { class: 'stack', style: 'margin-top:8px' },
+          h('p', { class: 'small muted', style: 'margin:0' }, 'A signed link straight to s3d that works without a key until it expires.'),
           h('label', { class: 'field' }, h('span', {}, 'Valid for'), expires),
           out,
           note,
-      h('details', { class: 'more-opts', open: !!savedEndpoint },
+          h('label', { class: 'field' }, h('span', {}, 'Your own public address'), endpoint, h('small', {}, 'The address where other people reach this s3d, e.g. a reverse proxy on your server. The link is remade when you change it.'))))),
-        h('summary', {}, 'Use my own public address'),
+    [{ label: 'Done', kind: 'primary', submit: true, result: true }]);
-        h('label', { class: 'field', style: 'margin-top:8px' }, endpoint, h('small', {}, 'The address where other people reach this s3d, e.g. a reverse proxy on your server. The link is remade when you change it.')))),
+  }
-    [{ label: 'Copy link', value: async () => { if (link) await copy(link, 'Link copied'); return false; } }, { label: 'Done', kind: 'primary', submit: true, result: true }]);
+
+  // A public link anyone can open, at navigate.st/sia: the file is read from
+  // your Sia account through Silent Mode, so it works with this computer off.
+  function siaLinkBlock({ key, prefix }) {
+    const days = h('select', {}, [['30', '1 month'], ['365', '1 year'], ['1825', '5 years']].map(([v, l]) => h('option', { value: v, selected: v === '365' }, l)));
+    const out = h('div');
+    const btn = h('button', { type: 'button', class: 'btn primary', onclick: async () => {
+      btn.disabled = true; put(btn, h('span', { class: 'spinner' }), ' Making the link…');
+      try {
+        const r = await api('POST', `${base}/sia-link`, { key, prefix, days: Number(days.value) });
+        put(out, secretRow('Public link', r.url), h('p', { class: 'small muted', style: 'margin:0' }, `Works until ${new Date(r.until * 1000).toLocaleDateString()}. Switch it off any time in Settings › Public links.`));
+        put(btn, 'Make another link');
+      } catch (e) { put(out, h('p', { class: 'error small', style: 'margin:0' }, e.message)); put(btn, 'Try again'); }
+      btn.disabled = false;
+    } }, 'Create public link');
+    return h('div', { class: 'stack' },
+      h('h3', { style: 'margin:0' }, 'Public link'),
+      h('p', { class: 'small muted', style: 'margin:0' }, prefix
+        ? 'A page with the files in this folder that anyone can open, on any device, even with this computer off. Files in sub-folders are not included.'
+        : 'Anyone with the link can open this file on any device, even with this computer off. It is read from your Sia account through navigate.st; only this file is shared.'),
+      h('div', { class: 'row' }, h('label', { class: 'row small' }, h('span', { class: 'muted' }, 'Works for'), days), btn),
+      out);
+  }
+
+  async function siaFolderLink(prefix) {
+    await modal(`Public link for ${prefix.slice(0, -1).split('/').pop()}`, siaLinkBlock({ prefix }), [{ label: 'Done', kind: 'primary', submit: true, result: true }]);
   }
 
   // On Silent Mode a link is a rule on the server, and the file is stored
@@ -1840,6 +1869,24 @@ function objectBrowser(head, body, user, bucket, prefix, userSel) {
   load();
 }
 
+// Settings › Public links: links made at navigate.st/sia, each can be switched off.
+async function publicLinksCard(card) {
+  let list = [];
+  try { list = await api('GET', '/api/sia-links'); } catch { card.remove(); return; }
+  if (!list.length) { card.remove(); return; }
+  put(card, h('h2', {}, 'Public links'),
+    h('p', { class: 'small muted' }, 'Links anyone can open at navigate.st/sia. Switching one off stops it at once.'),
+    h('table', {}, h('tbody', {}, list.map((l) => h('tr', {},
+      h('td', {}, h('a', { href: l.url, target: '_blank', rel: 'noopener' }, l.prefix ? `📁 ${l.bucket}/${l.prefix}` : `${l.bucket}/${l.key}`),
+        h('div', { class: 'small muted' }, `until ${new Date(l.until * 1000).toLocaleDateString()}${l.prefix ? ` · ${l.files} files` : ''}`)),
+      h('td', { class: 'actions' },
+        h('button', { class: 'btn small', onclick: () => copy(l.url, 'Link copied') }, 'Copy'), ' ',
+        h('button', { class: 'btn small danger', onclick: async () => {
+          if (!(await confirmModal('Switch this link off?', 'Anyone who has it will no longer be able to open it.', 'Switch off'))) return;
+          try { await api('DELETE', `/api/sia-links/${encodeURIComponent(l.id)}`); toast('Link switched off'); publicLinksCard(card); } catch (e) { fail(e); }
+        } }, 'Switch off')))))));
+}
+
 // ---------------------------------------------------------------- on Silent Mode
 
 const isHosted = () => state.status?.mode === 'hosted';
@@ -2260,6 +2307,9 @@ function settings(main) {
   const whereCard = h('div', { class: 'card', style: 'margin-bottom:16px' });
   main.append(h('div', { class: 'page-head' }, h('div', {}, h('h1', {}, 'Settings'), h('p', { class: 'muted' }, 'Where your drives live, and s3d.yml for s3d on this computer. s3d reads it at start, so a restart applies the changes.'))), whereCard, card, flushCard);
   hostedCard(whereCard);
+  const linksCard = h('div', { class: 'card', style: 'margin-top:16px' });
+  main.append(linksCard);
+  publicLinksCard(linksCard);
   // s3d.yml and the leftovers setting are for s3d on this computer; on Silent
   // Mode the server manages both (leftovers always go to Sia).
   if (isHosted()) {

bundled-addons/pithos/ui/index.html

@@ -30,7 +30,7 @@
 
 <div id="app" class="app" hidden>
   <aside class="sidebar">
-    <div class="brand"><img src="icon.svg" alt="" width="28" height="28"><span>Pithos</span><span class="here" id="here"></span></div>
+    <a class="brand" href="#/overview" title="Overview"><img src="icon.svg" alt="" width="28" height="28"><span>Pithos</span><span class="here" id="here"></span></a>
     <nav id="nav">
       <a href="#/account" data-view="account" data-title="Accounts" class="nav-account" title="Your Sia account: the storage everything here is kept in">
         <span class="na-label">Accounts</span>