Pithos: client-side encryption for hosted drives

Hosted drives sit on a server that holds the Sia connection key, so private
files must be unreadable there. Names are encrypted per path segment with
AES-SIV (deterministic, so folders and prefix listing still work on the
server) and bodies with chunked AES-256-GCM bound to drive and path, with
sizes computable both ways so uploads stream and range reads stay cheap.
This commit is contained in:
Local Dev 2026-10-04 01:20:05 +02:00
parent 76496fd0ee
commit 14f5ad6d74
2 changed files with 469 additions and 0 deletions

349
core/crypt.js Normal file
View file

@ -0,0 +1,349 @@
// Client-side encryption for drives hosted on Silent Mode. Everything here
// runs on the user's machine; the server only ever sees what comes out.
//
// Keys. One 32-byte master secret per person (from the Theseus vault, or a
// local key file on hosts without one). Each drive (bucket) gets its own keys
// from it, so a leaked drive key exposes one drive:
// HKDF(master, info "pithos/drive/v1/<bucket>") -> 64-byte name key + 32-byte body key
//
// Names. Every path segment is encrypted on its own with AES-SIV (RFC 5297),
// which is deterministic: the same name in the same folder always gives the
// same ciphertext, so prefix listing, folders and overwrite-by-name keep
// working on the server. The parent folder's plain path is associated data,
// so equal names in different folders do not look equal. An encrypted segment
// is "~" + base64url(SIV tag || ciphertext). SIV authenticates, so a segment
// that does not decrypt is simply a plain name (public files stay readable).
//
// Bodies. AES-256-GCM in 1 MiB chunks behind a 24-byte header:
// "PTE1" | salt (16) | log2(chunk size) (1) | 0 0 0
// The per-file key is HKDF(body key, salt). Chunk i's nonce is i (8 bytes BE)
// plus a last-chunk flag, so chunks cannot be reordered, dropped or cut off
// at the end. Each chunk's additional data is the header plus the drive and
// plain path, so the server cannot swap one file's contents for another's.
// Ciphertext size is a pure function of plaintext size and back, which keeps
// Content-Length known up front and lets range reads fetch only the chunks
// they need.
import crypto from 'node:crypto';
import { Readable } from 'node:stream';
export const HEADER_LEN = 24;
export const TAG_LEN = 16;
const MAGIC = Buffer.from('PTE1');
const LOG2_CHUNK = 20; // 1 MiB
// ---- AES-SIV (RFC 5297), AES-CMAC (RFC 4493) ---------------------------
const ZERO = Buffer.alloc(16);
function aesEcb(key, block) {
const c = crypto.createCipheriv(`aes-${key.length * 8}-ecb`, key, null);
c.setAutoPadding(false);
return Buffer.concat([c.update(block), c.final()]);
}
function dbl(b) {
const out = Buffer.alloc(16);
let carry = 0;
for (let i = 15; i >= 0; i--) {
out[i] = ((b[i] << 1) | carry) & 0xff;
carry = b[i] >> 7;
}
if (b[0] & 0x80) out[15] ^= 0x87;
return out;
}
function xor(a, b) {
const out = Buffer.alloc(a.length);
for (let i = 0; i < a.length; i++) out[i] = a[i] ^ b[i];
return out;
}
export function cmac(key, msg) {
const k1 = dbl(aesEcb(key, ZERO));
const k2 = dbl(k1);
const n = Math.max(1, Math.ceil(msg.length / 16));
const complete = msg.length > 0 && msg.length % 16 === 0;
const last = Buffer.alloc(16);
msg.copy(last, 0, (n - 1) * 16);
if (!complete) last[msg.length - (n - 1) * 16] = 0x80;
const m = Buffer.concat([msg.subarray(0, (n - 1) * 16), xor(last, complete ? k1 : k2)]);
const c = crypto.createCipheriv(`aes-${key.length * 8}-cbc`, key, ZERO);
c.setAutoPadding(false);
const out = Buffer.concat([c.update(m), c.final()]);
return out.subarray(out.length - 16);
}
function s2v(key, ads, plain) {
let d = cmac(key, ZERO);
for (const ad of ads) d = xor(dbl(d), cmac(key, ad));
let t;
if (plain.length >= 16) {
t = Buffer.from(plain);
const tail = t.length - 16;
for (let i = 0; i < 16; i++) t[tail + i] ^= d[i];
} else {
const padded = Buffer.alloc(16);
plain.copy(padded);
padded[plain.length] = 0x80;
t = xor(dbl(d), padded);
}
return cmac(key, t);
}
function sivCtr(key, v, data) {
const q = Buffer.from(v);
q[8] &= 0x7f;
q[12] &= 0x7f;
const c = crypto.createCipheriv(`aes-${key.length * 8}-ctr`, key, q);
return Buffer.concat([c.update(data), c.final()]);
}
// key: 32, 48 or 64 bytes (two AES keys). Returns tag || ciphertext.
export function sivEncrypt(key, ads, plain) {
const half = key.length / 2;
const v = s2v(key.subarray(0, half), ads, plain);
return Buffer.concat([v, sivCtr(key.subarray(half), v, plain)]);
}
// Returns the plaintext, or null when the tag does not match.
export function sivDecrypt(key, ads, data) {
if (data.length < 16) return null;
const half = key.length / 2;
const v = data.subarray(0, 16);
const plain = sivCtr(key.subarray(half), v, data.subarray(16));
return crypto.timingSafeEqual(s2v(key.subarray(0, half), ads, plain), v) ? plain : null;
}
// ---- keys ----------------------------------------------------------------
export function driveKeys(master, bucket) {
if (!Buffer.isBuffer(master) || master.length !== 32) throw new Error('the encryption key must be 32 bytes');
const okm = Buffer.from(crypto.hkdfSync('sha256', master, Buffer.alloc(0), `pithos/drive/v1/${bucket}`, 96));
return { bucket, name: okm.subarray(0, 64), body: okm.subarray(64) };
}
// ---- names -----------------------------------------------------------------
const b64u = (b) => b.toString('base64url');
function segmentAds(dk, parent) {
return [Buffer.from(dk.bucket, 'utf8'), Buffer.from(parent, 'utf8')];
}
export function encryptSegment(dk, parent, seg) {
return '~' + b64u(sivEncrypt(dk.name, segmentAds(dk, parent), Buffer.from(seg, 'utf8')));
}
// null when the segment is not one of ours (a plain name).
export function decryptSegment(dk, parent, seg) {
if (!seg.startsWith('~') || seg.length < 23) return null;
let raw;
try { raw = Buffer.from(seg.slice(1), 'base64url'); } catch { return null; }
if (b64u(raw) !== seg.slice(1)) return null;
const p = sivDecrypt(dk.name, segmentAds(dk, parent), raw);
return p ? p.toString('utf8') : null;
}
// "photos/2026/a.jpg" -> "~x/~y/~z"; a trailing "/" (folder marker or prefix)
// is kept. Empty segments are refused: S3 allows them but they make paths
// ambiguous.
export function encryptPath(dk, plain) {
if (!plain) return '';
const folder = plain.endsWith('/');
const segs = (folder ? plain.slice(0, -1) : plain).split('/');
const out = [];
let parent = '';
for (const s of segs) {
if (!s) throw new Error('empty folder names are not supported in encrypted drives');
out.push(encryptSegment(dk, parent, s));
parent += s + '/';
}
const key = out.join('/') + (folder ? '/' : '');
if (Buffer.byteLength(key) > 1024) throw new Error('this name is too long once encrypted (S3 allows 1024 bytes)');
return key;
}
// Returns { path, encrypted }. encrypted is true only if every segment was ours;
// a mix (a plain folder inside an encrypted one) reports false.
export function decryptPath(dk, key) {
if (!key) return { path: '', encrypted: false };
const folder = key.endsWith('/');
const segs = (folder ? key.slice(0, -1) : key).split('/');
const out = [];
let parent = '';
let all = true;
for (const s of segs) {
const d = decryptSegment(dk, parent, s);
if (d == null) all = false;
const plain = d ?? s;
out.push(plain);
parent += plain + '/';
}
return { path: out.join('/') + (folder ? '/' : ''), encrypted: all };
}
// ---- bodies ------------------------------------------------------------------
export function chunkSize(log2 = LOG2_CHUNK) { return 2 ** log2; }
export function cipherSize(plainSize, cs = chunkSize()) {
const chunks = Math.max(1, Math.ceil(plainSize / cs));
return HEADER_LEN + plainSize + chunks * TAG_LEN;
}
export function plainSize(cipherSizeBytes, cs = chunkSize()) {
const c = cipherSizeBytes - HEADER_LEN;
if (c < TAG_LEN) return null;
const chunks = Math.ceil(c / (cs + TAG_LEN));
const p = c - chunks * TAG_LEN;
return p >= 0 && cipherSize(p, cs) === cipherSizeBytes ? p : null;
}
export function isEncryptedHeader(buf) {
return buf.length >= HEADER_LEN && buf.subarray(0, 4).equals(MAGIC);
}
function parseHeader(header) {
if (!isEncryptedHeader(header)) throw new Error('not an encrypted Pithos file');
const log2 = header[20];
if (log2 < 12 || log2 > 24) throw new Error('unsupported chunk size in an encrypted file');
return { salt: header.subarray(4, 20), cs: 2 ** log2 };
}
function fileKey(dk, salt) {
return Buffer.from(crypto.hkdfSync('sha256', dk.body, salt, 'pithos/body/v1', 32));
}
function nonce(i, last) {
const n = Buffer.alloc(12);
n.writeBigUInt64BE(BigInt(i));
n[8] = last ? 1 : 0;
return n;
}
function aad(header, dk, plainPath) {
return Buffer.concat([header, Buffer.from(dk.bucket + '\0' + plainPath, 'utf8')]);
}
function newHeader() {
const h = Buffer.alloc(HEADER_LEN);
MAGIC.copy(h);
crypto.randomBytes(16).copy(h, 4);
h[20] = LOG2_CHUNK;
return h;
}
// Re-chunks an async iterable of buffers into exact `size`-byte pieces.
async function* rechunk(source, size) {
let parts = [];
let have = 0;
for await (const b of source) {
let buf = Buffer.isBuffer(b) ? b : Buffer.from(b);
while (buf.length) {
const take = Math.min(size - have, buf.length);
parts.push(buf.subarray(0, take));
have += take;
buf = buf.subarray(take);
if (have === size) { yield Buffer.concat(parts); parts = []; have = 0; }
}
}
if (have) yield Buffer.concat(parts);
}
// Encrypts a stream whose length is known (the upload's Content-Length).
// Returns { stream, size } so the ciphertext length can be sent up front.
export function encryptBody(dk, plainPath, source, plainLen) {
const header = newHeader();
const { salt, cs } = parseHeader(header);
const key = fileKey(dk, salt);
const ad = aad(header, dk, plainPath);
const chunks = Math.max(1, Math.ceil(plainLen / cs));
const src = Buffer.isBuffer(source) ? [source] : source;
async function* gen() {
yield header;
let i = 0;
let seen = 0;
for await (const piece of rechunk(src, cs)) {
seen += piece.length;
if (i >= chunks || seen > plainLen) throw new Error('the upload is longer than its Content-Length');
yield sealChunk(key, ad, i, i === chunks - 1, piece);
i++;
}
if (seen !== plainLen) throw new Error('the upload ended before its Content-Length');
if (plainLen === 0) yield sealChunk(key, ad, 0, true, Buffer.alloc(0));
}
return { stream: Readable.from(gen()), size: cipherSize(plainLen, cs) };
}
function sealChunk(key, ad, i, last, piece) {
const c = crypto.createCipheriv('aes-256-gcm', key, nonce(i, last));
c.setAAD(ad);
return Buffer.concat([c.update(piece), c.final(), c.getAuthTag()]);
}
function openChunk(key, ad, i, last, sealed) {
const d = crypto.createDecipheriv('aes-256-gcm', key, nonce(i, last));
d.setAAD(ad);
d.setAuthTag(sealed.subarray(sealed.length - TAG_LEN));
try {
return Buffer.concat([d.update(sealed.subarray(0, sealed.length - TAG_LEN)), d.final()]);
} catch {
throw new Error('this file was changed on the server or belongs to another drive or key');
}
}
// What to fetch for a plaintext byte range [start, end] (inclusive) of a file
// whose ciphertext is `cipherLen` bytes. Without a range, everything.
export function cipherRange(cipherLen, range, cs = chunkSize()) {
const plain = plainSize(cipherLen, cs);
if (plain == null) throw new Error('not an encrypted Pithos file (size does not match)');
const chunks = Math.max(1, Math.ceil(plain / cs));
let start = 0;
let end = plain - 1;
if (range) {
start = range.start;
end = Math.min(range.end ?? plain - 1, plain - 1);
if (start > end || start >= plain) return { plain, unsatisfiable: true };
}
const first = plain ? Math.floor(start / cs) : 0;
const lastChunk = plain ? Math.floor(end / cs) : 0;
return {
plain, start, end, chunks, first, lastChunk,
from: HEADER_LEN + first * (cs + TAG_LEN),
to: Math.min(cipherLen - 1, HEADER_LEN + (lastChunk + 1) * (cs + TAG_LEN) - 1),
};
}
// Decrypts chunks first..lastChunk from `source` (the ciphertext bytes
// r.from..r.to) and yields only plaintext bytes r.start..r.end.
export function decryptBody(dk, plainPath, header, source, r) {
const { salt, cs } = parseHeader(header);
const key = fileKey(dk, salt);
const ad = aad(header, dk, plainPath);
async function* gen() {
let i = r.first;
for await (const sealed of rechunk(source, cs + TAG_LEN)) {
if (i > r.lastChunk) break;
const plain = openChunk(key, ad, i, i === r.chunks - 1, sealed);
const base = i * cs;
const a = Math.max(0, r.start - base);
const b = Math.min(plain.length, r.end - base + 1);
if (b > a) yield plain.subarray(a, b);
i++;
}
if (i <= r.lastChunk && r.plain > 0) throw new Error('the encrypted file ended early');
}
return Readable.from(gen());
}
// "bytes=a-b" | "bytes=a-" | "bytes=-n" -> { start, end } against a known size.
export function parseRange(header, size) {
const m = /^bytes=(\d*)-(\d*)$/.exec(String(header || '').trim());
if (!m || (m[1] === '' && m[2] === '')) return null;
if (m[1] === '') {
const n = Number(m[2]);
return { start: Math.max(0, size - n), end: size - 1 };
}
return { start: Number(m[1]), end: m[2] === '' ? size - 1 : Number(m[2]) };
}

120
test/crypt.test.js Normal file
View file

@ -0,0 +1,120 @@
// Client-side encryption: published AES-CMAC / AES-SIV vectors, then the
// name and body formats round-tripped, tampered with and range-read.
import test from 'node:test';
import assert from 'node:assert/strict';
import crypto from 'node:crypto';
import { Readable } from 'node:stream';
import {
cmac, sivEncrypt, sivDecrypt, driveKeys, encryptPath, decryptPath,
encryptBody, decryptBody, cipherRange, cipherSize, plainSize, parseRange, HEADER_LEN, chunkSize,
} from '../core/crypt.js';
const hex = (s) => Buffer.from(s.replace(/\s+/g, ''), 'hex');
test('AES-CMAC matches RFC 4493', () => {
const k = hex('2b7e1516 28aed2a6 abf71588 09cf4f3c');
assert.equal(cmac(k, Buffer.alloc(0)).toString('hex'), 'bb1d6929e95937287fa37d129b756746');
assert.equal(cmac(k, hex('6bc1bee2 2e409f96 e93d7e11 7393172a')).toString('hex'), '070a16b46b4d4144f79bdd9dd04a287c');
const m40 = hex('6bc1bee2 2e409f96 e93d7e11 7393172a ae2d8a57 1e03ac9c 9eb76fac 45af8e51 30c81c46 a35ce411');
assert.equal(cmac(k, m40).toString('hex'), 'dfa66747de9ae63030ca32611497c827');
});
test('AES-SIV matches RFC 5297 A.1', () => {
const key = hex('fffefdfc fbfaf9f8 f7f6f5f4 f3f2f1f0 f0f1f2f3 f4f5f6f7 f8f9fafb fcfdfeff');
const ad = hex('10111213 14151617 18191a1b 1c1d1e1f 20212223 24252627');
const pt = hex('11223344 55667788 99aabbcc ddee');
const out = sivEncrypt(key, [ad], pt);
assert.equal(out.toString('hex'), '85632d07c6e8f37f950acd320a2ecc9340c02b9690c4dc04daef7f6afe5c');
assert.deepEqual(sivDecrypt(key, [ad], out), pt);
const bad = Buffer.from(out); bad[20] ^= 1;
assert.equal(sivDecrypt(key, [ad], bad), null);
});
const master = crypto.randomBytes(32);
const dk = driveKeys(master, 'photos');
test('paths: deterministic per folder, reversible, folders kept', () => {
const a = encryptPath(dk, 'trips/2026/beach.jpg');
assert.equal(a, encryptPath(dk, 'trips/2026/beach.jpg'));
assert.match(a, /^~[A-Za-z0-9_-]+\/~[A-Za-z0-9_-]+\/~[A-Za-z0-9_-]+$/);
assert.ok(!a.includes('beach') && !a.includes('trips'));
assert.deepEqual(decryptPath(dk, a), { path: 'trips/2026/beach.jpg', encrypted: true });
// A folder prefix lists its own children.
const folder = encryptPath(dk, 'trips/2026/');
assert.ok(a.startsWith(folder));
assert.deepEqual(decryptPath(dk, folder), { path: 'trips/2026/', encrypted: true });
// Same name, different folder: different ciphertext.
assert.notEqual(encryptPath(dk, 'x/a.txt').split('/')[1], encryptPath(dk, 'y/a.txt').split('/')[1]);
// Unicode survives.
assert.equal(decryptPath(dk, encryptPath(dk, 'Μουσική/ποίημα ✓.txt')).path, 'Μουσική/ποίημα ✓.txt');
});
test('paths: plain names, other drives and other keys stay plain', () => {
assert.deepEqual(decryptPath(dk, 'public/index.html'), { path: 'public/index.html', encrypted: false });
const enc = encryptPath(dk, 'secret.txt');
assert.equal(decryptPath(driveKeys(master, 'other'), enc).encrypted, false);
assert.equal(decryptPath(driveKeys(crypto.randomBytes(32), 'photos'), enc).encrypted, false);
assert.throws(() => encryptPath(dk, 'a//b'), /empty folder/);
});
async function collect(stream) {
const parts = [];
for await (const p of stream) parts.push(p);
return Buffer.concat(parts);
}
// Feeds the ciphertext in odd-sized pieces, like a network would.
function jagged(buf) {
const parts = [];
for (let i = 0; i < buf.length;) { const n = 1 + Math.floor(Math.random() * 70000); parts.push(buf.subarray(i, i + n)); i += n; }
return Readable.from(parts);
}
async function roundTrip(size, range) {
const plain = crypto.randomBytes(size);
const { stream, size: csize } = encryptBody(dk, 'a/b.bin', jagged(plain), size);
const ct = await collect(stream);
assert.equal(ct.length, csize);
assert.equal(plainSize(csize), size);
const r = cipherRange(ct.length, range);
const out = await collect(decryptBody(dk, 'a/b.bin', ct.subarray(0, HEADER_LEN), jagged(ct.subarray(r.from, r.to + 1)), r));
const want = range ? plain.subarray(range.start, Math.min(range.end, size - 1) + 1) : plain;
assert.ok(out.equals(want), `size ${size} range ${JSON.stringify(range)}`);
return ct;
}
test('bodies round-trip at chunk boundaries', async () => {
const cs = chunkSize();
for (const n of [0, 1, 15, cs - 1, cs, cs + 1, 3 * cs + 77]) await roundTrip(n);
});
test('bodies: range reads fetch only the chunks they need', async () => {
const cs = chunkSize();
await roundTrip(3 * cs + 500, { start: 0, end: 99 });
await roundTrip(3 * cs + 500, { start: cs - 10, end: cs + 10 });
await roundTrip(3 * cs + 500, { start: 3 * cs + 400, end: 9e15 });
const r = cipherRange(cipherSize(3 * cs + 500), { start: 2 * cs + 1, end: 2 * cs + 2 });
assert.equal(r.to - r.from + 1, cs + 16, 'one chunk');
assert.deepEqual(parseRange('bytes=-10', 100), { start: 90, end: 99 });
assert.deepEqual(parseRange('bytes=5-', 100), { start: 5, end: 99 });
assert.equal(cipherRange(cipherSize(10), { start: 50, end: 60 }).unsatisfiable, true);
});
test('bodies: tampering, truncation, swapping and length lies are caught', async () => {
const cs = chunkSize();
const size = 2 * cs + 10;
const plain = crypto.randomBytes(size);
const ct = await collect(encryptBody(dk, 'f.bin', plain, size).stream);
const full = cipherRange(ct.length);
const read = (buf, path = 'f.bin', key = dk) => collect(decryptBody(key, path, buf.subarray(0, HEADER_LEN), Readable.from([buf.subarray(HEADER_LEN)]), full));
const flipped = Buffer.from(ct); flipped[HEADER_LEN + cs + 40] ^= 1;
await assert.rejects(read(flipped), /changed on the server/);
await assert.rejects(read(ct, 'g.bin'), /changed on the server/, 'moved to another name');
await assert.rejects(read(ct, 'f.bin', driveKeys(master, 'other')), /changed on the server/);
// Dropping the last chunk: the previous one is not flagged last.
const cut = ct.subarray(0, HEADER_LEN + 2 * (cs + 16));
await assert.rejects(collect(decryptBody(dk, 'f.bin', cut.subarray(0, HEADER_LEN), Readable.from([cut.subarray(HEADER_LEN)]), full)), /ended early/);
await assert.rejects(collect(encryptBody(dk, 'x', Buffer.alloc(10), 9).stream), /longer than/);
await assert.rejects(collect(encryptBody(dk, 'x', Buffer.alloc(8), 9).stream), /ended before/);
});