Pithos: renames and free space work on Silent Mode too
Renaming refused on hosted drives because encrypted names and contents are bound to their path. The encrypting client now renames itself: readable files keep s3d's server-side copy, the rest are decrypted and encrypted again for their new place. A drive with shared folders or links is not renamed, since its rules belong to the old name. Free space is forwarded to the server, whose s3d answers for its own folder, instead of reporting nothing. space.js now builds on sia-account.js, so BLAKE2b and the indexer request signing exist once.
This commit is contained in:
Local Dev
parent
05a1941aa3
commit
8dad9f8b19
5 changed files with 71 additions and 104 deletions
|
|
@ -27,6 +27,7 @@ const FORWARD = [
|
|||
['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/],
|
||||
['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/],
|
||||
['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/],
|
||||
['GET', /^\/api\/space$/],
|
||||
];
|
||||
// Things that act on this computer's s3d and make no sense while drives live
|
||||
// on the server.
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import path from 'node:path';
|
|||
import crypto from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { S3Client, readBody, objectPath } from './s3.js';
|
||||
import { S3Client, S3Error, readBody, objectPath } from './s3.js';
|
||||
import {
|
||||
driveKeys, encryptPath, decryptPath, encryptBody, decryptBody, cipherRange, plainSize, parseRange, HEADER_LEN,
|
||||
} from './crypt.js';
|
||||
|
|
@ -271,6 +271,50 @@ export class EncryptingS3Client {
|
|||
return withStatus(out, status, headers);
|
||||
}
|
||||
|
||||
// Renames (the routes' renameObject / renamePrefix). Encrypted names and
|
||||
// contents are bound to their path and drive, so only a readable file
|
||||
// staying readable can use s3d's server-side copy; everything else streams
|
||||
// through here, decrypted and encrypted again for its new place.
|
||||
async renameObject(bucket, from, to, dstBucket = bucket) {
|
||||
if (from === to && dstBucket === bucket) return;
|
||||
const dk = await this.dk(bucket);
|
||||
const ek = encryptPath(dk, from);
|
||||
const head = await this.inner.request('HEAD', objectPath(bucket, ek));
|
||||
head.resume();
|
||||
const srcPlain = head.statusCode === 404 || head.statusCode === 403;
|
||||
if (srcPlain && await this.storeAsPlain(dstBucket, to)) {
|
||||
await this.inner.renameObject(bucket, from, to, dstBucket);
|
||||
// An older encrypted copy under the new name would shadow this one.
|
||||
await this.inner.deleteObject(dstBucket, encryptPath(await this.dk(dstBucket), to)).catch(() => {});
|
||||
return;
|
||||
}
|
||||
if (from.endsWith('/')) {
|
||||
await this.putObject(dstBucket, to, Buffer.alloc(0), { contentLength: 0 });
|
||||
} else {
|
||||
const src = await this.getObject(bucket, from);
|
||||
if (src.statusCode >= 300) { src.resume?.(); throw new Error(`could not read ${from}: HTTP ${src.statusCode}`); }
|
||||
await this.putObject(dstBucket, to, src, { contentType: guessType(to), contentLength: Number(src.headers['content-length']) });
|
||||
}
|
||||
await this.deleteObject(bucket, from);
|
||||
}
|
||||
|
||||
async renamePrefix(bucket, from, to, dstBucket = bucket) {
|
||||
// Sharing rules name their drive; a renamed drive would silently lose them.
|
||||
if (dstBucket !== bucket && this.encryptWrites && (await this.rules()).some((r) => r.bucket === bucket)) {
|
||||
throw new S3Error(409, 'Conflict', 'Stop sharing in this drive (Access) before renaming it; its shared folders and links belong to the old name.');
|
||||
}
|
||||
// Listed in full first, so the loop never walks into its own copies.
|
||||
const keys = [];
|
||||
let token;
|
||||
do {
|
||||
const page = await this.listObjects(bucket, { prefix: from, delimiter: '', token });
|
||||
for (const o of page.objects) keys.push(o.key);
|
||||
token = page.truncated ? page.nextToken : null;
|
||||
} while (token);
|
||||
for (const k of keys) await this.renameObject(bucket, k, to + k.slice(from.length), dstBucket);
|
||||
return keys.length;
|
||||
}
|
||||
|
||||
async deleteObject(bucket, key) {
|
||||
const dk = await this.dk(bucket);
|
||||
const ek = encryptPath(dk, key);
|
||||
|
|
|
|||
|
|
@ -412,8 +412,9 @@ export async function createPithos(opts = {}) {
|
|||
});
|
||||
// ---- renames: files, folders, drives -----------------------------------
|
||||
// Server-side copies, so they are quick and use no extra Sia storage.
|
||||
// Drives on Silent Mode are encrypted with keys tied to each name; their
|
||||
// client has no rename yet, and says so instead of breaking files.
|
||||
// Every client Pithos makes can rename; the check stays for any that
|
||||
// cannot (encrypted names and contents are tied to their path, so a raw
|
||||
// server-side copy would break them).
|
||||
async function renamer(user) {
|
||||
const s3 = await s3For(user);
|
||||
if (typeof s3.renamePrefix !== 'function') throw new HttpError(409, 'Renaming is not available for drives on Silent Mode yet.');
|
||||
|
|
@ -467,7 +468,7 @@ export async function createPithos(opts = {}) {
|
|||
// From the indexer (signed with s3d's app key). Cached for a minute.
|
||||
let spaceCache = { at: 0, value: null };
|
||||
route('GET', '/api/space', async () => {
|
||||
if (hosted.isHosted()) return { available: false, reason: 'hosted' };
|
||||
// On Silent Mode this call is forwarded: the server's s3d answers for its own folder.
|
||||
if (Date.now() - spaceCache.at < 60_000 && spaceCache.value) return spaceCache.value;
|
||||
try {
|
||||
const s = await accountSpace(cfg().directory);
|
||||
|
|
|
|||
120
core/space.js
120
core/space.js
|
|
@ -1,109 +1,29 @@
|
|||
// Storage left on the Sia account: the indexer's GET /account, signed with
|
||||
// the app key s3d registered at login (it lives in s3d's own database, and
|
||||
// is only read here to sign this one request; it is never sent anywhere).
|
||||
// The signature scheme is indexd's (api/app/auth.go requestHash + client.go
|
||||
// sign): BLAKE2b-256 over method, host, path and the expiry, Ed25519-signed,
|
||||
// passed as ?sv=<unix expiry>&sc=<pubkey>&ss=<signature> (URL-safe base64).
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
// Free space on the Sia account behind this s3d, as the indexer reports it.
|
||||
// The key reading, BLAKE2b-256 and request signing live in one place
|
||||
// (sia-account.js, blake2b.js); this keeps the names the space route and its
|
||||
// tests use.
|
||||
|
||||
// ---- BLAKE2b-256 (RFC 7693) ----------------------------------------------
|
||||
// Node only ships blake2b512, and Sia hashes are the 256-bit variant, which
|
||||
// is a different parameter block, not a truncation. Inputs here are tiny,
|
||||
// so plain BigInt arithmetic is fine.
|
||||
const M64 = (1n << 64n) - 1n;
|
||||
const IV = [
|
||||
0x6a09e667f3bcc908n, 0xbb67ae8584caa73bn, 0x3c6ef372fe94f82bn, 0xa54ff53a5f1d36f1n,
|
||||
0x510e527fade682d1n, 0x9b05688c2b3e6c1fn, 0x1f83d9abfb41bd6bn, 0x5be0cd19137e2179n,
|
||||
];
|
||||
const SIGMA = [
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3],
|
||||
[11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8],
|
||||
[9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13], [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9],
|
||||
[12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11], [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10],
|
||||
[6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5], [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0],
|
||||
];
|
||||
const rotr = (x, n) => ((x >> BigInt(n)) | (x << BigInt(64 - n))) & M64;
|
||||
import { readConnection, signRequest, accountInfo } from './sia-account.js';
|
||||
|
||||
export function blake2b256(data) {
|
||||
const outLen = 32;
|
||||
const h = IV.slice();
|
||||
h[0] ^= 0x01010000n ^ BigInt(outLen);
|
||||
const buf = Buffer.from(data);
|
||||
const blocks = Math.max(1, Math.ceil(buf.length / 128));
|
||||
for (let b = 0; b < blocks; b++) {
|
||||
const last = b === blocks - 1;
|
||||
const block = Buffer.alloc(128);
|
||||
buf.copy(block, 0, b * 128, Math.min(buf.length, (b + 1) * 128));
|
||||
const m = Array.from({ length: 16 }, (_, i) => block.readBigUInt64LE(i * 8));
|
||||
const t = BigInt(last ? buf.length : (b + 1) * 128);
|
||||
const v = [...h, ...IV];
|
||||
v[12] ^= t & M64;
|
||||
v[13] ^= t >> 64n;
|
||||
if (last) v[14] ^= M64;
|
||||
const G = (a, bb, c, d, x, y) => {
|
||||
v[a] = (v[a] + v[bb] + x) & M64; v[d] = rotr(v[d] ^ v[a], 32);
|
||||
v[c] = (v[c] + v[d]) & M64; v[bb] = rotr(v[bb] ^ v[c], 24);
|
||||
v[a] = (v[a] + v[bb] + y) & M64; v[d] = rotr(v[d] ^ v[a], 16);
|
||||
v[c] = (v[c] + v[d]) & M64; v[bb] = rotr(v[bb] ^ v[c], 63);
|
||||
};
|
||||
for (let r = 0; r < 12; r++) {
|
||||
const s = SIGMA[r % 10];
|
||||
G(0, 4, 8, 12, m[s[0]], m[s[1]]); G(1, 5, 9, 13, m[s[2]], m[s[3]]);
|
||||
G(2, 6, 10, 14, m[s[4]], m[s[5]]); G(3, 7, 11, 15, m[s[6]], m[s[7]]);
|
||||
G(0, 5, 10, 15, m[s[8]], m[s[9]]); G(1, 6, 11, 12, m[s[10]], m[s[11]]);
|
||||
G(2, 7, 8, 13, m[s[12]], m[s[13]]); G(3, 4, 9, 14, m[s[14]], m[s[15]]);
|
||||
}
|
||||
for (let i = 0; i < 8; i++) h[i] ^= v[i] ^ v[i + 8];
|
||||
}
|
||||
const out = Buffer.alloc(64);
|
||||
h.forEach((x, i) => out.writeBigUInt64LE(x, i * 8));
|
||||
return out.subarray(0, outLen);
|
||||
}
|
||||
export { blake2b256 } from './blake2b.js';
|
||||
|
||||
// ---- indexd request signing ---------------------------------------------------
|
||||
const PKCS8_ED25519 = Buffer.from('302e020100300506032b657004220420', 'hex');
|
||||
export const signIndexerUrl = (appKey, method, endpointUrl, validUntilSec) => signRequest(appKey, method, endpointUrl, validUntilSec);
|
||||
|
||||
export function signIndexerUrl(appKey, method, endpointUrl, validUntilSec) {
|
||||
const u = new URL(endpointUrl);
|
||||
const until = Buffer.alloc(8);
|
||||
until.writeBigUInt64LE(BigInt(validUntilSec));
|
||||
const hash = blake2b256(Buffer.concat([Buffer.from(method), Buffer.from(u.host), Buffer.from(u.pathname), until]));
|
||||
const seed = Buffer.from(appKey).subarray(0, 32); // a Sia private key is seed || public key
|
||||
const key = crypto.createPrivateKey({ key: Buffer.concat([PKCS8_ED25519, seed]), format: 'der', type: 'pkcs8' });
|
||||
const pub = crypto.createPublicKey(key).export({ format: 'der', type: 'spki' }).subarray(-32);
|
||||
const sig = crypto.sign(null, hash, key);
|
||||
const b64 = (b) => Buffer.from(b).toString('base64').replace(/\+/g, '-').replace(/\//g, '_');
|
||||
u.searchParams.set('sv', String(validUntilSec));
|
||||
u.searchParams.set('sc', b64(pub));
|
||||
u.searchParams.set('ss', b64(sig));
|
||||
return u;
|
||||
}
|
||||
|
||||
// ---- s3d's app key -------------------------------------------------------------
|
||||
export async function readAppKey(dataDir) {
|
||||
const file = path.join(dataDir, 's3d.db');
|
||||
if (!fs.existsSync(file)) return null;
|
||||
let DatabaseSync;
|
||||
try { ({ DatabaseSync } = await import('node:sqlite')); } catch { return null; }
|
||||
const db = new DatabaseSync(file, { readOnly: true });
|
||||
try {
|
||||
const row = db.prepare('SELECT app_key, indexer_url FROM global_settings LIMIT 1').get();
|
||||
if (!row || !row.app_key) return null;
|
||||
return { appKey: Buffer.from(row.app_key), indexerUrl: String(row.indexer_url || '') };
|
||||
} finally { db.close(); }
|
||||
const c = await readConnection(dataDir).catch(() => null);
|
||||
return c?.appKey && c.indexerUrl ? { appKey: c.appKey, indexerUrl: c.indexerUrl } : null;
|
||||
}
|
||||
|
||||
// { maxPinnedData, pinnedData, remainingStorage } in bytes, from the indexer.
|
||||
// { maxPinnedData, pinnedData, remainingStorage, indexer } in bytes, or null
|
||||
// when this s3d is not connected. Throws when the indexer does not answer.
|
||||
export async function accountSpace(dataDir, { fetchImpl = fetch } = {}) {
|
||||
const k = await readAppKey(dataDir);
|
||||
if (!k || !k.indexerUrl) return null;
|
||||
const base = k.indexerUrl.replace(/\/+$/, '');
|
||||
const url = signIndexerUrl(k.appKey, 'GET', `${base}/account`, Math.floor(Date.now() / 1000) + 120);
|
||||
k.appKey.fill(0);
|
||||
const res = await fetchImpl(url, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(15_000) });
|
||||
if (!res.ok) throw new Error(`indexer answered ${res.status}: ${(await res.text()).slice(0, 200)}`);
|
||||
const j = await res.json();
|
||||
return { maxPinnedData: Number(j.maxPinnedData), pinnedData: Number(j.pinnedData), remainingStorage: Number(j.remainingStorage), indexer: new URL(base).host };
|
||||
const a = await accountInfo(dataDir, { fetchImpl, timeoutMs: 15_000 });
|
||||
if (!a.connected) return null;
|
||||
if (a.error) throw new Error(a.error);
|
||||
return {
|
||||
maxPinnedData: Number(a.maxPinnedData),
|
||||
pinnedData: Number(a.pinnedData),
|
||||
remainingStorage: Number(a.remainingStorage),
|
||||
indexer: new URL(a.indexerUrl).host,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ const FORWARD = [
|
|||
['GET', /^\/api\/users$/], ['POST', /^\/api\/users$/], ['DELETE', /^\/api\/users\/[^/]+$/],
|
||||
['GET', /^\/api\/keys$/], ['POST', /^\/api\/keys$/], ['DELETE', /^\/api\/keys\/[^/]+$/],
|
||||
['GET', /^\/api\/account$/], ['PUT', /^\/api\/account\/label$/],
|
||||
['GET', /^\/api\/space$/],
|
||||
];
|
||||
const KEY_CHANGES = /^\/api\/(users|keys)/;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue