05a1941aa3
S3 has no rename, so files, folders and drives kept the name they were made with. s3d copies objects in its own database (the copy points at the same data on Sia), so a rename is a copy to the new name and a delete of the old: quick, and it uses no extra storage. A drive rename makes a new bucket, moves every file, rebuilds public access for the new name (a policy names its bucket) and removes the old one. Names that already exist are refused. Drives on Silent Mode encrypt with keys tied to each name, so their client gets no rename until it can re-encrypt. The drives page and Overview now show how much of the Sia account is used and how much is free. s3d does not report it; the indexer's GET /account does, signed with the app key s3d registered (read from its database for this one request, BLAKE2b-256 + Ed25519 as indexd expects).
259 lines
11 KiB
JavaScript
259 lines
11 KiB
JavaScript
// A small S3 client for s3d: AWS SigV4 (path-style, us-east-1, UNSIGNED-PAYLOAD
|
|
// for bodies so uploads stream straight through) plus the handful of bucket
|
|
// and object calls the console needs.
|
|
|
|
import crypto from 'node:crypto';
|
|
import http from 'node:http';
|
|
import https from 'node:https';
|
|
import { Readable } from 'node:stream';
|
|
|
|
export const REGION = 'us-east-1';
|
|
export const SERVICE = 's3';
|
|
|
|
const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex');
|
|
const hmac = (k, s) => crypto.createHmac('sha256', k).update(s).digest();
|
|
|
|
// RFC 3986 encoding as SigV4 wants it; '/' kept for object keys in paths.
|
|
export function uriEncode(str, keepSlash) {
|
|
const out = encodeURIComponent(str)
|
|
.replace(/[!'()*]/g, (c) => '%' + c.charCodeAt(0).toString(16).toUpperCase());
|
|
// (A "never matches" regex like /$^/ is not one: it matches the empty
|
|
// string, which turned every empty query value into "/".)
|
|
return keepSlash ? out.replace(/%2F/g, '/') : out;
|
|
}
|
|
|
|
export function canonicalQuery(query) {
|
|
return Object.keys(query).sort()
|
|
.map((k) => `${uriEncode(k)}=${uriEncode(String(query[k] ?? ''))}`)
|
|
.join('&');
|
|
}
|
|
|
|
export function amzDate(d = new Date()) {
|
|
return d.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '');
|
|
}
|
|
|
|
export function signingKey(secret, date) {
|
|
return hmac(hmac(hmac(hmac('AWS4' + secret, date), REGION), SERVICE), 'aws4_request');
|
|
}
|
|
|
|
export function objectPath(bucket, key) {
|
|
return '/' + uriEncode(bucket) + (key != null ? '/' + uriEncode(key, true) : '');
|
|
}
|
|
|
|
export class S3Client {
|
|
// endpoint: 'http://127.0.0.1:8000'
|
|
constructor({ endpoint, accessKeyId, secretKey }) {
|
|
this.endpoint = new URL(endpoint);
|
|
this.accessKeyId = accessKeyId;
|
|
this.secretKey = secretKey;
|
|
}
|
|
|
|
sign(method, path, query, headers, { now = amzDate(), payloadHash = 'UNSIGNED-PAYLOAD' } = {}) {
|
|
const date = now.slice(0, 8);
|
|
const h = { ...headers, host: this.endpoint.host, 'x-amz-date': now, 'x-amz-content-sha256': payloadHash };
|
|
const names = Object.keys(h).map((k) => k.toLowerCase()).sort();
|
|
const lower = Object.fromEntries(Object.entries(h).map(([k, v]) => [k.toLowerCase(), String(v).trim()]));
|
|
const signed = names.join(';');
|
|
const canonical = [method, path, canonicalQuery(query), names.map((n) => `${n}:${lower[n]}\n`).join(''), signed, payloadHash].join('\n');
|
|
const scope = `${date}/${REGION}/${SERVICE}/aws4_request`;
|
|
const toSign = ['AWS4-HMAC-SHA256', now, scope, sha256(canonical)].join('\n');
|
|
const sig = crypto.createHmac('sha256', signingKey(this.secretKey, date)).update(toSign).digest('hex');
|
|
lower.authorization = `AWS4-HMAC-SHA256 Credential=${this.accessKeyId}/${scope}, SignedHeaders=${signed}, Signature=${sig}`;
|
|
delete lower.host;
|
|
return lower;
|
|
}
|
|
|
|
// Presigned GET for share links. publicEndpoint lets the link name the
|
|
// address readers will use, which may differ from the loopback the GUI uses.
|
|
presign(bucket, key, expiresSec = 3600, publicEndpoint, now = amzDate()) {
|
|
const base = publicEndpoint ? new URL(publicEndpoint) : this.endpoint;
|
|
const date = now.slice(0, 8);
|
|
const scope = `${date}/${REGION}/${SERVICE}/aws4_request`;
|
|
const path = objectPath(bucket, key);
|
|
const query = {
|
|
'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
|
|
'X-Amz-Credential': `${this.accessKeyId}/${scope}`,
|
|
'X-Amz-Date': now,
|
|
'X-Amz-Expires': String(Math.min(Math.max(1, expiresSec | 0), 604800)),
|
|
'X-Amz-SignedHeaders': 'host',
|
|
};
|
|
const canonical = ['GET', path, canonicalQuery(query), `host:${base.host}\n`, 'host', 'UNSIGNED-PAYLOAD'].join('\n');
|
|
const toSign = ['AWS4-HMAC-SHA256', now, scope, sha256(canonical)].join('\n');
|
|
query['X-Amz-Signature'] = crypto.createHmac('sha256', signingKey(this.secretKey, date)).update(toSign).digest('hex');
|
|
return `${base.origin}${path}?${canonicalQuery(query)}`;
|
|
}
|
|
|
|
// Low-level request. body: Buffer | string | Readable | undefined.
|
|
// Resolves the raw http.IncomingMessage so callers can stream it.
|
|
request(method, path, { query = {}, headers = {}, body, payloadHash } = {}) {
|
|
const signedHeaders = this.sign(method, path, query, headers, payloadHash ? { payloadHash } : undefined);
|
|
const qs = canonicalQuery(query);
|
|
// The hosted gateway is https; a local s3d is plain http on loopback.
|
|
const transport = this.endpoint.protocol === 'https:' ? https : http;
|
|
return new Promise((resolve, reject) => {
|
|
const req = transport.request({
|
|
protocol: this.endpoint.protocol,
|
|
hostname: this.endpoint.hostname,
|
|
port: this.endpoint.port,
|
|
method,
|
|
path: path + (qs ? '?' + qs : ''),
|
|
headers: signedHeaders,
|
|
}, resolve);
|
|
req.on('error', reject);
|
|
if (body instanceof Readable) body.pipe(req);
|
|
else req.end(body);
|
|
});
|
|
}
|
|
|
|
async call(method, path, opts) {
|
|
const res = await this.request(method, path, opts);
|
|
const text = await readBody(res);
|
|
if (res.statusCode >= 300) throw s3Error(res.statusCode, text);
|
|
return { status: res.statusCode, headers: res.headers, text };
|
|
}
|
|
|
|
async listBuckets() {
|
|
const { text } = await this.call('GET', '/');
|
|
return blocks(text, 'Bucket').map((b) => ({ name: tag(b, 'Name'), created: tag(b, 'CreationDate') }));
|
|
}
|
|
|
|
createBucket(bucket) { return this.call('PUT', objectPath(bucket)); }
|
|
deleteBucket(bucket) { return this.call('DELETE', objectPath(bucket)); }
|
|
|
|
async listObjects(bucket, { prefix = '', delimiter = '/', token, max = 1000 } = {}) {
|
|
const query = { 'list-type': '2', prefix, delimiter, 'max-keys': String(max) };
|
|
if (token) query['continuation-token'] = token;
|
|
const { text } = await this.call('GET', objectPath(bucket), { query });
|
|
return {
|
|
folders: blocks(text, 'CommonPrefixes').map((b) => tag(b, 'Prefix')),
|
|
objects: blocks(text, 'Contents').map((b) => ({
|
|
key: tag(b, 'Key'), size: Number(tag(b, 'Size')), modified: tag(b, 'LastModified'), etag: tag(b, 'ETag'),
|
|
})),
|
|
truncated: tag(text, 'IsTruncated') === 'true',
|
|
nextToken: tag(text, 'NextContinuationToken'),
|
|
};
|
|
}
|
|
|
|
async deleteObject(bucket, key) { return this.call('DELETE', objectPath(bucket, key)); }
|
|
|
|
// Server-side copy. s3d copies the entry in its own database: the copy
|
|
// points at the same data on Sia, so nothing is uploaded again.
|
|
copyObject(srcBucket, srcKey, dstBucket, dstKey) {
|
|
return this.call('PUT', objectPath(dstBucket, dstKey), { headers: { 'x-amz-copy-source': objectPath(srcBucket, srcKey) } });
|
|
}
|
|
|
|
// Every key under prefix (no folder grouping), all pages.
|
|
async allKeys(bucket, prefix = '') {
|
|
const keys = [];
|
|
let token;
|
|
do {
|
|
const page = await this.listObjects(bucket, { prefix, delimiter: '', token });
|
|
keys.push(...page.objects.map((o) => o.key));
|
|
token = page.truncated ? page.nextToken : null;
|
|
} while (token);
|
|
return keys;
|
|
}
|
|
|
|
// S3 has no rename: copy to the new name, then delete the old one.
|
|
async renameObject(bucket, from, to, dstBucket = bucket) {
|
|
await this.copyObject(bucket, from, dstBucket, to);
|
|
await this.deleteObject(bucket, from);
|
|
}
|
|
|
|
// Rename a folder (or move a whole drive's contents): every key under
|
|
// `from` moves under `to`. Keys are listed first, so the loop never
|
|
// walks into the copies it makes. Returns the number moved.
|
|
async renamePrefix(bucket, from, to, dstBucket = bucket) {
|
|
const keys = await this.allKeys(bucket, from);
|
|
for (const k of keys) await this.renameObject(bucket, k, to + k.slice(from.length), dstBucket);
|
|
return keys.length;
|
|
}
|
|
|
|
// Deletes every key under prefix. Returns the number deleted.
|
|
async deletePrefix(bucket, prefix) {
|
|
let n = 0;
|
|
let token;
|
|
do {
|
|
const page = await this.listObjects(bucket, { prefix, delimiter: '', token });
|
|
for (const o of page.objects) { await this.deleteObject(bucket, o.key); n++; }
|
|
token = page.truncated ? page.nextToken : null;
|
|
} while (token);
|
|
return n;
|
|
}
|
|
|
|
putObject(bucket, key, body, { contentType, contentLength } = {}) {
|
|
const headers = {};
|
|
if (contentType) headers['content-type'] = contentType;
|
|
if (contentLength != null) headers['content-length'] = String(contentLength);
|
|
return this.call('PUT', objectPath(bucket, key), { headers, body });
|
|
}
|
|
|
|
getObject(bucket, key, { range } = {}) {
|
|
return this.request('GET', objectPath(bucket, key), { headers: range ? { range } : {} });
|
|
}
|
|
|
|
async getPolicy(bucket) {
|
|
const res = await this.request('GET', objectPath(bucket), { query: { policy: '' } });
|
|
const text = await readBody(res);
|
|
if (res.statusCode === 404) return null;
|
|
if (res.statusCode >= 300) throw s3Error(res.statusCode, text);
|
|
return JSON.parse(text);
|
|
}
|
|
|
|
putPolicy(bucket, policy) {
|
|
const body = JSON.stringify(policy);
|
|
return this.call('PUT', objectPath(bucket), { query: { policy: '' }, body, headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) } });
|
|
}
|
|
|
|
deletePolicy(bucket) { return this.call('DELETE', objectPath(bucket), { query: { policy: '' } }); }
|
|
}
|
|
|
|
// The only policy shape s3d accepts: anonymous read of objects and listing.
|
|
export function publicReadPolicy(bucket, { list = false } = {}) {
|
|
const statements = [{
|
|
Effect: 'Allow', Principal: '*', Action: ['s3:GetObject'], Resource: [`arn:aws:s3:::${bucket}/*`],
|
|
}];
|
|
if (list) statements.push({ Effect: 'Allow', Principal: '*', Action: ['s3:ListBucket'], Resource: [`arn:aws:s3:::${bucket}`] });
|
|
return { Version: '2012-10-17', Statement: statements };
|
|
}
|
|
|
|
export function readBody(res) {
|
|
return new Promise((resolve, reject) => {
|
|
const chunks = [];
|
|
res.on('data', (c) => chunks.push(c));
|
|
res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
|
res.on('error', reject);
|
|
});
|
|
}
|
|
|
|
export class S3Error extends Error {
|
|
constructor(status, code, message) { super(message || code); this.status = status; this.code = code; }
|
|
}
|
|
|
|
function s3Error(status, text) {
|
|
return new S3Error(status, tag(text, 'Code') || `HTTP ${status}`, tag(text, 'Message') || text.slice(0, 200) || `HTTP ${status}`);
|
|
}
|
|
|
|
// S3 responses are flat enough that tag extraction is all we need.
|
|
function blocks(xml, name) {
|
|
const re = new RegExp(`<${name}>([\\s\\S]*?)</${name}>`, 'g');
|
|
const out = [];
|
|
let m;
|
|
while ((m = re.exec(xml))) out.push(m[1]);
|
|
return out;
|
|
}
|
|
|
|
function tag(xml, name) {
|
|
const m = xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`));
|
|
return m ? unescapeXml(m[1]) : null;
|
|
}
|
|
|
|
function unescapeXml(s) {
|
|
return s.replace(/&(lt|gt|amp|quot|apos|#\d+|#x[0-9a-f]+);/gi, (_, e) => {
|
|
switch (e.toLowerCase()) {
|
|
case 'lt': return '<'; case 'gt': return '>'; case 'amp': return '&';
|
|
case 'quot': return '"'; case 'apos': return "'";
|
|
default: return String.fromCodePoint(e[1].toLowerCase() === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10));
|
|
}
|
|
});
|
|
}
|