pithos/sia-share/main.go

// sia-share serves files that people share from their own Sia account at
// navigate.st/sia/<id>/<name>, so anyone can open them in a browser.
//
// Pithos makes a Sia shared-object URL for a file (it can read that one
// object and nothing else in the account) and registers it here; this
// service keeps a short id for it and streams the file from Sia hosts on
// request, with byte ranges so audio and video can seek. The data stays in
// the sharer's account; reading it from hosts is paid by this service's own
// app on Silent Mode's indexer account.
//
//	sia-share register --data /var/lib/sia-share   one-time: prints an approval link
//	sia-share serve    --data /var/lib/sia-share   the HTTP service
package main

import (
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"html"
	"io"
	"log"
	"mime"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"go.sia.tech/core/types"
	sdk "go.sia.tech/siastorage"
	"go.uber.org/zap"
)

// The app's identity at the indexer. NEVER change it: keys registered under
// it stop working.
var appMeta = sdk.AppMetadata{
	ID:          mustHash("ed4ccccf5e8912c4e91e37ce5842ded282ffde68accd39fe1265a46903749dc2"),
	Name:        "Pithos share",
	Description: "Serves files shared from Pithos at navigate.st/sia",
	ServiceURL:  "https://navigate.st/sia",
}

func mustHash(s string) (h types.Hash256) {
	b, err := hex.DecodeString(s)
	if err != nil || len(b) != 32 {
		panic("bad app id")
	}
	copy(h[:], b)
	return
}

type config struct {
	Listen          string   `json:"listen"`
	Indexer         string   `json:"indexer"`
	PublicBase      string   `json:"publicBase"`      // https://navigate.st/sia
	AllowedIndexers []string `json:"allowedIndexers"` // hosts share URLs may point at
	PerIPPerDay     int      `json:"perIpPerDay"`     // new links per client address per day
	DailyBytes      int64    `json:"dailyBytesPerLink"`
	MaxFiles        int      `json:"maxFilesPerFolder"`
}

type file struct {
	Name string `json:"name"`
	URL  string `json:"url"` // shared-object URL, with #encryption_key
	Size int64  `json:"size"`
}

type link struct {
	ID         string `json:"id"`
	Name       string `json:"name"`
	Files      []file `json:"files"` // one entry for a single file
	Folder     bool   `json:"folder"`
	Created    int64  `json:"created"`
	DeleteHash string `json:"deleteHash"`
	Day        string `json:"day"`
	DayBytes   int64  `json:"dayBytes"`
}

type store struct {
	mu    sync.Mutex
	file  string
	links map[string]*link
	ips   map[string][]int64
}

func (s *store) load() error {
	s.links = map[string]*link{}
	s.ips = map[string][]int64{}
	b, err := os.ReadFile(s.file)
	if errors.Is(err, os.ErrNotExist) {
		return nil
	} else if err != nil {
		return err
	}
	return json.Unmarshal(b, &s.links)
}

func (s *store) save() error {
	b, _ := json.MarshalIndent(s.links, "", " ")
	tmp := s.file + ".tmp"
	if err := os.WriteFile(tmp, b, 0o600); err != nil {
		return err
	}
	return os.Rename(tmp, s.file)
}

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "usage: sia-share register|serve --data <dir> [--config <file>]")
		os.Exit(2)
	}
	fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
	dataDir := fs.String("data", "/var/lib/sia-share", "state folder (app key, links)")
	cfgFile := fs.String("config", "/etc/sia-share.json", "config file")
	fs.Parse(os.Args[2:])
	cfg := config{Listen: "127.0.0.1:8730", Indexer: "https://sia.storage", PublicBase: "https://navigate.st/sia",
		AllowedIndexers: []string{"sia.storage"}, PerIPPerDay: 30, DailyBytes: 20 << 30, MaxFiles: 1000}
	if b, err := os.ReadFile(*cfgFile); err == nil {
		if err := json.Unmarshal(b, &cfg); err != nil {
			log.Fatalf("config: %v", err)
		}
	}
	switch os.Args[1] {
	case "register":
		register(cfg, *dataDir)
	case "serve":
		serve(cfg, *dataDir)
	default:
		log.Fatalf("unknown command %q", os.Args[1])
	}
}

// register connects this service to the indexer as its own app. The person
// who owns the paying account opens the printed link once and approves it.
func register(cfg config, dataDir string) {
	keyFile := filepath.Join(dataDir, "app.key")
	if _, err := os.Stat(keyFile); err == nil {
		log.Fatalf("%s exists; delete it first to register again", keyFile)
	}
	if err := os.MkdirAll(dataDir, 0o700); err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	b := sdk.NewBuilder(cfg.Indexer, appMeta)
	approvalURL, err := b.RequestConnection(ctx)
	if err != nil {
		log.Fatalf("request connection: %v", err)
	}
	fmt.Printf("Open this link, sign in to the account that pays for downloads, and approve \"%s\":\n\n  %s\n\nWaiting for approval…\n", appMeta.Name, approvalURL)
	if err := b.WaitForApproval(ctx); err != nil {
		log.Fatalf("approval: %v", err)
	}
	phrase := sdk.NewSeedPhrase()
	s, err := b.Register(ctx, phrase)
	if err != nil {
		log.Fatalf("register: %v", err)
	}
	key := s.AppKey()
	if err := os.WriteFile(keyFile, []byte(hex.EncodeToString(key[:])), 0o600); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Registered. App key saved to %s (keep it private; it pays for downloads).\n", keyFile)
}

func loadAppKey(dataDir string) types.PrivateKey {
	b, err := os.ReadFile(filepath.Join(dataDir, "app.key"))
	if err != nil {
		log.Fatalf("no app key: run 'sia-share register' first (%v)", err)
	}
	raw, err := hex.DecodeString(strings.TrimSpace(string(b)))
	if err != nil || len(raw) != 64 {
		log.Fatal("app.key is not a 64-byte hex key")
	}
	return types.PrivateKey(raw)
}

var (
	sharedPath = regexp.MustCompile(`^/objects/[0-9a-f]{64}/shared$`)
	idRe       = regexp.MustCompile(`^[A-Za-z0-9_-]{6,32}$`)
)

func serve(cfg config, dataDir string) {
	appKey := loadAppKey(dataDir)
	client, err := sdk.NewBuilder(cfg.Indexer, appMeta).SDK(appKey, sdk.WithLogger(zap.NewNop()))
	if err != nil {
		log.Fatalf("connect to indexer: %v", err)
	}
	st := &store{file: filepath.Join(dataDir, "links.json")}
	if err := st.load(); err != nil {
		log.Fatalf("links: %v", err)
	}
	h := &handler{cfg: cfg, sdk: client, st: st}
	mux := http.NewServeMux()
	mux.HandleFunc("POST /api/sia/links", h.create)
	mux.HandleFunc("DELETE /api/sia/links/{id}", h.remove)
	mux.HandleFunc("GET /sia/{id}", h.get)
	mux.HandleFunc("HEAD /sia/{id}", h.get)
	mux.HandleFunc("GET /sia/{id}/{rest...}", h.get)
	mux.HandleFunc("HEAD /sia/{id}/{rest...}", h.get)
	srv := &http.Server{Addr: cfg.Listen, Handler: mux, ReadHeaderTimeout: 20 * time.Second}
	log.Printf("sia-share on %s for %s", cfg.Listen, cfg.PublicBase)
	log.Fatal(srv.ListenAndServe())
}

type handler struct {
	cfg config
	sdk *sdk.SDK
	st  *store
}

func jsonOut(w http.ResponseWriter, status int, v any) {
	w.Header().Set("content-type", "application/json; charset=utf-8")
	w.Header().Set("cache-control", "no-store")
	w.WriteHeader(status)
	json.NewEncoder(w).Encode(v)
}

func clientIP(r *http.Request) string {
	if xf := r.Header.Get("X-Forwarded-For"); xf != "" {
		return strings.TrimSpace(strings.Split(xf, ",")[0])
	}
	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	return host
}

func randomID(n int) string {
	b := make([]byte, n)
	rand.Read(b)
	return base64.RawURLEncoding.EncodeToString(b)
}

func cleanName(s string) string {
	s = strings.TrimSpace(strings.ReplaceAll(s, "\\", "/"))
	s = path.Base("/" + s)
	if s == "/" || s == "." || s == ".." || len(s) > 255 {
		return ""
	}
	return s
}

// sizeOf checks a shared-object URL against its indexer and returns the
// object's size. Only URLs on an allowed indexer, in the shared-object form
// and carrying a key, are accepted.
func (h *handler) sizeOf(ctx context.Context, raw string) (int64, error) {
	u, err := url.Parse(raw)
	if err != nil || u.Scheme != "https" || !sharedPath.MatchString(u.Path) {
		return 0, errors.New("not a Sia shared-object URL")
	}
	allowed := false
	for _, a := range h.cfg.AllowedIndexers {
		if strings.EqualFold(u.Host, a) {
			allowed = true
		}
	}
	if !allowed {
		return 0, fmt.Errorf("indexer %q is not accepted here", u.Host)
	}
	if v, _ := url.ParseQuery(u.Fragment); len(v.Get("encryption_key")) < 40 {
		return 0, errors.New("the URL has no encryption key")
	}
	q := *u
	q.Fragment = ""
	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, q.String(), nil)
	req.Header.Set("accept", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return 0, fmt.Errorf("indexer: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return 0, fmt.Errorf("indexer refused the link (%s); it may have expired", resp.Status)
	}
	var obj struct {
		Slabs []struct {
			Length int64 `json:"length"`
		} `json:"slabs"`
	}
	if err := json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(&obj); err != nil {
		return 0, fmt.Errorf("indexer: %w", err)
	}
	var n int64
	for _, s := range obj.Slabs {
		n += s.Length
	}
	return n, nil
}

// POST /api/sia/links {name, url} for one file, or {name, folder: true,
// files: [{name, url}]} for a folder. Answers {id, url, deleteToken}.
func (h *handler) create(w http.ResponseWriter, r *http.Request) {
	var body struct {
		Name   string `json:"name"`
		URL    string `json:"url"`
		Folder bool   `json:"folder"`
		Files  []file `json:"files"`
	}
	if err := json.NewDecoder(io.LimitReader(r.Body, 4<<20)).Decode(&body); err != nil {
		jsonOut(w, 400, map[string]string{"error": "invalid JSON"})
		return
	}
	ip := clientIP(r)
	now := time.Now().Unix()
	h.st.mu.Lock()
	recent := []int64{}
	for _, t := range h.st.ips[ip] {
		if now-t < 86400 {
			recent = append(recent, t)
		}
	}
	if h.cfg.PerIPPerDay > 0 && len(recent) >= h.cfg.PerIPPerDay {
		h.st.mu.Unlock()
		jsonOut(w, 429, map[string]string{"error": "too many new links from this address today"})
		return
	}
	h.st.mu.Unlock()

	files := body.Files
	if !body.Folder {
		files = []file{{Name: body.Name, URL: body.URL}}
	}
	if len(files) == 0 || len(files) > h.cfg.MaxFiles {
		jsonOut(w, 400, map[string]string{"error": fmt.Sprintf("between 1 and %d files", h.cfg.MaxFiles)})
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
	defer cancel()
	seen := map[string]bool{}
	for i := range files {
		files[i].Name = cleanName(files[i].Name)
		if files[i].Name == "" || seen[files[i].Name] {
			jsonOut(w, 400, map[string]string{"error": "every file needs a distinct name"})
			return
		}
		seen[files[i].Name] = true
		n, err := h.sizeOf(ctx, files[i].URL)
		if err != nil {
			jsonOut(w, 400, map[string]string{"error": fmt.Sprintf("%s: %v", files[i].Name, err)})
			return
		}
		files[i].Size = n
	}
	name := cleanName(body.Name)
	if name == "" {
		name = files[0].Name
	}
	token := randomID(24)
	sum := sha256.Sum256([]byte(token))
	l := &link{ID: randomID(6), Name: name, Files: files, Folder: body.Folder, Created: now, DeleteHash: hex.EncodeToString(sum[:])}

	h.st.mu.Lock()
	for h.st.links[l.ID] != nil {
		l.ID = randomID(6)
	}
	h.st.links[l.ID] = l
	h.st.ips[ip] = append(recent, now)
	err := h.st.save()
	h.st.mu.Unlock()
	if err != nil {
		jsonOut(w, 500, map[string]string{"error": "could not save the link"})
		return
	}
	public := h.cfg.PublicBase + "/" + l.ID + "/"
	if !l.Folder {
		public += url.PathEscape(files[0].Name)
	}
	log.Printf("[%s] new %s link %q (%d file(s))", l.ID, map[bool]string{true: "folder", false: "file"}[l.Folder], name, len(files))
	jsonOut(w, 200, map[string]any{"id": l.ID, "url": public, "deleteToken": token})
}

// DELETE /api/sia/links/{id} with header X-Delete-Token: the link stops working here.
func (h *handler) remove(w http.ResponseWriter, r *http.Request) {
	id := r.PathValue("id")
	sum := sha256.Sum256([]byte(r.Header.Get("X-Delete-Token")))
	h.st.mu.Lock()
	defer h.st.mu.Unlock()
	l := h.st.links[id]
	if l == nil || l.DeleteHash != hex.EncodeToString(sum[:]) {
		jsonOut(w, 404, map[string]string{"error": "no such link"})
		return
	}
	delete(h.st.links, id)
	h.st.save()
	jsonOut(w, 200, map[string]bool{"ok": true})
}

func (h *handler) get(w http.ResponseWriter, r *http.Request) {
	id := r.PathValue("id")
	if !idRe.MatchString(id) {
		http.NotFound(w, r)
		return
	}
	h.st.mu.Lock()
	l := h.st.links[id]
	h.st.mu.Unlock()
	if l == nil {
		notFound(w)
		return
	}
	rest := r.PathValue("rest")
	if l.Folder {
		if rest == "" {
			if !strings.HasSuffix(r.URL.Path, "/") {
				http.Redirect(w, r, id+"/", http.StatusMovedPermanently)
				return
			}
			folderPage(w, l)
			return
		}
		for _, f := range l.Files {
			if f.Name == rest {
				h.stream(w, r, l, f)
				return
			}
		}
		notFound(w)
		return
	}
	h.stream(w, r, l, l.Files[0])
}

func notFound(w http.ResponseWriter) {
	w.Header().Set("content-type", "text/html; charset=utf-8")
	w.WriteHeader(404)
	io.WriteString(w, `<!doctype html><meta charset="utf-8"><title>Not found</title><body style="font:16px system-ui;background:#0b100e;color:#e6ece9;display:grid;place-items:center;min-height:90vh"><p>This shared link does not exist, or its owner switched it off.</p>`)
}

func parseRange(hdr string, size int64) (off, length int64, partial, ok bool) {
	if hdr == "" {
		return 0, size, false, true
	}
	m := regexp.MustCompile(`^bytes=(\d*)-(\d*)$`).FindStringSubmatch(strings.TrimSpace(hdr))
	if m == nil {
		return 0, size, false, true // unsupported forms: whole file
	}
	if m[1] == "" { // suffix: last N bytes
		n, err := strconv.ParseInt(m[2], 10, 64)
		if err != nil || n <= 0 {
			return 0, 0, false, false
		}
		if n > size {
			n = size
		}
		return size - n, n, true, true
	}
	start, err := strconv.ParseInt(m[1], 10, 64)
	if err != nil || start >= size {
		return 0, 0, false, false
	}
	end := size - 1
	if m[2] != "" {
		if e, err := strconv.ParseInt(m[2], 10, 64); err == nil && e < end {
			end = e
		}
	}
	if end < start {
		return 0, 0, false, false
	}
	return start, end - start + 1, true, true
}

// stream sends one file from Sia, honouring Range, under the link's daily cap.
// Every file is served sandboxed: a shared HTML or SVG file cannot run script
// on navigate.st.
func (h *handler) stream(w http.ResponseWriter, r *http.Request, l *link, f file) {
	off, length, partial, ok := parseRange(r.Header.Get("Range"), f.Size)
	if !ok {
		w.Header().Set("content-range", fmt.Sprintf("bytes */%d", f.Size))
		w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
		return
	}
	today := time.Now().UTC().Format("2006-01-02")
	h.st.mu.Lock()
	if l.Day != today {
		l.Day, l.DayBytes = today, 0
	}
	over := h.cfg.DailyBytes > 0 && l.DayBytes+length > h.cfg.DailyBytes
	if !over && r.Method != http.MethodHead {
		l.DayBytes += length
	}
	h.st.mu.Unlock()
	if over {
		w.Header().Set("retry-after", "3600")
		http.Error(w, "this link has used today's download allowance; try again tomorrow", http.StatusTooManyRequests)
		return
	}
	ct := mime.TypeByExtension(strings.ToLower(path.Ext(f.Name)))
	if ct == "" {
		ct = "application/octet-stream"
	}
	hd := w.Header()
	hd.Set("content-type", ct)
	hd.Set("accept-ranges", "bytes")
	hd.Set("content-length", strconv.FormatInt(length, 10))
	hd.Set("content-disposition", mime.FormatMediaType("inline", map[string]string{"filename": f.Name}))
	hd.Set("content-security-policy", "sandbox")
	hd.Set("x-content-type-options", "nosniff")
	hd.Set("cache-control", "public, max-age=300")
	status := http.StatusOK
	if partial {
		hd.Set("content-range", fmt.Sprintf("bytes %d-%d/%d", off, off+length-1, f.Size))
		status = http.StatusPartialContent
	}
	if r.Method == http.MethodHead || length == 0 {
		w.WriteHeader(status)
		return
	}
	rc, err := h.sdk.DownloadSharedObject(r.Context(), f.URL, sdk.WithDownloadRange(uint64(off), uint64(length)))
	if err != nil {
		log.Printf("[%s] %s: %v", l.ID, f.Name, err)
		http.Error(w, "could not read the file from Sia right now", http.StatusBadGateway)
		return
	}
	defer rc.Close()
	w.WriteHeader(status)
	if _, err := io.Copy(w, rc); err != nil && !errors.Is(err, context.Canceled) {
		log.Printf("[%s] %s: stream: %v", l.ID, f.Name, err)
	}
}

func fmtSize(n int64) string {
	units := []string{"B", "KiB", "MiB", "GiB", "TiB"}
	f := float64(n)
	i := 0
	for f >= 1024 && i < len(units)-1 {
		f /= 1024
		i++
	}
	if i == 0 {
		return fmt.Sprintf("%d B", n)
	}
	return fmt.Sprintf("%.1f %s", f, units[i])
}

func folderPage(w http.ResponseWriter, l *link) {
	kind := func(n string) string {
		switch strings.ToLower(path.Ext(n)) {
		case ".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif":
			return "img"
		case ".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".flac", ".wav":
			return "audio"
		case ".mp4", ".webm", ".mov", ".m4v":
			return "video"
		}
		return ""
	}
	var rows strings.Builder
	for _, f := range l.Files {
		href := url.PathEscape(f.Name)
		media := ""
		switch k := kind(f.Name); k {
		case "img":
			media = fmt.Sprintf(`<img loading="lazy" src="%s" alt="">`, href)
		case "audio", "video":
			media = fmt.Sprintf(`<button type="button" class="play" data-src="%s" data-kind="%s">▶</button>`, href, k)
		}
		fmt.Fprintf(&rows, `<li>%s<a href="%s">%s</a><span class="sz">%s</span></li>`, media, href, html.EscapeString(f.Name), fmtSize(f.Size))
	}
	w.Header().Set("content-type", "text/html; charset=utf-8")
	w.Header().Set("content-security-policy", "default-src 'self'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; img-src 'self'; media-src 'self'; frame-ancestors 'none'")
	w.Header().Set("x-content-type-options", "nosniff")
	fmt.Fprintf(w, `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>%s · shared from Sia</title>
<style>body{margin:0;background:#0b100e;color:#e6ece9;font:15px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif}main{max-width:860px;margin:0 auto;padding:28px 18px 120px}
a{color:#3ddc97;text-decoration:none}a:hover{text-decoration:underline}.owner{color:#8e9c96;font-size:13px}h1{font-size:20px;margin:4px 0 18px;font-weight:600}
ul{list-style:none;padding:0;margin:0;border:1px solid rgba(255,255,255,.08);border-radius:10px;overflow:hidden}li{display:flex;align-items:center;gap:10px;padding:9px 14px;border-top:1px solid rgba(255,255,255,.06);min-width:0}li:first-child{border-top:none}
li a{flex:1;min-width:0;overflow-wrap:anywhere}li img{width:44px;height:44px;object-fit:cover;border-radius:6px;flex:none}.sz{color:#8e9c96;font-size:13px;white-space:nowrap}
.play{flex:none;width:32px;height:32px;border-radius:50%%;border:0;background:#3ddc97;color:#06231a;cursor:pointer;font-size:13px}
#player{position:fixed;left:0;right:0;bottom:0;background:#121a17;border-top:1px solid rgba(255,255,255,.08);padding:10px 16px;display:none}#player.on{display:block}#player audio,#player video{width:100%%;max-height:50vh}footer{color:#5f6b66;font-size:12.5px;margin-top:18px}</style></head>
<body><main><div class="owner">Shared from Sia</div><h1>%s</h1><ul>%s</ul>
<footer>Stored on the Sia network and shared with Pithos.</footer></main><div id="player"></div>
<script>document.addEventListener('click',function(e){var b=e.target.closest('.play');if(!b)return;var p=document.getElementById('player');var m=document.createElement(b.dataset.kind);m.src=b.dataset.src;m.controls=true;m.autoplay=true;p.innerHTML='';p.appendChild(m);p.className='on';});</script>
</body></html>`, html.EscapeString(l.Name), html.EscapeString(l.Name), rows.String())
}