cmd/tlog-torrent: experimental tool to generate .torrent files

Author: FiloSottile

cmd/tlog-torrent/main.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"io/fs"
+	"log/slog"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"filippo.io/sunlight"
+	"filippo.io/sunlight/internal/torrent"
+	"filippo.io/torchwood"
+	"golang.org/x/mod/sumdb/tlog"
+)
+
+func main() {
+	if len(os.Args) < 3 {
+		slog.Error("usage: tlog-torrent <path> <monitoring prefix>")
+		return
+	}
+
+	monitoringPrefix := strings.TrimSuffix(os.Args[2], "/")
+
+	root, err := os.OpenRoot(os.Args[1])
+	if err != nil {
+		slog.Error("failed to open root", "error", err)
+		return
+	}
+
+	checkpointBytes, err := fs.ReadFile(root.FS(), "checkpoint")
+	if err != nil {
+		slog.Error("failed to read checkpoint", "error", err)
+		return
+	}
+	checkpointBytes = checkpointBytes[:bytes.Index(checkpointBytes, []byte("\n\n"))+1]
+	checkpoint, err := torchwood.ParseCheckpoint(string(checkpointBytes))
+	if err != nil {
+		slog.Error("failed to parse checkpoint", "error", err, "checkpoint", string(checkpointBytes))
+		return
+	}
+
+	domain, path, _ := strings.Cut(checkpoint.Origin, "/")
+	parts := strings.Split(domain, ".")
+	slices.Reverse(parts)
+	parts = append(parts, strings.Split(path, "/")...)
+	reverse := strings.Join(parts, ".")
+
+	h := torrent.NewPieceHash(524288)
+	t := torrent.NewWriter(os.Stdout)
+	t.WriteDict(func(t *torrent.Writer) {
+		t.WriteString("comment")
+		t.WriteBytes(checkpointBytes)
+		t.WriteString("created by")
+		t.WriteString("tlog-torrent")
+		t.WriteString("creation date")
+		t.WriteInt64(time.Now().Unix())
+		t.WriteString("info")
+		t.WriteDict(func(t *torrent.Writer) {
+			t.WriteString("collections")
+			t.WriteList(func(t *torrent.Writer) {
+				t.WriteString(reverse)
+			})
+			t.WriteString("files")
+			t.WriteList(func(t *torrent.Writer) {
+				for n := range checkpoint.N / sunlight.TileWidth {
+					tile := sunlight.TilePath(tlog.Tile{
+						H: sunlight.TileHeight,
+						L: -1, N: n,
+						W: sunlight.TileWidth,
+					})
+					f, err := root.Open(tile)
+					if err != nil {
+						slog.Error("failed to open tile", "tile", tile, "error", err)
+						os.Exit(1)
+					}
+					length, err := io.Copy(h, f)
+					if err != nil {
+						slog.Error("failed to hash tile", "tile", tile, "error", err)
+						os.Exit(1)
+					}
+					if err := f.Close(); err != nil {
+						slog.Error("failed to close tile", "tile", tile, "error", err)
+						os.Exit(1)
+					}
+					t.WriteDict(func(t *torrent.Writer) {
+						t.WriteString("length")
+						t.WriteInt64(length)
+						t.WriteString("path")
+						t.WriteList(func(t *torrent.Writer) {
+							p := strings.TrimPrefix(tile, "tile/data/")
+							for _, part := range strings.Split(p, "/") {
+								t.WriteString(part)
+							}
+						})
+					})
+				}
+			})
+			t.WriteString("name")
+			t.WriteString("data")
+			t.WriteString("piece length")
+			t.WriteInt(524288)
+			t.WriteString("pieces")
+			t.WriteBytes(h.Pieces())
+			t.WriteString("update-url")
+			t.WriteString(monitoringPrefix + "/tile-data.torrent")
+		})
+		t.WriteString("title")
+		t.WriteString(checkpoint.Origin)
+		t.WriteString("url-list")
+		t.WriteList(func(t *torrent.Writer) {
+			t.WriteString(monitoringPrefix + "/tile/")
+		})
+	})
+}

internal/torrent/bencode.go

@@ -0,0 +1,86 @@
+package torrent
+
+import (
+	"fmt"
+	"io"
+)
+
+type Writer struct {
+	w      io.Writer
+	isDict bool
+	isKey  bool
+}
+
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{w: w}
+}
+
+func (w *Writer) WriteString(s string) {
+	fmt.Fprintf(w.w, "%d:%s", len(s), s)
+	if w.isKey {
+		w.isKey = false
+	} else if w.isDict {
+		w.isKey = true
+	}
+}
+
+func (w *Writer) WriteBytes(b []byte) {
+	fmt.Fprintf(w.w, "%d:", len(b))
+	w.w.Write(b)
+	if w.isKey {
+		w.isKey = false
+	} else if w.isDict {
+		w.isKey = true
+	}
+}
+
+func (w *Writer) WriteInt(i int) {
+	if w.isKey {
+		panic("int can't be a key")
+	}
+	fmt.Fprintf(w.w, "i%de", i)
+	if w.isDict {
+		w.isKey = true
+	}
+}
+
+func (w *Writer) WriteInt64(i int64) {
+	if w.isKey {
+		panic("int can't be a key")
+	}
+	fmt.Fprintf(w.w, "i%de", i)
+	if w.isDict {
+		w.isKey = true
+	}
+}
+
+func (w *Writer) WriteList(f func(*Writer)) {
+	if w.isKey {
+		panic("list can't be a key")
+	}
+	fmt.Fprintf(w.w, "l")
+	f(&Writer{w: w.w})
+	fmt.Fprintf(w.w, "e")
+	if w.isDict {
+		w.isKey = true
+	}
+}
+
+func (w *Writer) WriteDict(f func(*Writer)) {
+	if w.isKey {
+		panic("dict can't be a key")
+	}
+	fmt.Fprintf(w.w, "d")
+	ww := &Writer{}
+	ww.isDict = true
+	ww.isKey = true
+	ww.w = w.w
+	f(ww)
+	if !ww.isKey {
+		panic("missing value for key")
+	}
+	fmt.Fprintf(w.w, "e")
+	if w.isDict {
+		w.isKey = true
+	}
+}

internal/torrent/files.go

@@ -0,0 +1,40 @@
+package torrent
+
+import (
+	"crypto/sha1"
+	"hash"
+)
+
+type PieceHash struct {
+	size   int
+	idx    int
+	h      hash.Hash
+	pieces []byte
+}
+
+func NewPieceHash(size int) *PieceHash {
+	return &PieceHash{size: size, h: sha1.New()}
+}
+
+func (ph *PieceHash) Write(b []byte) (int, error) {
+	in := len(b)
+	for len(b) > 0 {
+		n := min(len(b), ph.size-ph.idx)
+		ph.h.Write(b[:n])
+		b = b[n:]
+		ph.idx += n
+		if ph.idx == ph.size {
+			ph.pieces = ph.h.Sum(ph.pieces)
+			ph.h.Reset()
+			ph.idx = 0
+		}
+	}
+	return in, nil
+}
+
+func (ph *PieceHash) Pieces() []byte {
+	if ph.idx > 0 {
+		return ph.h.Sum(ph.pieces[:len(ph.pieces):len(ph.pieces)])
+	}
+	return ph.pieces
+}