sunlight,cmd/skylight,internal/ctlog: add names tiles

client.go

@@ -5,12 +5,15 @@
 package sunlight
 
 import (
+	"bytes"
 	"context"
 	"crypto"
 	"crypto/sha256"
 	"crypto/x509"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"iter"
 	"log/slog"
 	"net/http"
@@ -30,7 +33,8 @@ import (
 type Client struct {
 	c   *torchwood.Client
 	f   *torchwood.TileFetcher
-	k   crypto.PublicKey
+	r   torchwood.TileReaderWithContext
+	cc  *ClientConfig
 	err error
 }
 
@@ -103,7 +107,7 @@ func NewClient(config *ClientConfig) (*Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &Client{c: client, f: fetcher, k: config.PublicKey}, nil
+	return &Client{c: client, f: fetcher, r: tileReader, cc: config}, nil
 }
 
 // Fetcher returns the underlying [torchwood.TileFetcher], which can be used to
@@ -189,7 +193,7 @@ func (c *Client) CheckInclusion(ctx context.Context, tree tlog.Tree, sct []byte)
 	if s.SCTVersion != ct.V1 {
 		return nil, nil, fmt.Errorf("sunlight: unsupported SCT version %d", s.SCTVersion)
 	}
-	spki, err := x509.MarshalPKIXPublicKey(c.k)
+	spki, err := x509.MarshalPKIXPublicKey(c.cc.PublicKey)
 	if err != nil {
 		return nil, nil, fmt.Errorf("sunlight: failed to marshal public key: %w", err)
 	}
@@ -217,7 +221,7 @@ func (c *Client) CheckInclusion(ctx context.Context, tree tlog.Tree, sct []byte)
 	if entry.Timestamp != int64(s.Timestamp) {
 		return nil, nil, fmt.Errorf("sunlight: SCT timestamp %d does not match entry timestamp %d", s.Timestamp, entry.Timestamp)
 	}
-	if err := tls.VerifySignature(c.k, entry.MerkleTreeLeaf(), tls.DigitallySigned(s.Signature)); err != nil {
+	if err := tls.VerifySignature(c.cc.PublicKey, entry.MerkleTreeLeaf(), tls.DigitallySigned(s.Signature)); err != nil {
 		return nil, nil, fmt.Errorf("sunlight: SCT signature verification failed: %w", err)
 	}
 	return entry, proof, nil
@@ -236,7 +240,7 @@ func (c *Client) Checkpoint(ctx context.Context) (torchwood.Checkpoint, *note.No
 	// origin line. We'll need witness-aware clients to enforce the origin line.
 	name, _, _ := strings.Cut(string(signedNote), "\n")
 
-	verifier, err := NewRFC6962Verifier(name, c.k)
+	verifier, err := NewRFC6962Verifier(name, c.cc.PublicKey)
 	if err != nil {
 		return torchwood.Checkpoint{}, nil, fmt.Errorf("sunlight: failed to create verifier for checkpoint: %w", err)
 	}
@@ -269,3 +273,112 @@ func (c *Client) Issuer(ctx context.Context, fp [32]byte) (*x509.Certificate, er
 	}
 	return x509.ParseCertificate(cert)
 }
+
+// UnauthenticatedTrimmedEntries returns an iterator that yields trimmed
+// entries, starting and ending at the given index. The first item in the
+// yielded pair is the overall entry index in the log, starting at start.
+//
+// Entries are NOT authenticated against a checkpoint and, if supported by the
+// log, are fetched through a more efficient protocol than [Client.Entries].
+// This method is only suitable for clients that don't participate in the
+// transparency ecosystem, and are only interested in a feed of names.
+//
+// Callers must check [Client.Err] after the iteration breaks.
+func (c *Client) UnauthenticatedTrimmedEntries(ctx context.Context, start, end int64) iter.Seq2[int64, *TrimmedEntry] {
+	c.err = nil
+	if start < 0 || end < 0 || start > end {
+		return func(func(int64, *TrimmedEntry) bool) {
+			c.err = fmt.Errorf("sunlight: invalid range %d-%d", start, end)
+		}
+	}
+
+	fallbackToDataTile := false
+	return func(yield func(int64, *TrimmedEntry) bool) {
+		for start < end {
+			N := start / TileWidth
+			W := int(min(end-N, TileWidth))
+
+			data, err := func() ([]byte, error) {
+				ctx := ctx
+				if c.cc.Timeout != 0 {
+					var cancel context.CancelFunc
+					ctx, cancel = context.WithTimeout(ctx, c.cc.Timeout)
+					defer cancel()
+				}
+
+				tiles := []tlog.Tile{{H: TileHeight, L: -2, N: N, W: W}}
+				if !fallbackToDataTile {
+					tdata, err := c.f.ReadTiles(ctx, tiles)
+					if err != nil {
+						if c.cc.Logger != nil {
+							c.cc.Logger.Info("failed to read names tile, falling back to data tiles",
+								"tile", TilePath(tiles[0]), "err", err)
+						}
+						fallbackToDataTile = true
+					} else {
+						return tdata[0], nil
+					}
+				}
+				if fallbackToDataTile {
+					tiles[0].L = -1
+					tdata, err := c.f.ReadTiles(ctx, tiles)
+					if err != nil {
+						return nil, err
+					}
+					return tdata[0], nil
+				}
+				panic("unreachable")
+			}()
+			if err != nil {
+				c.err = err
+				return
+			}
+
+			if fallbackToDataTile {
+				for len(data) > 0 {
+					var e *LogEntry
+					e, data, err = ReadTileLeaf(data)
+					if err != nil {
+						c.err = fmt.Errorf("failed to parse tile %d (size %d): %w", N, W, err)
+						return
+					}
+					te, err := e.TrimmedEntry()
+					if err != nil {
+						c.err = fmt.Errorf("failed to trim entry %d: %w", e.LeafIndex, err)
+						return
+					}
+
+					if e.LeafIndex < start {
+						continue
+					}
+					if !yield(e.LeafIndex, te) {
+						return
+					}
+				}
+			} else {
+				d := json.NewDecoder(bytes.NewReader(data))
+				i := N * TileWidth
+				for {
+					var te *TrimmedEntry
+					if err := d.Decode(&te); err == io.EOF {
+						break
+					} else if err != nil {
+						c.err = fmt.Errorf("failed to parse tile %d (size %d): %w", N, W, err)
+						return
+					}
+
+					if i < start {
+						i++
+						continue
+					}
+					if !yield(i, te) {
+						return
+					}
+					i++
+				}
+			}
+
+			start += int64(W)
+		}
+	}
+}

client_test.go

@@ -5,13 +5,19 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"filippo.io/sunlight"
 	"github.com/google/certificate-transparency-go/x509"
 	"golang.org/x/mod/sumdb/tlog"
 )
 
 func ExampleClient_Entries() {
+	// Note: clients that don't participate in the transparency ecosystem
+	// and are only interested in a feed of names can consider using the
+	// more efficient UnauthenticatedTrimmedEntries method instead.
+
 	block, _ := pem.Decode([]byte(`-----BEGIN PUBLIC KEY-----
 MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4i7AmqGoGHsorn/eyclTMjrAnM0J
 UUbyGJUxXqq1AjQ4qBC77wXkWt7s/HA8An2vrEBKIGQzqTjV8QIHrmpd4w==
@@ -118,3 +124,40 @@ ybky1bC4rbimZJIjvhnqMcMkf/I=
 		}
 	}
 }
+
+func ExampleClient_UnauthenticatedTrimmedEntries() {
+	// Important: UnauthenticatedTrimmedEntries does NOT verify the signed tree
+	// head. It is only suitable for clients that don't participate in the
+	// transparency ecosystem, and are only interested in a feed of names.
+
+	client, err := sunlight.NewClient(&sunlight.ClientConfig{
+		MonitoringPrefix: "https://navigli2025h2.skylight.geomys.org/",
+		UserAgent:        "ExampleClient (changeme@example.com, +https://example.com)",
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	var start int64
+	for {
+		checkpoint, err := client.Fetcher().ReadEndpoint(context.TODO(), "checkpoint")
+		if err != nil {
+			panic(err)
+		}
+
+		_, rest, _ := strings.Cut(string(checkpoint), "\n")
+		size, _, _ := strings.Cut(rest, "\n")
+		end, err := strconv.ParseInt(size, 10, 64)
+		if err != nil {
+			panic(err)
+		}
+
+		for i, entry := range client.UnauthenticatedTrimmedEntries(context.TODO(), end, start) {
+			fmt.Printf("%d: %s\n", i, entry.DNS)
+			start = i + 1
+		}
+		if err := client.Err(); err != nil {
+			panic(err)
+		}
+	}
+}

cmd/names-prism/names-prism.go

@@ -0,0 +1,133 @@
+// Command names-prism creates missing names tiles for existing data tiles.
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"log/slog"
+	"os"
+	"os/signal"
+	"strings"
+
+	"filippo.io/sunlight"
+	"filippo.io/sunlight/internal/stdlog"
+)
+
+func main() {
+	logger := slog.New(stdlog.Handler)
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer stop()
+
+	root, err := os.OpenRoot(os.Args[1])
+	if err != nil {
+		fatalError(logger, "failed to open local directory", "err", err)
+	}
+
+	var existing, created int
+	if err := fs.WalkDir(root.FS(), "tile/data", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+		if err := ctx.Err(); err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+
+		namesPath := "tile/names/" + strings.TrimPrefix(path, "tile/data/")
+
+		if d.IsDir() {
+			if path != "tile/data" {
+				root.Mkdir(namesPath, 0755)
+			}
+			return nil
+		}
+
+		if _, err := root.Stat(namesPath); err == nil {
+			existing++
+			return nil
+		} else if !os.IsNotExist(err) {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+
+		dataTile, err := fs.ReadFile(root.FS(), path)
+		if err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+		dataTile, err = decompress(dataTile)
+		if err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+		var namesTile []byte
+		for len(dataTile) > 0 {
+			var e *sunlight.LogEntry
+			e, dataTile, err = sunlight.ReadTileLeaf(dataTile)
+			if err != nil {
+				return fmt.Errorf("%s: %w", path, err)
+			}
+
+			if tl, err := e.TrimmedEntry(); err != nil {
+				return fmt.Errorf("%s: %w", path, err)
+			} else if line, err := json.Marshal(tl); err != nil {
+				return fmt.Errorf("%s: %w", path, err)
+			} else {
+				namesTile = append(namesTile, line...)
+				namesTile = append(namesTile, '\n')
+			}
+		}
+		namesTile, err = compress(namesTile)
+		if err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+
+		f, err := root.OpenFile(namesPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0444)
+		if err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+		if _, err := f.Write(namesTile); err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+		if err := f.Close(); err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+
+		created++
+		return nil
+	}); err != nil {
+		logger.Error("failed to walk tile directory", "err", err)
+	}
+
+	logger.Info("done", "existing", existing, "created", created)
+}
+
+func compress(data []byte) ([]byte, error) {
+	b := &bytes.Buffer{}
+	w := gzip.NewWriter(b)
+	if _, err := w.Write(data); err != nil {
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}
+
+const maxCompressRatio = 100
+
+func decompress(data []byte) ([]byte, error) {
+	r, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	maxSize := int64(len(data)) * maxCompressRatio
+	return io.ReadAll(io.LimitReader(r, maxSize))
+}
+
+func fatalError(logger *slog.Logger, msg string, args ...any) {
+	logger.Error(msg, args...)
+	os.Exit(1)
+}

cmd/skylight/skylight.go

@@ -366,14 +366,19 @@ func main() {
 				httpError(w, r, "tile", "invalid tile path", http.StatusBadRequest)
 				return
 			}
-			if tile.L == -1 {
+			switch tile.L {
+			case -1:
 				w.Header().Set("Content-Encoding", "gzip")
 				if tile.W < sunlight.TileWidth {
 					r = r.WithContext(context.WithValue(r.Context(), kindContextKey{}, "partial"))
 				} else {
 					r = r.WithContext(context.WithValue(r.Context(), kindContextKey{}, "data"))
 				}
-			} else {
+			case -2:
+				w.Header().Set("Content-Encoding", "gzip")
+				w.Header().Set("Content-Type", "application/jsonl; charset=utf-8")
+				r = r.WithContext(context.WithValue(r.Context(), kindContextKey{}, "names"))
+			default:
 				r = r.WithContext(context.WithValue(r.Context(), kindContextKey{}, "tile"))
 			}
 			handler.ServeHTTP(w, r)