Skip to content

Commit c4060b9

Browse files
committed
feat: multi-threaded slave (MTS) parallel DML apply
Implements LOGICAL_CLOCK-based parallel binlog event application, mirroring MySQL 5.7 MTS scheduling. With --num-workers=N, gh-ost applies DML events to the ghost table using N concurrent workers, significantly increasing throughput for high-write tables. Key components: - commitBarrier: dependency tracking via last_committed/sequence_number - mtsScheduleState: new-group detection and epoch reset handling - dmlCoordinator: transaction grouping and dependency-aware dispatch - dmlWorker: per-worker goroutine with independent DB connection - Deadlock-aware retry: immediate retry on errno 1213, 1s sleep on others - Monotonic coordinate update: prevents checkpoint regression when workers complete out of order Backward compatible: --num-workers=1 (default) uses the original single-threaded path with zero behavioral changes.
1 parent c636347 commit c4060b9

20 files changed

Lines changed: 2743 additions & 28 deletions

go/base/context.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,9 +274,28 @@ type MigrationContext struct {
274274
SkipMetadataLockCheck bool
275275
IsOpenMetadataLockInstruments bool
276276

277+
// MTS parallel apply configuration
278+
NumWorkers int
279+
BinlogHasLogicalTimestamps bool
280+
LogicalTimestampsDetected chan struct{}
281+
logicalTimestampsDetectOnce sync.Once
282+
277283
Log Logger
278284
}
279285

286+
// NotifyLogicalTimestampsDetection closes LogicalTimestampsDetected once so MTS
287+
// startup can proceed. found=true when binlog carries logical timestamps (MySQL 5.7+).
288+
func (mctx *MigrationContext) NotifyLogicalTimestampsDetection(found bool) {
289+
mctx.logicalTimestampsDetectOnce.Do(func() {
290+
if found {
291+
mctx.BinlogHasLogicalTimestamps = true
292+
}
293+
if mctx.LogicalTimestampsDetected != nil {
294+
close(mctx.LogicalTimestampsDetected)
295+
}
296+
})
297+
}
298+
280299
type Logger interface {
281300
Debug(args ...interface{})
282301
Debugf(format string, args ...interface{})

go/binlog/binlog_entry.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ import (
1313

1414
// BinlogEntry describes an entry in the binary log
1515
type BinlogEntry struct {
16-
Coordinates mysql.BinlogCoordinates
17-
DmlEvent *BinlogDMLEvent
16+
Coordinates mysql.BinlogCoordinates
17+
DmlEvent *BinlogDMLEvent
18+
LastCommitted int64 // logical timestamp of commit parent (0 = SEQ_UNINIT, unavailable)
19+
SequenceNumber int64 // monotonically increasing logical timestamp (0 = SEQ_UNINIT)
1820
}
1921

2022
// NewBinlogEntryAt creates an empty, ready to go BinlogEntry object

go/binlog/gomysql_reader.go

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func (gmr *GoMySQLReader) GetCurrentBinlogCoordinates() mysql.BinlogCoordinates
8585
return gmr.currentCoordinates.Clone()
8686
}
8787

88-
func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error {
88+
func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry, lastCommitted int64, sequenceNumber int64) error {
8989
currentCoords := gmr.GetCurrentBinlogCoordinates()
9090
dml := ToEventDML(ev.Header.EventType.String())
9191
if dml == NotDML {
@@ -98,6 +98,8 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent
9898
continue
9999
}
100100
binlogEntry := NewBinlogEntryAt(currentCoords)
101+
binlogEntry.LastCommitted = lastCommitted
102+
binlogEntry.SequenceNumber = sequenceNumber
101103
binlogEntry.DmlEvent = NewBinlogDMLEvent(
102104
string(rowsEvent.Table.Schema),
103105
string(rowsEvent.Table.Table),
@@ -130,6 +132,10 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent
130132

131133
// StreamEvents
132134
func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChannel chan<- *BinlogEntry) error {
135+
var currentLastCommitted int64
136+
var currentSequenceNumber int64
137+
logicalTimestampsDetected := false
138+
133139
for !canStopStreaming() {
134140
ev, err := gmr.binlogStreamer.GetEvent(context.Background())
135141
if err != nil {
@@ -156,6 +162,14 @@ func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChan
156162
if !gmr.migrationContext.UseGTIDs {
157163
continue
158164
}
165+
// Capture logical timestamps for MTS dependency tracking
166+
currentLastCommitted = event.LastCommitted
167+
currentSequenceNumber = event.SequenceNumber
168+
// Detect whether binlog contains logical timestamps (MySQL 5.7+)
169+
if !logicalTimestampsDetected && (event.LastCommitted > 0 || event.SequenceNumber > 0) {
170+
logicalTimestampsDetected = true
171+
gmr.migrationContext.NotifyLogicalTimestampsDetection(true)
172+
}
159173
sid, err := uuid.FromBytes(event.SID)
160174
if err != nil {
161175
return err
@@ -184,12 +198,13 @@ func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChan
184198
gmr.LastTrxCoords = gmr.currentCoordinates.Clone()
185199
}
186200
case *replication.RowsEvent:
187-
if err := gmr.handleRowsEvent(ev, event, entriesChannel); err != nil {
201+
if err := gmr.handleRowsEvent(ev, event, entriesChannel, currentLastCommitted, currentSequenceNumber); err != nil {
188202
return err
189203
}
190204
}
191205
}
192206
gmr.migrationContext.Log.Debugf("done streaming events")
207+
gmr.migrationContext.NotifyLogicalTimestampsDetection(logicalTimestampsDetected)
193208

194209
return nil
195210
}

go/cmd/gh-ost/main.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ func main() {
124124
exponentialBackoffMaxInterval := flag.Int64("exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between attempts when performing various operations with exponential backoff.")
125125
chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 10-100,000)")
126126
dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-1000)")
127+
numWorkers := flag.Int("num-workers", 1, "number of parallel DML apply workers (MTS mode). Requires MySQL 5.7+ with binlog logical timestamps. Default: 1 (single-threaded, backward compatible)")
127128
defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking")
128129
flag.BoolVar(&migrationContext.PanicOnWarnings, "panic-on-warnings", false, "Panic when SQL warnings are encountered when copying a batch indicating data loss")
129130
cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout) or attempting instant DDL")
@@ -377,6 +378,14 @@ func main() {
377378
migrationContext.SetChunkSize(*chunkSize)
378379
migrationContext.SetDMLBatchSize(*dmlBatchSize)
379380
migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis)
381+
if *numWorkers < 1 {
382+
migrationContext.Log.Warningf("invalid --num-workers=%d; using 1", *numWorkers)
383+
*numWorkers = 1
384+
}
385+
migrationContext.NumWorkers = *numWorkers
386+
if migrationContext.NumWorkers > 1 {
387+
migrationContext.LogicalTimestampsDetected = make(chan struct{})
388+
}
380389
migrationContext.SetThrottleQuery(*throttleQuery)
381390
migrationContext.SetThrottleHTTP(*throttleHTTP)
382391
migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors)

go/logic/applier.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,18 @@ func (apl *Applier) releaseMigrationLock() {
296296
apl.migrationLockConn = nil
297297
}
298298

299+
// adoptDMLQueryBuildersFrom shares read-only DML query builders from the primary applier.
300+
// The primary applier must have called prepareQueries() before MTS workers start.
301+
func (apl *Applier) adoptDMLQueryBuildersFrom(source *Applier) error {
302+
if source.dmlInsertQueryBuilder == nil {
303+
return fmt.Errorf("primary applier DML query builders are not prepared")
304+
}
305+
apl.dmlDeleteQueryBuilder = source.dmlDeleteQueryBuilder
306+
apl.dmlInsertQueryBuilder = source.dmlInsertQueryBuilder
307+
apl.dmlUpdateQueryBuilder = source.dmlUpdateQueryBuilder
308+
return nil
309+
}
310+
299311
func (apl *Applier) prepareQueries() (err error) {
300312
if apl.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder(
301313
apl.migrationContext.DatabaseName,
@@ -462,6 +474,12 @@ func (apl *Applier) AttemptInstantDDL() error {
462474
}, apl.migrationContext.MaxRetries(), apl.migrationContext.Log)
463475
}
464476

477+
// isDeadlockError checks whether the given error is a MySQL InnoDB deadlock (errno 1213).
478+
func isDeadlockError(err error) bool {
479+
var mysqlErr *drivermysql.MySQLError
480+
return errors.As(err, &mysqlErr) && mysqlErr.Number == 1213
481+
}
482+
465483
// retryOnLockWaitTimeout retries the given operation on MySQL lock wait timeout
466484
// (errno 1205). Non-timeout errors return immediately. This is used for instant
467485
// DDL attempts where the operation may be blocked by a long-running transaction.

go/logic/commit_barrier.go

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
Copyright 2025 GitHub Inc.
3+
See https://github.com/github/gh-ost/blob/master/LICENSE
4+
*/
5+
6+
package logic
7+
8+
import (
9+
"context"
10+
"sync"
11+
"sync/atomic"
12+
)
13+
14+
const seqUninit int64 = 0
15+
16+
// commitBarrier implements MTS LOGICAL_CLOCK dependency tracking using a
17+
// gap-free Low Water Mark (LWM), mirroring MySQL 8.0's GAQ scheduling.
18+
//
19+
// LWM invariant: all transactions with sequence_number <= lwm have completed.
20+
// This is equivalent to MySQL's find_lwm() + move_queue_head() in rpl_rli_pdb.cc.
21+
//
22+
// Reference: MySQL 8.0 sql/rpl_mta_submode.cc
23+
// - waitForDependency corresponds to wait_for_last_committed_trx()
24+
// - commit corresponds to Worker commit + GAQ LWM advancement
25+
// - waitForAllWorkers corresponds to wait_for_workers_to_finish()
26+
type commitBarrier struct {
27+
mu sync.Mutex
28+
lwm int64 // gap-free low water mark: all seq <= lwm are complete
29+
pending map[int64]bool // sequences > lwm that committed but aren't consecutive yet
30+
delegatedJobs atomic.Int64 // jobs dispatched but not yet completed
31+
cond *sync.Cond
32+
}
33+
34+
func newCommitBarrier() *commitBarrier {
35+
cb := &commitBarrier{
36+
lwm: seqUninit,
37+
pending: make(map[int64]bool),
38+
}
39+
cb.cond = sync.NewCond(&cb.mu)
40+
return cb
41+
}
42+
43+
// clockLeq implements MySQL's clock_leq: SEQ_UNINIT (0) is treated as the
44+
// minimum value in the clock domain.
45+
func clockLeq(a, b int64) bool {
46+
if a == seqUninit {
47+
return true
48+
}
49+
if b == seqUninit {
50+
return false
51+
}
52+
return a <= b
53+
}
54+
55+
// waitForDependency blocks until lwm >= lastCommitted.
56+
//
57+
// This matches MySQL's wait_for_last_committed_trx() which waits until the
58+
// low water mark advances past the parent transaction:
59+
//
60+
// while (!clock_leq(last_committed_arg, estimate_lwm_timestamp()))
61+
// wait(logical_clock_cond)
62+
//
63+
// Cross-table dependencies (parentSeenOnStream == false) are treated as
64+
// satisfied, matching MySQL's SEQ_UNINIT handling where undefined parents
65+
// don't block scheduling.
66+
func (cb *commitBarrier) waitForDependency(ctx context.Context, lastCommitted int64, parentSeenOnStream bool) {
67+
if lastCommitted == seqUninit || !parentSeenOnStream {
68+
return
69+
}
70+
cb.mu.Lock()
71+
defer cb.mu.Unlock()
72+
for !clockLeq(lastCommitted, cb.lwm) {
73+
if ctx.Err() != nil {
74+
return
75+
}
76+
cb.cond.Wait()
77+
}
78+
}
79+
80+
// commit records a transaction as complete and advances the LWM
81+
// as far as possible through consecutive completed sequences.
82+
//
83+
// This matches MySQL's move_queue_head() which dequeues jobs from
84+
// the GAQ head only while they are consecutively done:
85+
//
86+
// while (!empty()) {
87+
// if (ptr_g->done == 0) break; // gap — stop advancing
88+
// de_queue(&g); // remove from queue
89+
// lwm = g; // advance LWM
90+
// }
91+
func (cb *commitBarrier) commit(sequenceNumber int64) {
92+
if sequenceNumber == seqUninit {
93+
return
94+
}
95+
cb.mu.Lock()
96+
cb.pending[sequenceNumber] = true
97+
// Advance LWM through consecutive committed sequences
98+
for cb.pending[cb.lwm+1] {
99+
delete(cb.pending, cb.lwm+1)
100+
cb.lwm++
101+
}
102+
cb.mu.Unlock()
103+
cb.cond.Broadcast()
104+
}
105+
106+
// waitForAllWorkers blocks until all delegated jobs have completed (delegatedJobs == 0).
107+
//
108+
// Corresponds to MySQL: wait_for_workers_to_finish()
109+
func (cb *commitBarrier) waitForAllWorkers(ctx context.Context) {
110+
cb.mu.Lock()
111+
defer cb.mu.Unlock()
112+
for cb.delegatedJobs.Load() > 0 {
113+
if ctx.Err() != nil {
114+
return
115+
}
116+
cb.cond.Wait()
117+
}
118+
}
119+
120+
// addDelegatedJob increments the delegated job counter.
121+
func (cb *commitBarrier) addDelegatedJob() {
122+
cb.delegatedJobs.Add(1)
123+
}
124+
125+
// completeDelegatedJob decrements the delegated job counter and broadcasts wake signal.
126+
func (cb *commitBarrier) completeDelegatedJob() {
127+
cb.delegatedJobs.Add(-1)
128+
cb.cond.Broadcast()
129+
}
130+
131+
// getLWM returns the current gap-free low water mark (thread-safe).
132+
func (cb *commitBarrier) getLWM() int64 {
133+
cb.mu.Lock()
134+
defer cb.mu.Unlock()
135+
return cb.lwm
136+
}

0 commit comments

Comments
 (0)