Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/binaryen-c.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1371,7 +1371,8 @@ BinaryenExpressionRef BinaryenAtomicStore(BinaryenModuleRef module,
(Expression*)ptr,
(Expression*)value,
Type(type),
getMemoryName(module, memoryName)));
getMemoryName(module, memoryName),
MemoryOrder::SeqCst));
}
BinaryenExpressionRef BinaryenAtomicRMW(BinaryenModuleRef module,
BinaryenOp op,
Expand Down
10 changes: 6 additions & 4 deletions src/parser/contexts.h
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,8 @@ struct NullInstrParserCtx {
int,
bool,
MemoryIdxT*,
MemargT) {
MemargT,
MemoryOrder) {
return Ok{};
}
Result<> makeAtomicRMW(Index,
Expand Down Expand Up @@ -2255,12 +2256,13 @@ struct ParseDefsCtx : TypeParserCtx<ParseDefsCtx>, AnnotationParserCtx {
int bytes,
bool isAtomic,
Name* mem,
Memarg memarg) {
Memarg memarg,
MemoryOrder order) {
auto m = getMemory(pos, mem);
CHECK_ERR(m);
if (isAtomic) {
return withLoc(pos,
irBuilder.makeAtomicStore(bytes, memarg.offset, type, *m));
return withLoc(
pos, irBuilder.makeAtomicStore(bytes, memarg.offset, type, *m, order));
}
return withLoc(
pos, irBuilder.makeStore(bytes, memarg.offset, memarg.align, type, *m));
Expand Down
41 changes: 27 additions & 14 deletions src/parser/parsers.h
Original file line number Diff line number Diff line change
Expand Up @@ -850,17 +850,6 @@ Result<typename Ctx::MemTypeT> memtypeContinued(Ctx& ctx, Type addressType) {
return ctx.makeMemType(addressType, *limits, shared);
}

// memorder ::= '' | 'seqcst' | 'acqrel'
template<typename Ctx> Result<MemoryOrder> memorder(Ctx& ctx) {
if (ctx.in.takeKeyword("seqcst"sv)) {
return MemoryOrder::SeqCst;
}
if (ctx.in.takeKeyword("acqrel"sv)) {
return MemoryOrder::AcqRel;
}
return MemoryOrder::SeqCst;
}

// memorder ::= 'seqcst' | 'acqrel'
template<typename Ctx> MaybeResult<MemoryOrder> maybeMemOrder(Ctx& ctx) {
if (ctx.in.takeKeyword("seqcst"sv)) {
Expand All @@ -873,6 +862,13 @@ template<typename Ctx> MaybeResult<MemoryOrder> maybeMemOrder(Ctx& ctx) {
return {};
}

// memorder ::= '' | 'seqcst' | 'acqrel'
template<typename Ctx> Result<MemoryOrder> memorder(Ctx& ctx) {
auto order = maybeMemOrder(ctx);
CHECK_ERR(order);
return order ? *order : MemoryOrder::SeqCst;
}

// tabletype ::= (limits32 | 'i32' limits32 | 'i64' limit64) reftype
template<typename Ctx> Result<typename Ctx::TableTypeT> tabletype(Ctx& ctx) {
Type addressType = Type::i32;
Expand Down Expand Up @@ -1755,7 +1751,8 @@ Result<> makeLoad(Ctx& ctx,
CHECK_ERR(mem);

// We could only parse this when `isAtomic`, but this way gives a clearer
// error since `memIdx` can never be mistaken for a `memOrder`.
// error when a memorder is given for non-atomic operations
// since the next token can never be mistaken for a `memOrder`.
auto maybeOrder = maybeMemOrder(ctx);
CHECK_ERR(maybeOrder);

Expand Down Expand Up @@ -1787,10 +1784,26 @@ Result<> makeStore(Ctx& ctx,
bool isAtomic) {
auto mem = maybeMemidx(ctx);
CHECK_ERR(mem);

auto maybeOrder = maybeMemOrder(ctx);
CHECK_ERR(maybeOrder);

if (maybeOrder && !isAtomic) {
return Err{"Memory ordering can only be provided for atomic stores."};
}

auto arg = memarg(ctx, bytes);
CHECK_ERR(arg);
return ctx.makeStore(
pos, annotations, type, bytes, isAtomic, mem.getPtr(), *arg);
return ctx.makeStore(pos,
annotations,
type,
bytes,
isAtomic,
mem.getPtr(),
*arg,
maybeOrder ? *maybeOrder
: isAtomic ? MemoryOrder::SeqCst
: MemoryOrder::Unordered);
}

template<typename Ctx>
Expand Down
1 change: 1 addition & 0 deletions src/passes/Print.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,7 @@ struct PrintExpressionContents
}
restoreNormalColor(o);
printMemoryName(curr->memory, o, wasm);
printMemoryOrder(curr->order);
if (curr->offset) {
o << " offset=" << curr->offset;
}
Expand Down
30 changes: 18 additions & 12 deletions src/passes/SafeHeap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,22 @@ static Name getLoadName(Load* curr) {
}

static Name getStoreName(Store* curr) {
std::string ret = "SAFE_HEAP_STORE_";
ret += curr->valueType.toString();
ret += "_" + std::to_string(curr->bytes) + "_";
if (curr->isAtomic()) {
ret += "A";
} else {
ret += std::to_string(curr->align);
std::vector<std::string> parts{curr->valueType.toString(),
std::to_string(curr->bytes)};
switch (curr->order) {
case MemoryOrder::Unordered: {
parts.push_back(std::to_string(curr->align));
break;
}
case MemoryOrder::SeqCst: {
parts.push_back("SC");
break;
}
case MemoryOrder::AcqRel: {
parts.push_back("AR");
}
}
return ret;
return "SAFE_HEAP_STORE_" + String::join(parts, "_");
}

struct AccessInstrumenter : public WalkerPass<PostWalker<AccessInstrumenter>> {
Expand Down Expand Up @@ -287,10 +294,9 @@ struct SafeHeap : public Pass {
if (align > bytes) {
continue;
}
for (auto isAtomic : {true, false}) {
store.order =
isAtomic ? MemoryOrder::SeqCst : MemoryOrder::Unordered;
if (isAtomic &&
for (auto memoryOrder : memoryOrdersToGenerate) {
store.order = memoryOrder;
if (store.isAtomic() &&
!isPossibleAtomicOperation(
align, bytes, module->memories[0]->shared, valueType)) {
continue;
Expand Down
3 changes: 2 additions & 1 deletion src/tools/wasm-split/instrumenter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@ void Instrumenter::instrumentFuncs() {
builder.makeConstPtr(0, Type::i32),
builder.makeConst(uint32_t(1)),
Type::i32,
memoryName),
memoryName,
MemoryOrder::SeqCst),
func->body,
func->body->type);
++funcIdx;
Expand Down
8 changes: 6 additions & 2 deletions src/wasm-builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -451,9 +451,13 @@ class Builder {
Expression* ptr,
Expression* value,
Type type,
Name memory) {
Name memory,
MemoryOrder order) {
assert(order != MemoryOrder::Unordered &&
"Atomic stores can't be unordered");

Store* store = makeStore(bytes, offset, bytes, ptr, value, type, memory);
store->order = MemoryOrder::SeqCst;
store->order = order;
return store;
}
AtomicRMW* makeAtomicRMW(AtomicRMWOp op,
Expand Down
3 changes: 2 additions & 1 deletion src/wasm-ir-builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ class IRBuilder : public UnifiedExpressionVisitor<IRBuilder, Result<>> {
unsigned bytes, Address offset, unsigned align, Type type, Name mem);
Result<> makeAtomicLoad(
unsigned bytes, Address offset, Type type, Name mem, MemoryOrder order);
Result<> makeAtomicStore(unsigned bytes, Address offset, Type type, Name mem);
Result<> makeAtomicStore(
unsigned bytes, Address offset, Type type, Name mem, MemoryOrder order);
Result<> makeAtomicRMW(
AtomicRMWOp op, unsigned bytes, Address offset, Type type, Name mem);
Result<>
Expand Down
21 changes: 14 additions & 7 deletions src/wasm/wasm-binary.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3657,31 +3657,38 @@ Result<> WasmBinaryReader::readInst() {
}
case BinaryConsts::I32AtomicStore8: {
auto [mem, align, offset, memoryOrder] = getAtomicMemarg();
return builder.makeAtomicStore(1, offset, Type::i32, mem);
return builder.makeAtomicStore(
1, offset, Type::i32, mem, memoryOrder);
}
case BinaryConsts::I32AtomicStore16: {
auto [mem, align, offset, memoryOrder] = getAtomicMemarg();
return builder.makeAtomicStore(2, offset, Type::i32, mem);
return builder.makeAtomicStore(
2, offset, Type::i32, mem, memoryOrder);
}
case BinaryConsts::I32AtomicStore: {
auto [mem, align, offset, memoryOrder] = getAtomicMemarg();
return builder.makeAtomicStore(4, offset, Type::i32, mem);
return builder.makeAtomicStore(
4, offset, Type::i32, mem, memoryOrder);
}
case BinaryConsts::I64AtomicStore8: {
auto [mem, align, offset, memoryOrder] = getAtomicMemarg();
return builder.makeAtomicStore(1, offset, Type::i64, mem);
return builder.makeAtomicStore(
1, offset, Type::i64, mem, memoryOrder);
}
case BinaryConsts::I64AtomicStore16: {
auto [mem, align, offset, memoryOrder] = getAtomicMemarg();
return builder.makeAtomicStore(2, offset, Type::i64, mem);
return builder.makeAtomicStore(
2, offset, Type::i64, mem, memoryOrder);
}
case BinaryConsts::I64AtomicStore32: {
auto [mem, align, offset, memoryOrder] = getAtomicMemarg();
return builder.makeAtomicStore(4, offset, Type::i64, mem);
return builder.makeAtomicStore(
4, offset, Type::i64, mem, memoryOrder);
}
case BinaryConsts::I64AtomicStore: {
auto [mem, align, offset, memoryOrder] = getAtomicMemarg();
return builder.makeAtomicStore(8, offset, Type::i64, mem);
return builder.makeAtomicStore(
8, offset, Type::i64, mem, memoryOrder);
}

#define RMW(op) \
Expand Down
9 changes: 4 additions & 5 deletions src/wasm/wasm-ir-builder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1492,15 +1492,14 @@ Result<> IRBuilder::makeAtomicLoad(
return Ok{};
}

Result<> IRBuilder::makeAtomicStore(unsigned bytes,
Address offset,
Type type,
Name mem) {
Result<> IRBuilder::makeAtomicStore(
unsigned bytes, Address offset, Type type, Name mem, MemoryOrder order) {
Store curr;
curr.memory = mem;
curr.valueType = type;
CHECK_ERR(visitStore(&curr));
push(builder.makeAtomicStore(bytes, offset, curr.ptr, curr.value, type, mem));
push(builder.makeAtomicStore(
bytes, offset, curr.ptr, curr.value, type, mem, order));
return Ok{};
}

Expand Down
6 changes: 6 additions & 0 deletions src/wasm/wasm-validator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1147,6 +1147,12 @@ void FunctionValidator::visitStore(Store* curr) {
curr,
"Atomic store should be i32 or i64");
}
if (curr->order == MemoryOrder::AcqRel) {
shouldBeTrue(getModule()->features.hasRelaxedAtomics(),
curr,
"Acquire/release operations require relaxed atomics "
"[--enable-relaxed-atomics]");
}
if (curr->valueType == Type::v128) {
shouldBeTrue(getModule()->features.hasSIMD(),
curr,
Expand Down
46 changes: 33 additions & 13 deletions test/lit/basic/relaxed-atomics.wast
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,56 @@
;; RTRIP: (memory $0 23 256 shared)
(memory $0 23 256 shared)

;; RTRIP: (func $acqrel (type $0) (result i32)
;; RTRIP-NEXT: (i32.atomic.load acqrel
;; RTRIP: (func $acqrel (type $0)
;; RTRIP-NEXT: (i32.atomic.store acqrel
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: )
;; RTRIP-NEXT: (drop
;; RTRIP-NEXT: (i32.atomic.load acqrel
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: )
;; RTRIP-NEXT: )
;; RTRIP-NEXT: )
(func $acqrel (result i32)
(i32.atomic.load acqrel
(i32.const 1)
)
(func $acqrel
(i32.atomic.store acqrel (i32.const 1) (i32.const 1))
(drop
(i32.atomic.load acqrel
(i32.const 1)
))
)

;; Optional seqcst ordering is dropped in text output.
;; RTRIP: (func $seqcst (type $0) (result i32)
;; RTRIP: (func $seqcst (type $0)
;; RTRIP-NEXT: (i32.atomic.store
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: )
;; RTRIP-NEXT: (i32.atomic.store
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: )
;; RTRIP-NEXT: (drop
;; RTRIP-NEXT: (i32.atomic.load
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: )
;; RTRIP-NEXT: )
;; RTRIP-NEXT: (i32.atomic.load
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: (drop
;; RTRIP-NEXT: (i32.atomic.load
;; RTRIP-NEXT: (i32.const 1)
;; RTRIP-NEXT: )
;; RTRIP-NEXT: )
;; RTRIP-NEXT: )
(func $seqcst (result i32)
;; seqcst may be omitted for atomic loads, it's the default
(func $seqcst
(i32.atomic.store seqcst (i32.const 1) (i32.const 1))
(i32.atomic.store 0 seqcst (i32.const 1) (i32.const 1))
(drop (i32.atomic.load seqcst
(i32.const 1)
))
;; allows memory index before memory ordering immediate
(i32.atomic.load 0 seqcst
(drop
(i32.atomic.load 0 seqcst
(i32.const 1)
)
))
)
)
Loading
Loading