Skip to content

Commit b324c9f

Browse files
bognerbogner
authored
[DirectX] Move memset and memcpy handling to a new pass. NFC (#172921)
This introduces the DXILMemIntrinsics pass and moves memset and memcpy handling from DXILLegalize to here. We need to do this so that we can handle memory intrinsics before the DXILResourceAccess pass so that we can properly deal with arrays and large structures in resources.
1 parent f171b43 commit b324c9f

File tree

10 files changed

+234
-172
lines changed

10 files changed

+234
-172
lines changed

‎llvm/lib/Target/DirectX/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ add_llvm_target(DirectXCodeGen
2626
DXILForwardHandleAccesses.cpp
2727
DXILFlattenArrays.cpp
2828
DXILIntrinsicExpansion.cpp
29+
DXILMemIntrinsics.cpp
2930
DXILOpBuilder.cpp
3031
DXILOpLowering.cpp
3132
DXILPostOptimizationValidation.cpp
@@ -37,7 +38,7 @@ add_llvm_target(DirectXCodeGen
3738
DXILTranslateMetadata.cpp
3839
DXILRootSignature.cpp
3940
DXILLegalizePass.cpp
40-
41+
4142
LINK_COMPONENTS
4243
Analysis
4344
AsmPrinter

‎llvm/lib/Target/DirectX/DXILLegalizePass.cpp‎

Lines changed: 2 additions & 166 deletions
Original file line numberDiff line numberDiff line change
@@ -269,8 +269,8 @@ static bool upcastI8AllocasAndUses(Instruction &I,
269269
if (CastInst *Cast = dyn_cast<CastInst>(LU))
270270
Ty = Cast->getType();
271271
else if (CallInst *CI = dyn_cast<CallInst>(LU)) {
272-
if (CI->getIntrinsicID() == Intrinsic::memset)
273-
Ty = Type::getInt32Ty(CI->getContext());
272+
assert(CI->getIntrinsicID() != Intrinsic::memset &&
273+
"memset should have been eliminated in an earlier pass");
274274
}
275275

276276
if (!Ty)
@@ -346,168 +346,6 @@ downcastI64toI32InsertExtractElements(Instruction &I,
346346
return false;
347347
}
348348

349-
static void emitMemcpyExpansion(IRBuilder<> &Builder, Value *Dst, Value *Src,
350-
ConstantInt *Length) {
351-
352-
uint64_t ByteLength = Length->getZExtValue();
353-
// If length to copy is zero, no memcpy is needed.
354-
if (ByteLength == 0)
355-
return;
356-
357-
const DataLayout &DL = Builder.GetInsertBlock()->getModule()->getDataLayout();
358-
359-
auto GetArrTyFromVal = [](Value *Val) -> ArrayType * {
360-
assert(isa<AllocaInst>(Val) ||
361-
isa<GlobalVariable>(Val) &&
362-
"Expected Val to be an Alloca or Global Variable");
363-
if (auto *Alloca = dyn_cast<AllocaInst>(Val))
364-
return dyn_cast<ArrayType>(Alloca->getAllocatedType());
365-
if (auto *GlobalVar = dyn_cast<GlobalVariable>(Val))
366-
return dyn_cast<ArrayType>(GlobalVar->getValueType());
367-
return nullptr;
368-
};
369-
370-
ArrayType *DstArrTy = GetArrTyFromVal(Dst);
371-
assert(DstArrTy && "Expected Dst of memcpy to be a Pointer to an Array Type");
372-
if (auto *DstGlobalVar = dyn_cast<GlobalVariable>(Dst))
373-
assert(!DstGlobalVar->isConstant() &&
374-
"The Dst of memcpy must not be a constant Global Variable");
375-
[[maybe_unused]] ArrayType *SrcArrTy = GetArrTyFromVal(Src);
376-
assert(SrcArrTy && "Expected Src of memcpy to be a Pointer to an Array Type");
377-
378-
Type *DstElemTy = DstArrTy->getElementType();
379-
uint64_t DstElemByteSize = DL.getTypeStoreSize(DstElemTy);
380-
assert(DstElemByteSize > 0 && "Dst element type store size must be set");
381-
Type *SrcElemTy = SrcArrTy->getElementType();
382-
[[maybe_unused]] uint64_t SrcElemByteSize = DL.getTypeStoreSize(SrcElemTy);
383-
assert(SrcElemByteSize > 0 && "Src element type store size must be set");
384-
385-
// This assumption simplifies implementation and covers currently-known
386-
// use-cases for DXIL. It may be relaxed in the future if required.
387-
assert(DstElemTy == SrcElemTy &&
388-
"The element types of Src and Dst arrays must match");
389-
390-
[[maybe_unused]] uint64_t DstArrNumElems = DstArrTy->getArrayNumElements();
391-
assert(DstElemByteSize * DstArrNumElems >= ByteLength &&
392-
"Dst array size must be at least as large as the memcpy length");
393-
[[maybe_unused]] uint64_t SrcArrNumElems = SrcArrTy->getArrayNumElements();
394-
assert(SrcElemByteSize * SrcArrNumElems >= ByteLength &&
395-
"Src array size must be at least as large as the memcpy length");
396-
397-
uint64_t NumElemsToCopy = ByteLength / DstElemByteSize;
398-
assert(ByteLength % DstElemByteSize == 0 &&
399-
"memcpy length must be divisible by array element type");
400-
for (uint64_t I = 0; I < NumElemsToCopy; ++I) {
401-
SmallVector<Value *, 2> Indices = {Builder.getInt32(0),
402-
Builder.getInt32(I)};
403-
Value *SrcPtr = Builder.CreateInBoundsGEP(SrcArrTy, Src, Indices, "gep");
404-
Value *SrcVal = Builder.CreateLoad(SrcElemTy, SrcPtr);
405-
Value *DstPtr = Builder.CreateInBoundsGEP(DstArrTy, Dst, Indices, "gep");
406-
Builder.CreateStore(SrcVal, DstPtr);
407-
}
408-
}
409-
410-
static void emitMemsetExpansion(IRBuilder<> &Builder, Value *Dst, Value *Val,
411-
ConstantInt *SizeCI,
412-
DenseMap<Value *, Value *> &ReplacedValues) {
413-
[[maybe_unused]] const DataLayout &DL =
414-
Builder.GetInsertBlock()->getModule()->getDataLayout();
415-
[[maybe_unused]] uint64_t OrigSize = SizeCI->getZExtValue();
416-
417-
AllocaInst *Alloca = dyn_cast<AllocaInst>(Dst);
418-
419-
assert(Alloca && "Expected memset on an Alloca");
420-
assert(OrigSize == Alloca->getAllocationSize(DL)->getFixedValue() &&
421-
"Expected for memset size to match DataLayout size");
422-
423-
Type *AllocatedTy = Alloca->getAllocatedType();
424-
ArrayType *ArrTy = dyn_cast<ArrayType>(AllocatedTy);
425-
assert(ArrTy && "Expected Alloca for an Array Type");
426-
427-
Type *ElemTy = ArrTy->getElementType();
428-
uint64_t Size = ArrTy->getArrayNumElements();
429-
430-
[[maybe_unused]] uint64_t ElemSize = DL.getTypeStoreSize(ElemTy);
431-
432-
assert(ElemSize > 0 && "Size must be set");
433-
assert(OrigSize == ElemSize * Size && "Size in bytes must match");
434-
435-
Value *TypedVal = Val;
436-
437-
if (Val->getType() != ElemTy) {
438-
if (ReplacedValues[Val]) {
439-
// Note for i8 replacements if we know them we should use them.
440-
// Further if this is a constant ReplacedValues will return null
441-
// so we will stick to TypedVal = Val
442-
TypedVal = ReplacedValues[Val];
443-
444-
} else {
445-
// This case Val is a ConstantInt so the cast folds away.
446-
// However if we don't do the cast the store below ends up being
447-
// an i8.
448-
TypedVal = Builder.CreateIntCast(Val, ElemTy, false);
449-
}
450-
}
451-
452-
for (uint64_t I = 0; I < Size; ++I) {
453-
Value *Zero = Builder.getInt32(0);
454-
Value *Offset = Builder.getInt32(I);
455-
Value *Ptr = Builder.CreateGEP(ArrTy, Dst, {Zero, Offset}, "gep");
456-
Builder.CreateStore(TypedVal, Ptr);
457-
}
458-
}
459-
460-
// Expands the instruction `I` into corresponding loads and stores if it is a
461-
// memcpy call. In that case, the call instruction is added to the `ToRemove`
462-
// vector. `ReplacedValues` is unused.
463-
static bool legalizeMemCpy(Instruction &I,
464-
SmallVectorImpl<Instruction *> &ToRemove,
465-
DenseMap<Value *, Value *> &ReplacedValues) {
466-
467-
CallInst *CI = dyn_cast<CallInst>(&I);
468-
if (!CI)
469-
return false;
470-
471-
Intrinsic::ID ID = CI->getIntrinsicID();
472-
if (ID != Intrinsic::memcpy)
473-
return false;
474-
475-
IRBuilder<> Builder(&I);
476-
Value *Dst = CI->getArgOperand(0);
477-
Value *Src = CI->getArgOperand(1);
478-
ConstantInt *Length = dyn_cast<ConstantInt>(CI->getArgOperand(2));
479-
assert(Length && "Expected Length to be a ConstantInt");
480-
[[maybe_unused]] ConstantInt *IsVolatile =
481-
dyn_cast<ConstantInt>(CI->getArgOperand(3));
482-
assert(IsVolatile && "Expected IsVolatile to be a ConstantInt");
483-
assert(IsVolatile->getZExtValue() == 0 && "Expected IsVolatile to be false");
484-
emitMemcpyExpansion(Builder, Dst, Src, Length);
485-
ToRemove.push_back(CI);
486-
return true;
487-
}
488-
489-
static bool legalizeMemSet(Instruction &I,
490-
SmallVectorImpl<Instruction *> &ToRemove,
491-
DenseMap<Value *, Value *> &ReplacedValues) {
492-
493-
CallInst *CI = dyn_cast<CallInst>(&I);
494-
if (!CI)
495-
return false;
496-
497-
Intrinsic::ID ID = CI->getIntrinsicID();
498-
if (ID != Intrinsic::memset)
499-
return false;
500-
501-
IRBuilder<> Builder(&I);
502-
Value *Dst = CI->getArgOperand(0);
503-
Value *Val = CI->getArgOperand(1);
504-
ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(2));
505-
assert(Size && "Expected Size to be a ConstantInt");
506-
emitMemsetExpansion(Builder, Dst, Val, Size, ReplacedValues);
507-
ToRemove.push_back(CI);
508-
return true;
509-
}
510-
511349
static bool updateFnegToFsub(Instruction &I,
512350
SmallVectorImpl<Instruction *> &ToRemove,
513351
DenseMap<Value *, Value *> &) {
@@ -660,8 +498,6 @@ class DXILLegalizationPipeline {
660498
LegalizationPipeline[Stage1].push_back(fixI8UseChain);
661499
LegalizationPipeline[Stage1].push_back(legalizeGetHighLowi64Bytes);
662500
LegalizationPipeline[Stage1].push_back(legalizeFreeze);
663-
LegalizationPipeline[Stage1].push_back(legalizeMemCpy);
664-
LegalizationPipeline[Stage1].push_back(legalizeMemSet);
665501
LegalizationPipeline[Stage1].push_back(updateFnegToFsub);
666502
// Note: legalizeGetHighLowi64Bytes and
667503
// downcastI64toI32InsertExtractElements both modify extractelement, so they

‎llvm/lib/Target/DirectX/DXILMemIntrinsics.cpp‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
//===- DXILMemIntrinsics.cpp - Eliminate Memory Intrinsics ----------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "DXILMemIntrinsics.h"
10+
#include "DirectX.h"
11+
#include "llvm/IR/IRBuilder.h"
12+
#include "llvm/IR/IntrinsicInst.h"
13+
#include "llvm/IR/Module.h"
14+
15+
#define DEBUG_TYPE "dxil-mem-intrinsics"
16+
17+
using namespace llvm;
18+
19+
void expandMemSet(MemSetInst *MemSet) {
20+
IRBuilder<> Builder(MemSet);
21+
Value *Dst = MemSet->getDest();
22+
Value *Val = MemSet->getValue();
23+
ConstantInt *LengthCI = dyn_cast<ConstantInt>(MemSet->getLength());
24+
assert(LengthCI && "Expected length to be a ConstantInt");
25+
26+
[[maybe_unused]] const DataLayout &DL =
27+
Builder.GetInsertBlock()->getModule()->getDataLayout();
28+
[[maybe_unused]] uint64_t OrigLength = LengthCI->getZExtValue();
29+
30+
AllocaInst *Alloca = dyn_cast<AllocaInst>(Dst);
31+
32+
assert(Alloca && "Expected memset on an Alloca");
33+
assert(OrigLength == Alloca->getAllocationSize(DL)->getFixedValue() &&
34+
"Expected for memset size to match DataLayout size");
35+
36+
Type *AllocatedTy = Alloca->getAllocatedType();
37+
ArrayType *ArrTy = dyn_cast<ArrayType>(AllocatedTy);
38+
assert(ArrTy && "Expected Alloca for an Array Type");
39+
40+
Type *ElemTy = ArrTy->getElementType();
41+
uint64_t Size = ArrTy->getArrayNumElements();
42+
43+
[[maybe_unused]] uint64_t ElemSize = DL.getTypeStoreSize(ElemTy);
44+
45+
assert(ElemSize > 0 && "Size must be set");
46+
assert(OrigLength == ElemSize * Size && "Size in bytes must match");
47+
48+
Value *TypedVal = Val;
49+
50+
if (Val->getType() != ElemTy)
51+
TypedVal = Builder.CreateIntCast(Val, ElemTy, false);
52+
53+
for (uint64_t I = 0; I < Size; ++I) {
54+
Value *Zero = Builder.getInt32(0);
55+
Value *Offset = Builder.getInt32(I);
56+
Value *Ptr = Builder.CreateGEP(ArrTy, Dst, {Zero, Offset}, "gep");
57+
Builder.CreateStore(TypedVal, Ptr);
58+
}
59+
60+
MemSet->eraseFromParent();
61+
}
62+
63+
void expandMemCpy(MemCpyInst *MemCpy) {
64+
IRBuilder<> Builder(MemCpy);
65+
Value *Dst = MemCpy->getDest();
66+
Value *Src = MemCpy->getSource();
67+
ConstantInt *LengthCI = dyn_cast<ConstantInt>(MemCpy->getLength());
68+
assert(LengthCI && "Expected Length to be a ConstantInt");
69+
assert(!MemCpy->isVolatile() && "Handling for volatile not implemented");
70+
71+
uint64_t ByteLength = LengthCI->getZExtValue();
72+
// If length to copy is zero, no memcpy is needed.
73+
if (ByteLength == 0)
74+
return;
75+
76+
const DataLayout &DL = Builder.GetInsertBlock()->getModule()->getDataLayout();
77+
78+
auto GetArrTyFromVal = [](Value *Val) -> ArrayType * {
79+
assert(isa<AllocaInst>(Val) ||
80+
isa<GlobalVariable>(Val) &&
81+
"Expected Val to be an Alloca or Global Variable");
82+
if (auto *Alloca = dyn_cast<AllocaInst>(Val))
83+
return dyn_cast<ArrayType>(Alloca->getAllocatedType());
84+
if (auto *GlobalVar = dyn_cast<GlobalVariable>(Val))
85+
return dyn_cast<ArrayType>(GlobalVar->getValueType());
86+
return nullptr;
87+
};
88+
89+
ArrayType *DstArrTy = GetArrTyFromVal(Dst);
90+
assert(DstArrTy && "Expected Dst of memcpy to be a Pointer to an Array Type");
91+
if (auto *DstGlobalVar = dyn_cast<GlobalVariable>(Dst))
92+
assert(!DstGlobalVar->isConstant() &&
93+
"The Dst of memcpy must not be a constant Global Variable");
94+
[[maybe_unused]] ArrayType *SrcArrTy = GetArrTyFromVal(Src);
95+
assert(SrcArrTy && "Expected Src of memcpy to be a Pointer to an Array Type");
96+
97+
Type *DstElemTy = DstArrTy->getElementType();
98+
uint64_t DstElemByteSize = DL.getTypeStoreSize(DstElemTy);
99+
assert(DstElemByteSize > 0 && "Dst element type store size must be set");
100+
Type *SrcElemTy = SrcArrTy->getElementType();
101+
[[maybe_unused]] uint64_t SrcElemByteSize = DL.getTypeStoreSize(SrcElemTy);
102+
assert(SrcElemByteSize > 0 && "Src element type store size must be set");
103+
104+
// This assumption simplifies implementation and covers currently-known
105+
// use-cases for DXIL. It may be relaxed in the future if required.
106+
assert(DstElemTy == SrcElemTy &&
107+
"The element types of Src and Dst arrays must match");
108+
109+
[[maybe_unused]] uint64_t DstArrNumElems = DstArrTy->getArrayNumElements();
110+
assert(DstElemByteSize * DstArrNumElems >= ByteLength &&
111+
"Dst array size must be at least as large as the memcpy length");
112+
[[maybe_unused]] uint64_t SrcArrNumElems = SrcArrTy->getArrayNumElements();
113+
assert(SrcElemByteSize * SrcArrNumElems >= ByteLength &&
114+
"Src array size must be at least as large as the memcpy length");
115+
116+
uint64_t NumElemsToCopy = ByteLength / DstElemByteSize;
117+
assert(ByteLength % DstElemByteSize == 0 &&
118+
"memcpy length must be divisible by array element type");
119+
for (uint64_t I = 0; I < NumElemsToCopy; ++I) {
120+
SmallVector<Value *, 2> Indices = {Builder.getInt32(0),
121+
Builder.getInt32(I)};
122+
Value *SrcPtr = Builder.CreateInBoundsGEP(SrcArrTy, Src, Indices, "gep");
123+
Value *SrcVal = Builder.CreateLoad(SrcElemTy, SrcPtr);
124+
Value *DstPtr = Builder.CreateInBoundsGEP(DstArrTy, Dst, Indices, "gep");
125+
Builder.CreateStore(SrcVal, DstPtr);
126+
}
127+
128+
MemCpy->eraseFromParent();
129+
}
130+
131+
void expandMemMove(MemMoveInst *MemMove) {
132+
report_fatal_error("memmove expansion is not implemented yet.");
133+
}
134+
135+
static bool eliminateMemIntrinsics(Module &M) {
136+
bool HadMemIntrinsicUses = false;
137+
for (auto &F : make_early_inc_range(M.functions())) {
138+
Intrinsic::ID IID = F.getIntrinsicID();
139+
switch (IID) {
140+
case Intrinsic::memcpy:
141+
case Intrinsic::memcpy_inline:
142+
case Intrinsic::memmove:
143+
case Intrinsic::memset:
144+
case Intrinsic::memset_inline:
145+
break;
146+
default:
147+
continue;
148+
}
149+
for (User *U : make_early_inc_range(F.users())) {
150+
HadMemIntrinsicUses = true;
151+
if (auto *MemSet = dyn_cast<MemSetInst>(U))
152+
expandMemSet(MemSet);
153+
else if (auto *MemCpy = dyn_cast<MemCpyInst>(U))
154+
expandMemCpy(MemCpy);
155+
else if (auto *MemMove = dyn_cast<MemMoveInst>(U))
156+
expandMemMove(MemMove);
157+
else
158+
llvm_unreachable("Unhandled memory intrinsic");
159+
}
160+
assert(F.user_empty() && "Mem intrinsic not eliminated?");
161+
F.eraseFromParent();
162+
}
163+
return HadMemIntrinsicUses;
164+
}
165+
166+
PreservedAnalyses DXILMemIntrinsics::run(Module &M, ModuleAnalysisManager &) {
167+
if (eliminateMemIntrinsics(M))
168+
return PreservedAnalyses::none();
169+
return PreservedAnalyses::all();
170+
}
171+
172+
class DXILMemIntrinsicsLegacy : public ModulePass {
173+
public:
174+
bool runOnModule(Module &M) override { return eliminateMemIntrinsics(M); }
175+
DXILMemIntrinsicsLegacy() : ModulePass(ID) {}
176+
177+
static char ID; // Pass identification.
178+
};
179+
char DXILMemIntrinsicsLegacy::ID = 0;
180+
181+
INITIALIZE_PASS_BEGIN(DXILMemIntrinsicsLegacy, DEBUG_TYPE,
182+
"DXIL Memory Intrinsic Elimination", false, false)
183+
INITIALIZE_PASS_END(DXILMemIntrinsicsLegacy, DEBUG_TYPE,
184+
"DXIL Memory Intrinsic Elimination", false, false)
185+
186+
ModulePass *llvm::createDXILMemIntrinsicsLegacyPass() {
187+
return new DXILMemIntrinsicsLegacy();
188+
}

0 commit comments

Comments
 (0)