[WIP][Driver][SYCL] Link device libraries at compile time for 3rd party

clang/lib/Driver/Driver.cpp

@@ -5834,8 +5834,7 @@ class OffloadingActionBuilder final {
       // specific libs and doing a separate directory search.  Each toolchain
       // has their own getDeviceLibs that we can potentially use.
       DeviceLibraries =
-          SYCLTC.getDeviceLibNames(C.getDriver(), Args, TC->getTriple());
-
+          clang::driver::getSYCLDeviceLibNames(C.getDriver(), Args, *TC);
       for (const auto &DeviceLib : DeviceLibraries) {
         for (const auto &LLCandidate : LibLocCandidates) {
           SmallString<128> LibName(LLCandidate);

clang/lib/Driver/ToolChains/Clang.cpp

@@ -11593,90 +11593,10 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Add any SYCL offloading specific options to the clang-linker-wrapper
   if (C.hasOffloadToolChain<Action::OFK_SYCL>()) {
-
-    if (Args.hasArg(options::OPT_fsycl_link_EQ))
-      CmdArgs.push_back(Args.MakeArgString("--sycl-device-link"));
-
-    // -sycl-device-library-location=<dir> provides the location in which the
-    // SYCL device libraries can be found.
-    SmallString<128> DeviceLibDir(D.Dir);
-    llvm::sys::path::append(DeviceLibDir, "..", "lib");
-    // Check the library location candidates for the the libsycl-crt library
-    // and use that location.  Base the location on relative to driver if this
-    // is not resolved.
-    SmallVector<SmallString<128>, 4> LibLocCandidates;
     SYCLInstallationDetector SYCLInstallation(D, getToolChain().getTriple(),
                                               Args);
-    SYCLInstallation.getSYCLDeviceLibPath(LibLocCandidates);
-    SmallString<128> LibName("libsycl-crt");
-    bool IsNewOffload = D.getUseNewOffloadingDriver();
-    StringRef LibSuffix = TheTriple.isWindowsMSVCEnvironment()
-                              ? (IsNewOffload ? ".new.obj" : ".obj")
-                              : (IsNewOffload ? ".new.o" : ".o");
-    llvm::sys::path::replace_extension(LibName, LibSuffix);
-    for (const auto &LibLoc : LibLocCandidates) {
-      SmallString<128> FullLibName(LibLoc);
-      llvm::sys::path::append(FullLibName, LibName);
-      if (llvm::sys::fs::exists(FullLibName)) {
-        DeviceLibDir = LibLoc;
-        break;
-      }
-    }
-
-    // -sycl-device-libraries=<comma separated list> contains a list of
-    // file names for fat object files that contain SYCL device library bitcode
-    // necessary for SYCL offloading that will be linked to the user's device
-    // code. clang-linker-wrapper uses the value provided to
-    // -sycl-device-library-location=<dir> to construct the full paths of the
-    // device libraries.
-
-    // On the other hand, --bitcode-library=<triple>=<path to bc file> specifies
-    // one bitcode library to link in for a specific triple. Additionally, the
-    // path is *not* relative to the -sycl-device-library-location - the full
-    // path must be provided.
-    SmallVector<std::string, 4> BCLibList;
-
-    auto appendToList = [](SmallString<256> &List, const Twine &Arg) {
-      if (List.size() > 0)
-        List += ",";
-      List += Arg.str();
-    };
-
-    auto ToolChainRange = C.getOffloadToolChains<Action::OFK_SYCL>();
-    for (const auto &[Kind, TC] :
-         llvm::make_range(ToolChainRange.first, ToolChainRange.second)) {
-      llvm::Triple TargetTriple = TC->getTriple();
-      const toolchains::SYCLToolChain &SYCLTC =
-          static_cast<const toolchains::SYCLToolChain &>(*TC);
-      SmallVector<ToolChain::BitCodeLibraryInfo, 8> SYCLDeviceLibs;
-      // SPIR or SPIR-V device libraries are compiled into the device compile
-      // step.
-      if (!TargetTriple.isSPIROrSPIRV())
-        SYCLDeviceLibs.append(SYCLTC.getDeviceLibNames(D, Args, TargetTriple));
-      for (const auto &AddLib : SYCLDeviceLibs) {
-        if (llvm::sys::path::extension(AddLib.Path) == ".bc") {
-          SmallString<256> LibPath(DeviceLibDir);
-          llvm::sys::path::append(LibPath, AddLib.Path);
-          BCLibList.push_back(
-              (Twine(TC->getTriple().str()) + "=" + LibPath).str());
-          continue;
-        }
-      }
-
-      if (TC->getTriple().isNVPTX())
-        if (const char *LibSpirvFile = SYCLInstallation.findLibspirvPath(
-                TC->getTriple(), Args, *TC->getAuxTriple()))
-          BCLibList.push_back(
-              (Twine(TC->getTriple().str()) + "=" + LibSpirvFile).str());
-    }
-
-    if (BCLibList.size())
-      for (const std::string &Lib : BCLibList)
-        CmdArgs.push_back(
-            Args.MakeArgString(Twine("--bitcode-library=") + Lib));
-
-    CmdArgs.push_back(Args.MakeArgString(
-        Twine("-sycl-device-library-location=") + DeviceLibDir));
+    if (Args.hasArg(options::OPT_fsycl_link_EQ))
+      CmdArgs.push_back(Args.MakeArgString("--sycl-device-link"));
 
     if (C.getDriver().isSaveOffloadCodeEnabled()) {
       SmallString<128> DumpDir;
@@ -11759,6 +11679,7 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
     // -Xsycl-target-linker are forwarded using --device-linker.
     const toolchains::SYCLToolChain &SYCLTC =
         static_cast<const toolchains::SYCLToolChain &>(getToolChain());
+    auto ToolChainRange = C.getOffloadToolChains<Action::OFK_SYCL>();
     for (auto &ToolChainMember :
          llvm::make_range(ToolChainRange.first, ToolChainRange.second)) {
       const ToolChain *TC = ToolChainMember.second;

clang/lib/Driver/ToolChains/Cuda.cpp

@@ -1003,6 +1003,39 @@ void CudaToolChain::addClangTargetOptions(
       DriverArgs.hasArg(options::OPT_S))
     return;
 
+  // For SYCL offloading, add SYCL device libraries.
+  if (DeviceOffloadingKind == Action::OFK_SYCL) {
+    llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12> BCLibs;
+
+    llvm::SmallVector<llvm::SmallString<128>, 4> LibraryPaths;
+    SYCLInstallation.getSYCLDeviceLibPath(LibraryPaths);
+
+    // Get SYCL device library names for NVPTX target
+    llvm::SmallVector<BitCodeLibraryInfo, 8> DeviceLibs =
+        clang::driver::getSYCLDeviceLibNames(getDriver(), DriverArgs, *this);
+
+    // Create full path names to each device library
+    for (const auto &DeviceLib : DeviceLibs) {
+      bool DeviceLibFound = false;
+      for (const auto &LibraryPath : LibraryPaths) {
+        llvm::SmallString<128> FullLibName(LibraryPath);
+        llvm::sys::path::append(FullLibName, DeviceLib.Path);
+        if (llvm::sys::fs::exists(FullLibName)) {
+          BitCodeLibraryInfo BitCodeLibrary(
+              {FullLibName, DeviceLib.ShouldInternalize});
+          BCLibs.emplace_back(BitCodeLibrary);
+          DeviceLibFound = true;
+          break;
+        }
+      }
+      if (!DeviceLibFound)
+        getDriver().Diag(diag::err_drv_no_sycl_device_lib) << DeviceLib.Path;
+    }
+    for (const auto &BCLib : BCLibs) {
+      CC1Args.push_back("-mlink-builtin-bitcode");
+      CC1Args.push_back(DriverArgs.MakeArgString(BCLib.Path));
+    }
+  }
   std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch);
   if (LibDeviceFile.empty()) {
     getDriver().Diag(diag::err_drv_no_cuda_libdevice) << GpuArch;

clang/lib/Driver/ToolChains/HIPAMD.cpp

@@ -416,6 +416,34 @@ HIPAMDToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs,
 
   addDirectoryList(DriverArgs, LibraryPaths, "", "HIP_DEVICE_LIB_PATH");
 
+  // For SYCL offloading, add SYCL device libraries.
+  if (DeviceOffloadingKind == Action::OFK_SYCL) {
+    llvm::SmallVector<llvm::SmallString<128>, 4> LibraryPaths;
+    SYCLInstallation.getSYCLDeviceLibPath(LibraryPaths);
+
+    // Get SYCL device library names for NVPTX target
+    llvm::SmallVector<BitCodeLibraryInfo, 8> DeviceLibs =
+        clang::driver::getSYCLDeviceLibNames(getDriver(), DriverArgs, *this);
+
+    // Create full path names to each device library
+    for (const auto &DeviceLib : DeviceLibs) {
+      bool DeviceLibFound = false;
+      for (const auto &LibraryPath : LibraryPaths) {
+        llvm::SmallString<128> FullLibName(LibraryPath);
+        llvm::sys::path::append(FullLibName, DeviceLib.Path);
+        if (llvm::sys::fs::exists(FullLibName)) {
+          BitCodeLibraryInfo BitCodeLibrary(
+              {FullLibName, DeviceLib.ShouldInternalize});
+          BCLibs.emplace_back(BitCodeLibrary);
+          DeviceLibFound = true;
+          break;
+        }
+      }
+      if (!DeviceLibFound)
+        getDriver().Diag(diag::err_drv_no_sycl_device_lib) << DeviceLib.Path;
+    }
+  }
+
   // Maintain compatability with --hip-device-lib.
   auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
   if (!BCLibArgs.empty()) {

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -548,17 +548,18 @@ static void addSYCLDeviceSanitizerLibs(
 
 // Returns the list of SYCL device library names for the given target.
 SmallVector<ToolChain::BitCodeLibraryInfo, 8>
-SYCLToolChain::getDeviceLibNames(const Driver &D,
-                                 const llvm::opt::ArgList &Args,
-                                 const llvm::Triple &TargetTriple) const {
+clang::driver::getSYCLDeviceLibNames(const Driver &D,
+                                     const llvm::opt::ArgList &Args,
+                                     const ToolChain &TC) {
   SmallVector<ToolChain::BitCodeLibraryInfo, 8> LibraryList;
+  const llvm::Triple TargetTriple(TC.getTriple());
   bool NoOffloadLib =
       !Args.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib, true);
   // Default internalization to 'true' for these libraries, as they are
   // expected to link with -mlink-builtin-bitcode.
   auto addLibToList = [&LibraryList](StringRef LibName,
                                      bool Internalize = true) {
-    BitCodeLibraryInfo BitCodeLibrary({LibName, Internalize});
+    ToolChain::BitCodeLibraryInfo BitCodeLibrary({LibName, Internalize});
     LibraryList.emplace_back(BitCodeLibrary);
   };
   if (TargetTriple.isNVPTX()) {
@@ -576,7 +577,7 @@ SYCLToolChain::getDeviceLibNames(const Driver &D,
   // Ignore no-offloadlib for NativeCPU device library, it provides some
   // critical builtins which must be linked with user's device image.
   if (TargetTriple.isNativeCPU()) {
-    addLibToList("libsycl-nativecpu_utils.bc");
+    addLibToList("libsycl-nativecpu_utils.bc", false);
     return LibraryList;
   }
 
@@ -623,7 +624,7 @@ SYCLToolChain::getDeviceLibNames(const Driver &D,
       "libsycl-native-bfloat16"};
   bool NativeBfloatLibs;
   bool NeedBfloatLibs =
-      selectBfloatLibs(Args, TargetTriple, *this, NativeBfloatLibs);
+      selectBfloatLibs(Args, TargetTriple, TC, NativeBfloatLibs);
   if (NeedBfloatLibs && !NoOffloadLib) {
     // Add native or fallback bfloat16 library.
     if (NativeBfloatLibs)
@@ -1452,11 +1453,13 @@ void SYCLToolChain::addClangTargetOptions(
     SYCLInstallation.addLibspirvLinkArgs(getEffectiveTriple(), DriverArgs,
                                          HostTC.getTriple(), CC1Args);
   }
-  // Only link device libraries at compile time for SPIR/SPIRV targets.
-  // Other targets (NVPTX, AMD) link at link time via clang-linker-wrapper.
-  // This is only done with the new offloading model, to fit with LLVM community
-  // implementation.
-  if (!getDriver().getUseNewOffloadingDriver() || !getTriple().isSPIROrSPIRV())
+  // Only link device libraries at compile time for the new offloading model.
+  if (!getDriver().getUseNewOffloadingDriver())
+    return;
+
+  // --no-offloadlib does not pull in device libraries.
+  if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
+                          true))
     return;
 
   llvm::SmallVector<BitCodeLibraryInfo, 12> BCLibs;
@@ -1923,7 +1926,7 @@ SYCLToolChain::getDeviceLibs(
 
   // Formulate all of the device libraries needed for this compilation.
   SmallVector<BitCodeLibraryInfo, 8> DeviceLibs =
-      getDeviceLibNames(getDriver(), DriverArgs, getTriple());
+      clang::driver::getSYCLDeviceLibNames(getDriver(), DriverArgs, *this);
 
   // Create full path names to each device library.  If found, add to the list
   // of device libraries that will be linked against.

clang/lib/Driver/ToolChains/SYCL.h

@@ -21,6 +21,12 @@ namespace driver {
 // AOT compiler).
 StringRef mapIntelGPUArchName(StringRef ArchName);
 
+// Provides a vector of SYCL device library names for the given target triple.
+// This can be used by non-SYCL toolchains when SYCL offloading is enabled.
+llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 8>
+getSYCLDeviceLibNames(const Driver &D, const llvm::opt::ArgList &Args,
+                      const ToolChain &TC);
+
 class Command;
 
 namespace tools {
@@ -154,11 +160,6 @@ class LLVM_LIBRARY_VISIBILITY SYCLToolChain : public ToolChain {
                           llvm::opt::OptSpecifier Opt,
                           llvm::opt::OptSpecifier Opt_EQ,
                           StringRef Device) const;
-  // Provides a vector of device library names that are associated with the
-  // given triple and AOT information.
-  SmallVector<ToolChain::BitCodeLibraryInfo, 8>
-  getDeviceLibNames(const Driver &D, const llvm::opt::ArgList &Args,
-                    const llvm::Triple &TargetTriple) const;
 
   bool useIntegratedAs() const override { return true; }
   bool isPICDefault() const override {

clang/test/Driver/clang-linker-wrapper.cpp

@@ -7,7 +7,7 @@
 // RUN: %clang %s -fsycl -fsycl-targets=spir64_x86_64 -c --offload-new-driver -o %t_aot_cpu.o
 // RUN: %clang %s -fsycl -fsycl-targets=nvptx64-nvidia-cuda --cuda-gpu-arch=sm_50 -nocudalib -fno-sycl-libspirv -c --offload-new-driver -o %t_nvptx.o
 // RUN: %clang %s -fsycl -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx803 -fgpu-rdc -nogpulib -fno-sycl-libspirv -c --offload-new-driver -o %t_amdgcn.o
-// RUN: %clang %s -fsycl -fsycl-targets=native_cpu -fno-sycl-libspirv -c --offload-new-driver -o %t_native_cpu.o
+// RUN: %clang %s -fsycl -fsycl-targets=native_cpu -fno-sycl-libspirv --no-offloadlib -c --offload-new-driver -o %t_native_cpu.o
 //
 // Generate .o file as SYCL device library file.
 //
@@ -267,13 +267,13 @@
 // -------
 // Generate .o file as linker wrapper input.
 //
-// RUN: %clang %s -fsycl -fsycl-targets=native_cpu -fno-sycl-libspirv -c --offload-new-driver -o %t6.o
+// RUN: %clang %s -fsycl -fsycl-targets=native_cpu -fno-sycl-libspirv --no-offloadlib -c --offload-new-driver -o %t6.o
 //
-// RUN: clang-linker-wrapper "--host-triple=x86_64-unknown-linux-gnu" "-sycl-device-library-location=%S/Inputs/native_cpu" "--sycl-post-link-options=SYCL_POST_LINK_OPTIONS" "--linker-path=/usr/bin/ld" "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" %t6.o --dry-run 2>&1 | FileCheck -check-prefix=CHK-CMDS-NATIVE-CPU %s
+// RUN: clang-linker-wrapper "--host-triple=x86_64-unknown-linux-gnu" "--sycl-post-link-options=SYCL_POST_LINK_OPTIONS" "--linker-path=/usr/bin/ld" "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" %t6.o --dry-run 2>&1 | FileCheck -check-prefix=CHK-CMDS-NATIVE-CPU %s
 // CHK-CMDS-NATIVE-CPU: spirv-to-ir-wrapper{{.*}} --llvm-spirv-opts --spirv-preserve-auxdata --spirv-target-env=SPV-IR --spirv-builtin-format=global
 // CHK-CMDS-NATIVE-CPU-NEXT: llvm-link{{.*}} --suppress-warnings
 // CHK-CMDS-NATIVE-CPU-NEXT: sycl-post-link{{.*}} SYCL_POST_LINK_OPTIONS
-// CHK-CMDS-NATIVE-CPU-NEXT: clang{{.*}} --no-default-config -o [[OUT1:.*\.img]] -dumpdir a.out.native_cpu..img. --target=x86_64-unknown-linux-gnu -Wno-override-module -mllvm -sycl-native-cpu-backend -c {{.*}} -Xclang -mlink-bitcode-file -Xclang {{.*}}libsycl-nativecpu_utils.bc
+// CHK-CMDS-NATIVE-CPU-NEXT: clang{{.*}} --no-default-config -o [[OUT1:.*\.img]] -dumpdir a.out.native_cpu..img. --target=x86_64-unknown-linux-gnu -Wno-override-module -mllvm -sycl-native-cpu-backend -c
 // CHK-CMDS-NATIVE-CPU-NEXT:  offload-wrapper: output: [[OUT2:.*\.bc]], input: [[OUT1]]
 // CHK-CMDS-NATIVE-CPU-NEXT: clang{{.*}} --target=x86_64-unknown-linux-gnu -c -o [[OUT3:.*\.o]] [[OUT2]]
 // CHK-CMDS-NATIVE-CPU-NEXT: "{{.*}}/ld" -- HOST_LINKER_FLAGS -dynamic-linker HOST_DYN_LIB -o a.out [[OUT1]] [[OUT3]] {{.*\.o}}