nvcc中间链路故障

Question

当我使用CMake构建我的nvcc项目时，我一直在努力修复一个中间链接错误。我一直在升级以前的一个利用CUDA的项目，并且能够成功地从宿主代码中调用该库中的函数。当我试图从设备代码调用该库的函数时，我会得到中间链接错误。我用__device__和__host__描述符注释了所有函数。

顺便提一下，这是一个ROS项目，所以我使用了一些catkin CMake函数。

这是调用主机和设备函数的ParticleFilter代码片段：

#include <cuda.h>
#include <cuda_runtime.h>
#include <device_launch_parameters.h>
#include <curand_kernel.h>
#include <iostream>
#include <davinci_kinematics_cuda/davinci_fwd_kinematics.cuh>

__host__
ParticleFilter::ParticleFilter(const unsigned int numParticles, const std::vector<double> &initialJointState, const unsigned int threads,
        const unsigned int blocks) {

    /* random other work here */
    
    // This works fine (compiles and runs), it is calling host code from the other file
    kinematics = davinci_kinematics_cuda::Forward();
    std::cout << kinematics.fwd_kin_solve(initialJointState.data()).translation() << std::endl;
}

__global__
void printParticlesKernel(double *particles, const unsigned int numParticles, const unsigned int dimensions, const size_t pitch) {
    int locationStart = blockIdx.x * blockDim.x + threadIdx.x;
    int stride = blockDim.x * gridDim.x;

    // This fails, will not link
    davinci_kinematics_cuda::Forward kinematics = davinci_kinematics_cuda::Forward(); 

    for (int n = locationStart; n < numParticles; n += stride) {
        double *particle = (double*) ((char*) particles + n * pitch);
        
        /* random other work here */

        // this fails, will not link
        auto translation = kinematics.fwd_kin_solve(particle).translation();
        printf("%f %f %f\n", translation[0], translation[1], translation[2]);
    }
}

这是运动学文件中的：

#include <cuda.h>
#include <cuda_runtime.h>
#include <device_launch_parameters.h>

namespace davinci_kinematics_cuda {

    // use member fncs to compute and multiply successive transforms
    __host__ __device__
    Forward::Forward() {
        /* random initialization here */
    }

    __host__ __device__
    Eigen::Affine3d Forward::fwd_kin_solve(const double *q_vec, const unsigned int desired_joint) {
        /* other work here */
    }
}

这是与CMake文件相关的ParticleFilter部件。

cmake_minimum_required(VERSION 2.8.10)
project(tool_tracking LANGUAGES CUDA CXX)

# https://stackoverflow.com/questions/25748039/add-cuda-to-ros-package
find_package(CUDA REQUIRED) 

# set CUDA_NVCC_FLAGS as you would do with CXX/C FLAGS
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CURAND_FLAGS} -fPIC")
set(CUDA_SEPARABLE_COMPILATION ON)

find_package(catkin REQUIRED COMPONENTS
    message_generation
    roscpp
    std_msgs
    sensor_msgs
    geometry_msgs
    cwru_opencv_common
    tool_model
    cwru_davinci_control
    cwru_davinci_kinematics
    xform_utils
    tf
    tool_segmentation
)


catkin_package(
    INCLUDE_DIRS
        include
    LIBRARIES 
        tool_tracking_particle
    CATKIN_DEPENDS
        message_runtime
        std_msgs
        sensor_msgs
        geometry_msgs
        cwru_opencv_common
        tool_model
        cwru_davinci_control
        cwru_davinci_kinematics
        xform_utils
        tf
)

include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
include_directories(include ${catkin_INCLUDE_DIRS} tool_model_lib )

cuda_add_executable(test_particlefilter src/ParticleFilter.cu src/Particle.cu)
target_link_libraries(test_particlefilter tool_tracking_particle ${catkin_LIBRARIES} ${OpenCV_LIBRARIES} ${CUDA_LIBRARIES})

这是来自CMake的错误：

/usr/bin/cmake -H/home/ethan/catkin_ws/src/cwru_davinci_tool_tracking/tool_tracking -B/home/ethan/catkin_ws/build/tool_tracking --check-build-system CMakeFiles/Makefile.cmake 0
/usr/bin/cmake -E cmake_progress_start /home/ethan/catkin_ws/build/tool_tracking/CMakeFiles /home/ethan/catkin_ws/build/tool_tracking/CMakeFiles/progress.marks
/usr/bin/make -f CMakeFiles/Makefile2 all
make[1]: Entering directory '/home/ethan/catkin_ws/build/tool_tracking'
/usr/bin/make -f CMakeFiles/test_particlefilter.dir/build.make CMakeFiles/test_particlefilter.dir/depend
make[2]: Entering directory '/home/ethan/catkin_ws/build/tool_tracking'
[ 20%] Building NVCC intermediate link file CMakeFiles/test_particlefilter.dir/test_particlefilter_intermediate_link.o
/usr/local/cuda-11.0/bin/nvcc -lcudadevrt -m64 -ccbin /usr/bin/cc -dlink /home/ethan/catkin_ws/build/tool_tracking/CMakeFiles/test_particlefilter.dir/src/./test_particlefilter_generated_ParticleFilter.cu.o /home/ethan/catkin_ws/build/tool_tracking/CMakeFiles/test_particlefilter.dir/src/./test_particlefilter_generated_Particle.cu.o -o /home/ethan/catkin_ws/build/tool_tracking/CMakeFiles/test_particlefilter.dir/./test_particlefilter_intermediate_link.o -Xcompiler -fPIC
nvlink error   : Undefined reference to '_ZN23davinci_kinematics_cuda7ForwardC1Ev' in '/home/ethan/catkin_ws/build/tool_tracking/CMakeFiles/test_particlefilter.dir/src/./test_particlefilter_generated_ParticleFilter.cu.o'
nvlink error   : Undefined reference to '_ZN23davinci_kinematics_cuda7Forward13fwd_kin_solveEPKdj' in '/home/ethan/catkin_ws/build/tool_tracking/CMakeFiles/test_particlefilter.dir/src/./test_particlefilter_generated_ParticleFilter.cu.o'
CMakeFiles/test_particlefilter.dir/build.make:1468: recipe for target 'CMakeFiles/test_particlefilter.dir/test_particlefilter_intermediate_link.o' failed
make[2]: Leaving directory '/home/ethan/catkin_ws/build/tool_tracking'
make[2]: *** [CMakeFiles/test_particlefilter.dir/test_particlefilter_intermediate_link.o] Error 255
CMakeFiles/Makefile2:67: recipe for target 'CMakeFiles/test_particlefilter.dir/all' failed
make[1]: Leaving directory '/home/ethan/catkin_ws/build/tool_tracking'
make[1]: *** [CMakeFiles/test_particlefilter.dir/all] Error 2
Makefile:140: recipe for target 'all' failed
make: *** [all] Error 2

如何用未定义的引用修复错误？似乎是链接错误，但我还不太熟悉编译/链接过程，无法进一步排除故障。如果我需要张贴运动学文件的CMake，我也可以。

Answer 1

这是关键问题，也是对其他读者最有帮助的部分。默认情况下，Catkin将CMake配置为构建共享库，但CUDA可分离编译和nvlink只适用于静态库。您需要将您的CUDA库(在您的例子中，那些在cwru_davinci_kinematics中)设置为静态的，始终如此。可以通过将STATIC关键字添加到add_library调用中来做到这一点，如下所示：

add_library(my_cuda_lib STATIC source1.cu ...)

如果您在CMake中“链接”到一个与CUDA共享的库，它就会忽略它。这实际上是nvcc的记录行为。见此处：https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#libraries

设备链接器能够读取静态主机库格式(.a在Linux上，Mac，.lib在Windows上)。它忽略任何动态(.so或.dll)库。

这里的另一个主要教训是，将CMake设置为一个古老的版本必然会导致问题。在复制您的问题时，我被迫从源代码(它不在Ubuntu20.04LTS中)构建OpenCV 3，而且在引入CMP0074的3.12版本之前，没有一种很好的方法来覆盖特定包的搜索路径。

升级您的最低CMake版本。理想情况下，您可以升级到您在软件存储库中可以使用的最新版本，并将文件的最小值设置为该版本。与比~3.5更早的CMake版本兼容没有任何好处，我认为这将扩展到3.16 (Ubuntu20.04LTS中的版本)。既然你使用的是CUDA，3.18是最合适的。更糟糕的是，您的许多项目将最低设置在2.8.12以下；与此版本的兼容性将很快被CMake删除。

下面是我为使它构建在Ubuntu20.04LTS上所做的确切更改。我使用了以下构建脚本，放在ROS工作区中并从该工作区执行：

#!/usr/bin/bash

source /opt/ros/noetic/setup.bash

export CUDACXX=/usr/local/cuda/bin/nvcc
export OpenCV_ROOT=$(readlink -f opencv-install)

[ -f "$CUDACXX" ] || { echo "Invalid CUDACXX: $CUDACXX"; exit; }
[ -d "$OpenCV_ROOT" ] || { echo "Invalid OpenCV_ROOT: $OpenCV_ROOT"; exit; }

rm -rf build devel
catkin build tool_tracking --cmake-args \
    -Wno-dev \
    -DCMAKE_POLICY_DEFAULT_CMP0074=NEW \
    -DCMAKE_CUDA_ARCHITECTURES=75

opencv-install目录是通过构建自己的OpenCV 3创建的(因为Ubuntu20.04只有v4)。这方面的步骤是：

$ git clone -b 3.4.14 git@github.com:opencv/opencv.git
$ git clone -b 3.4.14 git@github.com:opencv/opencv_contrib.git
$ cmake -G Ninja -S opencv -B opencv-build/ -DOPENCV_EXTRA_MODULES_PATH=$(readlink -f opencv_contrib)/modules -DBUILD_opencv_cnn_3dobj=OFF -DBUILD_opencv_face=OFF -DBUILD_opencv_hdf=OFF -DBUILD_opencv_hfs=OFF -DBUILD_opencv_julia=OFF -DBUILD_opencv_matlab=OFF -DBUILD_opencv_ovis=OFF -DBUILD_opencv_reg=OFF -DBUILD_opencv_sfm=OFF -DBUILD_opencv_text=OFF -DBUILD_opencv_wechat_qrcode=OFF -DBUILD_opencv_ximgproc=OFF
$ cmake --build opencv-build
$ cmake --install opencv-build --prefix opencv-install

这将禁用具有重要/无关依赖关系的额外模块。

该脚本将环境变量OpenCV_ROOT设置为将CMake定向到此本地安装的OpenCV版本。因为文件中指定的CMake的最低版本是如此之低，所以我还必须设置CMAKE_POLICY_DEFAULT_CMP0074=NEW，这样才能遵守OpenCV_ROOT。

下面是我对您的CMake代码所做的更改：

src/cwru_davinci运动学/CMakeLists.txt

--- a/src/cwru_davinci_kinematics/CMakeLists.txt
+++ b/src/cwru_davinci_kinematics/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.10)
+cmake_minimum_required(VERSION 3.18)
 project(cwru_davinci_kinematics)
 
 #This is needed as part of the migration to ros jade and later
@@ -26,18 +26,16 @@ find_package(catkin REQUIRED COMPONENTS roscpp roslib roslint tf tf2 tf2_eigen)
 
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=gnu++0x")
 
-# https://stackoverflow.com/questions/25748039/add-cuda-to-ros-package
-find_package(CUDA) 
-message(STATUS "CUDA_FOUND=${CUDA_FOUND}")
-if(CUDA_FOUND)
-       message(STATUS "Found CUDA, setting nvcc compilation flags")
-       
-       # set CUDA_NVCC_FLAGS as you would do with CXX/C FLAGS         
-       set(CUDA_NVCC_FLAGS CACHE STRING "nvcc flags" FORCE)
-       set(CUDA_VERBOSE_BUILD ON CACHE BOOL "nvcc verbose" FORCE)
+include(CheckLanguage)
+check_language(CUDA)
+if (CMAKE_CUDA_COMPILER)
+  enable_language(CUDA)
+
        # fPIC fixes some linker issues with nvcc code / objects
-       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CURAND_FLAGS} -fPIC")
-       set(CUDA_SEPARABLE_COMPILATION ON)
+       set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -fPIC")
+       set(CMAKE_CUDA_SEPARABLE_COMPILATION ON)
+
+  find_package(CUDAToolkit REQUIRED)
 endif()
 
 include_directories(
@@ -48,7 +46,7 @@ include_directories(
   ${YAML_CPP_INCLUDE_DIRS}
 )
 
-if (CUDA_FOUND)
+if (CMAKE_CUDA_COMPILER)
        catkin_package(
                DEPENDS ${Eigen3_DEP}
                LIBRARIES 
@@ -82,14 +80,17 @@ target_link_libraries(davinci_kinematics
   davinci_kinematic_definitions
   )
 
-if (CUDA_FOUND)
-       cuda_add_library(davinci_kinematics_cuda src/davinci_fwd_kinematics.cu)
-       cuda_add_library(davinci_kinematics_definitions_cuda src/davinci_kinematic_definitions.cu)
-       
-       target_link_libraries(davinci_kinematics_cuda
-               ${catkin_LIBRARIES}
-               davinci_kinematics_definitions_cuda
-       )
+if (CMAKE_CUDA_COMPILER)
+  add_library(davinci_kinematics_cuda STATIC src/davinci_fwd_kinematics.cu)
+  add_library(davinci_kinematics_definitions_cuda STATIC src/davinci_kinematic_definitions.cu)
+
+  target_link_libraries(
+    davinci_kinematics_cuda
+    PRIVATE
+      CUDA::curand
+      ${catkin_LIBRARIES}
+      davinci_kinematics_definitions_cuda
+  )
 endif()
 
 # Examples

这里的重点是：

  add_library(davinci_kinematics_cuda STATIC src/davinci_fwd_kinematics.cu)
  add_library(davinci_kinematics_definitions_cuda STATIC src/davinci_kinematic_definitions.cu)

我还更新了这里的CMake代码，因为内置的CUDA语言支持已经相当先进了.

src/cwru_davinci_tool_tracking/tool_tracking/CMakeLists.txt

--- a/src/cwru_davinci_tool_tracking/tool_tracking/CMakeLists.txt
+++ b/src/cwru_davinci_tool_tracking/tool_tracking/CMakeLists.txt
@@ -1,18 +1,11 @@
-cmake_minimum_required(VERSION 2.8.10)
-project(tool_tracking LANGUAGES CUDA CXX)
+cmake_minimum_required(VERSION 3.18)
+project(tool_tracking LANGUAGES C CXX CUDA)
 
-# https://stackoverflow.com/questions/25748039/add-cuda-to-ros-package
-find_package(CUDA REQUIRED) 
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -fPIC")
+set(CMAKE_CUDA_SEPARABLE_COMPILATION ON)
 
-# set CUDA_NVCC_FLAGS as you would do with CXX/C FLAGS
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CURAND_FLAGS} -fPIC")
-set(CUDA_SEPARABLE_COMPILATION ON)
+find_package(OpenCV 3 REQUIRED)
 
-#find_package(catkin_simple REQUIRED)
-## Find catkin macros and libraries
-## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz)
-## is used, also find other catkin packages
-find_package(OpenCV REQUIRED)
 find_package(catkin REQUIRED COMPONENTS
        message_generation
        roscpp
@@ -28,11 +21,12 @@ find_package(catkin REQUIRED COMPONENTS
        tool_segmentation
 )
 
+find_package(CUDAToolkit REQUIRED)
 
 catkin_package(
        INCLUDE_DIRS
                include
-       LIBRARIES 
+       LIBRARIES
                tool_tracking_particle
        CATKIN_DEPENDS
                message_runtime
@@ -47,13 +41,7 @@ catkin_package(
                tf
 )
 
-include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
-include_directories(include ${catkin_INCLUDE_DIRS} tool_model_lib )
-
-#cuda_add_library(tool_tracking_particle src/ParticleFilter.cu src/Particle.cu)
-#add_executable(particle src/tracking_particle.cpp)
-#target_link_libraries(particle tool_tracking_particle ${catkin_LIBRARIES} ${OpenCV_LIBRARIES} davinci_kinematics_cuda 
-#                      davinci_kinematics_definitions_cuda)
-
-cuda_add_executable(test_particlefilter src/ParticleFilter.cu src/Particle.cu)
-target_link_libraries(test_particlefilter tool_tracking_particle ${catkin_LIBRARIES} ${OpenCV_LIBRARIES} ${CUDA_LIBRARIES})
+add_executable(test_particlefilter src/ParticleFilter.cu src/Particle.cu)
+target_include_directories(test_particlefilter SYSTEM PRIVATE ${OpenCV_INCLUDE_DIRS} ${catkin_INCLUDE_DIRS})
+target_include_directories(test_particlefilter PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_link_libraries(test_particlefilter PRIVATE ${catkin_LIBRARIES} ${OpenCV_LIBRARIES} CUDA::curand)

我还更新了这里的CMake代码，因为内置的CUDA语言支持已经相当先进了.

杂项变动

为了抑制警告，我在所有其他地方将最小的CMake版本从2.8.x跳到了3.0.2。我还将一个版本号3添加到所有没有它的find_package(OpenCV ...)调用中。

Boost不再有python3包；它现在只是python。我对src/vision_opencv/cv_bridge/CMakeLists.txt做了以下更改

--- a/src/vision_opencv/cv_bridge/CMakeLists.txt
+++ b/src/vision_opencv/cv_bridge/CMakeLists.txt
@@ -1,18 +1,15 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 3.0.2)
 project(cv_bridge)
 
 find_package(catkin REQUIRED COMPONENTS rosconsole sensor_msgs)
 
 if(NOT ANDROID)
   find_package(PythonLibs)
-  if(PYTHONLIBS_VERSION_STRING VERSION_LESS 3)
-    find_package(Boost REQUIRED python)
-  else()
-    find_package(Boost REQUIRED python3)
-  endif()
+  find_package(Boost REQUIRED python)
 else()
-find_package(Boost REQUIRED)
+  find_package(Boost REQUIRED)
 endif()
+
 find_package(OpenCV 3 REQUIRED
   COMPONENTS
     opencv_core