Adding CUDA and OpenCL support for HLT module
authortbreitne <timo.gunther.breitner@cern.ch>
Wed, 22 Jan 2014 18:31:27 +0000 (19:31 +0100)
committertbreitne <timo.gunther.breitner@cern.ch>
Mon, 3 Feb 2014 11:47:31 +0000 (12:47 +0100)
To enable GPU accelerated TPC tracking set

-DHLT_CUDA=ON

or

-DHLT_OPENCL=ON

This requires CUDA toolkit or AMD APP SDK installations, resp. In the
latter case, set $AMDAPPSDKROOT to the installation path.

HLT/CMakeLists.txt
HLT/TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPU.pkg [new file with mode: 0644]
HLT/TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPUOpenCL.pkg [new file with mode: 0644]

index 8559dc6..abd4eee 100644 (file)
@@ -5,8 +5,19 @@
 
 cmake_minimum_required(VERSION 2.8.4 FATAL_ERROR)
 
+option(HLT_CUDA   "Build HLT GPU tracker using CUDA"   OFF)
+option(HLT_OPENCL "Build HLT GPU tracker using OpenCL" OFF)
+
 include("CMakehltdoc.conf")
 
 file(GLOB PACKAGES CMake*.pkg)
 
+if(HLT_CUDA)
+  list(APPEND PACKAGES TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPU.pkg)
+endif(HLT_CUDA)
+
+if(HLT_OPENCL)
+  list(APPEND PACKAGES TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPUOpenCL.pkg)
+endif()
+
 ALICE_BuildModule()
diff --git a/HLT/TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPU.pkg b/HLT/TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPU.pkg
new file mode 100644 (file)
index 0000000..598c705
--- /dev/null
@@ -0,0 +1,109 @@
+# -*- mode: CMake -*- 
+# $Id$
+#--------------------------------------------------------------------------------#
+# Package File for AliHLTTPCCAGPU                                                #
+# Author : Johny Jose (johny.jose@cern.ch)                                       #
+# Variables Defined :                                                            #
+#                                                                                #
+# SRCS - C++ source files                                                        #
+# HDRS - C++ header files                                                        #
+# DHDR - ROOT Dictionary Linkdef header file                                     #
+# CSRCS - C source files                                                         #
+# CHDRS - C header files                                                         #
+# EINCLUDE - Include directories                                                 #
+# EDEFINE - Compiler definitions                                                 #
+# ELIBS - Extra libraries to link                                                #
+# ELIBSDIR - Extra library directories                                           #
+# PACKFFLAGS - Fortran compiler flags for package                                #
+# PACKCXXFLAGS - C++ compiler flags for package                                  #
+# PACKCFLAGS - C compiler flags for package                                      #
+# PACKSOFLAGS - Shared library linking flags                                     #
+# PACKLDFLAGS - Module linker flags                                              #
+# PACKBLIBS - Libraries to link (Executables only)                               #
+# EXPORT - Header files to be exported                                           #
+# CINTHDRS - Dictionary header files                                             #
+# CINTAUTOLINK - Set automatic dictionary generation                             #
+# ARLIBS - Archive Libraries and objects for linking (Executables only)          #
+# SHLIBS - Shared Libraries and objects for linking (Executables only)           #
+#--------------------------------------------------------------------------------#
+
+find_package ( CUDA )
+if ( NOT CUDA_FOUND )
+  #fail if CUDA wasn't found
+  message ( FATAL_ERROR "NVIDIA CUDA package not found. Please install or set HLT_CUDA=OFF" )
+endif ( NOT CUDA_FOUND )
+
+#set nvcc flags
+set ( CUDA_NVCC_FLAGS --use_fast_math --maxrregcount 64 -O4 -Xptxas -O4 -gencode arch=compute_20,code=sm_20 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_35,code=sm_35 )
+#for convenience
+set ( CAGPUDIR TPCLib/tracking-ca/cagpu )
+#add include path
+include_directories( ${CAGPUDIR} )
+#nvcc fails with std=c++11, remove it temporarily
+string ( FIND ${CMAKE_CXX_FLAGS} "-std=c++11" STDCXX11FOUND )
+if ( STDCXX11FOUND GREATER -1 )
+  string ( REPLACE "-std=c++11" "-std=c++98" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" )
+endif ( )
+#compile CUDA object file
+cuda_compile ( CAGPU_O ${CAGPUDIR}/AliHLTTPCCAGPUTrackerNVCC.cu )
+#set it back
+if ( STDCXX11FOUND GREATER -1 )
+  string ( REPLACE "-std=c++98" "-std=c++11" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" )
+endif ( )
+#end of CUDA specific stuff; from here on build usual library
+
+set ( CLASS_HDRS
+  AliHLTTPCCAGPUTrackerBase.h
+  AliHLTTPCCAGPUTrackerNVCC.h
+)
+
+set ( MODULE_SRCS 
+  AliHLTTPCCAGPUTrackerBase.cxx
+)
+
+set ( MODULE_HDRS
+  ${CLASS_HDRS}
+  AliHLTTPCCAGPUTrackerCommon.h
+)
+
+set ( MODULE_DHDR )
+
+set ( EINCLUDE HLT/${CAGPUDIR} HLT/TPCLib/tracking-ca HLT/BASE )
+
+#CUDA run-time and driver
+set ( ELIBS ${CUDA_LIBRARIES} -lcuda )
+
+include ("CMakehlt.conf")
+
+if (MODULE_SRCS)
+ALICE_Format (SRCS "${CAGPUDIR}/" "" "${MODULE_SRCS}")
+endif(MODULE_SRCS)
+if (CLASS_HDRS)
+ALICE_Format (CINTHDRS "${CAGPUDIR}/" "" "${CLASS_HDRS}")
+endif(CLASS_HDRS)
+if (MODULE_HDRS)
+ALICE_Format (HDRS "${CAGPUDIR}/" "" "${MODULE_HDRS}")
+endif(MODULE_HDRS)
+if (MODULE_DHDR)
+ALICE_Format (DHDR "${CAGPUDIR}/" "" "${MODULE_DHDR}")
+endif(MODULE_DHDR)
+
+#add CUDA object file(s) as library (link) dependency
+list(APPEND SRCS ${CAGPU_O})
+
+# bit of a hack, but otherwise dictionary compilation fails:
+# defines cudaError_t to make it known
+# defines __ROOT__; triggers "ifdef" in source code to circumvent broken cint behaviour with "volatile" keyword
+# not sure why __ROOT__ is used here, could just as well be something else
+set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/G__AliHLTTPCCAGPU.cxx PROPERTIES COMPILE_DEFINITIONS "cudaError_t=int;__ROOT__")
+
+if(NOT DHDR)
+  set(CINTAUTOLINK 1)
+endif(NOT DHDR)
+
+set ( EDEFINE  ${HLTDEFS} )
+set ( PACKCXXFLAGS  "${HLTCXXFLAGS} -Wshadow -Wall -O3")
+set ( PACKCFLAGS  ${HLTCLFAGS})
+set ( PACKDCXXFLAGS  ${HLTDCXXFLAGS})
+set ( PACKSOFLAGS ${HLTSOFLAGS})
+#set ( CINTFLAGS )
diff --git a/HLT/TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPUOpenCL.pkg b/HLT/TPCLib/tracking-ca/cagpu/CMakelibAliHLTTPCCAGPUOpenCL.pkg
new file mode 100644 (file)
index 0000000..c905df6
--- /dev/null
@@ -0,0 +1,128 @@
+# -*- mode: CMake -*- 
+# $Id$
+#--------------------------------------------------------------------------------#
+# Package File for AliHLTTPCCAGPUOpenCL                                          #
+# Author : Johny Jose (johny.jose@cern.ch)                                       #
+# Variables Defined :                                                            #
+#                                                                                #
+# SRCS - C++ source files                                                        #
+# HDRS - C++ header files                                                        #
+# DHDR - ROOT Dictionary Linkdef header file                                     #
+# CSRCS - C source files                                                         #
+# CHDRS - C header files                                                         #
+# EINCLUDE - Include directories                                                 #
+# EDEFINE - Compiler definitions                                                 #
+# ELIBS - Extra libraries to link                                                #
+# ELIBSDIR - Extra library directories                                           #
+# PACKFFLAGS - Fortran compiler flags for package                                #
+# PACKCXXFLAGS - C++ compiler flags for package                                  #
+# PACKCFLAGS - C compiler flags for package                                      #
+# PACKSOFLAGS - Shared library linking flags                                     #
+# PACKLDFLAGS - Module linker flags                                              #
+# PACKBLIBS - Libraries to link (Executables only)                               #
+# EXPORT - Header files to be exported                                           #
+# CINTHDRS - Dictionary header files                                             #
+# CINTAUTOLINK - Set automatic dictionary generation                             #
+# ARLIBS - Archive Libraries and objects for linking (Executables only)          #
+# SHLIBS - Shared Libraries and objects for linking (Executables only)           #
+#--------------------------------------------------------------------------------#
+
+
+# AMD APP SDK required for OpenCL tracker; it's using specific
+# extensions (currently) not provided by other vendors.
+# either set cmake variable via "-D$AMDAPPSDKROOT=/path/to/amdappsdkroot"
+# or via environment variable $AMDAPPSDKROOT
+if ( NOT AMDAPPSDKROOT )
+  set ( AMDAPPSDKROOT $ENV{AMDAPPSDKROOT} )
+endif()
+
+if ( NOT AMDAPPSDKROOT )
+  #fail if not set
+  message ( FATAL_ERROR "AMDAPPSDKROOT not set. Please install AMD APP SDK and set $AMDAPPSDKROOT or disable HLT_OPENCL." )
+endif()
+
+#convenience variables
+set ( CAGPUDIR TPCLib/tracking-ca/cagpu )
+
+# build the OpenCL compile wrapper:
+# -- checks the correct vendor implementation (AMD)
+# -- builds binary code (blob) for the found platform(s)
+add_executable ( opencl_compiler ${CAGPUDIR}/makefiles/makefile_opencl_compiler.cpp )
+target_link_libraries ( opencl_compiler -L${AMDAPPSDKROOT}/lib/x86_64 OpenCL )
+
+#convenience variables
+set (CL_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${CAGPUDIR}/AliHLTTPCCAGPUTrackerOpenCL.cl )
+set (CL_BIN ${CMAKE_CURRENT_BINARY_DIR}/AliHLTTPCCAGPUTrackerOpenCLCode.bin )
+set (CL_INC 
+  -I${CMAKE_CURRENT_SOURCE_DIR}/BASE 
+  -I${CMAKE_CURRENT_SOURCE_DIR}/TPCLib/tracking-ca 
+  -I${CMAKE_CURRENT_SOURCE_DIR}/${CAGPUDIR}
+)
+
+# executes OpenCL compiler wrapper to build binary object
+add_custom_command(
+  OUTPUT ${CL_BIN}
+  COMMAND opencl_compiler -output-file ${CL_BIN} ${CL_SRC} -- ${CL_INC} -x clc++
+  MAIN_DEPENDENCY ${CL_SRC}
+)
+
+# make cmake compile the assembler file
+set_source_files_properties(${CAGPUDIR}/makefiles/include.S PROPERTIES LANGUAGE CXX)
+# add proper dependency on included binary code
+set_source_files_properties(${CAGPUDIR}/makefiles/include.S PROPERTIES OBJECT_DEPENDS ${CL_BIN})
+
+
+#usual aliroot library stuff...
+set ( CLASS_HDRS
+  AliHLTTPCCAGPUTrackerBase.h
+  AliHLTTPCCAGPUTrackerOpenCL.h
+)
+
+set ( MODULE_SRCS 
+  AliHLTTPCCAGPUTrackerBase.cxx
+  AliHLTTPCCAGPUTrackerOpenCL.cxx
+  makefiles/include.S
+)
+
+set ( MODULE_HDRS
+  ${CLASS_HDRS}
+  AliHLTTPCCAGPUTrackerCommon.h
+  AliHLTTPCCAGPUTrackerOpenCLInternals.h
+)
+
+set ( MODULE_DHDR )
+
+set ( EINCLUDE ${AMDAPPSDKROOT}/include HLT/${CAGPUDIR} HLT/${CAGPUDIR}/makefiles HLT/TPCLib/tracking-ca HLT/BASE )
+
+set ( ELIBS -L$(AMDAPPSDKROOT)/lib/x86_64 -lOpenCL )
+
+include ("CMakehlt.conf")
+
+if (MODULE_SRCS)
+ALICE_Format (SRCS "${CAGPUDIR}/" "" "${MODULE_SRCS}")
+endif(MODULE_SRCS)
+if (CLASS_HDRS)
+ALICE_Format (CINTHDRS "${CAGPUDIR}/" "" "${CLASS_HDRS}")
+endif(CLASS_HDRS)
+if (MODULE_HDRS)
+ALICE_Format (HDRS "${CAGPUDIR}/" "" "${MODULE_HDRS}")
+endif(MODULE_HDRS)
+if (MODULE_DHDR)
+ALICE_Format (DHDR "${CAGPUDIR}/" "" "${MODULE_DHDR}")
+endif(MODULE_DHDR)
+
+# bit of a hack, but otherwise dictionary compilation fails:
+# defines __ROOT__; triggers "ifdef" in source code to circumvent broken cint behaviour with "volatile" keyword
+# not sure why __ROOT__ is used here, could just as well be something else
+set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/G__AliHLTTPCCAGPUOpenCL.cxx PROPERTIES COMPILE_DEFINITIONS "__ROOT__")
+
+if(NOT DHDR)
+  set(CINTAUTOLINK 1)
+endif(NOT DHDR)
+
+set ( EDEFINE  ${HLTDEFS} )
+set ( PACKCXXFLAGS  "${HLTCXXFLAGS} -Wshadow -Wall -O3")
+set ( PACKCFLAGS  ${HLTCLFAGS})
+set ( PACKDCXXFLAGS  ${HLTDCXXFLAGS})
+set ( PACKSOFLAGS ${HLTSOFLAGS})
+#set ( CINTFLAGS )