--- /dev/null
+macro(vc_add_target_property _target _prop _value)
+ get_target_property(_oldprop "${_target}" ${_prop})
+ if(NOT _oldprop)
+ set_target_properties("${_target}" PROPERTIES ${_prop} "${_value}")
+ else(NOT _oldprop)
+ set_target_properties("${_target}" PROPERTIES ${_prop} "${_oldprop} ${_value}")
+ endif(NOT _oldprop)
+endmacro()
+
+include_directories("${Vc_INCLUDE_DIR}" "${Vc_INCLUDE_DIR}/Vc" "${Vc_INCLUDE_DIR}/Vc/avx")
+
+if(USE_AVX)
+ add_library(Vc STATIC src/vector.cpp src/cpuid.cpp src/support.cpp src/avx/sorthelper.cpp)
+else()
+ set(_srcs src/vector.cpp src/cpuid.cpp src/support.cpp)
+
+ if(NOT Vc_AVX_INTRINSICS_BROKEN)
+ # we'd still like to have avx/sorthelper.cpp built in, but that requires compilation with -mavx (or a comparable flag)
+ foreach(_flag "-xAVX" "-mavx" "-arch:AVX")
+ check_cxx_compiler_flag("${_flag}" check_cxx_compiler_flag_${_flag})
+ if(check_cxx_compiler_flag_${_flag})
+ if(_flag STREQUAL "-xAVX")
+ set(_flag "${_flag} -diag-disable 10121") # disable the warning "overriding -xSSE4.2 with -xAVX"
+ endif()
+ set(_srcs ${_srcs} src/avx/sorthelper.cpp)
+ set_source_files_properties(src/avx/sorthelper.cpp PROPERTIES COMPILE_FLAGS "${_flag}")
+ break()
+ endif()
+ endforeach()
+ endif()
+ add_library(Vc STATIC ${_srcs})
+endif()
+string(REPLACE "-Weffc++" "" CXXFLAGS "${CXXFLAGS}")
+vc_add_target_property(Vc COMPILE_FLAGS "-DVC_COMPILE_LIB ${CXXFLAGS} -O3 ${Vc_DEFINITIONS}")
+vc_add_target_property(Vc LABELS "other")
+
+# vim: ft=cmake sw=3 et
--- /dev/null
+set(Vc_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/Vc/include")
+set(Vc_CMAKE_MODULES_DIR "${PROJECT_SOURCE_DIR}/Vc/cmake")
+include("${Vc_CMAKE_MODULES_DIR}/VcMacros.cmake")
+vc_set_preferred_compiler_flags()
+
+macro(ALICE_UseVc)
+ include_directories(SYSTEM "${Vc_INCLUDE_DIR}")
+ add_definitions(${Vc_DEFINITIONS})
+endmacro()
+
+# vim: ft=cmake sw=3 et
--- /dev/null
+get_filename_component(_currentDir "${CMAKE_CURRENT_LIST_FILE}" PATH)
+include("${_currentDir}/CheckCCompilerFlag.cmake")
+include("${_currentDir}/CheckCXXCompilerFlag.cmake")
+macro(AddCompilerFlag _flag)
+ string(REGEX REPLACE "[-+/:= ]" "_" _flag_esc "${_flag}")
+ check_c_compiler_flag("${_flag}" check_c_compiler_flag_${_flag_esc})
+ check_cxx_compiler_flag("${_flag}" check_cxx_compiler_flag_${_flag_esc})
+
+ set(_c_flags "CMAKE_C_FLAGS")
+ set(_cxx_flags "CMAKE_CXX_FLAGS")
+ if(${ARGC} EQUAL 2)
+ set(${ARGV1} "${check_cxx_compiler_flag_${_flag_esc}}")
+ elseif(${ARGC} GREATER 2)
+ set(state 0)
+ unset(_c_flags)
+ unset(_cxx_flags)
+ foreach(_arg ${ARGN})
+ if(_arg STREQUAL "C_FLAGS")
+ set(state 1)
+ elseif(_arg STREQUAL "CXX_FLAGS")
+ set(state 2)
+ elseif(_arg STREQUAL "C_RESULT")
+ set(state 3)
+ elseif(_arg STREQUAL "CXX_RESULT")
+ set(state 4)
+ elseif(state EQUAL 1)
+ set(_c_flags "${_arg}")
+ elseif(state EQUAL 2)
+ set(_cxx_flags "${_arg}")
+ elseif(state EQUAL 3)
+ set(${_arg} ${check_c_compiler_flag_${_flag_esc}})
+ elseif(state EQUAL 4)
+ set(${_arg} ${check_cxx_compiler_flag_${_flag_esc}})
+ else()
+ message(FATAL_ERROR "Syntax error for AddCompilerFlag")
+ endif()
+ endforeach()
+ endif()
+
+ if(check_c_compiler_flag_${_flag_esc} AND DEFINED _c_flags)
+ set(${_c_flags} "${${_c_flags}} ${_flag}")
+ endif()
+ if(check_cxx_compiler_flag_${_flag_esc} AND DEFINED _cxx_flags)
+ set(${_cxx_flags} "${${_cxx_flags}} ${_flag}")
+ endif()
+endmacro(AddCompilerFlag)
--- /dev/null
+# - Check whether the C compiler supports a given flag.
+# CHECK_C_COMPILER_FLAG(<flag> <var>)
+# <flag> - the compiler flag
+# <var> - variable to store the result
+# This internally calls the check_c_source_compiles macro.
+# See help for CheckCSourceCompiles for a listing of variables
+# that can modify the build.
+
+#=============================================================================
+# Copyright 2006-2009 Kitware, Inc.
+# Copyright 2006 Alexander Neundorf <neundorf@kde.org>
+# Copyright 2011 Matthias Kretz <kretz@kde.org>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distributed this file outside of CMake, substitute the full
+# License text for the above reference.)
+
+INCLUDE(CheckCSourceCompiles)
+
+MACRO (CHECK_C_COMPILER_FLAG _FLAG _RESULT)
+ SET(SAFE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}")
+ SET(CMAKE_REQUIRED_DEFINITIONS "${_FLAG}")
+ CHECK_C_SOURCE_COMPILES("int main() { return 0;}" ${_RESULT}
+ # Some compilers do not fail with a bad flag
+ FAIL_REGEX "argument unused during compilation" # clang
+ FAIL_REGEX "is valid for .* but not for C" # GNU
+ FAIL_REGEX "unrecognized .*option" # GNU
+ FAIL_REGEX "ignoring unknown option" # MSVC
+ FAIL_REGEX "[Uu]nknown option" # HP
+ FAIL_REGEX "[Ww]arning: [Oo]ption" # SunPro
+ FAIL_REGEX "command option .* is not recognized" # XL
+ FAIL_REGEX "WARNING: unknown flag:" # Open64
+ FAIL_REGEX " #10159: " # ICC
+ )
+ SET (CMAKE_REQUIRED_DEFINITIONS "${SAFE_CMAKE_REQUIRED_DEFINITIONS}")
+ENDMACRO (CHECK_C_COMPILER_FLAG)
+
--- /dev/null
+# - Check whether the CXX compiler supports a given flag.
+# CHECK_CXX_COMPILER_FLAG(<flag> <var>)
+# <flag> - the compiler flag
+# <var> - variable to store the result
+# This internally calls the check_cxx_source_compiles macro. See help
+# for CheckCXXSourceCompiles for a listing of variables that can
+# modify the build.
+
+#=============================================================================
+# Copyright 2006-2009 Kitware, Inc.
+# Copyright 2006 Alexander Neundorf <neundorf@kde.org>
+# Copyright 2011 Matthias Kretz <kretz@kde.org>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distributed this file outside of CMake, substitute the full
+# License text for the above reference.)
+
+INCLUDE(CheckCXXSourceCompiles)
+
+MACRO (CHECK_CXX_COMPILER_FLAG _FLAG _RESULT)
+ SET(SAFE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}")
+ SET(CMAKE_REQUIRED_DEFINITIONS "${_FLAG}")
+ CHECK_CXX_SOURCE_COMPILES("int main() { return 0;}" ${_RESULT}
+ # Some compilers do not fail with a bad flag
+ FAIL_REGEX "argument unused during compilation" # clang
+ FAIL_REGEX "is valid for .* but not for C\\\\+\\\\+" # GNU
+ FAIL_REGEX "unrecognized .*option" # GNU
+ FAIL_REGEX "ignoring unknown option" # MSVC
+ FAIL_REGEX "[Uu]nknown option" # HP
+ FAIL_REGEX "[Ww]arning: [Oo]ption" # SunPro
+ FAIL_REGEX "command option .* is not recognized" # XL
+ FAIL_REGEX "WARNING: unknown flag:" # Open64
+ FAIL_REGEX " #10159: " # ICC
+ )
+ SET (CMAKE_REQUIRED_DEFINITIONS "${SAFE_CMAKE_REQUIRED_DEFINITIONS}")
+ENDMACRO (CHECK_CXX_COMPILER_FLAG)
+
--- /dev/null
+get_filename_component(_currentDir "${CMAKE_CURRENT_LIST_FILE}" PATH)
+include("${_currentDir}/AddCompilerFlag.cmake")
+
+macro(_my_find _list _value _ret)
+ list(FIND ${_list} "${_value}" _found)
+ if(_found EQUAL -1)
+ set(${_ret} FALSE)
+ else(_found EQUAL -1)
+ set(${_ret} TRUE)
+ endif(_found EQUAL -1)
+endmacro(_my_find)
+
+macro(AutodetectHostArchitecture)
+ set(TARGET_ARCHITECTURE "generic")
+ set(Vc_ARCHITECTURE_FLAGS)
+ set(_vendor_id)
+ set(_cpu_family)
+ set(_cpu_model)
+ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ file(READ "/proc/cpuinfo" _cpuinfo)
+ string(REGEX REPLACE ".*vendor_id[ \t]*:[ \t]+([a-zA-Z0-9_-]+).*" "\\1" _vendor_id "${_cpuinfo}")
+ string(REGEX REPLACE ".*cpu family[ \t]*:[ \t]+([a-zA-Z0-9_-]+).*" "\\1" _cpu_family "${_cpuinfo}")
+ string(REGEX REPLACE ".*model[ \t]*:[ \t]+([a-zA-Z0-9_-]+).*" "\\1" _cpu_model "${_cpuinfo}")
+ string(REGEX REPLACE ".*flags[ \t]*:[ \t]+([a-zA-Z0-9_-]+).*" "\\1" _cpu_flags "${_cpuinfo}")
+ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+ exec_program("/usr/sbin/sysctl -n machdep.cpu.vendor" OUTPUT_VARIABLE _vendor_id)
+ exec_program("/usr/sbin/sysctl -n machdep.cpu.model" OUTPUT_VARIABLE _cpu_model)
+ exec_program("/usr/sbin/sysctl -n machdep.cpu.family" OUTPUT_VARIABLE _cpu_family)
+ exec_program("/usr/sbin/sysctl -n machdep.cpu.features" OUTPUT_VARIABLE _cpu_flags)
+ string(TOLOWER "${_cpu_flags}" _cpu_flags)
+ string(REPLACE "." "_" _cpu_flags "${_cpu_flags}")
+ elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+ get_filename_component(_vendor_id "[HKEY_LOCAL_MACHINE\\Hardware\\Description\\System\\CentralProcessor\\0;VendorIdentifier]" NAME CACHE)
+ get_filename_component(_cpu_id "[HKEY_LOCAL_MACHINE\\Hardware\\Description\\System\\CentralProcessor\\0;Identifier]" NAME CACHE)
+ mark_as_advanced(_vendor_id _cpu_id)
+ string(REGEX REPLACE ".* Family ([0-9]+) .*" "\\1" _cpu_family "${_cpu_id}")
+ string(REGEX REPLACE ".* Model ([0-9]+) .*" "\\1" _cpu_model "${_cpu_id}")
+ endif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ if(_vendor_id STREQUAL "GenuineIntel")
+ if(_cpu_family EQUAL 6)
+ # Any recent Intel CPU except NetBurst
+ if(_cpu_model EQUAL 46) # Xeon 7500 series
+ set(TARGET_ARCHITECTURE "westmere")
+ elseif(_cpu_model EQUAL 45) # Xeon TNG
+ set(TARGET_ARCHITECTURE "sandy-bridge")
+ elseif(_cpu_model EQUAL 44) # Xeon 5600 series
+ set(TARGET_ARCHITECTURE "westmere")
+ elseif(_cpu_model EQUAL 42) # Core TNG
+ set(TARGET_ARCHITECTURE "sandy-bridge")
+ elseif(_cpu_model EQUAL 37) # Core i7/i5/i3
+ set(TARGET_ARCHITECTURE "westmere")
+ elseif(_cpu_model EQUAL 31) # Core i7/i5
+ set(TARGET_ARCHITECTURE "westmere")
+ elseif(_cpu_model EQUAL 30) # Core i7/i5
+ set(TARGET_ARCHITECTURE "westmere")
+ elseif(_cpu_model EQUAL 29)
+ set(TARGET_ARCHITECTURE "penryn")
+ elseif(_cpu_model EQUAL 28)
+ set(TARGET_ARCHITECTURE "atom")
+ elseif(_cpu_model EQUAL 26)
+ set(TARGET_ARCHITECTURE "nehalem")
+ elseif(_cpu_model EQUAL 23)
+ set(TARGET_ARCHITECTURE "penryn")
+ elseif(_cpu_model EQUAL 15)
+ set(TARGET_ARCHITECTURE "merom")
+ elseif(_cpu_model EQUAL 14)
+ set(TARGET_ARCHITECTURE "core")
+ elseif(_cpu_model LESS 14)
+ message(WARNING "Your CPU (family ${_cpu_family}, model ${_cpu_model}) is not known. Auto-detection of optimization flags failed and will use the generic CPU settings with SSE2.")
+ set(TARGET_ARCHITECTURE "generic")
+ else()
+ message(WARNING "Your CPU (family ${_cpu_family}, model ${_cpu_model}) is not known. Auto-detection of optimization flags failed and will use the 65nm Core 2 CPU settings.")
+ set(TARGET_ARCHITECTURE "merom")
+ endif()
+ elseif(_cpu_family EQUAL 7) # Itanium (not supported)
+ message(WARNING "Your CPU (Itanium: family ${_cpu_family}, model ${_cpu_model}) is not supported by OptimizeForArchitecture.cmake.")
+ elseif(_cpu_family EQUAL 15) # NetBurst
+ list(APPEND _available_vector_units_list "sse" "sse2")
+ if(_cpu_model GREATER 2) # Not sure whether this must be 3 or even 4 instead
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3")
+ endif(_cpu_model GREATER 2)
+ endif(_cpu_family EQUAL 6)
+ elseif(_vendor_id STREQUAL "AuthenticAMD")
+ if(_cpu_family EQUAL 21) # 15h
+ set(TARGET_ARCHITECTURE "bulldozer")
+ elseif(_cpu_family EQUAL 20) # 14h
+ elseif(_cpu_family EQUAL 18) # 12h
+ elseif(_cpu_family EQUAL 16) # 10h
+ set(TARGET_ARCHITECTURE "barcelona")
+ elseif(_cpu_family EQUAL 15)
+ set(TARGET_ARCHITECTURE "k8")
+ if(_cpu_model GREATER 64) # I don't know the right number to put here. This is just a guess from the hardware I have access to
+ set(TARGET_ARCHITECTURE "k8-sse3")
+ endif(_cpu_model GREATER 64)
+ endif()
+ endif(_vendor_id STREQUAL "GenuineIntel")
+endmacro()
+
+macro(OptimizeForArchitecture)
+ set(TARGET_ARCHITECTURE "auto" CACHE STRING "CPU architecture to optimize for. Using an incorrect setting here can result in crashes of the resulting binary because of invalid instructions used.\nSetting the value to \"auto\" will try to optimize for the architecture where cmake is called.\nOther supported values are: \"generic\", \"core\", \"merom\" (65nm Core2), \"penryn\" (45nm Core2), \"nehalem\", \"westmere\", \"sandy-bridge\", \"atom\", \"k8\", \"k8-sse3\", \"barcelona\", \"istanbul\", \"magny-cours\", \"bulldozer\", \"interlagos\".")
+ set(_force)
+ if(NOT _last_target_arch STREQUAL "${TARGET_ARCHITECTURE}")
+ message(STATUS "target changed from \"${_last_target_arch}\" to \"${TARGET_ARCHITECTURE}\"")
+ set(_force FORCE)
+ endif()
+ set(_last_target_arch "${TARGET_ARCHITECTURE}" CACHE STRING "" FORCE)
+ mark_as_advanced(_last_target_arch)
+ string(TOLOWER "${TARGET_ARCHITECTURE}" TARGET_ARCHITECTURE)
+
+ set(_march_flag_list)
+ set(_available_vector_units_list)
+
+ if(TARGET_ARCHITECTURE STREQUAL "auto")
+ AutodetectHostArchitecture()
+ message(STATUS "Detected CPU: ${TARGET_ARCHITECTURE}")
+ endif(TARGET_ARCHITECTURE STREQUAL "auto")
+
+ if(TARGET_ARCHITECTURE STREQUAL "core")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3")
+ elseif(TARGET_ARCHITECTURE STREQUAL "merom")
+ list(APPEND _march_flag_list "merom")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3")
+ elseif(TARGET_ARCHITECTURE STREQUAL "penryn")
+ list(APPEND _march_flag_list "penryn")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3")
+ message(STATUS "Sadly the Penryn architecture exists in variants with SSE4.1 and without SSE4.1.")
+ if(_cpu_flags MATCHES "sse4_1")
+ message(STATUS "SSE4.1: enabled (auto-detected from this computer's CPU flags)")
+ list(APPEND _available_vector_units_list "sse4.1")
+ else()
+ message(STATUS "SSE4.1: disabled (auto-detected from this computer's CPU flags)")
+ endif()
+ elseif(TARGET_ARCHITECTURE STREQUAL "nehalem")
+ list(APPEND _march_flag_list "nehalem")
+ list(APPEND _march_flag_list "corei7")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4.1" "sse4.2")
+ elseif(TARGET_ARCHITECTURE STREQUAL "westmere")
+ list(APPEND _march_flag_list "westmere")
+ list(APPEND _march_flag_list "corei7")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4.1" "sse4.2")
+ elseif(TARGET_ARCHITECTURE STREQUAL "sandy-bridge")
+ list(APPEND _march_flag_list "sandybridge")
+ list(APPEND _march_flag_list "corei7-avx")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4.1" "sse4.2" "avx")
+ elseif(TARGET_ARCHITECTURE STREQUAL "atom")
+ list(APPEND _march_flag_list "atom")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3")
+ elseif(TARGET_ARCHITECTURE STREQUAL "k8")
+ list(APPEND _march_flag_list "k8")
+ list(APPEND _available_vector_units_list "sse" "sse2")
+ elseif(TARGET_ARCHITECTURE STREQUAL "k8-sse3")
+ list(APPEND _march_flag_list "k8-sse3")
+ list(APPEND _march_flag_list "k8")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3")
+ elseif(TARGET_ARCHITECTURE STREQUAL "interlagos")
+ list(APPEND _march_flag_list "bulldozer")
+ list(APPEND _march_flag_list "barcelona")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4a" "sse4.1" "sse4.2" "avx" "xop" "fma4")
+ elseif(TARGET_ARCHITECTURE STREQUAL "bulldozer")
+ list(APPEND _march_flag_list "bulldozer")
+ list(APPEND _march_flag_list "barcelona")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4a" "sse4.1" "sse4.2" "avx" "xop" "fma4")
+ elseif(TARGET_ARCHITECTURE STREQUAL "barcelona")
+ list(APPEND _march_flag_list "barcelona")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "sse4a")
+ elseif(TARGET_ARCHITECTURE STREQUAL "istanbul")
+ list(APPEND _march_flag_list "barcelona")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "sse4a")
+ elseif(TARGET_ARCHITECTURE STREQUAL "magny-cours")
+ list(APPEND _march_flag_list "barcelona")
+ list(APPEND _march_flag_list "core2")
+ list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "sse4a")
+ elseif(TARGET_ARCHITECTURE STREQUAL "generic")
+ list(APPEND _march_flag_list "generic")
+ else(TARGET_ARCHITECTURE STREQUAL "core")
+ message(FATAL_ERROR "Unknown target architecture: \"${TARGET_ARCHITECTURE}\". Please set TARGET_ARCHITECTURE to a supported value.")
+ endif(TARGET_ARCHITECTURE STREQUAL "core")
+
+ set(_disable_vector_unit_list)
+ set(_enable_vector_unit_list)
+ _my_find(_available_vector_units_list "sse2" SSE2_FOUND)
+ _my_find(_available_vector_units_list "sse3" SSE3_FOUND)
+ _my_find(_available_vector_units_list "ssse3" SSSE3_FOUND)
+ _my_find(_available_vector_units_list "sse4.1" SSE4_1_FOUND)
+ _my_find(_available_vector_units_list "sse4.2" SSE4_2_FOUND)
+ _my_find(_available_vector_units_list "sse4a" SSE4a_FOUND)
+ if(DEFINED Vc_AVX_INTRINSICS_BROKEN AND Vc_AVX_INTRINSICS_BROKEN)
+ UserWarning("AVX disabled per default because of old/broken compiler")
+ set(AVX_FOUND false)
+ set(XOP_FOUND false)
+ set(FMA4_FOUND false)
+ else()
+ _my_find(_available_vector_units_list "avx" AVX_FOUND)
+ _my_find(_available_vector_units_list "xop" XOP_FOUND)
+ _my_find(_available_vector_units_list "fma4" FMA4_FOUND)
+ endif()
+ set(USE_SSE2 ${SSE2_FOUND} CACHE BOOL "Use SSE2. If SSE2 instructions are not enabled the SSE implementation will be disabled." ${_force})
+ set(USE_SSE3 ${SSE3_FOUND} CACHE BOOL "Use SSE3. If SSE3 instructions are not enabled they will be emulated." ${_force})
+ set(USE_SSSE3 ${SSSE3_FOUND} CACHE BOOL "Use SSSE3. If SSSE3 instructions are not enabled they will be emulated." ${_force})
+ set(USE_SSE4_1 ${SSE4_1_FOUND} CACHE BOOL "Use SSE4.1. If SSE4.1 instructions are not enabled they will be emulated." ${_force})
+ set(USE_SSE4_2 ${SSE4_2_FOUND} CACHE BOOL "Use SSE4.2. If SSE4.2 instructions are not enabled they will be emulated." ${_force})
+ set(USE_SSE4a ${SSE4a_FOUND} CACHE BOOL "Use SSE4a. If SSE4a instructions are not enabled they will be emulated." ${_force})
+ set(USE_AVX ${AVX_FOUND} CACHE BOOL "Use AVX. This will double some of the vector sizes relative to SSE." ${_force})
+ set(USE_XOP ${XOP_FOUND} CACHE BOOL "Use XOP." ${_force})
+ set(USE_FMA4 ${FMA4_FOUND} CACHE BOOL "Use FMA4." ${_force})
+ mark_as_advanced(USE_SSE2 USE_SSE3 USE_SSSE3 USE_SSE4_1 USE_SSE4_2 USE_SSE4a USE_AVX USE_XOP USE_FMA4)
+ if(USE_SSE2)
+ list(APPEND _enable_vector_unit_list "sse2")
+ else(USE_SSE2)
+ list(APPEND _disable_vector_unit_list "sse2")
+ endif(USE_SSE2)
+ if(USE_SSE3)
+ list(APPEND _enable_vector_unit_list "sse3")
+ else(USE_SSE3)
+ list(APPEND _disable_vector_unit_list "sse3")
+ endif(USE_SSE3)
+ if(USE_SSSE3)
+ list(APPEND _enable_vector_unit_list "ssse3")
+ else(USE_SSSE3)
+ list(APPEND _disable_vector_unit_list "ssse3")
+ endif(USE_SSSE3)
+ if(USE_SSE4_1)
+ list(APPEND _enable_vector_unit_list "sse4.1")
+ else(USE_SSE4_1)
+ list(APPEND _disable_vector_unit_list "sse4.1")
+ endif(USE_SSE4_1)
+ if(USE_SSE4_2)
+ list(APPEND _enable_vector_unit_list "sse4.2")
+ else(USE_SSE4_2)
+ list(APPEND _disable_vector_unit_list "sse4.2")
+ endif(USE_SSE4_2)
+ if(USE_SSE4a)
+ list(APPEND _enable_vector_unit_list "sse4a")
+ else(USE_SSE4a)
+ list(APPEND _disable_vector_unit_list "sse4a")
+ endif(USE_SSE4a)
+ if(USE_AVX)
+ list(APPEND _enable_vector_unit_list "avx")
+ # we want SSE intrinsics to result in instructions using the VEX prefix.
+ # Otherwise integer ops (which require the older SSE intrinsics) would
+ # always have a large penalty.
+ list(APPEND _enable_vector_unit_list "sse2avx")
+ else(USE_AVX)
+ list(APPEND _disable_vector_unit_list "avx")
+ endif(USE_AVX)
+ if(USE_XOP)
+ list(APPEND _enable_vector_unit_list "xop")
+ else()
+ list(APPEND _disable_vector_unit_list "xop")
+ endif()
+ if(USE_FMA4)
+ list(APPEND _enable_vector_unit_list "fma4")
+ else()
+ list(APPEND _disable_vector_unit_list "fma4")
+ endif()
+ if(MSVC)
+ # MSVC on 32 bit can select /arch:SSE2 (since 2010 also /arch:AVX)
+ # MSVC on 64 bit cannot select anything (should have changed with MSVC 2010)
+ _my_find(_enable_vector_unit_list "avx" _avx)
+ set(_avx_flag FALSE)
+ if(_avx)
+ AddCompilerFlag("/arch:AVX" CXX_FLAGS Vc_ARCHITECTURE_FLAGS CXX_RESULT _avx_flag)
+ endif()
+ if(NOT _avx_flag)
+ _my_find(_enable_vector_unit_list "sse2" _found)
+ if(_found)
+ AddCompilerFlag("/arch:SSE2" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ endif()
+ endif()
+ foreach(_flag ${_enable_vector_unit_list})
+ string(TOUPPER "${_flag}" _flag)
+ string(REPLACE "." "_" _flag "__${_flag}__")
+ add_definitions("-D${_flag}")
+ endforeach(_flag)
+ elseif(CMAKE_CXX_COMPILER MATCHES "/(icpc|icc)$") # ICC (on Linux)
+ _my_find(_available_vector_units_list "avx" _found)
+ if(_found)
+ AddCompilerFlag("-xAVX" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ else(_found)
+ _my_find(_available_vector_units_list "sse4.2" _found)
+ if(_found)
+ AddCompilerFlag("-xSSE4.2" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ else(_found)
+ _my_find(_available_vector_units_list "sse4.1" _found)
+ if(_found)
+ AddCompilerFlag("-xSSE4.1" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ else(_found)
+ _my_find(_available_vector_units_list "ssse3" _found)
+ if(_found)
+ AddCompilerFlag("-xSSSE3" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ else(_found)
+ _my_find(_available_vector_units_list "sse3" _found)
+ if(_found)
+ # If the target host is an AMD machine then we still want to use -xSSE2 because the binary would refuse to run at all otherwise
+ _my_find(_march_flag_list "barcelona" _found)
+ if(NOT _found)
+ _my_find(_march_flag_list "k8-sse3" _found)
+ endif(NOT _found)
+ if(_found)
+ AddCompilerFlag("-xSSE2" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ else(_found)
+ AddCompilerFlag("-xSSE3" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ endif(_found)
+ else(_found)
+ _my_find(_available_vector_units_list "sse2" _found)
+ if(_found)
+ AddCompilerFlag("-xSSE2" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ endif(_found)
+ endif(_found)
+ endif(_found)
+ endif(_found)
+ endif(_found)
+ endif(_found)
+ else() # not MSVC and not ICC => GCC, Clang, Open64
+ foreach(_flag ${_march_flag_list})
+ AddCompilerFlag("-march=${_flag}" CXX_RESULT _good CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ if(_good)
+ break()
+ endif(_good)
+ endforeach(_flag)
+ foreach(_flag ${_enable_vector_unit_list})
+ AddCompilerFlag("-m${_flag}" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ endforeach(_flag)
+ foreach(_flag ${_disable_vector_unit_list})
+ AddCompilerFlag("-mno-${_flag}" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
+ endforeach(_flag)
+ endif()
+endmacro(OptimizeForArchitecture)
--- /dev/null
+macro(UserWarning _msg)
+ if("$ENV{DASHBOARD_TEST_FROM_CTEST}" STREQUAL "")
+ # developer (non-dashboard) build
+ message(WARNING "${_msg}")
+ else()
+ # dashboard build
+ message(STATUS "${_msg}")
+ endif()
+endmacro()
--- /dev/null
+# Macros for use with the Vc library. Vc can be found at http://code.compeng.uni-frankfurt.de/projects/vc
+#
+# The following macros are provided:
+# vc_determine_compiler
+# vc_set_preferred_compiler_flags
+#
+#=============================================================================
+# Copyright 2009-2012 Matthias Kretz <kretz@kde.org>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file CmakeCopyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+
+get_filename_component(_currentDir "${CMAKE_CURRENT_LIST_FILE}" PATH)
+include ("${_currentDir}/UserWarning.cmake")
+include ("${_currentDir}/AddCompilerFlag.cmake")
+include ("${_currentDir}/OptimizeForArchitecture.cmake")
+
+macro(vc_determine_compiler)
+ if(NOT DEFINED Vc_COMPILER_IS_INTEL)
+ set(Vc_COMPILER_IS_INTEL false)
+ set(Vc_COMPILER_IS_OPEN64 false)
+ set(Vc_COMPILER_IS_CLANG false)
+ set(Vc_COMPILER_IS_MSVC false)
+ set(Vc_COMPILER_IS_GCC false)
+ if(CMAKE_CXX_COMPILER MATCHES "/(icpc|icc)$")
+ set(Vc_COMPILER_IS_INTEL true)
+ exec_program(${CMAKE_C_COMPILER} ARGS -dumpversion OUTPUT_VARIABLE Vc_ICC_VERSION)
+ message(STATUS "Detected Compiler: Intel ${Vc_ICC_VERSION}")
+ elseif(CMAKE_CXX_COMPILER MATCHES "/(opencc|openCC)$")
+ set(Vc_COMPILER_IS_OPEN64 true)
+ message(STATUS "Detected Compiler: Open64")
+ elseif(CMAKE_CXX_COMPILER MATCHES "/clang\\+\\+$")
+ set(Vc_COMPILER_IS_CLANG true)
+ message(STATUS "Detected Compiler: Clang")
+ elseif(MSVC)
+ set(Vc_COMPILER_IS_MSVC true)
+ message(STATUS "Detected Compiler: MSVC")
+ elseif(CMAKE_COMPILER_IS_GNUCXX)
+ set(Vc_COMPILER_IS_GCC true)
+ exec_program(${CMAKE_C_COMPILER} ARGS -dumpversion OUTPUT_VARIABLE Vc_GCC_VERSION)
+ message(STATUS "Detected Compiler: GCC ${Vc_GCC_VERSION}")
+
+ # some distributions patch their GCC to return nothing or only major and minor version on -dumpversion.
+ # In that case we must extract the version number from --version.
+ if(NOT Vc_GCC_VERSION OR Vc_GCC_VERSION MATCHES "^[0-9]\\.[0-9]+$")
+ exec_program(${CMAKE_C_COMPILER} ARGS --version OUTPUT_VARIABLE Vc_GCC_VERSION)
+ string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" Vc_GCC_VERSION "${Vc_GCC_VERSION}")
+ message(STATUS "GCC Version from --version: ${Vc_GCC_VERSION}")
+ endif()
+
+ # some distributions patch their GCC to be API incompatible to what the FSF released. In
+ # those cases we require a macro to identify the distribution version
+ find_program(_lsb_release lsb_release)
+ mark_as_advanced(_lsb_release)
+ if(_lsb_release)
+ execute_process(COMMAND ${_lsb_release} -is OUTPUT_VARIABLE _distributor_id OUTPUT_STRIP_TRAILING_WHITESPACE)
+ execute_process(COMMAND ${_lsb_release} -rs OUTPUT_VARIABLE _distributor_release OUTPUT_STRIP_TRAILING_WHITESPACE)
+ string(TOUPPER "${_distributor_id}" _distributor_id)
+ if(_distributor_id STREQUAL "UBUNTU")
+ execute_process(COMMAND ${CMAKE_C_COMPILER} --version OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE _gcc_version)
+ string(REGEX MATCH "\\(.* ${Vc_GCC_VERSION}-([0-9]+).*\\)" _tmp "${_gcc_version}")
+ if(_tmp)
+ set(_patch ${CMAKE_MATCH_1})
+ string(REGEX MATCH "^([0-9]+)\\.([0-9]+)$" _tmp "${_distributor_release}")
+ execute_process(COMMAND printf 0x%x%02x%02x ${CMAKE_MATCH_1} ${CMAKE_MATCH_2} ${_patch} OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE _tmp)
+ set(Vc_DEFINITIONS "${Vc_DEFINITIONS} -D__GNUC_UBUNTU_VERSION__=${_tmp}")
+ endif()
+ endif()
+ endif()
+ else()
+ message(WARNING "Untested/-supported Compiler for use with Vc.\nPlease fill out the missing parts in the CMake scripts and submit a patch to http://code.compeng.uni-frankfurt.de/projects/vc")
+ endif()
+ endif()
+endmacro()
+
+macro(vc_set_gnu_buildtype_flags)
+ set(CMAKE_CXX_FLAGS_DEBUG "-g3" CACHE STRING "Flags used by the compiler during debug builds." FORCE)
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG" CACHE STRING "Flags used by the compiler during release minsize builds." FORCE)
+ set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "Flags used by the compiler during release builds (/MD /Ob1 /Oi /Ot /Oy /Gs will produce slightly less optimized but smaller files)." FORCE)
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -g" CACHE STRING "Flags used by the compiler during Release with Debug Info builds." FORCE)
+ set(CMAKE_C_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING "Flags used by the compiler during debug builds." FORCE)
+ set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}" CACHE STRING "Flags used by the compiler during release minsize builds." FORCE)
+ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "Flags used by the compiler during release builds (/MD /Ob1 /Oi /Ot /Oy /Gs will produce slightly less optimized but smaller files)." FORCE)
+ set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" CACHE STRING "Flags used by the compiler during Release with Debug Info builds." FORCE)
+ if(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+ set(ENABLE_STRICT_ALIASING true CACHE BOOL "Enables strict aliasing rules for more aggressive optimizations")
+ if(NOT ENABLE_STRICT_ALIASING)
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-strict-aliasing ")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-strict-aliasing ")
+ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -fno-strict-aliasing ")
+ set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-strict-aliasing ")
+ endif(NOT ENABLE_STRICT_ALIASING)
+ endif()
+endmacro()
+
+macro(vc_add_compiler_flag VAR _flag)
+ AddCompilerFlag("${_flag}" CXX_FLAGS ${VAR})
+endmacro()
+
+macro(vc_check_assembler)
+ if(APPLE)
+ if(NOT Vc_COMPILER_IS_CLANG)
+ message(WARNING "Apple does not provide an assembler with AVX support. Please use Clang instead of GCC.")
+ endif()
+ else(APPLE)
+ if(${ARGC} EQUAL 1)
+ set(_as "${ARGV1}")
+ else()
+ exec_program(${CMAKE_CXX_COMPILER} ARGS -print-prog-name=as OUTPUT_VARIABLE _as)
+ mark_as_advanced(_as)
+ endif()
+ if(NOT _as)
+ message(WARNING "Could not find 'as', the assembler used by GCC. Hoping everything will work out...")
+ else()
+ exec_program(${_as} ARGS --version OUTPUT_VARIABLE _as_version)
+ string(REGEX REPLACE "\\([^\\)]*\\)" "" _as_version "${_as_version}")
+ string(REGEX MATCH "[1-9]\\.[0-9]+(\\.[0-9]+)?" _as_version "${_as_version}")
+ if(_as_version VERSION_LESS "2.18.93")
+ message(WARNING "Your binutils is too old (${_as_version}). Some optimizations of Vc will be disabled.")
+ add_definitions(-DVC_NO_XGETBV) # old assembler doesn't know the xgetbv instruction
+ endif()
+ endif()
+ endif(APPLE)
+endmacro()
+
+macro(vc_check_fpmath)
+ # if compiling for 32 bit x86 we need to use the -mfpmath=sse since the x87 is broken by design
+ include (CheckCXXSourceRuns)
+ check_cxx_source_runs("int main() { return sizeof(void*) != 8; }" Vc_VOID_PTR_IS_64BIT)
+ if(NOT Vc_VOID_PTR_IS_64BIT)
+ exec_program(${CMAKE_C_COMPILER} ARGS -dumpmachine OUTPUT_VARIABLE _gcc_machine)
+ if(_gcc_machine MATCHES "[x34567]86")
+ vc_add_compiler_flag(Vc_DEFINITIONS "-mfpmath=sse")
+ endif(_gcc_machine MATCHES "[x34567]86")
+ endif()
+endmacro()
+
+macro(vc_set_preferred_compiler_flags)
+ vc_determine_compiler()
+
+ set(_add_warning_flags false)
+ set(_add_buildtype_flags false)
+ foreach(_arg ${ARGN})
+ if(_arg STREQUAL "WARNING_FLAGS")
+ set(_add_warning_flags true)
+ elseif(_arg STREQUAL "BUILDTYPE_FLAGS")
+ set(_add_buildtype_flags true)
+ endif()
+ endforeach()
+
+ set(Vc_SSE_INTRINSICS_BROKEN false)
+ set(Vc_AVX_INTRINSICS_BROKEN false)
+
+ if(Vc_COMPILER_IS_OPEN64)
+ ##################################################################################################
+ # Open64 #
+ ##################################################################################################
+ if(_add_warning_flags)
+ AddCompilerFlag("-W")
+ AddCompilerFlag("-Wall")
+ AddCompilerFlag("-Wimplicit")
+ AddCompilerFlag("-Wswitch")
+ AddCompilerFlag("-Wformat")
+ AddCompilerFlag("-Wchar-subscripts")
+ AddCompilerFlag("-Wparentheses")
+ AddCompilerFlag("-Wmultichar")
+ AddCompilerFlag("-Wtrigraphs")
+ AddCompilerFlag("-Wpointer-arith")
+ AddCompilerFlag("-Wcast-align")
+ AddCompilerFlag("-Wreturn-type")
+ AddCompilerFlag("-Wno-unused-function")
+ AddCompilerFlag("-ansi")
+ AddCompilerFlag("-pedantic")
+ AddCompilerFlag("-Wno-long-long")
+ AddCompilerFlag("-Wshadow")
+ AddCompilerFlag("-Wold-style-cast")
+ AddCompilerFlag("-Wno-variadic-macros")
+ endif()
+ if(_add_buildtype_flags)
+ vc_set_gnu_buildtype_flags()
+ endif()
+
+ vc_check_assembler()
+ elseif(Vc_COMPILER_IS_GCC)
+ ##################################################################################################
+ # GCC #
+ ##################################################################################################
+ if(_add_warning_flags)
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W -Wall -Wswitch -Wformat -Wchar-subscripts -Wparentheses -Wmultichar -Wtrigraphs -Wpointer-arith -Wcast-align -Wreturn-type -Wno-unused-function -ansi -pedantic -Wno-long-long -Wshadow")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wswitch -Wformat -Wchar-subscripts -Wparentheses -Wmultichar -Wtrigraphs -Wpointer-arith -Wcast-align -Wreturn-type -Wno-unused-function -ansi -pedantic -Wno-long-long -Wshadow")
+ AddCompilerFlag("-Wimplicit")
+ AddCompilerFlag("-Wold-style-cast")
+ AddCompilerFlag("-Wno-variadic-macros")
+ if(Vc_GCC_VERSION VERSION_GREATER "4.5.2" AND Vc_GCC_VERSION VERSION_LESS "4.6.4")
+ # GCC gives bogus "array subscript is above array bounds" warnings in math.cpp
+ AddCompilerFlag("-Wno-array-bounds")
+ endif()
+ endif()
+ vc_add_compiler_flag(Vc_DEFINITIONS "-Wabi")
+ vc_add_compiler_flag(Vc_DEFINITIONS "-fabi-version=0") # ABI version 4 is required to make __m128 and __m256 appear as different types. 0 should give us the latest version.
+
+ if(_add_buildtype_flags)
+ vc_set_gnu_buildtype_flags()
+ endif()
+
+ # GCC 4.5.[01] fail at inlining some functions, creating functions with a single instructions,
+ # thus creating a large overhead.
+ if(Vc_GCC_VERSION VERSION_LESS "4.5.2" AND NOT Vc_GCC_VERSION VERSION_LESS "4.5.0")
+ UserWarning("GCC 4.5.0 and 4.5.1 have problems with inlining correctly. Setting early-inlining-insns=12 as workaround.")
+ AddCompilerFlag("--param early-inlining-insns=12")
+ endif()
+
+ if(Vc_GCC_VERSION VERSION_LESS "4.4.6")
+ UserWarning("Your GCC is older than 4.4.6. This is known to cause problems/bugs. Please update to the latest GCC if you can.")
+ set(Vc_AVX_INTRINSICS_BROKEN true)
+ if(Vc_GCC_VERSION VERSION_LESS "4.3.0")
+ UserWarning("Your GCC is older than 4.3.0. It is unable to handle the full set of SSE2 intrinsics. All SSE code will be disabled. Please update to the latest GCC if you can.")
+ set(Vc_SSE_INTRINSICS_BROKEN true)
+ endif()
+ endif()
+
+ if(Vc_GCC_VERSION VERSION_LESS 4.5.0)
+ UserWarning("GCC 4.4.x shows false positives for -Wparentheses, thus we rather disable the warning.")
+ string(REPLACE " -Wparentheses " " " CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+ string(REPLACE " -Wparentheses " " " CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ set(Vc_DEFINITIONS "${Vc_DEFINITIONS} -Wno-parentheses")
+ elseif(Vc_GCC_VERSION VERSION_EQUAL 4.7.0)
+ UserWarning("GCC 4.7.0 miscompiles at -O3, adding -fno-predictive-commoning to the compiler flags as workaround")
+ set(Vc_DEFINITIONS "${Vc_DEFINITIONS} -fno-predictive-commoning")
+ endif()
+
+ vc_check_fpmath()
+ vc_check_assembler()
+ elseif(Vc_COMPILER_IS_INTEL)
+ ##################################################################################################
+ # Intel Compiler #
+ ##################################################################################################
+
+ if(_add_buildtype_flags)
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -DNDEBUG -O3")
+ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3")
+ set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -DNDEBUG -O3")
+
+ set(ALIAS_FLAGS "-no-ansi-alias")
+ if(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+ # default ICC to -no-ansi-alias because otherwise tests/utils_sse fails. So far I suspect a miscompilation...
+ set(ENABLE_STRICT_ALIASING false CACHE BOOL "Enables strict aliasing rules for more aggressive optimizations")
+ if(ENABLE_STRICT_ALIASING)
+ set(ALIAS_FLAGS "-ansi-alias")
+ endif(ENABLE_STRICT_ALIASING)
+ endif()
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ALIAS_FLAGS}")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ALIAS_FLAGS}")
+ endif()
+ vc_add_compiler_flag(Vc_DEFINITIONS "-diag-disable 913")
+ elseif(Vc_COMPILER_IS_MSVC)
+ if(_add_warning_flags)
+ AddCompilerFlag("/wd4800") # Disable warning "forcing value to bool"
+ AddCompilerFlag("/wd4996") # Disable warning about strdup vs. _strdup
+ AddCompilerFlag("/wd4244") # Disable warning "conversion from 'unsigned int' to 'float', possible loss of data"
+ AddCompilerFlag("/wd4146") # Disable warning "unary minus operator applied to unsigned type, result still unsigned"
+ AddCompilerFlag("/wd4227") # Disable warning "anachronism used : qualifiers on reference are ignored" (this is about 'restrict' usage on references, stupid MSVC)
+ AddCompilerFlag("/wd4722") # Disable warning "destructor never returns, potential memory leak" (warns about ~_UnitTest_Global_Object which we don't care about)
+ AddCompilerFlag("/wd4748") # Disable warning "/GS can not protect parameters and local variables from local buffer overrun because optimizations are disabled in function" (I don't get it)
+ add_definitions(-D_CRT_SECURE_NO_WARNINGS)
+ endif()
+
+ # MSVC does not support inline assembly on 64 bit! :(
+ # searching the help for xgetbv doesn't turn up anything. So just fall back to not supporting AVX on Windows :(
+ # TODO: apparently MSVC 2010 SP1 added _xgetbv
+ set(Vc_DEFINITIONS "${Vc_DEFINITIONS} -DVC_NO_XGETBV")
+
+ # get rid of the min/max macros
+ set(Vc_DEFINITIONS "${Vc_DEFINITIONS} -DNOMINMAX")
+ elseif(Vc_COMPILER_IS_CLANG)
+ # for now I don't know of any arguments I want to pass. -march and stuff is tried by OptimizeForArchitecture...
+
+ # disable these warnings because clang shows them for function overloads that were discarded via SFINAE
+ vc_add_compiler_flag(Vc_DEFINITIONS "-Wno-local-type-template-args")
+ vc_add_compiler_flag(Vc_DEFINITIONS "-Wno-unnamed-type-template-args")
+ endif()
+
+ OptimizeForArchitecture()
+ set(Vc_DEFINITIONS "${Vc_ARCHITECTURE_FLAGS} ${Vc_DEFINITIONS}")
+
+ set(VC_IMPL "auto" CACHE STRING "Force the Vc implementation globally to the selected instruction set. \"auto\" lets Vc use the best available instructions.")
+ if(NOT VC_IMPL STREQUAL "auto")
+ set(Vc_DEFINITIONS "${Vc_DEFINITIONS} -DVC_IMPL=${VC_IMPL}")
+ if(NOT VC_IMPL STREQUAL "Scalar")
+ set(_use_var "USE_${VC_IMPL}")
+ if(VC_IMPL STREQUAL "SSE")
+ set(_use_var "USE_SSE2")
+ endif()
+ if(NOT ${_use_var})
+ message(WARNING "The selected value for VC_IMPL (${VC_IMPL}) will not work because the relevant instructions are not enabled via compiler flags.")
+ endif()
+ endif()
+ endif()
+endmacro()
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VECIO_H
+#define VECIO_H
+
+#include "vector.h"
+#include "Memory"
+#include <iostream>
+
+#ifdef __GNUC__
+#include <unistd.h>
+#include <ext/stdio_sync_filebuf.h>
+#endif
+
+#include "internal/namespace.h"
+
+namespace
+{
+ namespace AnsiColor
+ {
+ struct Type { const char *data; };
+ static const Type green = { "\033[1;40;32m" };
+ static const Type yellow = { "\033[1;40;33m" };
+ static const Type blue = { "\033[1;40;34m" };
+ static const Type normal = { "\033[0m" };
+ } // namespace AnsiColor
+
+#ifdef __GNUC__
+ class hacked_ostream : public std::ostream
+ {
+ public:
+ using std::ostream::_M_streambuf;
+ };
+ static bool mayUseColor(const std::ostream &os)
+ __attribute__((__const__));
+ static bool mayUseColor(const std::ostream &os)
+ {
+ std::basic_streambuf<char> *hack1 = const_cast<std::basic_streambuf<char> *>(os.*(&hacked_ostream::_M_streambuf));
+ __gnu_cxx::stdio_sync_filebuf<char> *hack = dynamic_cast<__gnu_cxx::stdio_sync_filebuf<char> *>(hack1);
+ if (!hack) {
+ return false;
+ }
+ FILE *file = hack->file();
+ return 1 == isatty(fileno(file));
+ }
+#else
+ static bool mayUseColor(const std::ostream &) { return false; }
+#endif
+} // anonymous namespace
+
+namespace std
+{
+inline std::ostream &operator<<(std::ostream &out, const AnsiColor::Type &c)
+{
+ if (mayUseColor(out)) {
+ out << c.data;
+ }
+ return out;
+}
+
+template<typename T>
+inline std::ostream &operator<<(std::ostream &out, const VECTOR_NAMESPACE::Vector<T> &v)
+{
+ out << AnsiColor::green << "[";
+ out << v[0];
+ for (int i = 1; i < v.Size; ++i) {
+ out << ", " << v[i];
+ }
+ out << "]" << AnsiColor::normal;
+ return out;
+}
+
+template<>
+inline std::ostream &operator<<(std::ostream &out, const VECTOR_NAMESPACE::Vector<char> &v)
+{
+ out << AnsiColor::green << "[";
+ out << int(v[0]);
+ for (int i = 1; i < v.Size; ++i) {
+ out << ", " << int(v[i]);
+ }
+ out << "]" << AnsiColor::normal;
+ return out;
+}
+template<>
+inline std::ostream &operator<<(std::ostream &out, const VECTOR_NAMESPACE::Vector<unsigned char> &v)
+{
+ out << AnsiColor::green << "[";
+ out << int(v[0]);
+ for (int i = 1; i < v.Size; ++i) {
+ out << ", " << int(v[i]);
+ }
+ out << "]" << AnsiColor::normal;
+ return out;
+}
+
+#ifdef VC_HAVE_FMA
+template<typename T>
+inline std::ostream &operator<<(std::ostream &out, const VECTOR_NAMESPACE::VectorMultiplication<T> &v)
+{
+ return out << VECTOR_NAMESPACE::Vector<T>(v);
+}
+#endif
+
+#ifdef VC_IMPL_AVX
+template<unsigned int VectorSize, size_t RegisterWidth>
+inline std::ostream &operator<<(std::ostream &out, const VECTOR_NAMESPACE::Mask<VectorSize, RegisterWidth> &m)
+#else
+template<unsigned int VectorSize>
+inline std::ostream &operator<<(std::ostream &out, const VECTOR_NAMESPACE::Mask<VectorSize> &m)
+#endif
+{
+ out << AnsiColor::blue << "m[";
+ for (unsigned int i = 0; i < VectorSize; ++i) {
+ if (i > 0 && (i % 4) == 0) {
+ out << " ";
+ }
+ if ( m[i] ) {
+ out << AnsiColor::yellow << '1';
+ } else {
+ out << AnsiColor::blue << '0';
+ }
+ }
+ out << AnsiColor::blue << "]" << AnsiColor::normal;
+ return out;
+}
+#if VC_IMPL_SSE
+inline std::ostream &operator<<(std::ostream &out, const VECTOR_NAMESPACE::Float8Mask &m)
+{
+ out << AnsiColor::blue << "m[";
+ for (unsigned int i = 0; i < 8; ++i) {
+ if (i > 0 && (i % 4) == 0) {
+ out << " ";
+ }
+ if ( m[i] ) {
+ out << AnsiColor::yellow << '1';
+ } else {
+ out << AnsiColor::blue << '0';
+ }
+ }
+ out << AnsiColor::blue << "]" << AnsiColor::normal;
+ return out;
+}
+#endif
+
+template<typename V, typename Parent, typename RM>
+inline std::ostream &operator<<(std::ostream &out, const Vc::MemoryBase<V, Parent, 1, RM> &m )
+{
+ out << AnsiColor::blue << "{" << AnsiColor::normal;
+ for (unsigned int i = 0; i < m.vectorsCount(); ++i) {
+ out << V(m.vector(i));
+ }
+ out << AnsiColor::blue << "}" << AnsiColor::normal;
+ return out;
+}
+
+template<typename V, typename Parent, typename RM>
+inline std::ostream &operator<<(std::ostream &out, const Vc::MemoryBase<V, Parent, 2, RM> &m )
+{
+ out << AnsiColor::blue << "{" << AnsiColor::normal;
+ for (size_t i = 0; i < m.rowsCount(); ++i) {
+ if (i > 0) {
+ out << "\n ";
+ }
+ const size_t vcount = m[i].vectorsCount();
+ for (size_t j = 0; j < vcount; ++j) {
+ out << V(m[i].vector(j));
+ }
+ }
+ out << AnsiColor::blue << "}" << AnsiColor::normal;
+ return out;
+}
+} // namespace std
+
+#undef VECTOR_NAMESPACE
+
+#endif // VECIO_H
+
+// vim: ft=cpp
--- /dev/null
+
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef INCLUDE_VC_MEMORY
+#define INCLUDE_VC_MEMORY
+
+#include "vector.h"
+#include "common/memory.h"
+
+#endif // INCLUDE_VC_MEMORY
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_UTILS
+#define VC_UTILS
+
+#include "global.h"
+
+#if VC_IMPL_Scalar
+# define VECTOR_NAMESPACE Scalar
+#else
+# define VECTOR_NAMESPACE SSE
+#endif
+
+#include "common/deinterleave.h"
+
+#endif // VC_UTILS
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_VC
+#define VC_VC
+#include "vector.h"
+#include "IO"
+#include "Memory"
+#include "Utils"
+#endif // VC_VC
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef AVX_CASTS_H
+#define AVX_CASTS_H
+
+#include "intrinsics.h"
+#include "types.h"
+
+namespace Vc
+{
+namespace AVX
+{
+ template<typename T> static inline INTRINSIC_L T avx_cast(__m128 v) INTRINSIC_R;
+ template<typename T> static inline INTRINSIC_L T avx_cast(__m128i v) INTRINSIC_R;
+ template<typename T> static inline INTRINSIC_L T avx_cast(__m128d v) INTRINSIC_R;
+ template<typename T> static inline INTRINSIC_L T avx_cast(__m256 v) INTRINSIC_R;
+ template<typename T> static inline INTRINSIC_L T avx_cast(__m256i v) INTRINSIC_R;
+ template<typename T> static inline INTRINSIC_L T avx_cast(__m256d v) INTRINSIC_R;
+
+ // 128 -> 128
+ template<> inline __m128 INTRINSIC avx_cast(__m128 v) { return v; }
+ template<> inline __m128 INTRINSIC avx_cast(__m128i v) { return _mm_castsi128_ps(v); }
+ template<> inline __m128 INTRINSIC avx_cast(__m128d v) { return _mm_castpd_ps(v); }
+ template<> inline __m128i INTRINSIC avx_cast(__m128 v) { return _mm_castps_si128(v); }
+ template<> inline __m128i INTRINSIC avx_cast(__m128i v) { return v; }
+ template<> inline __m128i INTRINSIC avx_cast(__m128d v) { return _mm_castpd_si128(v); }
+ template<> inline __m128d INTRINSIC avx_cast(__m128 v) { return _mm_castps_pd(v); }
+ template<> inline __m128d INTRINSIC avx_cast(__m128i v) { return _mm_castsi128_pd(v); }
+ template<> inline __m128d INTRINSIC avx_cast(__m128d v) { return v; }
+
+ // 128 -> 256
+ template<> inline __m256 INTRINSIC avx_cast(__m128 v) { return _mm256_castps128_ps256(v); }
+ template<> inline __m256 INTRINSIC avx_cast(__m128i v) { return _mm256_castps128_ps256(_mm_castsi128_ps(v)); }
+ template<> inline __m256 INTRINSIC avx_cast(__m128d v) { return _mm256_castps128_ps256(_mm_castpd_ps(v)); }
+ template<> inline __m256i INTRINSIC avx_cast(__m128 v) { return _mm256_castsi128_si256(_mm_castps_si128(v)); }
+ template<> inline __m256i INTRINSIC avx_cast(__m128i v) { return _mm256_castsi128_si256(v); }
+ template<> inline __m256i INTRINSIC avx_cast(__m128d v) { return _mm256_castsi128_si256(_mm_castpd_si128(v)); }
+ template<> inline __m256d INTRINSIC avx_cast(__m128 v) { return _mm256_castpd128_pd256(_mm_castps_pd(v)); }
+ template<> inline __m256d INTRINSIC avx_cast(__m128i v) { return _mm256_castpd128_pd256(_mm_castsi128_pd(v)); }
+ template<> inline __m256d INTRINSIC avx_cast(__m128d v) { return _mm256_castpd128_pd256(v); }
+
+ // 256 -> 128
+ template<> inline __m128 INTRINSIC avx_cast(__m256 v) { return _mm256_castps256_ps128(v); }
+ template<> inline __m128 INTRINSIC avx_cast(__m256i v) { return _mm256_castps256_ps128(_mm256_castsi256_ps(v)); }
+ template<> inline __m128 INTRINSIC avx_cast(__m256d v) { return _mm256_castps256_ps128(_mm256_castpd_ps(v)); }
+ template<> inline __m128i INTRINSIC avx_cast(__m256 v) { return _mm256_castsi256_si128(_mm256_castps_si256(v)); }
+ template<> inline __m128i INTRINSIC avx_cast(__m256i v) { return _mm256_castsi256_si128(v); }
+ template<> inline __m128i INTRINSIC avx_cast(__m256d v) { return _mm256_castsi256_si128(_mm256_castpd_si256(v)); }
+ template<> inline __m128d INTRINSIC avx_cast(__m256 v) { return _mm256_castpd256_pd128(_mm256_castps_pd(v)); }
+ template<> inline __m128d INTRINSIC avx_cast(__m256i v) { return _mm256_castpd256_pd128(_mm256_castsi256_pd(v)); }
+ template<> inline __m128d INTRINSIC avx_cast(__m256d v) { return _mm256_castpd256_pd128(v); }
+
+ // 256 -> 256
+ template<> inline __m256 INTRINSIC avx_cast(__m256 v) { return v; }
+ template<> inline __m256 INTRINSIC avx_cast(__m256i v) { return _mm256_castsi256_ps(v); }
+ template<> inline __m256 INTRINSIC avx_cast(__m256d v) { return _mm256_castpd_ps(v); }
+ template<> inline __m256i INTRINSIC avx_cast(__m256 v) { return _mm256_castps_si256(v); }
+ template<> inline __m256i INTRINSIC avx_cast(__m256i v) { return v; }
+ template<> inline __m256i INTRINSIC avx_cast(__m256d v) { return _mm256_castpd_si256(v); }
+ template<> inline __m256d INTRINSIC avx_cast(__m256 v) { return _mm256_castps_pd(v); }
+ template<> inline __m256d INTRINSIC avx_cast(__m256i v) { return _mm256_castsi256_pd(v); }
+ template<> inline __m256d INTRINSIC avx_cast(__m256d v) { return v; }
+
+ // simplify splitting 256-bit registers in 128-bit registers
+ inline __m128 INTRINSIC lo128(__m256 v) { return avx_cast<__m128>(v); }
+ inline __m128d INTRINSIC lo128(__m256d v) { return avx_cast<__m128d>(v); }
+ inline __m128i INTRINSIC lo128(__m256i v) { return avx_cast<__m128i>(v); }
+ inline __m128 INTRINSIC hi128(__m256 v) { return _mm256_extractf128_ps(v, 1); }
+ inline __m128d INTRINSIC hi128(__m256d v) { return _mm256_extractf128_pd(v, 1); }
+ inline __m128i INTRINSIC hi128(__m256i v) { return _mm256_extractf128_si256(v, 1); }
+
+ // simplify combining 128-bit registers in 256-bit registers
+ inline __m256 INTRINSIC concat(__m128 a, __m128 b) { return _mm256_insertf128_ps (avx_cast<__m256 >(a), b, 1); }
+ inline __m256d INTRINSIC concat(__m128d a, __m128d b) { return _mm256_insertf128_pd (avx_cast<__m256d>(a), b, 1); }
+ inline __m256i INTRINSIC concat(__m128i a, __m128i b) { return _mm256_insertf128_si256(avx_cast<__m256i>(a), b, 1); }
+
+ template<typename From, typename To> struct StaticCastHelper {};
+ template<> struct StaticCastHelper<float , int > { static _M256I cast(const _M256 v) { return _mm256_cvttps_epi32(v); } };
+ template<> struct StaticCastHelper<double , int > { static _M256I cast(const _M256D v) { return avx_cast<_M256I>(_mm256_cvttpd_epi32(v)); } };
+ template<> struct StaticCastHelper<int , int > { static _M256I cast(const _M256I v) { return v; } };
+ template<> struct StaticCastHelper<unsigned int , int > { static _M256I cast(const _M256I v) { return v; } };
+ template<> struct StaticCastHelper<short , int > { static _M256I cast(const __m128i v) { return concat(_mm_srai_epi32(_mm_unpacklo_epi16(v, v), 16), _mm_srai_epi32(_mm_unpackhi_epi16(v, v), 16)); } };
+ template<> struct StaticCastHelper<float , unsigned int > { static _M256I cast(const _M256 v) {
+ return _mm256_castps_si256(_mm256_blendv_ps(
+ _mm256_castsi256_ps(_mm256_cvttps_epi32(v)),
+ _mm256_castsi256_ps(_mm256_add_epi32(_mm256_cvttps_epi32(_mm256_sub_ps(v, _mm256_set2power31_ps())), _mm256_set2power31_epu32())),
+ _mm256_cmpge_ps(v, _mm256_set2power31_ps())
+ ));
+
+ } };
+ template<> struct StaticCastHelper<double , unsigned int > { static _M256I cast(const _M256D v) { return avx_cast<_M256I>(_mm256_cvttpd_epi32(v)); } };
+ template<> struct StaticCastHelper<int , unsigned int > { static _M256I cast(const _M256I v) { return v; } };
+ template<> struct StaticCastHelper<unsigned int , unsigned int > { static _M256I cast(const _M256I v) { return v; } };
+ template<> struct StaticCastHelper<unsigned short, unsigned int > { static _M256I cast(const __m128i v) { return concat(_mm_srli_epi32(_mm_unpacklo_epi16(v, v), 16), _mm_srli_epi32(_mm_unpackhi_epi16(v, v), 16)); } };
+ template<> struct StaticCastHelper<float , float > { static _M256 cast(const _M256 v) { return v; } };
+ template<> struct StaticCastHelper<double , float > { static _M256 cast(const _M256D v) { return avx_cast<_M256>(_mm256_cvtpd_ps(v)); } };
+ template<> struct StaticCastHelper<int , float > { static _M256 cast(const _M256I v) { return _mm256_cvtepi32_ps(v); } };
+ template<> struct StaticCastHelper<unsigned int , float > { static _M256 cast(const _M256I v) {
+ return _mm256_blendv_ps(
+ _mm256_cvtepi32_ps(v),
+ _mm256_add_ps(_mm256_cvtepi32_ps(_mm256_sub_epi32(v, _mm256_set2power31_epu32())), _mm256_set2power31_ps()),
+ _mm256_castsi256_ps(_mm256_cmplt_epi32(v, _mm256_setzero_si256()))
+ );
+ } };
+ template<> struct StaticCastHelper<short , float > { static _M256 cast(const __m128i v) { return _mm256_cvtepi32_ps(StaticCastHelper<short, int>::cast(v)); } };
+ template<> struct StaticCastHelper<unsigned short, float > { static _M256 cast(const __m128i v) { return _mm256_cvtepi32_ps(StaticCastHelper<unsigned short, unsigned int>::cast(v)); } };
+ template<> struct StaticCastHelper<float , double > { static _M256D cast(const _M256 v) { return _mm256_cvtps_pd(avx_cast<__m128>(v)); } };
+ template<> struct StaticCastHelper<double , double > { static _M256D cast(const _M256D v) { return v; } };
+ template<> struct StaticCastHelper<int , double > { static _M256D cast(const _M256I v) { return _mm256_cvtepi32_pd(avx_cast<__m128i>(v)); } };
+ template<> struct StaticCastHelper<unsigned int , double > { static _M256D cast(const _M256I v) { return _mm256_cvtepi32_pd(avx_cast<__m128i>(v)); } };
+ template<> struct StaticCastHelper<int , short > { static __m128i cast(const _M256I v) { return _mm_packs_epi32(lo128(v), hi128(v)); } };
+ template<> struct StaticCastHelper<float , short > { static __m128i cast(const _M256 v) { return StaticCastHelper<int, short>::cast(StaticCastHelper<float, int>::cast(v)); } };
+ template<> struct StaticCastHelper<short , short > { static __m128i cast(const __m128i v) { return v; } };
+ template<> struct StaticCastHelper<unsigned short, short > { static __m128i cast(const __m128i v) { return v; } };
+ template<> struct StaticCastHelper<unsigned int , unsigned short> { static __m128i cast(const _M256I v) { return _mm_packus_epi32(lo128(v), hi128(v)); } };
+ template<> struct StaticCastHelper<float , unsigned short> { static __m128i cast(const _M256 v) { return StaticCastHelper<unsigned int, unsigned short>::cast(StaticCastHelper<float, unsigned int>::cast(v)); } };
+ template<> struct StaticCastHelper<short , unsigned short> { static __m128i cast(const __m128i v) { return v; } };
+ template<> struct StaticCastHelper<unsigned short, unsigned short> { static __m128i cast(const __m128i v) { return v; } };
+ template<> struct StaticCastHelper<sfloat , short > { static __m128i cast(const _M256 v) { return StaticCastHelper<int, short>::cast(StaticCastHelper<float, int>::cast(v)); } };
+ template<> struct StaticCastHelper<sfloat , unsigned short> { static __m128i cast(const _M256 v) { return StaticCastHelper<unsigned int, unsigned short>::cast(StaticCastHelper<float, unsigned int>::cast(v)); } };
+ template<> struct StaticCastHelper<short , sfloat > { static _M256 cast(const __m128i v) { return _mm256_cvtepi32_ps(StaticCastHelper<short, int>::cast(v)); } };
+ template<> struct StaticCastHelper<unsigned short, sfloat > { static _M256 cast(const __m128i v) { return _mm256_cvtepi32_ps(StaticCastHelper<unsigned short, unsigned int>::cast(v)); } };
+} // namespace AVX
+} // namespace Vc
+
+#endif // AVX_CASTS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_CONST_H
+#define VC_AVX_CONST_H
+
+#include <cstddef>
+#include "const_data.h"
+#include "macros.h"
+
+namespace Vc
+{
+namespace AVX
+{
+ template<typename T> class Vector;
+
+ template<typename T> struct IndexesFromZeroData;
+ template<> struct IndexesFromZeroData<int> {
+ static const int *address() { return reinterpret_cast<const int *>(&_IndexesFromZero32[0]); }
+ };
+ template<> struct IndexesFromZeroData<unsigned int> {
+ static const unsigned int *address() { return &_IndexesFromZero32[0]; }
+ };
+ template<> struct IndexesFromZeroData<short> {
+ static const short *address() { return reinterpret_cast<const short *>(&_IndexesFromZero16[0]); }
+ };
+ template<> struct IndexesFromZeroData<unsigned short> {
+ static const unsigned short *address() { return &_IndexesFromZero16[0]; }
+ };
+ template<> struct IndexesFromZeroData<signed char> {
+ static const signed char *address() { return reinterpret_cast<const signed char *>(&_IndexesFromZero8[0]); }
+ };
+ template<> struct IndexesFromZeroData<char> {
+ static const char *address() { return reinterpret_cast<const char *>(&_IndexesFromZero8[0]); }
+ };
+ template<> struct IndexesFromZeroData<unsigned char> {
+ static const unsigned char *address() { return &_IndexesFromZero8[0]; }
+ };
+
+ template<typename T> struct Const
+ {
+ typedef Vector<T> V;
+ typedef typename V::Mask M;
+
+ static inline V CONST_L _1_2pi() CONST_R { return V(c_sin<T>::data[0]); }
+ static inline V CONST_L _2pi() CONST_R { return V(c_sin<T>::data[1]); }
+ static inline V CONST_L _pi_2() CONST_R { return V(c_sin<T>::data[2]); }
+ static inline V CONST_L _pi() CONST_R { return V(c_sin<T>::data[3]); }
+ static inline V CONST_L _1_3fac() CONST_R { return V(c_sin<T>::data[4]); }
+ static inline V CONST_L _1_5fac() CONST_R { return V(c_sin<T>::data[5]); }
+ static inline V CONST_L _1_7fac() CONST_R { return V(c_sin<T>::data[6]); }
+ static inline V CONST_L _1_9fac() CONST_R { return V(c_sin<T>::data[7]); }
+
+ static inline M CONST_L exponentMask() CONST_R { return M(V(c_log<T>::d(1)).data()); }
+ static inline V CONST_L _1_2() CONST_R { return V(c_log<T>::d(18)); }
+ static inline V CONST_L _1_sqrt2() CONST_R { return V(c_log<T>::d(15)); }
+ static inline V CONST_L P(int i) CONST_R { return V(c_log<T>::d(2 + i)); }
+ static inline V CONST_L Q(int i) CONST_R { return V(c_log<T>::d(8 + i)); }
+ static inline V CONST_L min() CONST_R { return V(c_log<T>::d(14)); }
+ static inline V CONST_L ln2_small() CONST_R { return V(c_log<T>::d(17)); }
+ static inline V CONST_L ln2_large() CONST_R { return V(c_log<T>::d(16)); }
+ static inline V CONST_L neginf() CONST_R { return V(c_log<T>::d(13)); }
+ static inline V CONST_L log10_e() CONST_R { return V(c_log<T>::d(19)); }
+ static inline V CONST_L log2_e() CONST_R { return V(c_log<T>::d(20)); }
+ };
+
+ template<> struct Const<sfloat>
+ {
+ typedef sfloat_v V;
+ typedef V::Mask M;
+
+ static inline V CONST_L _1_2pi() CONST_R { return V(c_sin<float>::data[0]); }
+ static inline V CONST_L _2pi() CONST_R { return V(c_sin<float>::data[1]); }
+ static inline V CONST_L _pi_2() CONST_R { return V(c_sin<float>::data[2]); }
+ static inline V CONST_L _pi() CONST_R { return V(c_sin<float>::data[3]); }
+ static inline V CONST_L _1_3fac() CONST_R { return V(c_sin<float>::data[4]); }
+ static inline V CONST_L _1_5fac() CONST_R { return V(c_sin<float>::data[5]); }
+ static inline V CONST_L _1_7fac() CONST_R { return V(c_sin<float>::data[6]); }
+ static inline V CONST_L _1_9fac() CONST_R { return V(c_sin<float>::data[7]); }
+
+ static inline M CONST_L exponentMask() CONST_R { return M(V(c_log<float>::d(1)).data()); }
+ static inline V CONST_L _1_2() CONST_R { return V(c_log<float>::d(18)); }
+ static inline V CONST_L _1_sqrt2() CONST_R { return V(c_log<float>::d(15)); }
+ static inline V CONST_L P(int i) CONST_R { return V(c_log<float>::d(2 + i)); }
+ static inline V CONST_L Q(int i) CONST_R { return V(c_log<float>::d(8 + i)); }
+ static inline V CONST_L min() CONST_R { return V(c_log<float>::d(14)); }
+ static inline V CONST_L ln2_small() CONST_R { return V(c_log<float>::d(17)); }
+ static inline V CONST_L ln2_large() CONST_R { return V(c_log<float>::d(16)); }
+ static inline V CONST_L neginf() CONST_R { return V(c_log<float>::d(13)); }
+ static inline V CONST_L log10_e() CONST_R { return V(c_log<float>::d(19)); }
+ static inline V CONST_L log2_e() CONST_R { return V(c_log<float>::d(20)); }
+ };
+} // namespace AVX
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_AVX_CONST_H
--- /dev/null
+/* This file is part of the Vc library. {{{
+
+ Copyright (C) 2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+}}}*/
+
+#ifndef VC_AVX_CONST_DATA_H
+#define VC_AVX_CONST_DATA_H
+
+#include "macros.h"
+namespace Vc
+{
+namespace AVX
+{
+
+ALIGN(64) extern const unsigned int _IndexesFromZero32[8];
+ALIGN(16) extern const unsigned short _IndexesFromZero16[8];
+ALIGN(16) extern const unsigned char _IndexesFromZero8[16];
+
+struct STRUCT_ALIGN1(64) c_general
+{
+ static const float oneFloat;
+ static const unsigned int absMaskFloat[2];
+ static const unsigned int signMaskFloat[2];
+ static const unsigned short minShort[2];
+ static const unsigned short one16[2];
+ static const float _2power31;
+ static const double oneDouble;
+ static const unsigned long long frexpMask;
+} STRUCT_ALIGN2(64);
+
+template<typename T> struct c_sin
+{
+ ALIGN(64) static const T data[];
+};
+
+template<typename T> struct c_log
+{
+ typedef float floatAlias MAY_ALIAS;
+ static inline float d(int i) { return *reinterpret_cast<const floatAlias *>(&data[i]); }
+ ALIGN(64) static const unsigned int data[];
+};
+
+template<> struct c_log<double>
+{
+ enum VectorSize { Size = 16 / sizeof(double) };
+ typedef double doubleAlias MAY_ALIAS;
+ static inline double d(int i) { return *reinterpret_cast<const doubleAlias *>(&data[i]); }
+ ALIGN(64) static const unsigned long long data[];
+};
+
+} // namespace AVX
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_AVX_CONST_DATA_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_DEBUG_H
+#define VC_AVX_DEBUG_H
+
+#ifndef NDEBUG
+#include "vectorbase.h"
+#include <iostream>
+#include <iomanip>
+#endif
+
+namespace Vc
+{
+namespace AVX
+{
+
+#ifdef NDEBUG
+class DebugStream
+{
+ public:
+ DebugStream(const char *, const char *, int) {}
+ template<typename T> inline DebugStream &operator<<(const T &) { return *this; }
+};
+#else
+class DebugStream
+{
+ private:
+ template<typename T, typename V> static void printVector(V _x)
+ {
+ enum { Size = sizeof(V) / sizeof(T) };
+ union { V v; T m[Size]; } x = { _x };
+ std::cerr << '[' << std::setprecision(24) << x.m[0];
+ for (int i = 1; i < Size; ++i) {
+ std::cerr << ", " << std::setprecision(24) << x.m[i];
+ }
+ std::cerr << ']';
+ }
+ public:
+ DebugStream(const char *func, const char *file, int line)
+ {
+ std::cerr << "\033[1;40;33mDEBUG: " << file << ':' << line << ' ' << func << ' ';
+ }
+
+ template<typename T> DebugStream &operator<<(const T &x) { std::cerr << x; return *this; }
+
+ DebugStream &operator<<(__m128 x) {
+ printVector<float, __m128>(x);
+ return *this;
+ }
+ DebugStream &operator<<(__m256 x) {
+ printVector<float, __m256>(x);
+ return *this;
+ }
+ DebugStream &operator<<(__m128d x) {
+ printVector<double, __m128d>(x);
+ return *this;
+ }
+ DebugStream &operator<<(__m256d x) {
+ printVector<double, __m256d>(x);
+ return *this;
+ }
+ DebugStream &operator<<(__m128i x) {
+ printVector<unsigned int, __m128i>(x);
+ return *this;
+ }
+ DebugStream &operator<<(__m256i x) {
+ printVector<unsigned int, __m256i>(x);
+ return *this;
+ }
+
+ ~DebugStream()
+ {
+ std::cerr << "\033[0m" << std::endl;
+ }
+};
+#endif
+
+#define VC_DEBUG ::Vc::AVX::DebugStream(__PRETTY_FUNCTION__, __FILE__, __LINE__)
+
+} // namespace AVX
+} // namespace Vc
+
+#endif // VC_AVX_DEBUG_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "macros.h"
+
+namespace Vc
+{
+namespace AVX
+{
+
+inline void deinterleave(double_v &VC_RESTRICT a, double_v &VC_RESTRICT b, double_v &VC_RESTRICT c)
+{ // estimated latency (AVX): 4.5 cycles
+ const __m256d tmp0 = Mem::shuffle128<X0, Y1>(a.data(), b.data());
+ const __m256d tmp1 = Mem::shuffle128<X1, Y0>(a.data(), c.data());
+ const __m256d tmp2 = Mem::shuffle128<X0, Y1>(b.data(), c.data());
+ a.data() = Mem::shuffle<X0, Y1, X2, Y3>(tmp0, tmp1);
+ b.data() = Mem::shuffle<X1, Y0, X3, Y2>(tmp0, tmp2);
+ c.data() = Mem::shuffle<X0, Y1, X2, Y3>(tmp1, tmp2);
+}
+
+inline void deinterleave(float_v &VC_RESTRICT a, float_v &VC_RESTRICT b, float_v &VC_RESTRICT c)
+{
+ // abc abc abc
+ // a = [a0 b0 c0 a1 b1 c1 a2 b2] 332 = 211+121
+ // b = [c2 a3 b3 c3 a4 b4 c4 a5] 323 = 112+211
+ // c = [b5 c5 a6 b6 c6 a7 b7 c7] 233 = 121+112
+ const __m256 ac0 = Mem::shuffle128<X0, Y0>(a.data(), c.data()); // a0 b0 c0 a1 b5 c5 a6 b6
+ const __m256 ac1 = Mem::shuffle128<X1, Y1>(a.data(), c.data()); // b1 c1 a2 b2 c6 a7 b7 c7
+
+ __m256 tmp0 = Mem::blend<X0, Y1, X2, X3, Y4, X5, X6, Y7>( ac0, b.data());
+ tmp0 = Mem::blend<X0, X1, Y2, X3, X4, Y5, X6, X7>(tmp0, ac1); // a0 a3 a2 a1 a4 a7 a6 a5
+ __m256 tmp1 = Mem::blend<X0, X1, Y2, X3, X4, Y5, X6, X7>( ac0, b.data());
+ tmp1 = Mem::blend<Y0, X1, X2, Y3, X4, X5, Y6, X7>(tmp1, ac1); // b1 b0 b3 b2 b5 b4 b7 b6
+ __m256 tmp2 = Mem::blend<Y0, X1, X2, Y3, X4, X5, Y6, X7>( ac0, b.data());
+ tmp2 = Mem::blend<X0, Y1, X2, X3, Y4, X5, X6, Y7>(tmp2, ac1); // c2 c1 c0 c3 c6 c5 c4 c7
+
+ a.data() = Mem::permute<X0, X3, X2, X1>(tmp0);
+ b.data() = Mem::permute<X1, X0, X3, X2>(tmp1);
+ c.data() = Mem::permute<X2, X1, X0, X3>(tmp2);
+}
+
+inline void deinterleave(int_v &VC_RESTRICT a, int_v &VC_RESTRICT b, int_v &VC_RESTRICT c)
+{
+ deinterleave(reinterpret_cast<float_v &>(a), reinterpret_cast<float_v &>(b),
+ reinterpret_cast<float_v &>(c));
+}
+
+inline void deinterleave(uint_v &VC_RESTRICT a, uint_v &VC_RESTRICT b, uint_v &VC_RESTRICT c)
+{
+ deinterleave(reinterpret_cast<float_v &>(a), reinterpret_cast<float_v &>(b),
+ reinterpret_cast<float_v &>(c));
+}
+
+inline void deinterleave(Vector<short> &VC_RESTRICT a, Vector<short> &VC_RESTRICT b,
+ Vector<short> &VC_RESTRICT c)
+{
+ // abc abc abc
+ // a = [a0 b0 c0 a1 b1 c1 a2 b2] 332 = 211+121
+ // b = [c2 a3 b3 c3 a4 b4 c4 a5] 323 = 112+211
+ // c = [b5 c5 a6 b6 c6 a7 b7 c7] 233 = 121+112
+ __m128i ac0 = _mm_unpacklo_epi64(a.data(), c.data()); // a0 b0 c0 a1 b5 c5 a6 b6
+ __m128i ac1 = _mm_unpackhi_epi64(a.data(), c.data()); // b1 c1 a2 b2 c6 a7 b7 c7
+
+ __m128i tmp0 = Mem::blend<X0, Y1, X2, X3, Y4, X5, X6, Y7>( ac0, b.data());
+ tmp0 = Mem::blend<X0, X1, Y2, X3, X4, Y5, X6, X7>(tmp0, ac1); // a0 a3 a2 a1 a4 a7 a6 a5
+ __m128i tmp1 = Mem::blend<X0, X1, Y2, X3, X4, Y5, X6, X7>( ac0, b.data());
+ tmp1 = Mem::blend<Y0, X1, X2, Y3, X4, X5, Y6, X7>(tmp1, ac1); // b1 b0 b3 b2 b5 b4 b7 b6
+ __m128i tmp2 = Mem::blend<Y0, X1, X2, Y3, X4, X5, Y6, X7>( ac0, b.data());
+ tmp2 = Mem::blend<X0, Y1, X2, X3, Y4, X5, X6, Y7>(tmp2, ac1); // c2 c1 c0 c3 c6 c5 c4 c7
+
+ a.data() = Mem::permuteHi<X4, X7, X6, X5>(Mem::permuteLo<X0, X3, X2, X1>(tmp0));
+ b.data() = Mem::permuteHi<X5, X4, X7, X6>(Mem::permuteLo<X1, X0, X3, X2>(tmp1));
+ c.data() = Mem::permuteHi<X6, X5, X4, X7>(Mem::permuteLo<X2, X1, X0, X3>(tmp2));
+}
+
+inline void deinterleave(Vector<unsigned short> &VC_RESTRICT a, Vector<unsigned short> &VC_RESTRICT b,
+ Vector<unsigned short> &VC_RESTRICT c)
+{
+ deinterleave(reinterpret_cast<Vector<short> &>(a), reinterpret_cast<Vector<short> &>(b),
+ reinterpret_cast<Vector<short> &>(c));
+}
+
+inline void deinterleave(Vector<float> &a, Vector<float> &b)
+{
+ // a7 a6 a5 a4 a3 a2 a1 a0
+ // b7 b6 b5 b4 b3 b2 b1 b0
+ const _M256 tmp0 = Reg::permute128<Y0, X0>(a.data(), b.data()); // b3 b2 b1 b0 a3 a2 a1 a0
+ const _M256 tmp1 = Reg::permute128<Y1, X1>(a.data(), b.data()); // b7 b6 b5 b4 a7 a6 a5 a4
+
+ const _M256 tmp2 = _mm256_unpacklo_ps(tmp0, tmp1); // b5 b1 b4 b0 a5 a1 a4 a0
+ const _M256 tmp3 = _mm256_unpackhi_ps(tmp0, tmp1); // b7 b3 b6 b2 a7 a3 a6 a2
+
+ a.data() = _mm256_unpacklo_ps(tmp2, tmp3); // b6 b4 b2 b0 a6 a4 a2 a0
+ b.data() = _mm256_unpackhi_ps(tmp2, tmp3); // b7 b5 b3 b1 a7 a5 a3 a1
+}
+
+inline void deinterleave(Vector<short> &a, Vector<short> &b)
+{
+ __m128i tmp0 = _mm_unpacklo_epi16(a.data(), b.data()); // a0 a4 b0 b4 a1 a5 b1 b5
+ __m128i tmp1 = _mm_unpackhi_epi16(a.data(), b.data()); // a2 a6 b2 b6 a3 a7 b3 b7
+ __m128i tmp2 = _mm_unpacklo_epi16(tmp0, tmp1); // a0 a2 a4 a6 b0 b2 b4 b6
+ __m128i tmp3 = _mm_unpackhi_epi16(tmp0, tmp1); // a1 a3 a5 a7 b1 b3 b5 b7
+ a.data() = _mm_unpacklo_epi16(tmp2, tmp3);
+ b.data() = _mm_unpackhi_epi16(tmp2, tmp3);
+}
+
+inline void deinterleave(Vector<unsigned short> &a, Vector<unsigned short> &b)
+{
+ __m128i tmp0 = _mm_unpacklo_epi16(a.data(), b.data()); // a0 a4 b0 b4 a1 a5 b1 b5
+ __m128i tmp1 = _mm_unpackhi_epi16(a.data(), b.data()); // a2 a6 b2 b6 a3 a7 b3 b7
+ __m128i tmp2 = _mm_unpacklo_epi16(tmp0, tmp1); // a0 a2 a4 a6 b0 b2 b4 b6
+ __m128i tmp3 = _mm_unpackhi_epi16(tmp0, tmp1); // a1 a3 a5 a7 b1 b3 b5 b7
+ a.data() = _mm_unpacklo_epi16(tmp2, tmp3);
+ b.data() = _mm_unpackhi_epi16(tmp2, tmp3);
+}
+
+} // namespace AVX
+
+
+namespace Internal
+{
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ float_v &a, float_v &b, const float *m, A align)
+{
+ a.load(m, align);
+ b.load(m + float_v::Size, align);
+ Vc::AVX::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ float_v &a, float_v &b, const short *m, A align)
+{
+ const __m256i tmp = Vc::AVX::VectorHelper<__m256i>::load(m, align);
+ a.data() = _mm256_cvtepi32_ps(Vc::AVX::concat(
+ _mm_srai_epi32(_mm_slli_epi32(AVX::lo128(tmp), 16), 16),
+ _mm_srai_epi32(_mm_slli_epi32(AVX::hi128(tmp), 16), 16)));
+ b.data() = _mm256_cvtepi32_ps(Vc::AVX::concat(
+ _mm_srai_epi32(AVX::lo128(tmp), 16),
+ _mm_srai_epi32(AVX::hi128(tmp), 16)));
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ float_v &a, float_v &b, const unsigned short *m, A align)
+{
+ const __m256i tmp = Vc::AVX::VectorHelper<__m256i>::load(m, align);
+ a.data() = _mm256_cvtepi32_ps(Vc::AVX::concat(
+ _mm_blend_epi16(AVX::lo128(tmp), _mm_setzero_si128(), 0xaa),
+ _mm_blend_epi16(AVX::hi128(tmp), _mm_setzero_si128(), 0xaa)));
+ b.data() = _mm256_cvtepi32_ps(Vc::AVX::concat(
+ _mm_srli_epi32(AVX::lo128(tmp), 16),
+ _mm_srli_epi32(AVX::hi128(tmp), 16)));
+}
+
+template<typename A, typename MemT> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ sfloat_v &_a, sfloat_v &_b, const MemT *m, A align)
+{
+ float_v &a = reinterpret_cast<float_v &>(_a);
+ float_v &b = reinterpret_cast<float_v &>(_b);
+ HelperImpl<Vc::AVXImpl>::deinterleave(a, b, m, align);
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ double_v &a, double_v &b, const double *m, A align)
+{
+ a.load(m, align);
+ b.load(m + double_v::Size, align);
+
+ __m256d tmp0 = Mem::shuffle128<Vc::X0, Vc::Y0>(a.data(), b.data()); // b1 b0 a1 a0
+ __m256d tmp1 = Mem::shuffle128<Vc::X1, Vc::Y1>(a.data(), b.data()); // b3 b2 a3 a2
+
+ a.data() = _mm256_unpacklo_pd(tmp0, tmp1); // b2 b0 a2 a0
+ b.data() = _mm256_unpackhi_pd(tmp0, tmp1); // b3 b1 a3 a1
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ int_v &a, int_v &b, const int *m, A align)
+{
+ a.load(m, align);
+ b.load(m + int_v::Size, align);
+
+ const __m256 tmp0 = AVX::avx_cast<__m256>(Mem::shuffle128<Vc::X0, Vc::Y0>(a.data(), b.data()));
+ const __m256 tmp1 = AVX::avx_cast<__m256>(Mem::shuffle128<Vc::X1, Vc::Y1>(a.data(), b.data()));
+
+ const __m256 tmp2 = _mm256_unpacklo_ps(tmp0, tmp1); // b5 b1 b4 b0 a5 a1 a4 a0
+ const __m256 tmp3 = _mm256_unpackhi_ps(tmp0, tmp1); // b7 b3 b6 b2 a7 a3 a6 a2
+
+ a.data() = AVX::avx_cast<__m256i>(_mm256_unpacklo_ps(tmp2, tmp3)); // b6 b4 b2 b0 a6 a4 a2 a0
+ b.data() = AVX::avx_cast<__m256i>(_mm256_unpackhi_ps(tmp2, tmp3)); // b7 b5 b3 b1 a7 a5 a3 a1
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ int_v &a, int_v &b, const short *m, A align)
+{
+ const __m256i tmp = Vc::AVX::VectorHelper<__m256i>::load(m, align);
+ a.data() = Vc::AVX::concat(
+ _mm_srai_epi32(_mm_slli_epi32(AVX::lo128(tmp), 16), 16),
+ _mm_srai_epi32(_mm_slli_epi32(AVX::hi128(tmp), 16), 16));
+ b.data() = Vc::AVX::concat(
+ _mm_srai_epi32(AVX::lo128(tmp), 16),
+ _mm_srai_epi32(AVX::hi128(tmp), 16));
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ uint_v &a, uint_v &b, const unsigned int *m, A align)
+{
+ a.load(m, align);
+ b.load(m + uint_v::Size, align);
+
+ const __m256 tmp0 = AVX::avx_cast<__m256>(Mem::shuffle128<Vc::X0, Vc::Y0>(a.data(), b.data()));
+ const __m256 tmp1 = AVX::avx_cast<__m256>(Mem::shuffle128<Vc::X1, Vc::Y1>(a.data(), b.data()));
+
+ const __m256 tmp2 = _mm256_unpacklo_ps(tmp0, tmp1); // b5 b1 b4 b0 a5 a1 a4 a0
+ const __m256 tmp3 = _mm256_unpackhi_ps(tmp0, tmp1); // b7 b3 b6 b2 a7 a3 a6 a2
+
+ a.data() = AVX::avx_cast<__m256i>(_mm256_unpacklo_ps(tmp2, tmp3)); // b6 b4 b2 b0 a6 a4 a2 a0
+ b.data() = AVX::avx_cast<__m256i>(_mm256_unpackhi_ps(tmp2, tmp3)); // b7 b5 b3 b1 a7 a5 a3 a1
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ uint_v &a, uint_v &b, const unsigned short *m, A align)
+{
+ const __m256i tmp = Vc::AVX::VectorHelper<__m256i>::load(m, align);
+ a.data() = Vc::AVX::concat(
+ _mm_srli_epi32(_mm_slli_epi32(AVX::lo128(tmp), 16), 16),
+ _mm_srli_epi32(_mm_slli_epi32(AVX::hi128(tmp), 16), 16));
+ b.data() = Vc::AVX::concat(
+ _mm_srli_epi32(AVX::lo128(tmp), 16),
+ _mm_srli_epi32(AVX::hi128(tmp), 16));
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ short_v &a, short_v &b, const short *m, A align)
+{
+ a.load(m, align);
+ b.load(m + short_v::Size, align);
+ Vc::AVX::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::AVXImpl>::deinterleave(
+ ushort_v &a, ushort_v &b, const unsigned short *m, A align)
+{
+ a.load(m, align);
+ b.load(m + ushort_v::Size, align);
+ Vc::AVX::deinterleave(a, b);
+}
+
+// only support M == V::EntryType -> no specialization
+template<typename V, typename M, typename A>
+inline void HelperImpl<Vc::AVXImpl>::deinterleave(V &VC_RESTRICT a, V &VC_RESTRICT b,
+ V &VC_RESTRICT c, const M *VC_RESTRICT memory, A align)
+{
+ a.load(&memory[0 * V::Size], align);
+ b.load(&memory[1 * V::Size], align);
+ c.load(&memory[2 * V::Size], align);
+ Vc::AVX::deinterleave(a, b, c);
+}
+
+} // namespace Internal
+} // namespace Vc
--- /dev/null
+#ifdef __GNUC__
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x1.data()));
+}
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x1.data()));
+}
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x2.data()), "x"(x1.data()));
+}
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T6> &x6, const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x6.data()), "x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T6> &x6, Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x6.data()), "+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T7> &x7, const Vector<T6> &x6, const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x7.data()), "x"(x6.data()), "x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T7> &x7, Vector<T6> &x6, Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x7.data()), "+x"(x6.data()), "+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T8> &x8, const Vector<T7> &x7, const Vector<T6> &x6, const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x8.data()), "x"(x7.data()), "x"(x6.data()), "x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T8> &x8, Vector<T7> &x7, Vector<T6> &x6, Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x8.data()), "+x"(x7.data()), "+x"(x6.data()), "+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+#elif defined(VC_MSVC)
+#pragma optimize("g", off)
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T6> &/*x6*/, const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T6> &/*x6*/, Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T7> &/*x7*/, const Vector<T6> &/*x6*/, const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T7> &/*x7*/, Vector<T6> &/*x6*/, Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T8> &/*x8*/, const Vector<T7> &/*x7*/, const Vector<T6> &/*x6*/, const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T8> &/*x8*/, Vector<T7> &/*x7*/, Vector<T6> &/*x6*/, Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#else
+#error "forceToRegisters unsupported on this compiler"
+#endif
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_HELPERIMPL_H
+#define VC_AVX_HELPERIMPL_H
+
+#include "macros.h"
+
+namespace Vc
+{
+namespace Internal
+{
+
+template<> struct HelperImpl<Vc::AVXImpl>
+{
+ typedef AVX::Vector<float> float_v;
+ typedef AVX::Vector<sfloat> sfloat_v;
+ typedef AVX::Vector<double> double_v;
+ typedef AVX::Vector<int> int_v;
+ typedef AVX::Vector<unsigned int> uint_v;
+ typedef AVX::Vector<short> short_v;
+ typedef AVX::Vector<unsigned short> ushort_v;
+
+ template<typename A> static void deinterleave(float_v &, float_v &, const float *, A);
+ template<typename A> static void deinterleave(float_v &, float_v &, const short *, A);
+ template<typename A> static void deinterleave(float_v &, float_v &, const unsigned short *, A);
+
+ template<typename A, typename MemT> static void deinterleave(sfloat_v &, sfloat_v &, const MemT *, A);
+
+ template<typename A> static void deinterleave(double_v &, double_v &, const double *, A);
+
+ template<typename A> static void deinterleave(int_v &, int_v &, const int *, A);
+ template<typename A> static void deinterleave(int_v &, int_v &, const short *, A);
+
+ template<typename A> static void deinterleave(uint_v &, uint_v &, const unsigned int *, A);
+ template<typename A> static void deinterleave(uint_v &, uint_v &, const unsigned short *, A);
+
+ template<typename A> static void deinterleave(short_v &, short_v &, const short *, A);
+
+ template<typename A> static void deinterleave(ushort_v &, ushort_v &, const unsigned short *, A);
+
+ template<typename V, typename M, typename A>
+ static inline void deinterleave(V &VC_RESTRICT a, V &VC_RESTRICT b,
+ V &VC_RESTRICT c, const M *VC_RESTRICT memory, A align);
+
+ template<typename V, typename M, typename A>
+ static inline void deinterleave(V &VC_RESTRICT a, V &VC_RESTRICT b,
+ V &VC_RESTRICT c, V &VC_RESTRICT d,
+ const M *VC_RESTRICT memory, A align);
+
+ template<typename V, typename M, typename A>
+ static inline void deinterleave(V &VC_RESTRICT a, V &VC_RESTRICT b,
+ V &VC_RESTRICT c, V &VC_RESTRICT d, V &VC_RESTRICT e,
+ const M *VC_RESTRICT memory, A align);
+
+ template<typename V, typename M, typename A>
+ static inline void deinterleave(V &VC_RESTRICT a, V &VC_RESTRICT b,
+ V &VC_RESTRICT c, V &VC_RESTRICT d, V &VC_RESTRICT e,
+ V &VC_RESTRICT f, const M *VC_RESTRICT memory, A align);
+
+ template<typename V, typename M, typename A>
+ static inline void deinterleave(V &VC_RESTRICT a, V &VC_RESTRICT b,
+ V &VC_RESTRICT c, V &VC_RESTRICT d, V &VC_RESTRICT e,
+ V &VC_RESTRICT f, V &VC_RESTRICT g, V &VC_RESTRICT h,
+ const M *VC_RESTRICT memory, A align);
+
+ static inline void ALWAYS_INLINE_L prefetchForOneRead(const void *addr) ALWAYS_INLINE_R;
+ static inline void ALWAYS_INLINE_L prefetchForModify(const void *addr) ALWAYS_INLINE_R;
+ static inline void ALWAYS_INLINE_L prefetchClose(const void *addr) ALWAYS_INLINE_R;
+ static inline void ALWAYS_INLINE_L prefetchMid(const void *addr) ALWAYS_INLINE_R;
+ static inline void ALWAYS_INLINE_L prefetchFar(const void *addr) ALWAYS_INLINE_R;
+
+ template<Vc::MallocAlignment A>
+ static inline ALWAYS_INLINE_L void *malloc(size_t n) ALWAYS_INLINE_R;
+ static inline ALWAYS_INLINE_L void free(void *p) ALWAYS_INLINE_R;
+};
+
+} // namespace Internal
+} // namespace Vc
+
+#include "undomacros.h"
+#include "deinterleave.tcc"
+#include "prefetches.tcc"
+#include "helperimpl.tcc"
+
+#endif // VC_AVX_HELPERIMPL_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_HELPERIMPL_TCC
+#define VC_AVX_HELPERIMPL_TCC
+
+namespace Vc
+{
+namespace Internal
+{
+
+template<size_t X>
+static inline size_t nextMultipleOf(size_t value)
+{
+ const size_t offset = value % X;
+ if ( offset > 0 ) {
+ return value + X - offset;
+ }
+ return value;
+}
+
+template<Vc::MallocAlignment A>
+inline ALWAYS_INLINE void *HelperImpl<AVXImpl>::malloc(size_t n)
+{
+ switch (A) {
+ case Vc::AlignOnVector:
+ return _mm_malloc(nextMultipleOf<Vc::AVX::VectorAlignment>(n), Vc::AVX::VectorAlignment);
+ case Vc::AlignOnCacheline:
+ // TODO: hardcoding 64 is not such a great idea
+ return _mm_malloc(nextMultipleOf<64>(n), 64);
+ case Vc::AlignOnPage:
+ // TODO: hardcoding 4096 is not such a great idea
+ return _mm_malloc(nextMultipleOf<4096>(n), 4096);
+ default:
+#ifndef NDEBUG
+ abort();
+#endif
+ return _mm_malloc(n, 8);
+ }
+}
+
+inline ALWAYS_INLINE void HelperImpl<AVXImpl>::free(void *p)
+{
+ _mm_free(p);
+}
+
+} // namespace Internal
+} // namespace Vc
+
+#endif // VC_AVX_HELPERIMPL_TCC
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_INTRINSICS_H
+#define VC_AVX_INTRINSICS_H
+
+#include "../common/windows_fix_intrin.h"
+
+// AVX
+#include <immintrin.h>
+
+#if defined(VC_CLANG) && VC_CLANG < 0x30100
+// _mm_permute_ps is broken: http://llvm.org/bugs/show_bug.cgi?id=12401
+#undef _mm_permute_ps
+#define _mm_permute_ps(A, C) __extension__ ({ \
+ __m128 __A = (A); \
+ (__m128)__builtin_shufflevector((__v4sf)__A, (__v4sf) _mm_setzero_ps(), \
+ (C) & 0x3, ((C) & 0xc) >> 2, \
+ ((C) & 0x30) >> 4, ((C) & 0xc0) >> 6); })
+#endif
+
+#include <Vc/global.h>
+#include "const_data.h"
+#include "macros.h"
+#include <cstdlib>
+
+#if defined(VC_CLANG) || defined(VC_MSVC) || (defined(VC_GCC) && !defined(__OPTIMIZE__))
+#define VC_REQUIRES_MACRO_FOR_IMMEDIATE_ARGUMENT
+#endif
+
+#if defined(VC_CLANG) && VC_CLANG <= 0x30000
+// _mm_alignr_epi8 doesn't specify its return type, thus breaking overload resolution
+#undef _mm_alignr_epi8
+#define _mm_alignr_epi8(a, b, n) ((__m128i)__builtin_ia32_palignr128((a), (b), (n)))
+#endif
+
+namespace Vc
+{
+namespace AVX
+{
+#if defined(VC_GNU_ASM) && !defined(NVALGRIND)
+ static inline __m128i CONST _mm_setallone() { __m128i r; __asm__("pcmpeqb %0,%0":"=x"(r)); return r; }
+#else
+ static inline __m128i CONST _mm_setallone() { __m128i r = _mm_setzero_si128(); return _mm_cmpeq_epi8(r, r); }
+#endif
+ static inline __m128i CONST _mm_setallone_si128() { return _mm_setallone(); }
+ static inline __m128d CONST _mm_setallone_pd() { return _mm_castsi128_pd(_mm_setallone()); }
+ static inline __m128 CONST _mm_setallone_ps() { return _mm_castsi128_ps(_mm_setallone()); }
+
+ static inline __m128i CONST _mm_setone_epi8 () { return _mm_set1_epi8(1); }
+ static inline __m128i CONST _mm_setone_epu8 () { return _mm_setone_epi8(); }
+ static inline __m128i CONST _mm_setone_epi16() { return _mm_castps_si128(_mm_broadcast_ss(reinterpret_cast<const float *>(c_general::one16))); }
+ static inline __m128i CONST _mm_setone_epu16() { return _mm_setone_epi16(); }
+ static inline __m128i CONST _mm_setone_epi32() { return _mm_castps_si128(_mm_broadcast_ss(reinterpret_cast<const float *>(&_IndexesFromZero32[1]))); }
+
+#if defined(VC_GNU_ASM) && !defined(NVALGRIND)
+ static inline __m256 CONST _mm256_setallone() { __m256 r; __asm__("vcmpps $8,%0,%0,%0":"=x"(r)); return r; }
+#else
+ static inline __m256 CONST _mm256_setallone() { __m256 r = _mm256_setzero_ps(); return _mm256_cmp_ps(r, r, _CMP_EQ_UQ); }
+#endif
+ static inline __m256i CONST _mm256_setallone_si256() { return _mm256_castps_si256(_mm256_setallone()); }
+ static inline __m256d CONST _mm256_setallone_pd() { return _mm256_castps_pd(_mm256_setallone()); }
+ static inline __m256 CONST _mm256_setallone_ps() { return _mm256_setallone(); }
+
+ static inline __m256i CONST _mm256_setone_epi8 () { return _mm256_set1_epi8(1); }
+ static inline __m256i CONST _mm256_setone_epu8 () { return _mm256_setone_epi8(); }
+ static inline __m256i CONST _mm256_setone_epi16() { return _mm256_castps_si256(_mm256_broadcast_ss(reinterpret_cast<const float *>(c_general::one16))); }
+ static inline __m256i CONST _mm256_setone_epu16() { return _mm256_setone_epi16(); }
+ static inline __m256i CONST _mm256_setone_epi32() { return _mm256_castps_si256(_mm256_broadcast_ss(reinterpret_cast<const float *>(&_IndexesFromZero32[1]))); }
+ static inline __m256i CONST _mm256_setone_epu32() { return _mm256_setone_epi32(); }
+
+ static inline __m256 CONST _mm256_setone_ps() { return _mm256_broadcast_ss(&c_general::oneFloat); }
+ static inline __m256d CONST _mm256_setone_pd() { return _mm256_broadcast_sd(&c_general::oneDouble); }
+
+ static inline __m256d CONST _mm256_setabsmask_pd() { return _mm256_broadcast_sd(reinterpret_cast<const double *>(&c_general::absMaskFloat[0])); }
+ static inline __m256 CONST _mm256_setabsmask_ps() { return _mm256_broadcast_ss(reinterpret_cast<const float *>(&c_general::absMaskFloat[1])); }
+ static inline __m256d CONST _mm256_setsignmask_pd(){ return _mm256_broadcast_sd(reinterpret_cast<const double *>(&c_general::signMaskFloat[0])); }
+ static inline __m256 CONST _mm256_setsignmask_ps(){ return _mm256_broadcast_ss(reinterpret_cast<const float *>(&c_general::signMaskFloat[1])); }
+
+ static inline __m256 CONST _mm256_set2power31_ps() { return _mm256_broadcast_ss(&c_general::_2power31); }
+ static inline __m256i CONST _mm256_set2power31_epu32() { return _mm256_castps_si256(_mm256_broadcast_ss(reinterpret_cast<const float *>(&c_general::signMaskFloat[1]))); }
+
+ //X static inline __m256i CONST _mm256_setmin_epi8 () { return _mm256_slli_epi8 (_mm256_setallone_si256(), 7); }
+ static inline __m128i CONST _mm_setmin_epi16() { return _mm_castps_si128(_mm_broadcast_ss(reinterpret_cast<const float *>(c_general::minShort))); }
+ static inline __m128i CONST _mm_setmin_epi32() { return _mm_castps_si128(_mm_broadcast_ss(reinterpret_cast<const float *>(&c_general::signMaskFloat[1]))); }
+ static inline __m256i CONST _mm256_setmin_epi16() { return _mm256_castps_si256(_mm256_broadcast_ss(reinterpret_cast<const float *>(c_general::minShort))); }
+ static inline __m256i CONST _mm256_setmin_epi32() { return _mm256_castps_si256(_mm256_broadcast_ss(reinterpret_cast<const float *>(&c_general::signMaskFloat[1]))); }
+
+#ifdef VC_REQUIRES_MACRO_FOR_IMMEDIATE_ARGUMENT
+#define _mm_extract_epu8 _mm_extract_epi8
+#define _mm_extract_epu16 _mm_extract_epi16
+#define _mm_extract_epu32 _mm_extract_epi32
+#else
+ static inline unsigned char INTRINSIC CONST _mm_extract_epu8(__m128i x, const int i) { return _mm_extract_epi8(x, i); }
+ static inline unsigned short INTRINSIC CONST _mm_extract_epu16(__m128i x, const int i) { return _mm_extract_epi16(x, i); }
+ static inline unsigned int INTRINSIC CONST _mm_extract_epu32(__m128i x, const int i) { return _mm_extract_epi32(x, i); }
+#endif
+
+ /////////////////////// COMPARE OPS ///////////////////////
+ static inline __m256d INTRINSIC CONST _mm256_cmpeq_pd (__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_EQ_OQ); }
+ static inline __m256d INTRINSIC CONST _mm256_cmpneq_pd (__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_NEQ_UQ); }
+ static inline __m256d INTRINSIC CONST _mm256_cmplt_pd (__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_LT_OS); }
+ static inline __m256d INTRINSIC CONST _mm256_cmpnlt_pd (__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_NLT_US); }
+ static inline __m256d INTRINSIC CONST _mm256_cmple_pd (__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_LE_OS); }
+ static inline __m256d INTRINSIC CONST _mm256_cmpnle_pd (__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_NLE_US); }
+ static inline __m256d INTRINSIC CONST _mm256_cmpord_pd (__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_ORD_Q); }
+ static inline __m256d INTRINSIC CONST _mm256_cmpunord_pd(__m256d a, __m256d b) { return _mm256_cmp_pd(a, b, _CMP_UNORD_Q); }
+
+ static inline __m256 INTRINSIC CONST _mm256_cmpeq_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_EQ_OQ); }
+ static inline __m256 INTRINSIC CONST _mm256_cmpneq_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_NEQ_UQ); }
+ static inline __m256 INTRINSIC CONST _mm256_cmplt_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_LT_OS); }
+ static inline __m256 INTRINSIC CONST _mm256_cmpnlt_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_NLT_US); }
+ static inline __m256 INTRINSIC CONST _mm256_cmpge_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_NLT_US); }
+ static inline __m256 INTRINSIC CONST _mm256_cmple_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_LE_OS); }
+ static inline __m256 INTRINSIC CONST _mm256_cmpnle_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_NLE_US); }
+ static inline __m256 INTRINSIC CONST _mm256_cmpgt_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_NLE_US); }
+ static inline __m256 INTRINSIC CONST _mm256_cmpord_ps (__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_ORD_Q); }
+ static inline __m256 INTRINSIC CONST _mm256_cmpunord_ps(__m256 a, __m256 b) { return _mm256_cmp_ps(a, b, _CMP_UNORD_Q); }
+
+ static inline __m128i _mm_cmplt_epu16(__m128i a, __m128i b) {
+ return _mm_cmplt_epi16(_mm_xor_si128(a, _mm_setmin_epi16()), _mm_xor_si128(b, _mm_setmin_epi16()));
+ }
+ static inline __m128i _mm_cmpgt_epu16(__m128i a, __m128i b) {
+ return _mm_cmpgt_epi16(_mm_xor_si128(a, _mm_setmin_epi16()), _mm_xor_si128(b, _mm_setmin_epi16()));
+ }
+
+ /////////////////////// INTEGER OPS ///////////////////////
+#define AVX_TO_SSE_2(name) \
+ static inline __m256i INTRINSIC CONST _mm256_##name(__m256i a0, __m256i b0) { \
+ __m128i a1 = _mm256_extractf128_si256(a0, 1); \
+ __m128i b1 = _mm256_extractf128_si256(b0, 1); \
+ __m128i r0 = _mm_##name(_mm256_castsi256_si128(a0), _mm256_castsi256_si128(b0)); \
+ __m128i r1 = _mm_##name(a1, b1); \
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1); \
+ }
+#define AVX_TO_SSE_2_si128_si256(name) \
+ static inline __m256i INTRINSIC CONST _mm256_##name##_si256(__m256i a0, __m256i b0) { \
+ __m128i a1 = _mm256_extractf128_si256(a0, 1); \
+ __m128i b1 = _mm256_extractf128_si256(b0, 1); \
+ __m128i r0 = _mm_##name##_si128(_mm256_castsi256_si128(a0), _mm256_castsi256_si128(b0)); \
+ __m128i r1 = _mm_##name##_si128(a1, b1); \
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1); \
+ }
+#define AVX_TO_SSE_1(name) \
+ static inline __m256i INTRINSIC CONST _mm256_##name(__m256i a0) { \
+ __m128i a1 = _mm256_extractf128_si256(a0, 1); \
+ __m128i r0 = _mm_##name(_mm256_castsi256_si128(a0)); \
+ __m128i r1 = _mm_##name(a1); \
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1); \
+ }
+#define AVX_TO_SSE_1i(name) \
+ static inline __m256i INTRINSIC CONST _mm256_##name(__m256i a0, const int i) { \
+ __m128i a1 = _mm256_extractf128_si256(a0, 1); \
+ __m128i r0 = _mm_##name(_mm256_castsi256_si128(a0), i); \
+ __m128i r1 = _mm_##name(a1, i); \
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1); \
+ }
+
+ AVX_TO_SSE_2(cmpeq_epi8)
+ AVX_TO_SSE_2(cmpeq_epi16)
+ AVX_TO_SSE_2(cmpeq_epi32)
+ AVX_TO_SSE_2(cmplt_epi8)
+ AVX_TO_SSE_2(cmplt_epi16)
+ AVX_TO_SSE_2(cmplt_epi32)
+ AVX_TO_SSE_2(cmpgt_epi8)
+ AVX_TO_SSE_2(cmpgt_epi16)
+ AVX_TO_SSE_2(cmpgt_epi32)
+
+#ifndef VC_ICC
+ // ICC ships the Integer intrinsics inside the AVX1 header these days.
+
+#ifdef VC_REQUIRES_MACRO_FOR_IMMEDIATE_ARGUMENT
+# define _mm256_srli_si256(a, i) \
+ _mm256_insertf128_si256( \
+ _mm256_castsi128_si256(_mm_srli_si128(_mm256_castsi256_si128((a)), i)), \
+ _mm_srli_si128(_mm256_extractf128_si256((a), 1), i), 1);
+# define _mm256_slli_si256(a, i) \
+ _mm256_insertf128_si256( \
+ _mm256_castsi128_si256( _mm_slli_si128(_mm256_castsi256_si128((a)), i)), \
+ _mm_slli_si128(_mm256_extractf128_si256((a), 1), i), 1);
+#else
+ static inline __m256i INTRINSIC CONST _mm256_srli_si256(__m256i a0, const int i) {
+ const __m128i r0 = _mm_srli_si128(_mm256_castsi256_si128(a0), i);
+ const __m128i r1 = _mm_srli_si128(_mm256_extractf128_si256(a0, 1), i);
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1);
+ }
+ static inline __m256i INTRINSIC CONST _mm256_slli_si256(__m256i a0, const int i) {
+ const __m128i r0 = _mm_slli_si128(_mm256_castsi256_si128(a0), i);
+ const __m128i r1 = _mm_slli_si128(_mm256_extractf128_si256(a0, 1), i);
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1);
+ }
+#endif
+
+ static inline __m256i INTRINSIC CONST _mm256_and_si256(__m256i x, __m256i y) {
+ return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(x), _mm256_castsi256_ps(y)));
+ }
+ static inline __m256i INTRINSIC CONST _mm256_andnot_si256(__m256i x, __m256i y) {
+ return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(x), _mm256_castsi256_ps(y)));
+ }
+ static inline __m256i INTRINSIC CONST _mm256_or_si256(__m256i x, __m256i y) {
+ return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(x), _mm256_castsi256_ps(y)));
+ }
+ static inline __m256i INTRINSIC CONST _mm256_xor_si256(__m256i x, __m256i y) {
+ return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(x), _mm256_castsi256_ps(y)));
+ }
+
+ AVX_TO_SSE_2(packs_epi16)
+ AVX_TO_SSE_2(packs_epi32)
+ AVX_TO_SSE_2(packus_epi16)
+ AVX_TO_SSE_2(unpackhi_epi8)
+ AVX_TO_SSE_2(unpackhi_epi16)
+ AVX_TO_SSE_2(unpackhi_epi32)
+ AVX_TO_SSE_2(unpackhi_epi64)
+ AVX_TO_SSE_2(unpacklo_epi8)
+ AVX_TO_SSE_2(unpacklo_epi16)
+ AVX_TO_SSE_2(unpacklo_epi32)
+ AVX_TO_SSE_2(unpacklo_epi64)
+ AVX_TO_SSE_2(add_epi8)
+ AVX_TO_SSE_2(add_epi16)
+ AVX_TO_SSE_2(add_epi32)
+ AVX_TO_SSE_2(add_epi64)
+ AVX_TO_SSE_2(adds_epi8)
+ AVX_TO_SSE_2(adds_epi16)
+ AVX_TO_SSE_2(adds_epu8)
+ AVX_TO_SSE_2(adds_epu16)
+ AVX_TO_SSE_2(sub_epi8)
+ AVX_TO_SSE_2(sub_epi16)
+ AVX_TO_SSE_2(sub_epi32)
+ AVX_TO_SSE_2(sub_epi64)
+ AVX_TO_SSE_2(subs_epi8)
+ AVX_TO_SSE_2(subs_epi16)
+ AVX_TO_SSE_2(subs_epu8)
+ AVX_TO_SSE_2(subs_epu16)
+ AVX_TO_SSE_2(madd_epi16)
+ AVX_TO_SSE_2(mulhi_epi16)
+ AVX_TO_SSE_2(mullo_epi16)
+ AVX_TO_SSE_2(mul_epu32)
+ AVX_TO_SSE_1i(slli_epi16)
+ AVX_TO_SSE_1i(slli_epi32)
+ AVX_TO_SSE_1i(slli_epi64)
+ AVX_TO_SSE_1i(srai_epi16)
+ AVX_TO_SSE_1i(srai_epi32)
+ AVX_TO_SSE_1i(srli_epi16)
+ AVX_TO_SSE_1i(srli_epi32)
+ AVX_TO_SSE_1i(srli_epi64)
+ AVX_TO_SSE_2(sll_epi16)
+ AVX_TO_SSE_2(sll_epi32)
+ AVX_TO_SSE_2(sll_epi64)
+ AVX_TO_SSE_2(sra_epi16)
+ AVX_TO_SSE_2(sra_epi32)
+ AVX_TO_SSE_2(srl_epi16)
+ AVX_TO_SSE_2(srl_epi32)
+ AVX_TO_SSE_2(srl_epi64)
+ AVX_TO_SSE_2(max_epi16)
+ AVX_TO_SSE_2(max_epu8)
+ AVX_TO_SSE_2(min_epi16)
+ AVX_TO_SSE_2(min_epu8)
+ inline int INTRINSIC CONST _mm256_movemask_epi8(__m256i a0)
+ {
+ __m128i a1 = _mm256_extractf128_si256(a0, 1);
+ return (_mm_movemask_epi8(a1) << 16) | _mm_movemask_epi8(_mm256_castsi256_si128(a0));
+ }
+ AVX_TO_SSE_2(mulhi_epu16)
+ // shufflehi_epi16
+ // shufflelo_epi16 (__m128i __A, const int __mask)
+ // shuffle_epi32 (__m128i __A, const int __mask)
+ // maskmoveu_si128 (__m128i __A, __m128i __B, char *__C)
+ AVX_TO_SSE_2(avg_epu8)
+ AVX_TO_SSE_2(avg_epu16)
+ AVX_TO_SSE_2(sad_epu8)
+ // stream_si32 (int *__A, int __B)
+ // stream_si128 (__m128i *__A, __m128i __B)
+ // cvtsi32_si128 (int __A)
+ // cvtsi64_si128 (long long __A)
+ // cvtsi64x_si128 (long long __A)
+ AVX_TO_SSE_2(hadd_epi16)
+ AVX_TO_SSE_2(hadd_epi32)
+ AVX_TO_SSE_2(hadds_epi16)
+ AVX_TO_SSE_2(hsub_epi16)
+ AVX_TO_SSE_2(hsub_epi32)
+ AVX_TO_SSE_2(hsubs_epi16)
+ AVX_TO_SSE_2(maddubs_epi16)
+ AVX_TO_SSE_2(mulhrs_epi16)
+ AVX_TO_SSE_2(shuffle_epi8)
+ AVX_TO_SSE_2(sign_epi8)
+ AVX_TO_SSE_2(sign_epi16)
+ AVX_TO_SSE_2(sign_epi32)
+ // alignr_epi8(__m128i __X, __m128i __Y, const int __N)
+ AVX_TO_SSE_1(abs_epi8)
+ AVX_TO_SSE_1(abs_epi16)
+ AVX_TO_SSE_1(abs_epi32)
+#if !defined(VC_REQUIRES_MACRO_FOR_IMMEDIATE_ARGUMENT)
+ __m256i inline INTRINSIC CONST _mm256_blend_epi16(__m256i a0, __m256i b0, const int m) {
+ __m128i a1 = _mm256_extractf128_si256(a0, 1);
+ __m128i b1 = _mm256_extractf128_si256(b0, 1);
+ __m128i r0 = _mm_blend_epi16(_mm256_castsi256_si128(a0), _mm256_castsi256_si128(b0), m & 0xff);
+ __m128i r1 = _mm_blend_epi16(a1, b1, m >> 8);
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1);
+ }
+#else
+# define _mm256_blend_epi16(a0, b0, m) \
+ _mm256_insertf128_si256( \
+ _mm256_castsi128_si256( \
+ _mm_blend_epi16( \
+ _mm256_castsi256_si128(a0), _mm256_castsi256_si128(b0), m & 0xff)), \
+ _mm_blend_epi16(_mm256_extractf128_si256(a0, 1), _mm256_extractf128_si256(b0, 1), m >> 8);, 1)
+#endif
+ inline __m256i INTRINSIC CONST _mm256_blendv_epi8(__m256i a0, __m256i b0, __m256i m0) {
+ __m128i a1 = _mm256_extractf128_si256(a0, 1);
+ __m128i b1 = _mm256_extractf128_si256(b0, 1);
+ __m128i m1 = _mm256_extractf128_si256(m0, 1);
+ __m128i r0 = _mm_blendv_epi8(_mm256_castsi256_si128(a0), _mm256_castsi256_si128(b0), _mm256_castsi256_si128(m0));
+ __m128i r1 = _mm_blendv_epi8(a1, b1, m1);
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(r0), r1, 1);
+ }
+ AVX_TO_SSE_2(cmpeq_epi64)
+ AVX_TO_SSE_2(min_epi8)
+ AVX_TO_SSE_2(max_epi8)
+ AVX_TO_SSE_2(min_epu16)
+ AVX_TO_SSE_2(max_epu16)
+ AVX_TO_SSE_2(min_epi32)
+ AVX_TO_SSE_2(max_epi32)
+ AVX_TO_SSE_2(min_epu32)
+ AVX_TO_SSE_2(max_epu32)
+ AVX_TO_SSE_2(mullo_epi32)
+ AVX_TO_SSE_2(mul_epi32)
+#if !defined(VC_CLANG) || VC_CLANG > 0x30100
+ // clang is missing _mm_minpos_epu16 from smmintrin.h
+ // http://llvm.org/bugs/show_bug.cgi?id=12399
+ AVX_TO_SSE_1(minpos_epu16)
+#endif
+ AVX_TO_SSE_1(cvtepi8_epi32)
+ AVX_TO_SSE_1(cvtepi16_epi32)
+ AVX_TO_SSE_1(cvtepi8_epi64)
+ AVX_TO_SSE_1(cvtepi32_epi64)
+ AVX_TO_SSE_1(cvtepi16_epi64)
+ AVX_TO_SSE_1(cvtepi8_epi16)
+ AVX_TO_SSE_1(cvtepu8_epi32)
+ AVX_TO_SSE_1(cvtepu16_epi32)
+ AVX_TO_SSE_1(cvtepu8_epi64)
+ AVX_TO_SSE_1(cvtepu32_epi64)
+ AVX_TO_SSE_1(cvtepu16_epi64)
+ AVX_TO_SSE_1(cvtepu8_epi16)
+ AVX_TO_SSE_2(packus_epi32)
+ // mpsadbw_epu8 (__m128i __X, __m128i __Y, const int __M)
+ // stream_load_si128 (__m128i *__X)
+ AVX_TO_SSE_2(cmpgt_epi64)
+#endif
+
+//X static inline __m256i _mm256_cmplt_epu8 (__m256i a, __m256i b) { return _mm256_cmplt_epi8 (
+//X _mm256_xor_si256(a, _mm256_setmin_epi8 ()), _mm256_xor_si256(b, _mm256_setmin_epi8 ())); }
+//X static inline __m256i _mm256_cmpgt_epu8 (__m256i a, __m256i b) { return _mm256_cmpgt_epi8 (
+//X _mm256_xor_si256(a, _mm256_setmin_epi8 ()), _mm256_xor_si256(b, _mm256_setmin_epi8 ())); }
+ static inline __m256i CONST _mm256_cmplt_epu32(__m256i a, __m256i b) {
+ a = _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(_mm256_setmin_epi32())));
+ b = _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(_mm256_setmin_epi32())));
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(
+ _mm_cmplt_epi32(_mm256_castsi256_si128(a), _mm256_castsi256_si128(b))),
+ _mm_cmplt_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)), 1);
+ }
+ static inline __m256i CONST _mm256_cmpgt_epu32(__m256i a, __m256i b) {
+ a = _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(_mm256_setmin_epi32())));
+ b = _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(_mm256_setmin_epi32())));
+ return _mm256_insertf128_si256(_mm256_castsi128_si256(
+ _mm_cmpgt_epi32(_mm256_castsi256_si128(a), _mm256_castsi256_si128(b))),
+ _mm_cmpgt_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1)), 1);
+ }
+
+ static inline void INTRINSIC _mm256_maskstore(float *mem, const __m256 mask, const __m256 v) {
+#ifndef VC_MM256_MASKSTORE_WRONG_MASK_TYPE
+ _mm256_maskstore_ps(mem, _mm256_castps_si256(mask), v);
+#else
+ _mm256_maskstore_ps(mem, mask, v);
+#endif
+ }
+ static inline void INTRINSIC _mm256_maskstore(double *mem, const __m256d mask, const __m256d v) {
+#ifndef VC_MM256_MASKSTORE_WRONG_MASK_TYPE
+ _mm256_maskstore_pd(mem, _mm256_castpd_si256(mask), v);
+#else
+ _mm256_maskstore_pd(mem, mask, v);
+#endif
+ }
+ static inline void INTRINSIC _mm256_maskstore(int *mem, const __m256i mask, const __m256i v) {
+#ifndef VC_MM256_MASKSTORE_WRONG_MASK_TYPE
+ _mm256_maskstore_ps(reinterpret_cast<float *>(mem), mask, _mm256_castsi256_ps(v));
+#else
+ _mm256_maskstore_ps(reinterpret_cast<float *>(mem), _mm256_castsi256_ps(mask), _mm256_castsi256_ps(v));
+#endif
+ }
+ static inline void INTRINSIC _mm256_maskstore(unsigned int *mem, const __m256i mask, const __m256i v) {
+ _mm256_maskstore(reinterpret_cast<int *>(mem), mask, v);
+ }
+} // namespace AVX
+} // namespace Vc
+
+#include "shuffle.h"
+
+#endif // VC_AVX_INTRINSICS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_LIMITS_H
+#define VC_AVX_LIMITS_H
+
+#include "intrinsics.h"
+#include "types.h"
+
+namespace std
+{
+#define _VC_NUM_LIM(T, _max, _min) \
+template<> struct numeric_limits<Vc::AVX::Vector<T> > : public numeric_limits<T> \
+{ \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> max() _VC_NOEXCEPT { return _max; } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> min() _VC_NOEXCEPT { return _min; } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> lowest() _VC_NOEXCEPT { return min(); } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> epsilon() _VC_NOEXCEPT { return Vc::AVX::Vector<T>::Zero(); } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> round_error() _VC_NOEXCEPT { return Vc::AVX::Vector<T>::Zero(); } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> infinity() _VC_NOEXCEPT { return Vc::AVX::Vector<T>::Zero(); } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> quiet_NaN() _VC_NOEXCEPT { return Vc::AVX::Vector<T>::Zero(); } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> signaling_NaN() _VC_NOEXCEPT { return Vc::AVX::Vector<T>::Zero(); } \
+ static inline INTRINSIC CONST Vc::AVX::Vector<T> denorm_min() _VC_NOEXCEPT { return Vc::AVX::Vector<T>::Zero(); } \
+}
+_VC_NUM_LIM(unsigned short, Vc::AVX::_mm_setallone_si128(), _mm_setzero_si128());
+_VC_NUM_LIM( short, _mm_srli_epi16(Vc::AVX::_mm_setallone_si128(), 1), Vc::AVX::_mm_setmin_epi16());
+_VC_NUM_LIM( unsigned int, Vc::AVX::_mm256_setallone_si256(), _mm256_setzero_si256());
+_VC_NUM_LIM( int, Vc::AVX::_mm256_srli_epi32(Vc::AVX::_mm256_setallone_si256(), 1), Vc::AVX::_mm256_setmin_epi32());
+#undef _VC_NUM_LIM
+
+} // namespace std
+
+#endif // VC_AVX_LIMITS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "../common/macros.h"
+
+#ifndef VC_AVX_MACROS_H
+#define VC_AVX_MACROS_H
+#undef VC_AVX_UNDOMACROS_H
+
+#ifndef _M256
+# define _M256 __m256
+#endif
+
+#ifndef _M256I
+# define _M256I __m256i
+#endif
+
+#ifndef _M256D
+# define _M256D __m256d
+#endif
+
+#endif // VC_AVX_MACROS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_MASK_H
+#define VC_AVX_MASK_H
+
+#include "intrinsics.h"
+#include "../common/bitscanintrinsics.h"
+
+namespace Vc
+{
+namespace AVX
+{
+
+template<unsigned int VectorSize> class Mask<VectorSize, 32u>
+{
+ friend class Mask<4u, 32u>; // double_v
+ friend class Mask<8u, 32u>; // float_v, (u)int_v
+ friend class Mask<8u, 16u>; // (u)short_v
+ friend class Mask<16u, 16u>; // (u)char_v
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(32)
+
+ // abstracts the way Masks are passed to functions, it can easily be changed to const ref here
+#if defined VC_MSVC && defined _WIN32
+ typedef const Mask<VectorSize, 32u> &AsArg;
+#else
+ typedef Mask<VectorSize, 32u> AsArg;
+#endif
+
+ inline Mask() {}
+ inline Mask(const __m256 &x) : k(x) {}
+ inline Mask(const __m256d &x) : k(_mm256_castpd_ps(x)) {}
+ inline Mask(const __m256i &x) : k(_mm256_castsi256_ps(x)) {}
+ inline explicit Mask(VectorSpecialInitializerZero::ZEnum) : k(_mm256_setzero_ps()) {}
+ inline explicit Mask(VectorSpecialInitializerOne::OEnum) : k(_mm256_setallone_ps()) {}
+ inline explicit Mask(bool b) : k(b ? _mm256_setallone_ps() : _mm256_setzero_ps()) {}
+ inline Mask(const Mask &rhs) : k(rhs.k) {}
+ inline Mask(const Mask<VectorSize, 16u> &rhs) : k(avx_cast<__m256>(concat(
+ _mm_unpacklo_epi16(rhs.dataI(), rhs.dataI()),
+ _mm_unpackhi_epi16(rhs.dataI(), rhs.dataI())))) {}
+ inline Mask(const Mask<VectorSize * 2, 32u> &m);
+ inline Mask(const Mask<VectorSize / 2, 32u> &m);
+
+ inline bool operator==(const Mask &rhs) const { return 0 != _mm256_testc_ps(k, rhs.k); }
+ inline bool operator!=(const Mask &rhs) const { return 0 == _mm256_testc_ps(k, rhs.k); }
+
+ inline Mask operator&&(const Mask &rhs) const { return _mm256_and_ps(k, rhs.k); }
+ inline Mask operator& (const Mask &rhs) const { return _mm256_and_ps(k, rhs.k); }
+ inline Mask operator||(const Mask &rhs) const { return _mm256_or_ps (k, rhs.k); }
+ inline Mask operator| (const Mask &rhs) const { return _mm256_or_ps (k, rhs.k); }
+ inline Mask operator^ (const Mask &rhs) const { return _mm256_xor_ps(k, rhs.k); }
+ inline Mask operator!() const { return _mm256_andnot_ps(data(), _mm256_setallone_ps()); }
+
+ inline Mask &operator&=(const Mask &rhs) { k = _mm256_and_ps(k, rhs.k); return *this; }
+ inline Mask &operator|=(const Mask &rhs) { k = _mm256_or_ps (k, rhs.k); return *this; }
+
+ // no need for expression template optimizations because cmp(n)eq for floats are not bitwise
+ // compares
+ inline bool isFull () const { return 0 != _mm256_testc_ps(k, _mm256_setallone_ps()); }
+ inline bool isEmpty() const { return 0 != _mm256_testz_ps(k, k); }
+ inline bool isMix () const { return 0 != _mm256_testnzc_ps(k, _mm256_setallone_ps()); }
+
+#ifndef VC_NO_AUTOMATIC_BOOL_FROM_MASK
+ inline operator bool() const { return isFull(); }
+#endif
+
+ inline int CONST_L shiftMask() const CONST_R;
+ int CONST_L toInt() const CONST_R;
+
+ inline _M256 data () const { return k; }
+ inline _M256I dataI() const { return _mm256_castps_si256(k); }
+ inline _M256D dataD() const { return _mm256_castps_pd(k); }
+
+ bool operator[](int index) const;
+
+ int count() const;
+ int firstOne() const;
+
+ private:
+ _M256 k;
+};
+
+template<unsigned int VectorSize> class Mask<VectorSize, 16u>
+{
+ friend class Mask<4u, 32u>; // double_v
+ friend class Mask<8u, 32u>; // float_v, (u)int_v
+ friend class Mask<8u, 16u>; // (u)short_v
+ friend class Mask<16u, 16u>; // (u)char_v
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(16)
+
+ // abstracts the way Masks are passed to functions, it can easily be changed to const ref here
+#if defined VC_MSVC && defined _WIN32
+ typedef const Mask<VectorSize, 16u> &AsArg;
+#else
+ typedef Mask<VectorSize, 16u> AsArg;
+#endif
+
+ inline Mask() {}
+ inline Mask(const __m128 &x) : k(x) {}
+ inline Mask(const __m128d &x) : k(_mm_castpd_ps(x)) {}
+ inline Mask(const __m128i &x) : k(_mm_castsi128_ps(x)) {}
+ inline explicit Mask(VectorSpecialInitializerZero::ZEnum) : k(_mm_setzero_ps()) {}
+ inline explicit Mask(VectorSpecialInitializerOne::OEnum) : k(_mm_setallone_ps()) {}
+ inline explicit Mask(bool b) : k(b ? _mm_setallone_ps() : _mm_setzero_ps()) {}
+ inline Mask(const Mask &rhs) : k(rhs.k) {}
+ inline Mask(const Mask<VectorSize, 32u> &rhs) : k(avx_cast<__m128>(
+ _mm_packs_epi32(avx_cast<__m128i>(rhs.data()), _mm256_extractf128_si256(rhs.dataI(), 1)))) {}
+ inline Mask(const Mask<VectorSize / 2, 16u> *a) : k(avx_cast<__m128>(
+ _mm_packs_epi16(a[0].dataI(), a[1].dataI()))) {}
+
+ inline bool operator==(const Mask &rhs) const { return 0 != _mm_testc_si128(dataI(), rhs.dataI()); }
+ inline bool operator!=(const Mask &rhs) const { return 0 == _mm_testc_si128(dataI(), rhs.dataI()); }
+
+ inline Mask operator&&(const Mask &rhs) const { return _mm_and_ps(k, rhs.k); }
+ inline Mask operator& (const Mask &rhs) const { return _mm_and_ps(k, rhs.k); }
+ inline Mask operator||(const Mask &rhs) const { return _mm_or_ps (k, rhs.k); }
+ inline Mask operator| (const Mask &rhs) const { return _mm_or_ps (k, rhs.k); }
+ inline Mask operator^ (const Mask &rhs) const { return _mm_xor_ps(k, rhs.k); }
+ inline Mask operator!() const { return _mm_andnot_ps(data(), _mm_setallone_ps()); }
+
+ inline Mask &operator&=(const Mask &rhs) { k = _mm_and_ps(k, rhs.k); return *this; }
+ inline Mask &operator|=(const Mask &rhs) { k = _mm_or_ps (k, rhs.k); return *this; }
+
+ // TODO: use expression templates to optimize (v1 == v2).isFull() and friends
+ inline bool isFull () const { return 0 != _mm_testc_si128(dataI(), _mm_setallone_si128()); }
+ inline bool isEmpty() const { return 0 != _mm_testz_si128(dataI(), dataI()); }
+ inline bool isMix () const { return 0 != _mm_testnzc_si128(dataI(), _mm_setallone_si128()); }
+
+#ifndef VC_NO_AUTOMATIC_BOOL_FROM_MASK
+ inline operator bool() const { return isFull(); }
+#endif
+
+ inline int CONST_L shiftMask() const CONST_R;
+ int CONST_L toInt() const CONST_R;
+
+ inline __m128 data () const { return k; }
+ inline __m128i dataI() const { return avx_cast<__m128i>(k); }
+ inline __m128d dataD() const { return avx_cast<__m128d>(k); }
+
+ bool operator[](int index) const;
+
+ int count() const;
+ int firstOne() const;
+
+ private:
+ __m128 k;
+};
+
+struct ForeachHelper
+{
+ size_t mask;
+ bool brk;
+ inline ForeachHelper(size_t _mask) : mask(_mask), brk(false) {}
+ inline bool outer() const { return mask != 0; }
+ inline bool inner() { return (brk = !brk); }
+ inline size_t next() {
+#ifdef VC_GNU_ASM
+ const size_t bit = __builtin_ctzl(mask);
+ __asm__("btr %1,%0" : "+r"(mask) : "r"(bit));
+#else
+ const size_t bit = _bit_scan_forward(mask);
+ mask &= ~(1 << bit);
+#endif
+ return bit;
+ }
+};
+
+#define Vc_foreach_bit(_it_, _mask_) \
+ for (Vc::AVX::ForeachHelper _Vc_foreach_bit_helper((_mask_).toInt()); _Vc_foreach_bit_helper.outer(); ) \
+ for (_it_ = _Vc_foreach_bit_helper.next(); _Vc_foreach_bit_helper.inner(); )
+
+} // namespace AVX
+} // namespace Vc
+
+#include "mask.tcc"
+
+#endif // VC_AVX_MASK_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+namespace Vc
+{
+namespace AVX
+{
+
+template<> inline Mask<4, 32>::Mask(const Mask<8, 32> &m)
+ : k(concat(_mm_unpacklo_ps(lo128(m.data()), lo128(m.data())),
+ _mm_unpackhi_ps(lo128(m.data()), lo128(m.data()))))
+{
+}
+
+template<> inline Mask<8, 32>::Mask(const Mask<4, 32> &m)
+ // aabb ccdd -> abcd 0000
+ : k(concat(Mem::shuffle<X0, X2, Y0, Y2>(lo128(m.data()), hi128(m.data())),
+ _mm_setzero_ps()))
+{
+}
+
+template<unsigned int Size> inline int Mask<Size, 32u>::shiftMask() const
+{
+ return _mm256_movemask_epi8(dataI());
+}
+template<unsigned int Size> inline int Mask<Size, 16u>::shiftMask() const
+{
+ return _mm_movemask_epi8(dataI());
+}
+
+template<> inline int Mask< 4, 32>::toInt() const { return _mm256_movemask_pd(dataD()); }
+template<> inline int Mask< 8, 32>::toInt() const { return _mm256_movemask_ps(data ()); }
+template<> inline int Mask< 8, 16>::toInt() const { return _mm_movemask_epi8(_mm_packs_epi16(dataI(), _mm_setzero_si128())); }
+template<> inline int Mask<16, 16>::toInt() const { return _mm_movemask_epi8(dataI()); }
+
+template<> inline bool Mask< 4, 32>::operator[](int index) const { return toInt() & (1 << index); }
+template<> inline bool Mask< 8, 32>::operator[](int index) const { return toInt() & (1 << index); }
+template<> inline bool Mask< 8, 16>::operator[](int index) const { return shiftMask() & (1 << 2 * index); }
+template<> inline bool Mask<16, 16>::operator[](int index) const { return toInt() & (1 << index); }
+
+template<unsigned int Size> inline int Mask<Size, 32u>::count() const { return _mm_popcnt_u32(toInt()); }
+template<unsigned int Size> inline int Mask<Size, 16u>::count() const { return _mm_popcnt_u32(toInt()); }
+template<unsigned int Size> inline int Mask<Size, 32u>::firstOne() const { return _bit_scan_forward(toInt()); }
+template<unsigned int Size> inline int Mask<Size, 16u>::firstOne() const { return _bit_scan_forward(toInt()); }
+
+} // namespace AVX
+} // namespace Vc
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_MATH_H
+#define VC_AVX_MATH_H
+
+#include "const.h"
+#include "limits.h"
+#include "macros.h"
+
+namespace Vc
+{
+namespace AVX
+{
+ /**
+ * splits \p v into exponent and mantissa, the sign is kept with the mantissa
+ *
+ * The return value will be in the range [0.5, 1.0[
+ * The \p e value will be an integer defining the power-of-two exponent
+ */
+ inline double_v frexp(double_v::AsArg v, int_v *e) {
+ const __m256d exponentBits = Const<double>::exponentMask().dataD();
+ const __m256d exponentPart = _mm256_and_pd(v.data(), exponentBits);
+ e->data() = _mm256_sub_epi32(_mm256_srli_epi64(avx_cast<__m256i>(exponentPart), 52), _mm256_set1_epi32(0x3fe));
+ const __m256d exponentMaximized = _mm256_or_pd(v.data(), exponentBits);
+ double_v ret = _mm256_and_pd(exponentMaximized, _mm256_broadcast_sd(reinterpret_cast<const double *>(&c_general::frexpMask)));
+ double_m zeroMask = v == double_v::Zero();
+ ret(isnan(v) || !isfinite(v) || zeroMask) = v;
+ e->setZero(zeroMask.data());
+ return ret;
+ }
+ inline float_v frexp(float_v::AsArg v, int_v *e) {
+ const __m256 exponentBits = Const<float>::exponentMask().data();
+ const __m256 exponentPart = _mm256_and_ps(v.data(), exponentBits);
+ e->data() = _mm256_sub_epi32(_mm256_srli_epi32(avx_cast<__m256i>(exponentPart), 23), _mm256_set1_epi32(0x7e));
+ const __m256 exponentMaximized = _mm256_or_ps(v.data(), exponentBits);
+ float_v ret = _mm256_and_ps(exponentMaximized, avx_cast<__m256>(_mm256_set1_epi32(0xbf7fffffu)));
+ ret(isnan(v) || !isfinite(v) || v == float_v::Zero()) = v;
+ e->setZero(v == float_v::Zero());
+ return ret;
+ }
+ inline sfloat_v frexp(sfloat_v::AsArg v, short_v *e) {
+ const __m256 exponentBits = Const<float>::exponentMask().data();
+ const __m256 exponentPart = _mm256_and_ps(v.data(), exponentBits);
+ e->data() = _mm_sub_epi16(_mm_packs_epi32(_mm_srli_epi32(avx_cast<__m128i>(exponentPart), 23),
+ _mm_srli_epi32(avx_cast<__m128i>(hi128(exponentPart)), 23)), _mm_set1_epi16(0x7e));
+ const __m256 exponentMaximized = _mm256_or_ps(v.data(), exponentBits);
+ sfloat_v ret = _mm256_and_ps(exponentMaximized, avx_cast<__m256>(_mm256_set1_epi32(0xbf7fffffu)));
+ ret(isnan(v) || !isfinite(v) || v == sfloat_v::Zero()) = v;
+ e->setZero(v == sfloat_v::Zero());
+ return ret;
+ }
+
+ /* -> x * 2^e
+ * x == NaN -> NaN
+ * x == (-)inf -> (-)inf
+ */
+ inline double_v ldexp(double_v::AsArg v, int_v::AsArg _e) {
+ int_v e = _e;
+ e.setZero((v == double_v::Zero()).dataI());
+ const __m256i exponentBits = _mm256_slli_epi64(e.data(), 52);
+ return avx_cast<__m256d>(_mm256_add_epi64(avx_cast<__m256i>(v.data()), exponentBits));
+ }
+ inline float_v ldexp(float_v::AsArg v, int_v::AsArg _e) {
+ int_v e = _e;
+ e.setZero(static_cast<int_m>(v == float_v::Zero()));
+ return (v.reinterpretCast<int_v>() + (e << 23)).reinterpretCast<float_v>();
+ }
+ inline sfloat_v ldexp(sfloat_v::AsArg v, short_v::AsArg _e) {
+ short_v e = _e;
+ e.setZero(static_cast<short_m>(v == sfloat_v::Zero()));
+ e = e << (23 - 16);
+ const __m256i exponentBits = concat(_mm_unpacklo_epi16(_mm_setzero_si128(), e.data()),
+ _mm_unpackhi_epi16(_mm_setzero_si128(), e.data()));
+ return (v.reinterpretCast<int_v>() + int_v(exponentBits)).reinterpretCast<sfloat_v>();
+ }
+
+ static inline float_v floor(float_v::AsArg v) { return _mm256_floor_ps(v.data()); }
+ static inline sfloat_v floor(sfloat_v::AsArg v) { return _mm256_floor_ps(v.data()); }
+ static inline double_v floor(double_v::AsArg v) { return _mm256_floor_pd(v.data()); }
+
+ static inline float_v ceil(float_v::AsArg v) { return _mm256_ceil_ps(v.data()); }
+ static inline sfloat_v ceil(sfloat_v::AsArg v) { return _mm256_ceil_ps(v.data()); }
+ static inline double_v ceil(double_v::AsArg v) { return _mm256_ceil_pd(v.data()); }
+} // namespace AVX
+} // namespace Vc
+
+#include "undomacros.h"
+#define VC__USE_NAMESPACE AVX
+#include "../common/trigonometric.h"
+#define VC__USE_NAMESPACE AVX
+#include "../common/logarithm.h"
+#define VC__USE_NAMESPACE AVX
+#include "../common/exponential.h"
+#undef VC__USE_NAMESPACE
+
+#endif // VC_AVX_MATH_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010, 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_PREFETCHES_TCC
+#define VC_AVX_PREFETCHES_TCC
+
+namespace Vc
+{
+namespace Internal
+{
+
+inline ALWAYS_INLINE void HelperImpl<Vc::AVXImpl>::prefetchForOneRead(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_NTA);
+}
+inline ALWAYS_INLINE void HelperImpl<Vc::AVXImpl>::prefetchClose(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T0);
+}
+inline ALWAYS_INLINE void HelperImpl<Vc::AVXImpl>::prefetchMid(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T1);
+}
+inline ALWAYS_INLINE void HelperImpl<Vc::AVXImpl>::prefetchFar(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T2);
+}
+inline ALWAYS_INLINE void HelperImpl<Vc::AVXImpl>::prefetchForModify(const void *addr)
+{
+#ifdef __3dNOW__
+ _m_prefetchw(const_cast<void *>(addr));
+#else
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T0);
+#endif
+}
+
+} // namespace Internal
+} // namespace Vc
+
+#endif // VC_AVX_PREFETCHES_TCC
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_SHUFFLE_H
+#define VC_AVX_SHUFFLE_H
+
+#include "../sse/shuffle.h"
+
+namespace Vc
+{
+ namespace Mem
+ {
+ template<VecPos L, VecPos H> static inline __m256 ALWAYS_INLINE CONST shuffle128(__m256 x, __m256 y) {
+ VC_STATIC_ASSERT(L >= X0 && H >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(L <= Y1 && H <= Y1, Incorrect_Range);
+ return _mm256_permute2f128_ps(x, y, (L < Y0 ? L : L - Y0 + 2) + (H < Y0 ? H : H - Y0 + 2) * (1 << 4));
+ }
+ template<VecPos L, VecPos H> static inline __m256i ALWAYS_INLINE CONST shuffle128(__m256i x, __m256i y) {
+ VC_STATIC_ASSERT(L >= X0 && H >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(L <= Y1 && H <= Y1, Incorrect_Range);
+ return _mm256_permute2f128_si256(x, y, (L < Y0 ? L : L - Y0 + 2) + (H < Y0 ? H : H - Y0 + 2) * (1 << 4));
+ }
+ template<VecPos L, VecPos H> static inline __m256d ALWAYS_INLINE CONST shuffle128(__m256d x, __m256d y) {
+ VC_STATIC_ASSERT(L >= X0 && H >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(L <= Y1 && H <= Y1, Incorrect_Range);
+ return _mm256_permute2f128_pd(x, y, (L < Y0 ? L : L - Y0 + 2) + (H < Y0 ? H : H - Y0 + 2) * (1 << 4));
+ }
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m256d ALWAYS_INLINE CONST permute(__m256d x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X2 && Dst3 >= X2, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X1 && Dst1 <= X1 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm256_permute_pd(x, Dst0 + Dst1 * 2 + (Dst2 - X2) * 4 + (Dst3 - X2) * 8);
+ }
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m256 ALWAYS_INLINE CONST permute(__m256 x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm256_permute_ps(x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m256i ALWAYS_INLINE CONST permute(__m256i x) {
+ return _mm256_castps_si256(permute<Dst0, Dst1, Dst2, Dst3>(_mm256_castsi256_ps(x)));
+ }
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m256d ALWAYS_INLINE CONST shuffle(__m256d x, __m256d y) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= Y0 && Dst2 >= X2 && Dst3 >= Y2, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X1 && Dst1 <= Y1 && Dst2 <= X3 && Dst3 <= Y3, Incorrect_Range);
+ return _mm256_shuffle_pd(x, y, Dst0 + (Dst1 - Y0) * 2 + (Dst2 - X2) * 4 + (Dst3 - Y2) * 8);
+ }
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m256 ALWAYS_INLINE CONST shuffle(__m256 x, __m256 y) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= Y0 && Dst3 >= Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= Y3 && Dst3 <= Y3, Incorrect_Range);
+ return _mm256_shuffle_ps(x, y, Dst0 + Dst1 * 4 + (Dst2 - Y0) * 16 + (Dst3 - Y0) * 64);
+ }
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3, VecPos Dst4, VecPos Dst5, VecPos Dst6, VecPos Dst7>
+ static inline __m256 ALWAYS_INLINE CONST blend(__m256 x, __m256 y) {
+ VC_STATIC_ASSERT(Dst0 == X0 || Dst0 == Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst1 == X1 || Dst1 == Y1, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst2 == X2 || Dst2 == Y2, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst3 == X3 || Dst3 == Y3, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst4 == X4 || Dst4 == Y4, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst5 == X5 || Dst5 == Y5, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst6 == X6 || Dst6 == Y6, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst7 == X7 || Dst7 == Y7, Incorrect_Range);
+ return _mm256_blend_ps(x, y,
+ (Dst0 / Y0) * 1 + (Dst1 / Y1) * 2 +
+ (Dst2 / Y2) * 4 + (Dst3 / Y3) * 8 +
+ (Dst4 / Y4) * 16 + (Dst5 / Y5) * 32 +
+ (Dst6 / Y6) * 64 + (Dst7 / Y7) *128
+ );
+ }
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3, VecPos Dst4, VecPos Dst5, VecPos Dst6, VecPos Dst7>
+ static inline __m256i ALWAYS_INLINE CONST blend(__m256i x, __m256i y) {
+ return _mm256_castps_si256(blend<Dst0, Dst1, Dst2, Dst3, Dst4, Dst5, Dst6, Dst7>(_mm256_castsi256_ps(x), _mm256_castsi256_ps(y)));
+ }
+ template<VecPos Dst> struct ScaleForBlend { enum { Value = Dst >= X4 ? Dst - X4 + Y0 : Dst }; };
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3, VecPos Dst4, VecPos Dst5, VecPos Dst6, VecPos Dst7>
+ static inline __m256 ALWAYS_INLINE CONST permute(__m256 x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst0 <= X7, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst1 >= X0 && Dst1 <= X7, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst2 >= X0 && Dst2 <= X7, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst3 >= X0 && Dst3 <= X7, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst4 >= X0 && Dst4 <= X7, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst5 >= X0 && Dst5 <= X7, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst6 >= X0 && Dst6 <= X7, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst7 >= X0 && Dst7 <= X7, Incorrect_Range);
+ if (Dst0 + X4 == Dst4 && Dst1 + X4 == Dst5 && Dst2 + X4 == Dst6 && Dst3 + X4 == Dst7) {
+ return permute<Dst0, Dst1, Dst2, Dst3>(x);
+ }
+ const __m128 loIn = _mm256_castps256_ps128(x);
+ const __m128 hiIn = _mm256_extractf128_ps(x, 1);
+ __m128 lo, hi;
+
+ if (Dst0 < X4 && Dst1 < X4 && Dst2 < X4 && Dst3 < X4) {
+ lo = _mm_permute_ps(loIn, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ } else if (Dst0 >= X4 && Dst1 >= X4 && Dst2 >= X4 && Dst3 >= X4) {
+ lo = _mm_permute_ps(hiIn, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ } else if (Dst0 < X4 && Dst1 < X4 && Dst2 >= X4 && Dst3 >= X4) {
+ lo = shuffle<Dst0, Dst1, Dst2 - X4 + Y0, Dst3 - X4 + Y0>(loIn, hiIn);
+ } else if (Dst0 >= X4 && Dst1 >= X4 && Dst2 < X4 && Dst3 < X4) {
+ lo = shuffle<Dst0 - X4, Dst1 - X4, Dst2 + Y0, Dst3 + Y0>(hiIn, loIn);
+ } else if (Dst0 == X0 && Dst1 == X4 && Dst2 == X1 && Dst3 == X5) {
+ lo = _mm_unpacklo_ps(loIn, hiIn);
+ } else if (Dst0 == X4 && Dst1 == X0 && Dst2 == X5 && Dst3 == X1) {
+ lo = _mm_unpacklo_ps(hiIn, loIn);
+ } else if (Dst0 == X2 && Dst1 == X6 && Dst2 == X3 && Dst3 == X7) {
+ lo = _mm_unpackhi_ps(loIn, hiIn);
+ } else if (Dst0 == X6 && Dst1 == X2 && Dst2 == X7 && Dst3 == X3) {
+ lo = _mm_unpackhi_ps(hiIn, loIn);
+ } else if (Dst0 % X4 == 0 && Dst1 % X4 == 1 && Dst2 % X4 == 2 && Dst3 % X4 == 3) {
+ lo = blend<ScaleForBlend<Dst0>::Value, ScaleForBlend<Dst1>::Value,
+ ScaleForBlend<Dst2>::Value, ScaleForBlend<Dst3>::Value>(loIn, hiIn);
+ }
+
+ if (Dst4 >= X4 && Dst5 >= X4 && Dst6 >= X4 && Dst7 >= X4) {
+ hi = _mm_permute_ps(hiIn, (Dst4 - X4) + (Dst5 - X4) * 4 + (Dst6 - X4) * 16 + (Dst7 - X4) * 64);
+ } else if (Dst4 < X4 && Dst5 < X4 && Dst6 < X4 && Dst7 < X4) {
+ hi = _mm_permute_ps(loIn, (Dst4 - X4) + (Dst5 - X4) * 4 + (Dst6 - X4) * 16 + (Dst7 - X4) * 64);
+ } else if (Dst4 < X4 && Dst5 < X4 && Dst6 >= X4 && Dst7 >= X4) {
+ hi = shuffle<Dst4, Dst5, Dst6 - X4 + Y0, Dst7 - X4 + Y0>(loIn, hiIn);
+ } else if (Dst4 >= X4 && Dst5 >= X4 && Dst6 < X4 && Dst7 < X4) {
+ hi = shuffle<Dst4 - X4, Dst5 - X4, Dst6 + Y0, Dst7 + Y0>(hiIn, loIn);
+ } else if (Dst4 == X0 && Dst5 == X4 && Dst6 == X1 && Dst7 == X5) {
+ hi = _mm_unpacklo_ps(loIn, hiIn);
+ } else if (Dst4 == X4 && Dst5 == X0 && Dst6 == X5 && Dst7 == X1) {
+ hi = _mm_unpacklo_ps(hiIn, loIn);
+ } else if (Dst4 == X2 && Dst5 == X6 && Dst6 == X3 && Dst7 == X7) {
+ hi = _mm_unpackhi_ps(loIn, hiIn);
+ } else if (Dst4 == X6 && Dst5 == X2 && Dst6 == X7 && Dst7 == X3) {
+ hi = _mm_unpackhi_ps(hiIn, loIn);
+ } else if (Dst4 % X4 == 0 && Dst5 % X4 == 1 && Dst6 % X4 == 2 && Dst7 % X4 == 3) {
+ hi = blend<ScaleForBlend<Dst4>::Value, ScaleForBlend<Dst5>::Value,
+ ScaleForBlend<Dst6>::Value, ScaleForBlend<Dst7>::Value>(loIn, hiIn);
+ }
+
+ return _mm256_insertf128_ps(_mm256_castps128_ps256(lo), hi, 1);
+ }
+ } // namespace Mem
+
+ // little endian has the lo bits on the right and high bits on the left
+ // with vectors this becomes greatly confusing:
+ // Mem: abcd
+ // Reg: dcba
+ //
+ // The shuffles and permutes above use memory ordering. The ones below use register ordering:
+ namespace Reg
+ {
+ template<VecPos H, VecPos L> static inline __m256 ALWAYS_INLINE CONST permute128(__m256 x, __m256 y) {
+ VC_STATIC_ASSERT(L >= X0 && H >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(L <= Y1 && H <= Y1, Incorrect_Range);
+ return _mm256_permute2f128_ps(x, y, (L < Y0 ? L : L - Y0 + 2) + (H < Y0 ? H : H - Y0 + 2) * (1 << 4));
+ }
+ template<VecPos H, VecPos L> static inline __m256i ALWAYS_INLINE CONST permute128(__m256i x, __m256i y) {
+ VC_STATIC_ASSERT(L >= X0 && H >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(L <= Y1 && H <= Y1, Incorrect_Range);
+ return _mm256_permute2f128_si256(x, y, (L < Y0 ? L : L - Y0 + 2) + (H < Y0 ? H : H - Y0 + 2) * (1 << 4));
+ }
+ template<VecPos H, VecPos L> static inline __m256d ALWAYS_INLINE CONST permute128(__m256d x, __m256d y) {
+ VC_STATIC_ASSERT(L >= X0 && H >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(L <= Y1 && H <= Y1, Incorrect_Range);
+ return _mm256_permute2f128_pd(x, y, (L < Y0 ? L : L - Y0 + 2) + (H < Y0 ? H : H - Y0 + 2) * (1 << 4));
+ }
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m256d ALWAYS_INLINE CONST permute(__m256d x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X2 && Dst3 >= X2, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X1 && Dst1 <= X1 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm256_permute_pd(x, Dst0 + Dst1 * 2 + (Dst2 - X2) * 4 + (Dst3 - X2) * 8);
+ }
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m256 ALWAYS_INLINE CONST permute(__m256 x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm256_permute_ps(x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+ template<VecPos Dst1, VecPos Dst0> static inline __m128d ALWAYS_INLINE CONST permute(__m128d x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X1 && Dst1 <= X1, Incorrect_Range);
+ return _mm_permute_pd(x, Dst0 + Dst1 * 2);
+ }
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m128 ALWAYS_INLINE CONST permute(__m128 x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm_permute_ps(x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m256d ALWAYS_INLINE CONST shuffle(__m256d x, __m256d y) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= Y0 && Dst2 >= X2 && Dst3 >= Y2, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X1 && Dst1 <= Y1 && Dst2 <= X3 && Dst3 <= Y3, Incorrect_Range);
+ return _mm256_shuffle_pd(x, y, Dst0 + (Dst1 - Y0) * 2 + (Dst2 - X2) * 4 + (Dst3 - Y2) * 8);
+ }
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m256 ALWAYS_INLINE CONST shuffle(__m256 x, __m256 y) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= Y0 && Dst3 >= Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= Y3 && Dst3 <= Y3, Incorrect_Range);
+ return _mm256_shuffle_ps(x, y, Dst0 + Dst1 * 4 + (Dst2 - Y0) * 16 + (Dst3 - Y0) * 64);
+ }
+ } // namespace Reg
+} // namespace Vc
+
+#endif // VC_AVX_SHUFFLE_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_SORTHELPER_H
+#define VC_AVX_SORTHELPER_H
+
+#include "types.h"
+
+namespace Vc
+{
+namespace AVX
+{
+template<typename T> struct SortHelper
+{
+ typedef typename VectorTypeHelper<T>::Type VectorType;
+ static VectorType sort(VectorType);
+ static void sort(VectorType &, VectorType &);
+};
+} // namespace AVX
+} // namespace Vc
+
+#endif // VC_AVX_SORTHELPER_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef AVX_TYPES_H
+#define AVX_TYPES_H
+
+#include "intrinsics.h"
+#include "../common/storage.h"
+#include "macros.h"
+
+#define VC_DOUBLE_V_SIZE 4
+#define VC_FLOAT_V_SIZE 8
+#define VC_SFLOAT_V_SIZE 8
+#define VC_INT_V_SIZE 8
+#define VC_UINT_V_SIZE 8
+#define VC_SHORT_V_SIZE 8
+#define VC_USHORT_V_SIZE 8
+
+#include "../common/types.h"
+
+namespace Vc
+{
+namespace AVX
+{
+ template<typename T> class Vector;
+
+ template<unsigned int VectorSize, size_t RegisterWidth> class Mask;
+
+ template<typename T> struct VectorHelper {};
+ template<typename T> struct GatherHelper;
+ template<typename T> struct ScatterHelper;
+
+ template<typename T> struct IndexTypeHelper;
+ template<> struct IndexTypeHelper< char > { typedef unsigned char Type; };
+ template<> struct IndexTypeHelper<unsigned char > { typedef unsigned char Type; };
+ template<> struct IndexTypeHelper< short> { typedef unsigned short Type; };
+ template<> struct IndexTypeHelper<unsigned short> { typedef unsigned short Type; };
+ template<> struct IndexTypeHelper< int > { typedef unsigned int Type; };
+ template<> struct IndexTypeHelper<unsigned int > { typedef unsigned int Type; };
+ template<> struct IndexTypeHelper< float> { typedef unsigned int Type; };
+ template<> struct IndexTypeHelper< sfloat> { typedef unsigned short Type; };
+ template<> struct IndexTypeHelper< double> { typedef unsigned int Type; }; // _M128I based int32 would be nice
+
+ template<typename T> struct VectorTypeHelper;
+ template<> struct VectorTypeHelper< char > { typedef __m128i Type; };
+ template<> struct VectorTypeHelper<unsigned char > { typedef __m128i Type; };
+ template<> struct VectorTypeHelper< short> { typedef __m128i Type; };
+ template<> struct VectorTypeHelper<unsigned short> { typedef __m128i Type; };
+ template<> struct VectorTypeHelper< int > { typedef _M256I Type; };
+ template<> struct VectorTypeHelper<unsigned int > { typedef _M256I Type; };
+ template<> struct VectorTypeHelper< float> { typedef _M256 Type; };
+ template<> struct VectorTypeHelper< sfloat> { typedef _M256 Type; };
+ template<> struct VectorTypeHelper< double> { typedef _M256D Type; };
+
+ template<typename T> struct HasVectorDivisionHelper { enum { Value = 1 }; };
+ //template<> struct HasVectorDivisionHelper<unsigned int> { enum { Value = 0 }; };
+
+ template<typename T> struct VectorHelperSize;
+
+#ifdef VC_MSVC
+ // MSVC's __declspec(align(#)) only works with numbers, no enums or sizeof allowed ;(
+ template<size_t size> class _VectorAlignedBaseHack;
+ template<> class STRUCT_ALIGN1( 8) _VectorAlignedBaseHack< 8> {} STRUCT_ALIGN2( 8);
+ template<> class STRUCT_ALIGN1(16) _VectorAlignedBaseHack<16> {} STRUCT_ALIGN2(16);
+ template<> class STRUCT_ALIGN1(32) _VectorAlignedBaseHack<32> {} STRUCT_ALIGN2(32);
+ template<> class STRUCT_ALIGN1(64) _VectorAlignedBaseHack<64> {} STRUCT_ALIGN2(64);
+ template<typename V = Vector<float> >
+ class VectorAlignedBaseT : public _VectorAlignedBaseHack<sizeof(V)>
+ {
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(sizeof(V))
+ };
+#else
+ template<typename V = Vector<float> >
+ class STRUCT_ALIGN1(sizeof(V)) VectorAlignedBaseT
+ {
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(sizeof(V))
+ } STRUCT_ALIGN2(sizeof(V));
+#endif
+} // namespace AVX
+} // namespace Vc
+
+#endif // AVX_TYPES_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_UNDOMACROS_H
+#define VC_AVX_UNDOMACROS_H
+#undef VC_AVX_MACROS_H
+
+#endif // VC_AVX_UNDOMACROS_H
+
+#include "../common/undomacros.h"
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef AVX_VECTOR_H
+#define AVX_VECTOR_H
+
+#include "intrinsics.h"
+#include "vectorhelper.h"
+#include "mask.h"
+#include "writemaskedvector.h"
+#include "sorthelper.h"
+#include <algorithm>
+#include <cmath>
+#include "../common/aliasingentryhelper.h"
+#include "../common/memoryfwd.h"
+#include "macros.h"
+
+#ifdef isfinite
+#undef isfinite
+#endif
+#ifdef isnan
+#undef isnan
+#endif
+
+namespace Vc
+{
+namespace AVX
+{
+enum VectorAlignmentEnum { VectorAlignment = 32 };
+
+template<typename T> class Vector
+{
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(32)
+
+ typedef typename VectorTypeHelper<T>::Type VectorType;
+ typedef typename DetermineEntryType<T>::Type EntryType;
+ enum Constants {
+ Size = sizeof(VectorType) / sizeof(EntryType),
+ HasVectorDivision = HasVectorDivisionHelper<T>::Value
+ };
+ typedef Vector<typename IndexTypeHelper<T>::Type> IndexType;
+ typedef typename Vc::AVX::Mask<Size, sizeof(VectorType)> Mask;
+ typedef typename Mask::AsArg MaskArg;
+ typedef Vc::Memory<Vector<T>, Size> Memory;
+#ifdef VC_PASSING_VECTOR_BY_VALUE_IS_BROKEN
+ typedef const Vector<T> &AsArg;
+#else
+ typedef Vector<T> AsArg;
+#endif
+
+ protected:
+ // helper that specializes on VectorType
+ typedef VectorHelper<VectorType> HV;
+
+ // helper that specializes on T
+ typedef VectorHelper<T> HT;
+
+ // cast any m256/m128 to VectorType
+ static inline VectorType INTRINSIC _cast(__m128 v) { return avx_cast<VectorType>(v); }
+ static inline VectorType INTRINSIC _cast(__m128i v) { return avx_cast<VectorType>(v); }
+ static inline VectorType INTRINSIC _cast(__m128d v) { return avx_cast<VectorType>(v); }
+ static inline VectorType INTRINSIC _cast(__m256 v) { return avx_cast<VectorType>(v); }
+ static inline VectorType INTRINSIC _cast(__m256i v) { return avx_cast<VectorType>(v); }
+ static inline VectorType INTRINSIC _cast(__m256d v) { return avx_cast<VectorType>(v); }
+
+ typedef Common::VectorMemoryUnion<VectorType, EntryType> StorageType;
+ StorageType d;
+
+ public:
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // uninitialized
+ inline Vector() {}
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // constants
+ explicit Vector(VectorSpecialInitializerZero::ZEnum);
+ explicit Vector(VectorSpecialInitializerOne::OEnum);
+ explicit Vector(VectorSpecialInitializerIndexesFromZero::IEnum);
+ static Vector Zero();
+ static Vector One();
+ static Vector IndexesFromZero();
+ static Vector Random();
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // internal: required to enable returning objects of VectorType
+ inline Vector(const VectorType &x) : d(x) {}
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // static_cast / copy ctor
+ template<typename T2> explicit Vector(Vector<T2> x);
+
+ // implicit cast
+ template<typename OtherT> inline INTRINSIC_L Vector &operator=(const Vector<OtherT> &x) INTRINSIC_R;
+
+ // copy assignment
+ inline Vector &operator=(AsArg v) { d.v() = v.d.v(); return *this; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // broadcast
+ explicit Vector(EntryType a);
+ template<typename TT> inline INTRINSIC Vector(TT x, VC_EXACT_TYPE(TT, EntryType, void *) = 0) : d(HT::set(x)) {}
+ inline Vector &operator=(EntryType a) { d.v() = HT::set(a); return *this; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // load ctors
+ explicit inline INTRINSIC_L
+ Vector(const EntryType *x) INTRINSIC_R;
+ template<typename Alignment> inline INTRINSIC_L
+ Vector(const EntryType *x, Alignment align) INTRINSIC_R;
+ template<typename OtherT> explicit inline INTRINSIC_L
+ Vector(const OtherT *x) INTRINSIC_R;
+ template<typename OtherT, typename Alignment> inline INTRINSIC_L
+ Vector(const OtherT *x, Alignment align) INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // load member functions
+ inline INTRINSIC_L
+ void load(const EntryType *mem) INTRINSIC_R;
+ template<typename Alignment> inline INTRINSIC_L
+ void load(const EntryType *mem, Alignment align) INTRINSIC_R;
+ template<typename OtherT> inline INTRINSIC_L
+ void load(const OtherT *mem) INTRINSIC_R;
+ template<typename OtherT, typename Alignment> inline INTRINSIC_L
+ void load(const OtherT *mem, Alignment align) INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // expand/merge 1 float_v <=> 2 double_v XXX rationale? remove it for release? XXX
+ explicit inline Vector(const Vector<typename HT::ConcatType> *a);
+ inline void expand(Vector<typename HT::ConcatType> *x) const;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // zeroing
+ inline void setZero();
+ inline void setZero(const Mask &k);
+
+ void setQnan();
+ void setQnan(MaskArg k);
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // stores
+ inline void store(EntryType *mem) const;
+ inline void store(EntryType *mem, const Mask &mask) const;
+ template<typename A> inline void store(EntryType *mem, A align) const;
+ template<typename A> inline void store(EntryType *mem, const Mask &mask, A align) const;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // swizzles
+ inline const Vector<T> INTRINSIC_L CONST_L &abcd() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L cdab() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L badc() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L aaaa() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L bbbb() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L cccc() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dddd() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L bcad() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L bcda() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dabc() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L acbd() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dbca() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dcba() const INTRINSIC_R CONST_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // gathers
+ template<typename IndexT> Vector(const EntryType *mem, const IndexT *indexes);
+ template<typename IndexT> Vector(const EntryType *mem, Vector<IndexT> indexes);
+ template<typename IndexT> Vector(const EntryType *mem, const IndexT *indexes, MaskArg mask);
+ template<typename IndexT> Vector(const EntryType *mem, Vector<IndexT> indexes, MaskArg mask);
+ template<typename S1, typename IT> Vector(const S1 *array, const EntryType S1::* member1, IT indexes);
+ template<typename S1, typename IT> Vector(const S1 *array, const EntryType S1::* member1, IT indexes, MaskArg mask);
+ template<typename S1, typename S2, typename IT> Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes);
+ template<typename S1, typename S2, typename IT> Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes, MaskArg mask);
+ template<typename S1, typename IT1, typename IT2> Vector(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes);
+ template<typename S1, typename IT1, typename IT2> Vector(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes, MaskArg mask);
+ template<typename Index> void gather(const EntryType *mem, Index indexes);
+ template<typename Index> void gather(const EntryType *mem, Index indexes, MaskArg mask);
+#ifdef VC_USE_SET_GATHERS
+ template<typename IT> void gather(const EntryType *mem, Vector<IT> indexes, MaskArg mask);
+#endif
+ template<typename S1, typename IT> void gather(const S1 *array, const EntryType S1::* member1, IT indexes);
+ template<typename S1, typename IT> void gather(const S1 *array, const EntryType S1::* member1, IT indexes, MaskArg mask);
+ template<typename S1, typename S2, typename IT> void gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes);
+ template<typename S1, typename S2, typename IT> void gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes, MaskArg mask);
+ template<typename S1, typename IT1, typename IT2> void gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes);
+ template<typename S1, typename IT1, typename IT2> void gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes, MaskArg mask);
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // scatters
+ template<typename Index> void scatter(EntryType *mem, Index indexes) const;
+ template<typename Index> void scatter(EntryType *mem, Index indexes, MaskArg mask) const;
+ template<typename S1, typename IT> void scatter(S1 *array, EntryType S1::* member1, IT indexes) const;
+ template<typename S1, typename IT> void scatter(S1 *array, EntryType S1::* member1, IT indexes, MaskArg mask) const;
+ template<typename S1, typename S2, typename IT> void scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, IT indexes) const;
+ template<typename S1, typename S2, typename IT> void scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, IT indexes, MaskArg mask) const;
+ template<typename S1, typename IT1, typename IT2> void scatter(S1 *array, EntryType *S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes) const;
+ template<typename S1, typename IT1, typename IT2> void scatter(S1 *array, EntryType *S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes, MaskArg mask) const;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ //prefix
+ inline Vector ALWAYS_INLINE &operator++() { data() = VectorHelper<T>::add(data(), VectorHelper<T>::one()); return *this; }
+ //postfix
+ inline Vector ALWAYS_INLINE operator++(int) { const Vector<T> r = *this; data() = VectorHelper<T>::add(data(), VectorHelper<T>::one()); return r; }
+
+ inline Common::AliasingEntryHelper<StorageType> INTRINSIC operator[](int index) {
+#if defined(VC_GCC) && VC_GCC >= 0x40300 && VC_GCC < 0x40400
+ ::Vc::Warnings::_operator_bracket_warning();
+#endif
+ return d.m(index);
+ }
+ inline EntryType ALWAYS_INLINE operator[](int index) const {
+ return d.m(index);
+ }
+
+ inline Vector ALWAYS_INLINE operator~() const { return VectorHelper<VectorType>::andnot_(data(), VectorHelper<VectorType>::allone()); }
+ inline Vector<typename NegateTypeHelper<T>::Type> operator-() const;
+
+#define OP1(fun) \
+ inline Vector fun() const { return Vector<T>(VectorHelper<T>::fun(data())); } \
+ inline Vector &fun##_eq() { data() = VectorHelper<T>::fun(data()); return *this; }
+ OP1(sqrt)
+ OP1(abs)
+#undef OP1
+
+#define OP(symbol, fun) \
+ inline Vector ALWAYS_INLINE &operator symbol##=(const Vector<T> &x) { data() = VectorHelper<T>::fun(data(), x.data()); return *this; } \
+ inline Vector ALWAYS_INLINE &operator symbol##=(EntryType x) { return operator symbol##=(Vector(x)); } \
+ inline Vector ALWAYS_INLINE operator symbol(const Vector<T> &x) const { return Vector<T>(VectorHelper<T>::fun(data(), x.data())); } \
+ template<typename TT> inline VC_EXACT_TYPE(TT, EntryType, Vector) ALWAYS_INLINE operator symbol(TT x) const { return operator symbol(Vector(x)); }
+
+ OP(+, add)
+ OP(-, sub)
+ OP(*, mul)
+#undef OP
+ inline Vector &operator/=(EntryType x);
+ template<typename TT> inline PURE_L VC_EXACT_TYPE(TT, EntryType, Vector) operator/(TT x) const PURE_R;
+ inline Vector &operator/=(const Vector<T> &x);
+ inline Vector operator/ (const Vector<T> &x) const;
+
+ // bitwise ops
+#define OP_VEC(op) \
+ inline Vector<T> ALWAYS_INLINE_L &operator op##=(AsArg x) ALWAYS_INLINE_R; \
+ inline Vector<T> ALWAYS_INLINE_L operator op (AsArg x) const ALWAYS_INLINE_R;
+#define OP_ENTRY(op) \
+ inline ALWAYS_INLINE Vector<T> &operator op##=(EntryType x) { return operator op##=(Vector(x)); } \
+ template<typename TT> inline ALWAYS_INLINE VC_EXACT_TYPE(TT, EntryType, Vector) operator op(TT x) const { return operator op(Vector(x)); }
+ VC_ALL_BINARY(OP_VEC)
+ VC_ALL_BINARY(OP_ENTRY)
+ VC_ALL_SHIFTS(OP_VEC)
+#undef OP_VEC
+#undef OP_ENTRY
+
+ inline Vector<T> ALWAYS_INLINE_L &operator>>=(int x) ALWAYS_INLINE_R;
+ inline Vector<T> ALWAYS_INLINE_L &operator<<=(int x) ALWAYS_INLINE_R;
+ inline Vector<T> ALWAYS_INLINE_L operator>>(int x) const ALWAYS_INLINE_R;
+ inline Vector<T> ALWAYS_INLINE_L operator<<(int x) const ALWAYS_INLINE_R;
+
+#define OPcmp(symbol, fun) \
+ inline Mask ALWAYS_INLINE operator symbol(AsArg x) const { return VectorHelper<T>::fun(data(), x.data()); } \
+ template<typename TT> inline VC_EXACT_TYPE(TT, EntryType, Mask) ALWAYS_INLINE operator symbol(TT x) const { return operator symbol(Vector(x)); }
+
+ OPcmp(==, cmpeq)
+ OPcmp(!=, cmpneq)
+ OPcmp(>=, cmpnlt)
+ OPcmp(>, cmpnle)
+ OPcmp(<, cmplt)
+ OPcmp(<=, cmple)
+#undef OPcmp
+
+ inline void multiplyAndAdd(const Vector<T> &factor, const Vector<T> &summand) {
+ VectorHelper<T>::multiplyAndAdd(data(), factor, summand);
+ }
+
+ inline void assign( const Vector<T> &v, const Mask &mask ) {
+ const VectorType k = avx_cast<VectorType>(mask.data());
+ data() = VectorHelper<VectorType>::blend(data(), v.data(), k);
+ }
+
+ template<typename V2> inline V2 staticCast() const { return V2(*this); }
+ template<typename V2> inline V2 reinterpretCast() const { return avx_cast<typename V2::VectorType>(data()); }
+
+ inline WriteMaskedVector<T> ALWAYS_INLINE operator()(const Mask &k) { return WriteMaskedVector<T>(this, k); }
+
+ /**
+ * \return \p true This vector was completely filled. m2 might be 0 or != 0. You still have
+ * to test this.
+ * \p false This vector was not completely filled. m2 is all 0.
+ */
+ //inline bool pack(Mask &m1, Vector<T> &v2, Mask &m2) {
+ //return VectorHelper<T>::pack(data(), m1.data, v2.data(), m2.data);
+ //}
+
+ inline VectorType &data() { return d.v(); }
+ inline const VectorType &data() const { return d.v(); }
+
+ inline EntryType min() const { return VectorHelper<T>::min(data()); }
+ inline EntryType max() const { return VectorHelper<T>::max(data()); }
+ inline EntryType product() const { return VectorHelper<T>::mul(data()); }
+ inline EntryType sum() const { return VectorHelper<T>::add(data()); }
+ inline EntryType min(MaskArg m) const;
+ inline EntryType max(MaskArg m) const;
+ inline EntryType product(MaskArg m) const;
+ inline EntryType sum(MaskArg m) const;
+
+ inline Vector sorted() const { return SortHelper<T>::sort(data()); }
+
+ template<typename F> void callWithValuesSorted(F &f) {
+ EntryType value = d.m(0);
+ f(value);
+ for (int i = 1; i < Size; ++i) {
+ if (d.m(i) != value) {
+ value = d.m(i);
+ f(value);
+ }
+ }
+ }
+
+ template<typename F> inline void INTRINSIC call(const F &f) const {
+ for_all_vector_entries(i,
+ f(EntryType(d.m(i)));
+ );
+ }
+ template<typename F> inline void INTRINSIC call(F &f) const {
+ for_all_vector_entries(i,
+ f(EntryType(d.m(i)));
+ );
+ }
+
+ template<typename F> inline void INTRINSIC call(const F &f, const Mask &mask) const {
+ Vc_foreach_bit(size_t i, mask) {
+ f(EntryType(d.m(i)));
+ }
+ }
+ template<typename F> inline void INTRINSIC call(F &f, const Mask &mask) const {
+ Vc_foreach_bit(size_t i, mask) {
+ f(EntryType(d.m(i)));
+ }
+ }
+
+ template<typename F> inline Vector<T> INTRINSIC apply(const F &f) const {
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = f(EntryType(d.m(i)));
+ );
+ return r;
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(F &f) const {
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = f(EntryType(d.m(i)));
+ );
+ return r;
+ }
+
+ template<typename F> inline Vector<T> INTRINSIC apply(const F &f, const Mask &mask) const {
+ Vector<T> r(*this);
+ Vc_foreach_bit (size_t i, mask) {
+ r.d.m(i) = f(EntryType(r.d.m(i)));
+ }
+ return r;
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(F &f, const Mask &mask) const {
+ Vector<T> r(*this);
+ Vc_foreach_bit (size_t i, mask) {
+ r.d.m(i) = f(EntryType(r.d.m(i)));
+ }
+ return r;
+ }
+
+ template<typename IndexT> inline void INTRINSIC fill(EntryType (&f)(IndexT)) {
+ for_all_vector_entries(i,
+ d.m(i) = f(i);
+ );
+ }
+ inline void INTRINSIC fill(EntryType (&f)()) {
+ for_all_vector_entries(i,
+ d.m(i) = f();
+ );
+ }
+
+ inline INTRINSIC_L Vector copySign(AsArg reference) const INTRINSIC_R;
+ inline INTRINSIC_L Vector exponent() const INTRINSIC_R;
+};
+
+typedef Vector<double> double_v;
+typedef Vector<float> float_v;
+typedef Vector<sfloat> sfloat_v;
+typedef Vector<int> int_v;
+typedef Vector<unsigned int> uint_v;
+typedef Vector<short> short_v;
+typedef Vector<unsigned short> ushort_v;
+typedef double_v::Mask double_m;
+typedef float_v::Mask float_m;
+typedef sfloat_v::Mask sfloat_m;
+typedef int_v::Mask int_m;
+typedef uint_v::Mask uint_m;
+typedef short_v::Mask short_m;
+typedef ushort_v::Mask ushort_m;
+
+template<typename T> class SwizzledVector : public Vector<T> {};
+
+static inline int_v min(const int_v &x, const int_v &y) { return _mm256_min_epi32(x.data(), y.data()); }
+static inline uint_v min(const uint_v &x, const uint_v &y) { return _mm256_min_epu32(x.data(), y.data()); }
+static inline short_v min(const short_v &x, const short_v &y) { return _mm_min_epi16(x.data(), y.data()); }
+static inline ushort_v min(const ushort_v &x, const ushort_v &y) { return _mm_min_epu16(x.data(), y.data()); }
+static inline float_v min(const float_v &x, const float_v &y) { return _mm256_min_ps(x.data(), y.data()); }
+static inline sfloat_v min(const sfloat_v &x, const sfloat_v &y) { return _mm256_min_ps(x.data(), y.data()); }
+static inline double_v min(const double_v &x, const double_v &y) { return _mm256_min_pd(x.data(), y.data()); }
+static inline int_v max(const int_v &x, const int_v &y) { return _mm256_max_epi32(x.data(), y.data()); }
+static inline uint_v max(const uint_v &x, const uint_v &y) { return _mm256_max_epu32(x.data(), y.data()); }
+static inline short_v max(const short_v &x, const short_v &y) { return _mm_max_epi16(x.data(), y.data()); }
+static inline ushort_v max(const ushort_v &x, const ushort_v &y) { return _mm_max_epu16(x.data(), y.data()); }
+static inline float_v max(const float_v &x, const float_v &y) { return _mm256_max_ps(x.data(), y.data()); }
+static inline sfloat_v max(const sfloat_v &x, const sfloat_v &y) { return _mm256_max_ps(x.data(), y.data()); }
+static inline double_v max(const double_v &x, const double_v &y) { return _mm256_max_pd(x.data(), y.data()); }
+
+ template<typename T> static inline Vector<T> sqrt (const Vector<T> &x) { return VectorHelper<T>::sqrt(x.data()); }
+ template<typename T> static inline Vector<T> rsqrt(const Vector<T> &x) { return VectorHelper<T>::rsqrt(x.data()); }
+ template<typename T> static inline Vector<T> abs (const Vector<T> &x) { return VectorHelper<T>::abs(x.data()); }
+ template<typename T> static inline Vector<T> reciprocal(const Vector<T> &x) { return VectorHelper<T>::reciprocal(x.data()); }
+ template<typename T> static inline Vector<T> round(const Vector<T> &x) { return VectorHelper<T>::round(x.data()); }
+
+ template<typename T> static inline typename Vector<T>::Mask isfinite(const Vector<T> &x) { return VectorHelper<T>::isFinite(x.data()); }
+ template<typename T> static inline typename Vector<T>::Mask isnan(const Vector<T> &x) { return VectorHelper<T>::isNaN(x.data()); }
+
+#include "forceToRegisters.tcc"
+} // namespace AVX
+} // namespace Vc
+
+#include "vector.tcc"
+#include "math.h"
+#include "undomacros.h"
+
+#endif // AVX_VECTOR_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "limits.h"
+#include "const.h"
+#include "macros.h"
+
+namespace Vc
+{
+ALIGN(64) extern unsigned int RandomState[16];
+
+namespace AVX
+{
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// constants {{{1
+template<typename T> inline ALWAYS_INLINE Vector<T>::Vector(VectorSpecialInitializerZero::ZEnum) : d(HT::zero()) {}
+template<typename T> inline ALWAYS_INLINE Vector<T>::Vector(VectorSpecialInitializerOne::OEnum) : d(HT::one()) {}
+template<typename T> inline ALWAYS_INLINE Vector<T>::Vector(VectorSpecialInitializerIndexesFromZero::IEnum)
+ : d(HV::load(IndexesFromZeroData<T>::address(), Aligned)) {}
+
+template<typename T> inline Vector<T> INTRINSIC CONST Vector<T>::Zero() { return HT::zero(); }
+template<typename T> inline Vector<T> INTRINSIC CONST Vector<T>::One() { return HT::one(); }
+template<typename T> inline Vector<T> INTRINSIC CONST Vector<T>::IndexesFromZero() { return HV::load(IndexesFromZeroData<T>::address(), Aligned); }
+
+template<typename T> template<typename T2> inline ALWAYS_INLINE Vector<T>::Vector(Vector<T2> x)
+ : d(StaticCastHelper<T2, T>::cast(x.data())) {}
+
+template<typename T> inline ALWAYS_INLINE Vector<T>::Vector(EntryType x) : d(HT::set(x)) {}
+template<> inline ALWAYS_INLINE Vector<double>::Vector(EntryType x) : d(_mm256_set1_pd(x)) {}
+
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// load ctors {{{1
+template<typename T> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *x) { load(x); }
+template<typename T> template<typename A> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *x, A a) { load(x, a); }
+template<typename T> template<typename OtherT> inline ALWAYS_INLINE Vector<T>::Vector(const OtherT *x) { load(x); }
+template<typename T> template<typename OtherT, typename A> inline ALWAYS_INLINE Vector<T>::Vector(const OtherT *x, A a) { load(x, a); }
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// load member functions {{{1
+template<typename T> inline void INTRINSIC Vector<T>::load(const EntryType *mem)
+{
+ load(mem, Aligned);
+}
+
+template<typename T> template<typename A> inline void INTRINSIC Vector<T>::load(const EntryType *mem, A align)
+{
+ d.v() = HV::load(mem, align);
+}
+
+template<typename T> template<typename OtherT> inline void INTRINSIC Vector<T>::load(const OtherT *mem)
+{
+ load(mem, Aligned);
+}
+
+// LoadHelper {{{2
+template<typename DstT, typename SrcT, typename Flags> struct LoadHelper;
+
+// float {{{2
+template<typename Flags> struct LoadHelper<float, double, Flags> {
+ static __m256 load(const double *mem, Flags f)
+ {
+ return concat(_mm256_cvtpd_ps(VectorHelper<__m256d>::load(&mem[0], f)),
+ _mm256_cvtpd_ps(VectorHelper<__m256d>::load(&mem[4], f)));
+ }
+};
+template<typename Flags> struct LoadHelper<float, unsigned int, Flags> {
+ static __m256 load(const unsigned int *mem, Flags f)
+ {
+ return StaticCastHelper<unsigned int, float>::cast(VectorHelper<__m256i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, int, Flags> {
+ static __m256 load(const int *mem, Flags f)
+ {
+ return StaticCastHelper<int, float>::cast(VectorHelper<__m256i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, unsigned short, Flags> {
+ static __m256 load(const unsigned short *mem, Flags f)
+ {
+ return StaticCastHelper<unsigned short, float>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, short, Flags> {
+ static __m256 load(const short *mem, Flags f)
+ {
+ return StaticCastHelper<short, float>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, unsigned char, Flags> {
+ static __m256 load(const unsigned char *mem, Flags f)
+ {
+ return StaticCastHelper<unsigned int, float>::cast(LoadHelper<unsigned int, unsigned char, Flags>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, signed char, Flags> {
+ static __m256 load(const signed char *mem, Flags f)
+ {
+ return StaticCastHelper<int, float>::cast(LoadHelper<int, signed char, Flags>::load(mem, f));
+ }
+};
+
+template<typename SrcT, typename Flags> struct LoadHelper<sfloat, SrcT, Flags> : public LoadHelper<float, SrcT, Flags> {};
+
+// int {{{2
+template<typename Flags> struct LoadHelper<int, unsigned int, Flags> {
+ static __m256i load(const unsigned int *mem, Flags f)
+ {
+ return VectorHelper<__m256i>::load(mem, f);
+ }
+};
+template<typename Flags> struct LoadHelper<int, unsigned short, Flags> {
+ static __m256i load(const unsigned short *mem, Flags f)
+ {
+ return StaticCastHelper<unsigned short, unsigned int>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<int, short, Flags> {
+ static __m256i load(const short *mem, Flags f)
+ {
+ return StaticCastHelper<short, int>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<int, unsigned char, Flags> {
+ static __m256i load(const unsigned char *mem, Flags)
+ {
+ // the only available streaming load loads 16 bytes - twice as much as we need => can't use
+ // it, or we risk an out-of-bounds read and an unaligned load exception
+ const __m128i epu8 = _mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem));
+ const __m128i epu16 = _mm_cvtepu8_epi16(epu8);
+ return StaticCastHelper<unsigned short, unsigned int>::cast(epu16);
+ }
+};
+template<typename Flags> struct LoadHelper<int, signed char, Flags> {
+ static __m256i load(const signed char *mem, Flags)
+ {
+ // the only available streaming load loads 16 bytes - twice as much as we need => can't use
+ // it, or we risk an out-of-bounds read and an unaligned load exception
+ const __m128i epi8 = _mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem));
+ const __m128i epi16 = _mm_cvtepi8_epi16(epi8);
+ return StaticCastHelper<short, int>::cast(epi16);
+ }
+};
+
+// unsigned int {{{2
+template<typename Flags> struct LoadHelper<unsigned int, unsigned short, Flags> {
+ static __m256i load(const unsigned short *mem, Flags f)
+ {
+ return StaticCastHelper<unsigned short, unsigned int>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<unsigned int, unsigned char, Flags> {
+ static __m256i load(const unsigned char *mem, Flags)
+ {
+ // the only available streaming load loads 16 bytes - twice as much as we need => can't use
+ // it, or we risk an out-of-bounds read and an unaligned load exception
+ const __m128i epu8 = _mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem));
+ const __m128i epu16 = _mm_cvtepu8_epi16(epu8);
+ return StaticCastHelper<unsigned short, unsigned int>::cast(epu16);
+ }
+};
+
+// short {{{2
+template<typename Flags> struct LoadHelper<short, unsigned short, Flags> {
+ static __m128i load(const unsigned short *mem, Flags f)
+ {
+ return StaticCastHelper<unsigned short, short>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<short, unsigned char, Flags> {
+ static __m128i load(const unsigned char *mem, Flags)
+ {
+ // the only available streaming load loads 16 bytes - twice as much as we need => can't use
+ // it, or we risk an out-of-bounds read and an unaligned load exception
+ const __m128i epu8 = _mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem));
+ return _mm_cvtepu8_epi16(epu8);
+ }
+};
+template<typename Flags> struct LoadHelper<short, signed char, Flags> {
+ static __m128i load(const signed char *mem, Flags)
+ {
+ // the only available streaming load loads 16 bytes - twice as much as we need => can't use
+ // it, or we risk an out-of-bounds read and an unaligned load exception
+ const __m128i epi8 = _mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem));
+ return _mm_cvtepi8_epi16(epi8);
+ }
+};
+
+// unsigned short {{{2
+template<typename Flags> struct LoadHelper<unsigned short, unsigned char, Flags> {
+ static __m128i load(const unsigned char *mem, Flags)
+ {
+ // the only available streaming load loads 16 bytes - twice as much as we need => can't use
+ // it, or we risk an out-of-bounds read and an unaligned load exception
+ const __m128i epu8 = _mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem));
+ return _mm_cvtepu8_epi16(epu8);
+ }
+};
+
+// general load, implemented via LoadHelper {{{2
+template<typename DstT> template<typename SrcT, typename Flags> inline void INTRINSIC Vector<DstT>::load(const SrcT *x, Flags f)
+{
+ d.v() = LoadHelper<DstT, SrcT, Flags>::load(x, f);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// zeroing {{{1
+template<typename T> inline void INTRINSIC Vector<T>::setZero()
+{
+ data() = HV::zero();
+}
+template<typename T> inline void INTRINSIC Vector<T>::setZero(const Mask &k)
+{
+ data() = HV::andnot_(avx_cast<VectorType>(k.data()), data());
+}
+
+template<> inline void INTRINSIC Vector<double>::setQnan()
+{
+ data() = _mm256_setallone_pd();
+}
+template<> inline void INTRINSIC Vector<double>::setQnan(MaskArg k)
+{
+ data() = _mm256_or_pd(data(), k.dataD());
+}
+template<> inline void INTRINSIC Vector<float>::setQnan()
+{
+ data() = _mm256_setallone_ps();
+}
+template<> inline void INTRINSIC Vector<float>::setQnan(MaskArg k)
+{
+ data() = _mm256_or_ps(data(), k.data());
+}
+template<> inline void INTRINSIC Vector<sfloat>::setQnan()
+{
+ data() = _mm256_setallone_ps();
+}
+template<> inline void INTRINSIC Vector<sfloat>::setQnan(MaskArg k)
+{
+ data() = _mm256_or_ps(data(), k.data());
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// stores {{{1
+template<typename T> inline void INTRINSIC Vector<T>::store(EntryType *mem) const
+{
+ HV::store(mem, data(), Aligned);
+}
+template<typename T> inline void INTRINSIC Vector<T>::store(EntryType *mem, const Mask &mask) const
+{
+ HV::store(mem, data(), avx_cast<VectorType>(mask.data()), Aligned);
+}
+template<typename T> template<typename A> inline void INTRINSIC Vector<T>::store(EntryType *mem, A align) const
+{
+ HV::store(mem, data(), align);
+}
+template<typename T> template<typename A> inline void INTRINSIC Vector<T>::store(EntryType *mem, const Mask &mask, A align) const
+{
+ HV::store(mem, data(), avx_cast<VectorType>(mask.data()), align);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// expand/merge 1 float_v <=> 2 double_v XXX rationale? remove it for release? XXX {{{1
+template<typename T> inline ALWAYS_INLINE FLATTEN Vector<T>::Vector(const Vector<typename HT::ConcatType> *a)
+ : d(a[0])
+{
+}
+template<> inline ALWAYS_INLINE FLATTEN Vector<float>::Vector(const Vector<HT::ConcatType> *a)
+ : d(concat(_mm256_cvtpd_ps(a[0].data()), _mm256_cvtpd_ps(a[1].data())))
+{
+}
+template<> inline ALWAYS_INLINE FLATTEN Vector<short>::Vector(const Vector<HT::ConcatType> *a)
+ : d(_mm_packs_epi32(lo128(a->data()), hi128(a->data())))
+{
+}
+template<> inline ALWAYS_INLINE FLATTEN Vector<unsigned short>::Vector(const Vector<HT::ConcatType> *a)
+ : d(_mm_packus_epi32(lo128(a->data()), hi128(a->data())))
+{
+}
+template<typename T> inline void ALWAYS_INLINE FLATTEN Vector<T>::expand(Vector<typename HT::ConcatType> *x) const
+{
+ x[0] = *this;
+}
+template<> inline void ALWAYS_INLINE FLATTEN Vector<float>::expand(Vector<HT::ConcatType> *x) const
+{
+ x[0].data() = _mm256_cvtps_pd(lo128(d.v()));
+ x[1].data() = _mm256_cvtps_pd(hi128(d.v()));
+}
+template<> inline void ALWAYS_INLINE FLATTEN Vector<short>::expand(Vector<HT::ConcatType> *x) const
+{
+ x[0].data() = concat(_mm_cvtepi16_epi32(d.v()),
+ _mm_cvtepi16_epi32(_mm_unpackhi_epi64(d.v(), d.v())));
+}
+template<> inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::expand(Vector<HT::ConcatType> *x) const
+{
+ x[0].data() = concat(_mm_cvtepu16_epi32(d.v()),
+ _mm_cvtepu16_epi32(_mm_unpackhi_epi64(d.v(), d.v())));
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// swizzles {{{1
+template<typename T> inline const Vector<T> INTRINSIC CONST &Vector<T>::abcd() const { return *this; }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::cdab() const { return Mem::permute<X2, X3, X0, X1>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::badc() const { return Mem::permute<X1, X0, X3, X2>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::aaaa() const { return Mem::permute<X0, X0, X0, X0>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::bbbb() const { return Mem::permute<X1, X1, X1, X1>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::cccc() const { return Mem::permute<X2, X2, X2, X2>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dddd() const { return Mem::permute<X3, X3, X3, X3>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::bcad() const { return Mem::permute<X1, X2, X0, X3>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::bcda() const { return Mem::permute<X1, X2, X3, X0>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dabc() const { return Mem::permute<X3, X0, X1, X2>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::acbd() const { return Mem::permute<X0, X2, X1, X3>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dbca() const { return Mem::permute<X3, X1, X2, X0>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dcba() const { return Mem::permute<X3, X2, X1, X0>(data()); }
+
+template<> inline const double_v INTRINSIC CONST Vector<double>::cdab() const { return Mem::shuffle128<X1, X0>(data(), data()); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::badc() const { return Mem::permute<X1, X0, X3, X2>(data()); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::aaaa() const { const double &tmp = d.m(0); return _mm256_broadcast_sd(&tmp); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::bbbb() const { const double &tmp = d.m(1); return _mm256_broadcast_sd(&tmp); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::cccc() const { const double &tmp = d.m(2); return _mm256_broadcast_sd(&tmp); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::dddd() const { const double &tmp = d.m(3); return _mm256_broadcast_sd(&tmp); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::bcad() const { return Mem::shuffle<X1, Y0, X2, Y3>(Mem::shuffle128<X0, X0>(data(), data()), Mem::shuffle128<X1, X1>(data(), data())); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::bcda() const { return Mem::shuffle<X1, Y0, X3, Y2>(data(), Mem::shuffle128<X1, X0>(data(), data())); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::dabc() const { return Mem::shuffle<X1, Y0, X3, Y2>(Mem::shuffle128<X1, X0>(data(), data()), data()); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::acbd() const { return Mem::shuffle<X0, Y0, X3, Y3>(Mem::shuffle128<X0, X0>(data(), data()), Mem::shuffle128<X1, X1>(data(), data())); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::dbca() const { return Mem::shuffle<X1, Y1, X2, Y2>(Mem::shuffle128<X1, X1>(data(), data()), Mem::shuffle128<X0, X0>(data(), data())); }
+template<> inline const double_v INTRINSIC CONST Vector<double>::dcba() const { return cdab().badc(); }
+
+#define VC_SWIZZLES_16BIT_IMPL(T) \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::cdab() const { return Mem::permute<X2, X3, X0, X1, X6, X7, X4, X5>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::badc() const { return Mem::permute<X1, X0, X3, X2, X5, X4, X7, X6>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::aaaa() const { return Mem::permute<X0, X0, X0, X0, X4, X4, X4, X4>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::bbbb() const { return Mem::permute<X1, X1, X1, X1, X5, X5, X5, X5>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::cccc() const { return Mem::permute<X2, X2, X2, X2, X6, X6, X6, X6>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dddd() const { return Mem::permute<X3, X3, X3, X3, X7, X7, X7, X7>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::bcad() const { return Mem::permute<X1, X2, X0, X3, X5, X6, X4, X7>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::bcda() const { return Mem::permute<X1, X2, X3, X0, X5, X6, X7, X4>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dabc() const { return Mem::permute<X3, X0, X1, X2, X7, X4, X5, X6>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::acbd() const { return Mem::permute<X0, X2, X1, X3, X4, X6, X5, X7>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dbca() const { return Mem::permute<X3, X1, X2, X0, X7, X5, X6, X4>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dcba() const { return Mem::permute<X3, X2, X1, X0, X7, X6, X5, X4>(data()); }
+VC_SWIZZLES_16BIT_IMPL(short)
+VC_SWIZZLES_16BIT_IMPL(unsigned short)
+#undef VC_SWIZZLES_16BIT_IMPL
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// division {{{1
+template<typename T> inline Vector<T> &Vector<T>::operator/=(EntryType x)
+{
+ if (HasVectorDivision) {
+ return operator/=(Vector<T>(x));
+ }
+ for_all_vector_entries(i,
+ d.m(i) /= x;
+ );
+ return *this;
+}
+template<typename T> template<typename TT> inline PURE VC_EXACT_TYPE(TT, typename DetermineEntryType<T>::Type, Vector<T>) Vector<T>::operator/(TT x) const
+{
+ if (HasVectorDivision) {
+ return operator/(Vector<T>(x));
+ }
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = d.m(i) / x;
+ );
+ return r;
+}
+// per default fall back to scalar division
+template<typename T> inline Vector<T> &Vector<T>::operator/=(const Vector<T> &x)
+{
+ for_all_vector_entries(i,
+ d.m(i) /= x.d.m(i);
+ );
+ return *this;
+}
+
+template<typename T> inline Vector<T> PURE Vector<T>::operator/(const Vector<T> &x) const
+{
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = d.m(i) / x.d.m(i);
+ );
+ return r;
+}
+// specialize division on type
+static inline __m256i INTRINSIC CONST divInt(__m256i a, __m256i b) {
+ const __m256d lo1 = _mm256_cvtepi32_pd(lo128(a));
+ const __m256d lo2 = _mm256_cvtepi32_pd(lo128(b));
+ const __m256d hi1 = _mm256_cvtepi32_pd(hi128(a));
+ const __m256d hi2 = _mm256_cvtepi32_pd(hi128(b));
+ return concat(
+ _mm256_cvttpd_epi32(_mm256_div_pd(lo1, lo2)),
+ _mm256_cvttpd_epi32(_mm256_div_pd(hi1, hi2))
+ );
+}
+template<> inline Vector<int> &Vector<int>::operator/=(const Vector<int> &x)
+{
+ d.v() = divInt(d.v(), x.d.v());
+ return *this;
+}
+template<> inline Vector<int> PURE Vector<int>::operator/(const Vector<int> &x) const
+{
+ return divInt(d.v(), x.d.v());
+}
+static inline __m256i CONST divUInt(__m256i a, __m256i b) {
+ __m256d loa = _mm256_cvtepi32_pd(lo128(a));
+ __m256d hia = _mm256_cvtepi32_pd(hi128(a));
+ __m256d lob = _mm256_cvtepi32_pd(lo128(b));
+ __m256d hib = _mm256_cvtepi32_pd(hi128(b));
+ // if a >= 2^31 then after conversion to double it will contain a negative number (i.e. a-2^32)
+ // to get the right number back we have to add 2^32 where a >= 2^31
+ loa = _mm256_add_pd(loa, _mm256_and_pd(_mm256_cmp_pd(loa, _mm256_setzero_pd(), _CMP_LT_OS), _mm256_set1_pd(4294967296.)));
+ hia = _mm256_add_pd(hia, _mm256_and_pd(_mm256_cmp_pd(hia, _mm256_setzero_pd(), _CMP_LT_OS), _mm256_set1_pd(4294967296.)));
+ // we don't do the same for b because division by b >= 2^31 should be a seldom corner case and
+ // we rather want the standard stuff fast
+ //
+ // there is one remaining problem: a >= 2^31 and b == 1
+ // in that case the return value would be 2^31
+ return avx_cast<__m256i>(_mm256_blendv_ps(avx_cast<__m256>(concat(
+ _mm256_cvttpd_epi32(_mm256_div_pd(loa, lob)),
+ _mm256_cvttpd_epi32(_mm256_div_pd(hia, hib))
+ )), avx_cast<__m256>(a), avx_cast<__m256>(concat(
+ _mm_cmpeq_epi32(lo128(b), _mm_setone_epi32()),
+ _mm_cmpeq_epi32(hi128(b), _mm_setone_epi32())))));
+}
+template<> inline Vector<unsigned int> ALWAYS_INLINE &Vector<unsigned int>::operator/=(const Vector<unsigned int> &x)
+{
+ d.v() = divUInt(d.v(), x.d.v());
+ return *this;
+}
+template<> inline Vector<unsigned int> ALWAYS_INLINE PURE Vector<unsigned int>::operator/(const Vector<unsigned int> &x) const
+{
+ return divUInt(d.v(), x.d.v());
+}
+template<typename T> static inline __m128i CONST divShort(__m128i a, __m128i b)
+{
+ const __m256 r = _mm256_div_ps(StaticCastHelper<T, float>::cast(a),
+ StaticCastHelper<T, float>::cast(b));
+ return StaticCastHelper<float, T>::cast(r);
+}
+template<> inline Vector<short> ALWAYS_INLINE &Vector<short>::operator/=(const Vector<short> &x)
+{
+ d.v() = divShort<short>(d.v(), x.d.v());
+ return *this;
+}
+template<> inline Vector<short> ALWAYS_INLINE PURE Vector<short>::operator/(const Vector<short> &x) const
+{
+ return divShort<short>(d.v(), x.d.v());
+}
+template<> inline Vector<unsigned short> ALWAYS_INLINE &Vector<unsigned short>::operator/=(const Vector<unsigned short> &x)
+{
+ d.v() = divShort<unsigned short>(d.v(), x.d.v());
+ return *this;
+}
+template<> inline Vector<unsigned short> ALWAYS_INLINE PURE Vector<unsigned short>::operator/(const Vector<unsigned short> &x) const
+{
+ return divShort<unsigned short>(d.v(), x.d.v());
+}
+template<> inline Vector<float> INTRINSIC &Vector<float>::operator/=(const Vector<float> &x)
+{
+ d.v() = _mm256_div_ps(d.v(), x.d.v());
+ return *this;
+}
+template<> inline Vector<float> INTRINSIC PURE Vector<float>::operator/(const Vector<float> &x) const
+{
+ return _mm256_div_ps(d.v(), x.d.v());
+}
+template<> inline Vector<double> INTRINSIC &Vector<double>::operator/=(const Vector<double> &x)
+{
+ d.v() = _mm256_div_pd(d.v(), x.d.v());
+ return *this;
+}
+template<> inline Vector<double> INTRINSIC PURE Vector<double>::operator/(const Vector<double> &x) const
+{
+ return _mm256_div_pd(d.v(), x.d.v());
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// integer ops {{{1
+#define OP_IMPL(T, symbol) \
+template<> inline Vector<T> &Vector<T>::operator symbol##=(AsArg x) \
+{ \
+ for_all_vector_entries(i, d.m(i) symbol##= x.d.m(i); ); \
+ return *this; \
+} \
+template<> inline Vector<T> Vector<T>::operator symbol(AsArg x) const \
+{ \
+ Vector<T> r; \
+ for_all_vector_entries(i, r.d.m(i) = d.m(i) symbol x.d.m(i); ); \
+ return r; \
+}
+OP_IMPL(int, <<)
+OP_IMPL(int, >>)
+OP_IMPL(unsigned int, <<)
+OP_IMPL(unsigned int, >>)
+OP_IMPL(short, <<)
+OP_IMPL(short, >>)
+OP_IMPL(unsigned short, <<)
+OP_IMPL(unsigned short, >>)
+#undef OP_IMPL
+
+template<typename T> inline Vector<T> &Vector<T>::operator>>=(int shift) {
+ d.v() = VectorHelper<T>::shiftRight(d.v(), shift);
+ return *static_cast<Vector<T> *>(this);
+}
+template<typename T> inline Vector<T> Vector<T>::operator>>(int shift) const {
+ return VectorHelper<T>::shiftRight(d.v(), shift);
+}
+template<typename T> inline Vector<T> &Vector<T>::operator<<=(int shift) {
+ d.v() = VectorHelper<T>::shiftLeft(d.v(), shift);
+ return *static_cast<Vector<T> *>(this);
+}
+template<typename T> inline Vector<T> Vector<T>::operator<<(int shift) const {
+ return VectorHelper<T>::shiftLeft(d.v(), shift);
+}
+
+#define OP_IMPL(T, symbol, fun) \
+ template<> inline Vector<T> &Vector<T>::operator symbol##=(AsArg x) { d.v() = HV::fun(d.v(), x.d.v()); return *this; } \
+ template<> inline Vector<T> Vector<T>::operator symbol(AsArg x) const { return Vector<T>(HV::fun(d.v(), x.d.v())); }
+ OP_IMPL(int, &, and_)
+ OP_IMPL(int, |, or_)
+ OP_IMPL(int, ^, xor_)
+ OP_IMPL(unsigned int, &, and_)
+ OP_IMPL(unsigned int, |, or_)
+ OP_IMPL(unsigned int, ^, xor_)
+ OP_IMPL(short, &, and_)
+ OP_IMPL(short, |, or_)
+ OP_IMPL(short, ^, xor_)
+ OP_IMPL(unsigned short, &, and_)
+ OP_IMPL(unsigned short, |, or_)
+ OP_IMPL(unsigned short, ^, xor_)
+ OP_IMPL(float, &, and_)
+ OP_IMPL(float, |, or_)
+ OP_IMPL(float, ^, xor_)
+ OP_IMPL(sfloat, &, and_)
+ OP_IMPL(sfloat, |, or_)
+ OP_IMPL(sfloat, ^, xor_)
+ OP_IMPL(double, &, and_)
+ OP_IMPL(double, |, or_)
+ OP_IMPL(double, ^, xor_)
+#undef OP_IMPL
+
+// operators {{{1
+#include "../common/operators.h"
+// gathers {{{1
+// Better implementation (hopefully) with _mm256_set_
+//X template<typename T> template<typename Index> Vector<T>::Vector(const EntryType *mem, const Index *indexes)
+//X {
+//X for_all_vector_entries(int i,
+//X d.m(i) = mem[indexes[i]];
+//X );
+//X }
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, const IndexT *indexes)
+{
+ gather(mem, indexes);
+}
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, Vector<IndexT> indexes)
+{
+ gather(mem, indexes);
+}
+
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, const IndexT *indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(mem, indexes, mask);
+}
+
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, Vector<IndexT> indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(mem, indexes, mask);
+}
+
+template<typename T> template<typename S1, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ gather(array, member1, indexes);
+}
+template<typename T> template<typename S1, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType S1::* member1, IT indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(array, member1, indexes, mask);
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ gather(array, member1, member2, indexes);
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(array, member1, member2, indexes, mask);
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ gather(array, ptrMember1, outerIndexes, innerIndexes);
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(array, ptrMember1, outerIndexes, innerIndexes, mask);
+}
+
+template<typename T, size_t Size> struct IndexSizeChecker { static void check() {} };
+template<typename T, size_t Size> struct IndexSizeChecker<Vector<T>, Size>
+{
+ static void check() {
+ VC_STATIC_ASSERT(Vector<T>::Size >= Size, IndexVector_must_have_greater_or_equal_number_of_entries);
+ }
+};
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const EntryType *mem, Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm256_setr_pd(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const EntryType *mem, Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm256_setr_ps(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<sfloat>::gather(const EntryType *mem, Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm256_setr_ps(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const EntryType *mem, Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm256_setr_epi32(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const EntryType *mem, Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm256_setr_epi32(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const EntryType *mem, Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_epi16(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const EntryType *mem, Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_epi16(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+
+#ifdef VC_USE_SET_GATHERS
+template<typename T> template<typename IT> inline void ALWAYS_INLINE Vector<T>::gather(const EntryType *mem, Vector<IT> indexes, MaskArg mask)
+{
+ IndexSizeChecker<Vector<IT>, Size>::check();
+ indexes.setZero(!mask);
+ (*this)(mask) = Vector<T>(mem, indexes);
+}
+#endif
+
+#ifdef VC_USE_BSF_GATHERS
+#define VC_MASKED_GATHER \
+ int bits = mask.toInt(); \
+ while (bits) { \
+ const int i = _bit_scan_forward(bits); \
+ bits &= ~(1 << i); /* btr? */ \
+ d.m(i) = ith_value(i); \
+ }
+#elif defined(VC_USE_POPCNT_BSF_GATHERS)
+#define VC_MASKED_GATHER \
+ unsigned int bits = mask.toInt(); \
+ unsigned int low, high = 0; \
+ switch (_mm_popcnt_u32(bits)) { \
+ case 8: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ high = (1 << high); \
+ case 7: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ d.m(low) = ith_value(low); \
+ case 6: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ high = (1 << high); \
+ case 5: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ d.m(low) = ith_value(low); \
+ case 4: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ high = (1 << high); \
+ case 3: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ d.m(low) = ith_value(low); \
+ case 2: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ case 1: \
+ low = _bit_scan_forward(bits); \
+ d.m(low) = ith_value(low); \
+ case 0: \
+ break; \
+ }
+#else
+#define VC_MASKED_GATHER \
+ if (mask.isEmpty()) { \
+ return; \
+ } \
+ for_all_vector_entries(i, \
+ if (mask[i]) d.m(i) = ith_value(i); \
+ );
+#endif
+
+template<typename T> template<typename Index>
+inline void INTRINSIC Vector<T>::gather(const EntryType *mem, Index indexes, MaskArg mask)
+{
+ IndexSizeChecker<Index, Size>::check();
+#define ith_value(_i_) (mem[indexes[_i_]])
+ VC_MASKED_GATHER
+#undef ith_value
+}
+
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_pd(array[indexes[0]].*(member1), array[indexes[1]].*(member1),
+ array[indexes[2]].*(member1), array[indexes[3]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_ps(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<sfloat>::gather(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_ps(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_epi32(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_epi32(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const S1 *array, const EntryType S1::* member1, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<typename T> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<T>::gather(const S1 *array, const EntryType S1::* member1, IT indexes, MaskArg mask)
+{
+ IndexSizeChecker<IT, Size>::check();
+#define ith_value(_i_) (array[indexes[_i_]].*(member1))
+ VC_MASKED_GATHER
+#undef ith_value
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_pd(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2),
+ array[indexes[2]].*(member1).*(member2), array[indexes[3]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_ps(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<sfloat>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_ps(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_epi32(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm256_setr_epi32(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<typename T> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<T>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, IT indexes, MaskArg mask)
+{
+ IndexSizeChecker<IT, Size>::check();
+#define ith_value(_i_) (array[indexes[_i_]].*(member1).*(member2))
+ VC_MASKED_GATHER
+#undef ith_value
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm256_setr_pd((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm256_setr_ps((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<sfloat>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm256_setr_ps((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm256_setr_epi32((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm256_setr_epi32((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_epi16((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_epi16((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<typename T> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<T>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes, MaskArg mask)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+#define ith_value(_i_) (array[outerIndexes[_i_]].*(ptrMember1))[innerIndexes[_i_]]
+ VC_MASKED_GATHER
+#undef ith_value
+}
+
+#undef VC_MASKED_GATHER
+#ifdef VC_USE_BSF_SCATTERS
+#define VC_MASKED_SCATTER \
+ int bits = mask.toInt(); \
+ while (bits) { \
+ const int i = _bit_scan_forward(bits); \
+ bits ^= (1 << i); /* btr? */ \
+ ith_value(i) = d.m(i); \
+ }
+#elif defined(VC_USE_POPCNT_BSF_SCATTERS)
+#define VC_MASKED_SCATTER \
+ unsigned int bits = mask.toInt(); \
+ unsigned int low, high = 0; \
+ switch (_mm_popcnt_u32(bits)) { \
+ case 8: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ high = (1 << high); \
+ case 7: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ ith_value(low) = d.m(low); \
+ case 6: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ high = (1 << high); \
+ case 5: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ ith_value(low) = d.m(low); \
+ case 4: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ high = (1 << high); \
+ case 3: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ ith_value(low) = d.m(low); \
+ case 2: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ case 1: \
+ low = _bit_scan_forward(bits); \
+ ith_value(low) = d.m(low); \
+ case 0: \
+ break; \
+ }
+#else
+#define VC_MASKED_SCATTER \
+ if (mask.isEmpty()) { \
+ return; \
+ } \
+ for_all_vector_entries(i, \
+ if (mask[i]) ith_value(i) = d.m(i); \
+ );
+#endif
+
+template<typename T> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(EntryType *mem, Index indexes) const
+{
+ for_all_vector_entries(i,
+ mem[indexes[i]] = d.m(i);
+ );
+}
+template<typename T> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(EntryType *mem, Index indexes, MaskArg mask) const
+{
+#define ith_value(_i_) mem[indexes[_i_]]
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+template<typename T> template<typename S1, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType S1::* member1, IT indexes) const
+{
+ for_all_vector_entries(i,
+ array[indexes[i]].*(member1) = d.m(i);
+ );
+}
+template<typename T> template<typename S1, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType S1::* member1, IT indexes, MaskArg mask) const
+{
+#define ith_value(_i_) array[indexes[_i_]].*(member1)
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, IT indexes) const
+{
+ for_all_vector_entries(i,
+ array[indexes[i]].*(member1).*(member2) = d.m(i);
+ );
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, IT indexes, MaskArg mask) const
+{
+#define ith_value(_i_) array[indexes[_i_]].*(member1).*(member2)
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType *S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes) const
+{
+ for_all_vector_entries(i,
+ (array[innerIndexes[i]].*(ptrMember1))[outerIndexes[i]] = d.m(i);
+ );
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType *S1::* ptrMember1, IT1 outerIndexes, IT2 innerIndexes, MaskArg mask) const
+{
+#define ith_value(_i_) (array[outerIndexes[_i_]].*(ptrMember1))[innerIndexes[_i_]]
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// operator- {{{1
+template<> inline Vector<double> PURE ALWAYS_INLINE FLATTEN Vector<double>::operator-() const
+{
+ return _mm256_xor_pd(d.v(), _mm256_setsignmask_pd());
+}
+template<> inline Vector<float> PURE ALWAYS_INLINE FLATTEN Vector<float>::operator-() const
+{
+ return _mm256_xor_ps(d.v(), _mm256_setsignmask_ps());
+}
+template<> inline Vector<sfloat> PURE ALWAYS_INLINE FLATTEN Vector<sfloat>::operator-() const
+{
+ return _mm256_xor_ps(d.v(), _mm256_setsignmask_ps());
+}
+template<> inline Vector<int> PURE ALWAYS_INLINE FLATTEN Vector<int>::operator-() const
+{
+ return _mm256_sign_epi32(d.v(), _mm256_setallone_si256());
+}
+template<> inline Vector<int> PURE ALWAYS_INLINE FLATTEN Vector<unsigned int>::operator-() const
+{
+ return _mm256_sign_epi32(d.v(), _mm256_setallone_si256());
+}
+template<> inline Vector<short> PURE ALWAYS_INLINE FLATTEN Vector<short>::operator-() const
+{
+ return _mm_sign_epi16(d.v(), _mm_setallone_si128());
+}
+template<> inline Vector<short> PURE ALWAYS_INLINE FLATTEN Vector<unsigned short>::operator-() const
+{
+ return _mm_sign_epi16(d.v(), _mm_setallone_si128());
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// horizontal ops {{{1
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::min(MaskArg m) const
+{
+ Vector<T> tmp = std::numeric_limits<Vector<T> >::max();
+ tmp(m) = *this;
+ return tmp.min();
+}
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::max(MaskArg m) const
+{
+ Vector<T> tmp = std::numeric_limits<Vector<T> >::min();
+ tmp(m) = *this;
+ return tmp.max();
+}
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::product(MaskArg m) const
+{
+ Vector<T> tmp(VectorSpecialInitializerOne::One);
+ tmp(m) = *this;
+ return tmp.product();
+}
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::sum(MaskArg m) const
+{
+ Vector<T> tmp(VectorSpecialInitializerZero::Zero);
+ tmp(m) = *this;
+ return tmp.sum();
+}//}}}
+// copySign {{{1
+template<> inline Vector<float> INTRINSIC Vector<float>::copySign(Vector<float>::AsArg reference) const
+{
+ return _mm256_or_ps(
+ _mm256_and_ps(reference.d.v(), _mm256_setsignmask_ps()),
+ _mm256_and_ps(d.v(), _mm256_setabsmask_ps())
+ );
+}
+template<> inline Vector<sfloat> INTRINSIC Vector<sfloat>::copySign(Vector<sfloat>::AsArg reference) const
+{
+ return _mm256_or_ps(
+ _mm256_and_ps(reference.d.v(), _mm256_setsignmask_ps()),
+ _mm256_and_ps(d.v(), _mm256_setabsmask_ps())
+ );
+}
+template<> inline Vector<double> INTRINSIC Vector<double>::copySign(Vector<double>::AsArg reference) const
+{
+ return _mm256_or_pd(
+ _mm256_and_pd(reference.d.v(), _mm256_setsignmask_pd()),
+ _mm256_and_pd(d.v(), _mm256_setabsmask_pd())
+ );
+}//}}}1
+// exponent {{{1
+template<> inline Vector<float> INTRINSIC Vector<float>::exponent() const
+{
+ VC_ASSERT((*this > 0.f).isFull());
+ __m128i tmp0 = _mm_srli_epi32(avx_cast<__m128i>(d.v()), 23);
+ __m128i tmp1 = _mm_srli_epi32(avx_cast<__m128i>(hi128(d.v())), 23);
+ tmp0 = _mm_sub_epi32(tmp0, _mm_set1_epi32(0x7f));
+ tmp1 = _mm_sub_epi32(tmp1, _mm_set1_epi32(0x7f));
+ return _mm256_cvtepi32_ps(concat(tmp0, tmp1));
+}
+template<> inline Vector<sfloat> INTRINSIC Vector<sfloat>::exponent() const
+{
+ VC_ASSERT((*this > 0.f).isFull());
+ __m128i tmp0 = _mm_srli_epi32(avx_cast<__m128i>(d.v()), 23);
+ __m128i tmp1 = _mm_srli_epi32(avx_cast<__m128i>(hi128(d.v())), 23);
+ tmp0 = _mm_sub_epi32(tmp0, _mm_set1_epi32(0x7f));
+ tmp1 = _mm_sub_epi32(tmp1, _mm_set1_epi32(0x7f));
+ return _mm256_cvtepi32_ps(concat(tmp0, tmp1));
+}
+template<> inline Vector<double> INTRINSIC Vector<double>::exponent() const
+{
+ VC_ASSERT((*this > 0.).isFull());
+ __m128i tmp0 = _mm_srli_epi64(avx_cast<__m128i>(d.v()), 52);
+ __m128i tmp1 = _mm_srli_epi64(avx_cast<__m128i>(hi128(d.v())), 52);
+ tmp0 = _mm_sub_epi32(tmp0, _mm_set1_epi32(0x3ff));
+ tmp1 = _mm_sub_epi32(tmp1, _mm_set1_epi32(0x3ff));
+ return _mm256_cvtepi32_pd(avx_cast<__m128i>(Mem::shuffle<X0, X2, Y0, Y2>(avx_cast<__m128>(tmp0), avx_cast<__m128>(tmp1))));
+}
+// }}}1
+// Random {{{1
+static inline ALWAYS_INLINE void _doRandomStep(Vector<unsigned int> &state0,
+ Vector<unsigned int> &state1)
+{
+ state0.load(&Vc::RandomState[0]);
+ state1.load(&Vc::RandomState[uint_v::Size]);
+ (state1 * 0xdeece66du + 11).store(&Vc::RandomState[uint_v::Size]);
+ uint_v(_mm256_xor_si256((state0 * 0xdeece66du + 11).data(), _mm256_srli_epi32(state1.data(), 16))).store(&Vc::RandomState[0]);
+}
+
+template<typename T> inline ALWAYS_INLINE Vector<T> Vector<T>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ return state0.reinterpretCast<Vector<T> >();
+}
+
+template<> inline ALWAYS_INLINE Vector<float> Vector<float>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ return HT::sub(HV::or_(_cast(_mm256_srli_epi32(state0.data(), 2)), HT::one()), HT::one());
+}
+
+template<> inline ALWAYS_INLINE Vector<sfloat> Vector<sfloat>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ return HT::sub(HV::or_(_cast(_mm256_srli_epi32(state0.data(), 2)), HT::one()), HT::one());
+}
+
+template<> inline ALWAYS_INLINE Vector<double> Vector<double>::Random()
+{
+ const __m256i state = VectorHelper<__m256i>::load(&Vc::RandomState[0], Vc::Aligned);
+ for (size_t k = 0; k < 8; k += 2) {
+ typedef unsigned long long uint64 MAY_ALIAS;
+ const uint64 stateX = *reinterpret_cast<const uint64 *>(&Vc::RandomState[k]);
+ *reinterpret_cast<uint64 *>(&Vc::RandomState[k]) = (stateX * 0x5deece66dull + 11);
+ }
+ return (Vector<double>(_cast(_mm256_srli_epi64(state, 12))) | One()) - One();
+}
+// }}}1
+} // namespace AVX
+} // namespace Vc
+
+#include "undomacros.h"
+
+// vim: foldmethod=marker
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef AVX_VECTORHELPER_H
+#define AVX_VECTORHELPER_H
+
+#include <limits>
+#include "types.h"
+#include "intrinsics.h"
+#include "casts.h"
+
+namespace Vc
+{
+namespace AVX
+{
+#define OP0(name, code) static inline VectorType name() { return code; }
+#define OP1(name, code) static inline VectorType name(const VectorType &a) { return code; }
+#define OP2(name, code) static inline VectorType name(const VectorType &a, const VectorType &b) { return code; }
+#define OP3(name, code) static inline VectorType name(const VectorType &a, const VectorType &b, const VectorType &c) { return code; }
+
+ template<> struct VectorHelper<_M256>
+ {
+ typedef _M256 VectorType;
+ template<typename A> static VectorType load(const float *x, A) PURE;
+ static void store(float *mem, const VectorType x, AlignedFlag);
+ static void store(float *mem, const VectorType x, UnalignedFlag);
+ static void store(float *mem, const VectorType x, StreamingAndAlignedFlag);
+ static void store(float *mem, const VectorType x, StreamingAndUnalignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, AlignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, UnalignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag);
+
+ static VectorType cdab(VectorType x) { return _mm256_permute_ps(x, _MM_SHUFFLE(2, 3, 0, 1)); }
+ static VectorType badc(VectorType x) { return _mm256_permute_ps(x, _MM_SHUFFLE(1, 0, 3, 2)); }
+ static VectorType aaaa(VectorType x) { return _mm256_permute_ps(x, _MM_SHUFFLE(0, 0, 0, 0)); }
+ static VectorType bbbb(VectorType x) { return _mm256_permute_ps(x, _MM_SHUFFLE(1, 1, 1, 1)); }
+ static VectorType cccc(VectorType x) { return _mm256_permute_ps(x, _MM_SHUFFLE(2, 2, 2, 2)); }
+ static VectorType dddd(VectorType x) { return _mm256_permute_ps(x, _MM_SHUFFLE(3, 3, 3, 3)); }
+ static VectorType dacb(VectorType x) { return _mm256_permute_ps(x, _MM_SHUFFLE(3, 0, 2, 1)); }
+
+ OP0(allone, _mm256_setallone_ps())
+ OP0(zero, _mm256_setzero_ps())
+ OP2(or_, _mm256_or_ps(a, b))
+ OP2(xor_, _mm256_xor_ps(a, b))
+ OP2(and_, _mm256_and_ps(a, b))
+ OP2(andnot_, _mm256_andnot_ps(a, b))
+ OP3(blend, _mm256_blendv_ps(a, b, c))
+ };
+
+ template<> struct VectorHelper<_M256D>
+ {
+ typedef _M256D VectorType;
+ template<typename A> static VectorType load(const double *x, A) PURE;
+ static void store(double *mem, const VectorType x, AlignedFlag);
+ static void store(double *mem, const VectorType x, UnalignedFlag);
+ static void store(double *mem, const VectorType x, StreamingAndAlignedFlag);
+ static void store(double *mem, const VectorType x, StreamingAndUnalignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, AlignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, UnalignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag);
+
+ static VectorType cdab(VectorType x) { return _mm256_permute_pd(x, 5); }
+ static VectorType badc(VectorType x) { return _mm256_permute2f128_pd(x, x, 1); }
+ // aaaa bbbb cccc dddd specialized in vector.tcc
+ static VectorType dacb(VectorType x) {
+ const __m128d cb = avx_cast<__m128d>(_mm_alignr_epi8(avx_cast<__m128i>(lo128(x)),
+ avx_cast<__m128i>(hi128(x)), sizeof(double))); // XXX: lo and hi swapped?
+ const __m128d da = _mm_blend_pd(lo128(x), hi128(x), 0 + 2); // XXX: lo and hi swapped?
+ return concat(cb, da);
+ }
+
+ OP0(allone, _mm256_setallone_pd())
+ OP0(zero, _mm256_setzero_pd())
+ OP2(or_, _mm256_or_pd(a, b))
+ OP2(xor_, _mm256_xor_pd(a, b))
+ OP2(and_, _mm256_and_pd(a, b))
+ OP2(andnot_, _mm256_andnot_pd(a, b))
+ OP3(blend, _mm256_blendv_pd(a, b, c))
+ };
+
+ template<> struct VectorHelper<_M256I>
+ {
+ typedef _M256I VectorType;
+ template<typename T> static VectorType load(const T *x, AlignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, UnalignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, StreamingAndAlignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, StreamingAndUnalignedFlag) PURE;
+ template<typename T> static void store(T *mem, const VectorType x, AlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, UnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, StreamingAndAlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, StreamingAndUnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, AlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, UnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag);
+
+ static VectorType cdab(VectorType x) { return avx_cast<VectorType>(_mm256_permute_ps(avx_cast<__m256>(x), _MM_SHUFFLE(2, 3, 0, 1))); }
+ static VectorType badc(VectorType x) { return avx_cast<VectorType>(_mm256_permute_ps(avx_cast<__m256>(x), _MM_SHUFFLE(1, 0, 3, 2))); }
+ static VectorType aaaa(VectorType x) { return avx_cast<VectorType>(_mm256_permute_ps(avx_cast<__m256>(x), _MM_SHUFFLE(0, 0, 0, 0))); }
+ static VectorType bbbb(VectorType x) { return avx_cast<VectorType>(_mm256_permute_ps(avx_cast<__m256>(x), _MM_SHUFFLE(1, 1, 1, 1))); }
+ static VectorType cccc(VectorType x) { return avx_cast<VectorType>(_mm256_permute_ps(avx_cast<__m256>(x), _MM_SHUFFLE(2, 2, 2, 2))); }
+ static VectorType dddd(VectorType x) { return avx_cast<VectorType>(_mm256_permute_ps(avx_cast<__m256>(x), _MM_SHUFFLE(3, 3, 3, 3))); }
+ static VectorType dacb(VectorType x) { return avx_cast<VectorType>(_mm256_permute_ps(avx_cast<__m256>(x), _MM_SHUFFLE(3, 0, 2, 1))); }
+
+ OP0(allone, _mm256_setallone_si256())
+ OP0(zero, _mm256_setzero_si256())
+ OP2(or_, _mm256_or_si256(a, b))
+ OP2(xor_, _mm256_xor_si256(a, b))
+ OP2(and_, _mm256_and_si256(a, b))
+ OP2(andnot_, _mm256_andnot_si256(a, b))
+ OP3(blend, _mm256_blendv_epi8(a, b, c))
+ };
+
+ template<> struct VectorHelper<__m128i>
+ {
+ typedef __m128i VectorType;
+ template<typename T> static VectorType load(const T *x, AlignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, UnalignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, StreamingAndAlignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, StreamingAndUnalignedFlag) PURE;
+ template<typename T> static void store(T *mem, const VectorType x, AlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, UnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, StreamingAndAlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, StreamingAndUnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, AlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, UnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag);
+
+ static VectorType cdab(VectorType x) { return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, _MM_SHUFFLE(2, 3, 0, 1)), _MM_SHUFFLE(2, 3, 0, 1)); }
+ static VectorType badc(VectorType x) { return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, _MM_SHUFFLE(1, 0, 3, 2)), _MM_SHUFFLE(1, 0, 3, 2)); }
+ static VectorType aaaa(VectorType x) { return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, _MM_SHUFFLE(0, 0, 0, 0)), _MM_SHUFFLE(0, 0, 0, 0)); }
+ static VectorType bbbb(VectorType x) { return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, _MM_SHUFFLE(1, 1, 1, 1)), _MM_SHUFFLE(1, 1, 1, 1)); }
+ static VectorType cccc(VectorType x) { return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, _MM_SHUFFLE(2, 2, 2, 2)), _MM_SHUFFLE(2, 2, 2, 2)); }
+ static VectorType dddd(VectorType x) { return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, _MM_SHUFFLE(3, 3, 3, 3)), _MM_SHUFFLE(3, 3, 3, 3)); }
+ static VectorType dacb(VectorType x) { return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, _MM_SHUFFLE(3, 0, 2, 1)), _MM_SHUFFLE(3, 0, 2, 1)); }
+
+ OP0(allone, _mm_setallone_si128())
+ OP0(zero, _mm_setzero_si128())
+ OP2(or_, _mm_or_si128(a, b))
+ OP2(xor_, _mm_xor_si128(a, b))
+ OP2(and_, _mm_and_si128(a, b))
+ OP2(andnot_, _mm_andnot_si128(a, b))
+ OP3(blend, _mm_blendv_epi8(a, b, c))
+ };
+#undef OP1
+#undef OP2
+#undef OP3
+
+#define OP1(op) \
+ static inline VectorType INTRINSIC CONST op(const VectorType &a) { return CAT(_mm256_##op##_, SUFFIX)(a); }
+#define OP(op) \
+ static inline VectorType INTRINSIC CONST op(const VectorType &a, const VectorType &b) { return CAT(_mm256_##op##_ , SUFFIX)(a, b); }
+#define OP_(op) \
+ static inline VectorType INTRINSIC CONST op(const VectorType &a, const VectorType &b) { return CAT(_mm256_##op , SUFFIX)(a, b); }
+#define OPx(op, op2) \
+ static inline VectorType INTRINSIC CONST op(const VectorType &a, const VectorType &b) { return CAT(_mm256_##op2##_, SUFFIX)(a, b); }
+#define OPcmp(op) \
+ static inline VectorType INTRINSIC CONST cmp##op(const VectorType &a, const VectorType &b) { return CAT(_mm256_cmp##op##_, SUFFIX)(a, b); }
+#define OP_CAST_(op) \
+ static inline VectorType INTRINSIC CONST op(const VectorType &a, const VectorType &b) { return CAT(_mm256_castps_, SUFFIX)( \
+ _mm256_##op##ps(CAT(CAT(_mm256_cast, SUFFIX), _ps)(a), \
+ CAT(CAT(_mm256_cast, SUFFIX), _ps)(b))); \
+ }
+#define MINMAX \
+ static inline VectorType INTRINSIC CONST min(VectorType a, VectorType b) { return CAT(_mm256_min_, SUFFIX)(a, b); } \
+ static inline VectorType INTRINSIC CONST max(VectorType a, VectorType b) { return CAT(_mm256_max_, SUFFIX)(a, b); }
+
+ template<> struct VectorHelper<double> {
+ typedef _M256D VectorType;
+ typedef double EntryType;
+ typedef double ConcatType;
+#define SUFFIX pd
+
+ static inline VectorType notMaskedToZero(VectorType a, _M256 mask) { return CAT(_mm256_and_, SUFFIX)(_mm256_castps_pd(mask), a); }
+ static inline VectorType set(const double a) { return CAT(_mm256_set1_, SUFFIX)(a); }
+ static inline VectorType set(const double a, const double b, const double c, const double d) {
+ return CAT(_mm256_set_, SUFFIX)(a, b, c, d);
+ }
+ static inline VectorType zero() { return CAT(_mm256_setzero_, SUFFIX)(); }
+ static inline VectorType one() { return CAT(_mm256_setone_, SUFFIX)(); }// set(1.); }
+
+ static inline void multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) { v1 = add(mul(v1, v2), v3); }
+ static inline VectorType mul(VectorType a, VectorType b, _M256 _mask) {
+ _M256D mask = _mm256_castps_pd(_mask);
+ return _mm256_or_pd(
+ _mm256_and_pd(mask, _mm256_mul_pd(a, b)),
+ _mm256_andnot_pd(mask, a)
+ );
+ }
+ static inline VectorType div(VectorType a, VectorType b, _M256 _mask) {
+ _M256D mask = _mm256_castps_pd(_mask);
+ return _mm256_or_pd(
+ _mm256_and_pd(mask, _mm256_div_pd(a, b)),
+ _mm256_andnot_pd(mask, a)
+ );
+ }
+
+ OP(add) OP(sub) OP(mul)
+ OPcmp(eq) OPcmp(neq)
+ OPcmp(lt) OPcmp(nlt)
+ OPcmp(le) OPcmp(nle)
+
+ OP1(sqrt)
+ static inline VectorType rsqrt(VectorType x) {
+ return _mm256_div_pd(one(), sqrt(x));
+ }
+ static inline VectorType reciprocal(VectorType x) {
+ return _mm256_div_pd(one(), x);
+ }
+ static inline VectorType isNaN(VectorType x) {
+ return _mm256_cmpunord_pd(x, x);
+ }
+ static inline VectorType isFinite(VectorType x) {
+ return _mm256_cmpord_pd(x, _mm256_mul_pd(zero(), x));
+ }
+ static inline VectorType abs(const VectorType a) {
+ return CAT(_mm256_and_, SUFFIX)(a, _mm256_setabsmask_pd());
+ }
+
+ MINMAX
+ static inline EntryType min(VectorType a) {
+ __m128d b = _mm_min_pd(avx_cast<__m128d>(a), _mm256_extractf128_pd(a, 1));
+ b = _mm_min_sd(b, _mm_unpackhi_pd(b, b));
+ return _mm_cvtsd_f64(b);
+ }
+ static inline EntryType max(VectorType a) {
+ __m128d b = _mm_max_pd(avx_cast<__m128d>(a), _mm256_extractf128_pd(a, 1));
+ b = _mm_max_sd(b, _mm_unpackhi_pd(b, b));
+ return _mm_cvtsd_f64(b);
+ }
+ static inline EntryType mul(VectorType a) {
+ __m128d b = _mm_mul_pd(avx_cast<__m128d>(a), _mm256_extractf128_pd(a, 1));
+ b = _mm_mul_sd(b, _mm_shuffle_pd(b, b, _MM_SHUFFLE2(0, 1)));
+ return _mm_cvtsd_f64(b);
+ }
+ static inline EntryType add(VectorType a) {
+ __m128d b = _mm_add_pd(avx_cast<__m128d>(a), _mm256_extractf128_pd(a, 1));
+ b = _mm_hadd_pd(b, b); // or: b = _mm_add_sd(b, _mm256_shuffle_pd(b, b, _MM_SHUFFLE2(0, 1)));
+ return _mm_cvtsd_f64(b);
+ }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) {
+ return _mm256_round_pd(a, _MM_FROUND_NINT);
+ }
+ };
+
+ template<> struct VectorHelper<float> {
+ typedef float EntryType;
+ typedef _M256 VectorType;
+ typedef double ConcatType;
+#define SUFFIX ps
+
+ static inline VectorType notMaskedToZero(VectorType a, _M256 mask) { return CAT(_mm256_and_, SUFFIX)(mask, a); }
+ static inline VectorType set(const float a) { return CAT(_mm256_set1_, SUFFIX)(a); }
+ static inline VectorType set(const float a, const float b, const float c, const float d,
+ const float e, const float f, const float g, const float h) {
+ return CAT(_mm256_set_, SUFFIX)(a, b, c, d, e, f, g, h); }
+ static inline VectorType zero() { return CAT(_mm256_setzero_, SUFFIX)(); }
+ static inline VectorType one() { return CAT(_mm256_setone_, SUFFIX)(); }// set(1.f); }
+ static inline _M256 concat(_M256D a, _M256D b) { return _mm256_insertf128_ps(avx_cast<_M256>(_mm256_cvtpd_ps(a)), _mm256_cvtpd_ps(b), 1); }
+
+ static inline void multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) { v1 = add(mul(v1, v2), v3); }
+ static inline VectorType mul(VectorType a, VectorType b, _M256 mask) {
+ return _mm256_or_ps(
+ _mm256_and_ps(mask, _mm256_mul_ps(a, b)),
+ _mm256_andnot_ps(mask, a)
+ );
+ }
+ static inline VectorType div(VectorType a, VectorType b, _M256 mask) {
+ return _mm256_or_ps(
+ _mm256_and_ps(mask, _mm256_div_ps(a, b)),
+ _mm256_andnot_ps(mask, a)
+ );
+ }
+
+ OP(add) OP(sub) OP(mul)
+ OPcmp(eq) OPcmp(neq)
+ OPcmp(lt) OPcmp(nlt)
+ OPcmp(le) OPcmp(nle)
+
+ OP1(sqrt) OP1(rsqrt)
+ static inline VectorType isNaN(VectorType x) {
+ return _mm256_cmpunord_ps(x, x);
+ }
+ static inline VectorType isFinite(VectorType x) {
+ return _mm256_cmpord_ps(x, _mm256_mul_ps(zero(), x));
+ }
+ static inline VectorType reciprocal(VectorType x) {
+ return _mm256_rcp_ps(x);
+ }
+ static inline VectorType abs(const VectorType a) {
+ return CAT(_mm256_and_, SUFFIX)(a, _mm256_setabsmask_ps());
+ }
+
+ MINMAX
+ static inline EntryType min(VectorType a) {
+ __m128 b = _mm_min_ps(avx_cast<__m128>(a), _mm256_extractf128_ps(a, 1));
+ b = _mm_min_ps(b, _mm_movehl_ps(b, b)); // b = min(a0, a2), min(a1, a3), min(a2, a2), min(a3, a3)
+ b = _mm_min_ss(b, _mm_shuffle_ps(b, b, _MM_SHUFFLE(1, 1, 1, 1))); // b = min(a0, a1), a1, a2, a3
+ return _mm_cvtss_f32(b);
+ }
+ static inline EntryType max(VectorType a) {
+ __m128 b = _mm_max_ps(avx_cast<__m128>(a), _mm256_extractf128_ps(a, 1));
+ b = _mm_max_ps(b, _mm_movehl_ps(b, b)); // b = max(a0, a2), max(a1, a3), max(a2, a2), max(a3, a3)
+ b = _mm_max_ss(b, _mm_shuffle_ps(b, b, _MM_SHUFFLE(1, 1, 1, 1))); // b = max(a0, a1), a1, a2, a3
+ return _mm_cvtss_f32(b);
+ }
+ static inline EntryType mul(VectorType a) {
+ __m128 b = _mm_mul_ps(avx_cast<__m128>(a), _mm256_extractf128_ps(a, 1));
+ b = _mm_mul_ps(b, _mm_shuffle_ps(b, b, _MM_SHUFFLE(0, 1, 2, 3)));
+ b = _mm_mul_ss(b, _mm_shuffle_ps(b, b, _MM_SHUFFLE(3, 2, 0, 1)));
+ return _mm_cvtss_f32(b);
+ }
+ static inline EntryType add(VectorType a) {
+ __m128 b = _mm_add_ps(avx_cast<__m128>(a), _mm256_extractf128_ps(a, 1));
+ b = _mm_add_ps(b, _mm_shuffle_ps(b, b, _MM_SHUFFLE(0, 1, 2, 3)));
+ b = _mm_add_ss(b, _mm_shuffle_ps(b, b, _MM_SHUFFLE(3, 2, 0, 1)));
+ return _mm_cvtss_f32(b);
+ }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) {
+ return _mm256_round_ps(a, _MM_FROUND_NINT);
+ }
+ };
+
+ template<> struct VectorHelper<sfloat> : public VectorHelper<float> {};
+
+ template<> struct VectorHelper<int> {
+ typedef int EntryType;
+ typedef _M256I VectorType;
+ typedef long long ConcatType;
+#define SUFFIX si256
+
+ OP_(or_) OP_(and_) OP_(xor_)
+ static inline VectorType INTRINSIC CONST zero() { return CAT(_mm256_setzero_, SUFFIX)(); }
+ static inline VectorType INTRINSIC CONST notMaskedToZero(VectorType a, _M256 mask) { return CAT(_mm256_and_, SUFFIX)(_mm256_castps_si256(mask), a); }
+#undef SUFFIX
+#define SUFFIX epi32
+ static inline VectorType INTRINSIC CONST one() { return CAT(_mm256_setone_, SUFFIX)(); }
+
+ static inline VectorType INTRINSIC CONST set(const int a) { return CAT(_mm256_set1_, SUFFIX)(a); }
+ static inline VectorType INTRINSIC CONST set(const int a, const int b, const int c, const int d,
+ const int e, const int f, const int g, const int h) {
+ return CAT(_mm256_set_, SUFFIX)(a, b, c, d, e, f, g, h); }
+
+ static inline void INTRINSIC CONST multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) { v1 = add(mul(v1, v2), v3); }
+
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm256_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm256_srai_, SUFFIX)(a, shift);
+ }
+ OP1(abs)
+
+ MINMAX
+ static inline EntryType INTRINSIC CONST min(VectorType a) {
+ __m128i b = _mm_min_epi32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_min_epi32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_min_epi32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2))); // using lo_epi16 for speed here
+ return _mm_cvtsi128_si32(b);
+ }
+ static inline EntryType INTRINSIC CONST max(VectorType a) {
+ __m128i b = _mm_max_epi32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_max_epi32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_max_epi32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2))); // using lo_epi16 for speed here
+ return _mm_cvtsi128_si32(b);
+ }
+ static inline EntryType INTRINSIC CONST add(VectorType a) {
+ __m128i b = _mm_add_epi32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_add_epi32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_add_epi32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(b);
+ }
+ static inline EntryType INTRINSIC CONST mul(VectorType a) {
+ __m128i b = _mm_mullo_epi32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_mullo_epi32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_mullo_epi32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(b);
+ }
+
+ static inline VectorType INTRINSIC CONST mul(VectorType a, VectorType b) { return _mm256_mullo_epi32(a, b); }
+
+ OP(add) OP(sub)
+ OPcmp(eq)
+ OPcmp(lt)
+ OPcmp(gt)
+ static inline VectorType INTRINSIC CONST cmpneq(const VectorType &a, const VectorType &b) { _M256I x = cmpeq(a, b); return _mm256_andnot_si256(x, _mm256_setallone_si256()); }
+ static inline VectorType INTRINSIC CONST cmpnlt(const VectorType &a, const VectorType &b) { _M256I x = cmplt(a, b); return _mm256_andnot_si256(x, _mm256_setallone_si256()); }
+ static inline VectorType INTRINSIC CONST cmple (const VectorType &a, const VectorType &b) { _M256I x = cmpgt(a, b); return _mm256_andnot_si256(x, _mm256_setallone_si256()); }
+ static inline VectorType INTRINSIC CONST cmpnle(const VectorType &a, const VectorType &b) { return cmpgt(a, b); }
+#undef SUFFIX
+ static inline VectorType INTRINSIC CONST round(VectorType a) { return a; }
+ };
+
+ template<> struct VectorHelper<unsigned int> {
+ typedef unsigned int EntryType;
+ typedef _M256I VectorType;
+ typedef unsigned long long ConcatType;
+#define SUFFIX si256
+ OP_CAST_(or_) OP_CAST_(and_) OP_CAST_(xor_)
+ static inline VectorType INTRINSIC CONST zero() { return CAT(_mm256_setzero_, SUFFIX)(); }
+ static inline VectorType INTRINSIC CONST notMaskedToZero(VectorType a, _M256 mask) { return CAT(_mm256_and_, SUFFIX)(_mm256_castps_si256(mask), a); }
+
+#undef SUFFIX
+#define SUFFIX epu32
+ static inline VectorType INTRINSIC CONST one() { return CAT(_mm256_setone_, SUFFIX)(); }
+
+ MINMAX
+ static inline EntryType INTRINSIC CONST min(VectorType a) {
+ __m128i b = _mm_min_epu32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_min_epu32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_min_epu32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2))); // using lo_epi16 for speed here
+ return _mm_cvtsi128_si32(b);
+ }
+ static inline EntryType INTRINSIC CONST max(VectorType a) {
+ __m128i b = _mm_max_epu32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_max_epu32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_max_epu32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2))); // using lo_epi16 for speed here
+ return _mm_cvtsi128_si32(b);
+ }
+ static inline EntryType INTRINSIC CONST add(VectorType a) {
+ __m128i b = _mm_add_epi32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_add_epi32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_add_epi32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(b);
+ }
+ static inline EntryType INTRINSIC CONST mul(VectorType a) {
+ __m128i b = _mm_mullo_epi32(avx_cast<__m128i>(a), _mm256_extractf128_si256(a, 1));
+ b = _mm_mullo_epi32(b, _mm_shuffle_epi32(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ b = _mm_mullo_epi32(b, _mm_shufflelo_epi16(b, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(b);
+ }
+
+ static inline VectorType INTRINSIC CONST mul(VectorType a, VectorType b) { return _mm256_mullo_epi32(a, b); }
+
+#undef SUFFIX
+#define SUFFIX epi32
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm256_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm256_srli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType INTRINSIC CONST set(const unsigned int a) { return CAT(_mm256_set1_, SUFFIX)(a); }
+ static inline VectorType INTRINSIC CONST set(const unsigned int a, const unsigned int b, const unsigned int c, const unsigned int d,
+ const unsigned int e, const unsigned int f, const unsigned int g, const unsigned int h) {
+ return CAT(_mm256_set_, SUFFIX)(a, b, c, d, e, f, g, h); }
+
+ OP(add) OP(sub)
+ OPcmp(eq)
+ static inline VectorType INTRINSIC CONST cmpneq(const VectorType &a, const VectorType &b) { return _mm256_andnot_si256(cmpeq(a, b), _mm256_setallone_si256()); }
+
+#ifndef USE_INCORRECT_UNSIGNED_COMPARE
+ static inline VectorType INTRINSIC CONST cmplt(const VectorType &a, const VectorType &b) {
+ return _mm256_cmplt_epu32(a, b);
+ }
+ static inline VectorType INTRINSIC CONST cmpgt(const VectorType &a, const VectorType &b) {
+ return _mm256_cmpgt_epu32(a, b);
+ }
+#else
+ OPcmp(lt)
+ OPcmp(gt)
+#endif
+ static inline VectorType INTRINSIC CONST cmpnlt(const VectorType &a, const VectorType &b) { return _mm256_andnot_si256(cmplt(a, b), _mm256_setallone_si256()); }
+ static inline VectorType INTRINSIC CONST cmple (const VectorType &a, const VectorType &b) { return _mm256_andnot_si256(cmpgt(a, b), _mm256_setallone_si256()); }
+ static inline VectorType INTRINSIC CONST cmpnle(const VectorType &a, const VectorType &b) { return cmpgt(a, b); }
+
+#undef SUFFIX
+ static inline VectorType INTRINSIC CONST round(VectorType a) { return a; }
+ };
+
+ template<> struct VectorHelper<signed short> {
+ typedef VectorTypeHelper<signed short>::Type VectorType;
+ typedef signed short EntryType;
+ typedef int ConcatType;
+
+ static inline VectorType INTRINSIC CONST or_(VectorType a, VectorType b) { return _mm_or_si128(a, b); }
+ static inline VectorType INTRINSIC CONST and_(VectorType a, VectorType b) { return _mm_and_si128(a, b); }
+ static inline VectorType INTRINSIC CONST xor_(VectorType a, VectorType b) { return _mm_xor_si128(a, b); }
+ static inline VectorType INTRINSIC CONST zero() { return _mm_setzero_si128(); }
+ static inline VectorType INTRINSIC CONST notMaskedToZero(VectorType a, __m128 mask) { return _mm_and_si128(_mm_castps_si128(mask), a); }
+
+#define SUFFIX epi16
+ static inline VectorType INTRINSIC CONST one() { return CAT(_mm_setone_, SUFFIX)(); }
+
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm_srai_, SUFFIX)(a, shift);
+ }
+ static inline VectorType INTRINSIC CONST set(const EntryType a) { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType INTRINSIC CONST set(const EntryType a, const EntryType b, const EntryType c, const EntryType d,
+ const EntryType e, const EntryType f, const EntryType g, const EntryType h) {
+ return CAT(_mm_set_, SUFFIX)(a, b, c, d, e, f, g, h);
+ }
+
+ static inline void INTRINSIC CONST multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) {
+ v1 = add(mul(v1, v2), v3); }
+
+ static inline VectorType INTRINSIC CONST abs(VectorType a) { return _mm_abs_epi16(a); }
+ static inline VectorType INTRINSIC CONST mul(VectorType a, VectorType b) { return _mm_mullo_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST min(VectorType a, VectorType b) { return _mm_min_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST max(VectorType a, VectorType b) { return _mm_max_epi16(a, b); }
+
+ static inline EntryType INTRINSIC CONST min(VectorType a) {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = min(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType INTRINSIC CONST max(VectorType a) {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = max(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType INTRINSIC CONST mul(VectorType a) {
+ a = mul(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType INTRINSIC CONST add(VectorType a) {
+ a = add(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+
+ static inline VectorType INTRINSIC CONST add(VectorType a, VectorType b) { return _mm_add_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST sub(VectorType a, VectorType b) { return _mm_sub_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST cmpeq(VectorType a, VectorType b) { return _mm_cmpeq_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST cmplt(VectorType a, VectorType b) { return _mm_cmplt_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST cmpgt(VectorType a, VectorType b) { return _mm_cmpgt_epi16(a, b); }
+ static inline VectorType cmpneq(const VectorType &a, const VectorType &b) { __m128i x = cmpeq(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmpnlt(const VectorType &a, const VectorType &b) { __m128i x = cmplt(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmple (const VectorType &a, const VectorType &b) { __m128i x = cmpgt(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmpnle(const VectorType &a, const VectorType &b) { return cmpgt(a, b); }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) { return a; }
+ };
+
+ template<> struct VectorHelper<unsigned short> {
+ typedef VectorTypeHelper<unsigned short>::Type VectorType;
+ typedef unsigned short EntryType;
+ typedef unsigned int ConcatType;
+
+ static inline VectorType INTRINSIC CONST or_(VectorType a, VectorType b) { return _mm_or_si128(a, b); }
+ static inline VectorType INTRINSIC CONST and_(VectorType a, VectorType b) { return _mm_and_si128(a, b); }
+ static inline VectorType INTRINSIC CONST xor_(VectorType a, VectorType b) { return _mm_xor_si128(a, b); }
+ static inline VectorType INTRINSIC CONST zero() { return _mm_setzero_si128(); }
+ static inline VectorType INTRINSIC CONST notMaskedToZero(VectorType a, __m128 mask) { return _mm_and_si128(_mm_castps_si128(mask), a); }
+ static inline VectorType INTRINSIC CONST one() { return _mm_setone_epu16(); }
+
+ static inline VectorType INTRINSIC CONST mul(VectorType a, VectorType b) { return _mm_mullo_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST min(VectorType a, VectorType b) { return _mm_min_epu16(a, b); }
+ static inline VectorType INTRINSIC CONST max(VectorType a, VectorType b) { return _mm_max_epu16(a, b); }
+
+#define SUFFIX epi16
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm_srli_, SUFFIX)(a, shift);
+ }
+ static inline EntryType INTRINSIC CONST min(VectorType a) {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = min(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType INTRINSIC CONST max(VectorType a) {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = max(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType INTRINSIC CONST mul(VectorType a) {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = mul(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType INTRINSIC CONST add(VectorType a) {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = add(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline VectorType INTRINSIC CONST set(const EntryType a) { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType INTRINSIC CONST set(const EntryType a, const EntryType b, const EntryType c,
+ const EntryType d, const EntryType e, const EntryType f,
+ const EntryType g, const EntryType h) {
+ return CAT(_mm_set_, SUFFIX)(a, b, c, d, e, f, g, h);
+ }
+
+ static inline VectorType INTRINSIC CONST add(VectorType a, VectorType b) { return _mm_add_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST sub(VectorType a, VectorType b) { return _mm_sub_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST cmpeq(VectorType a, VectorType b) { return _mm_cmpeq_epi16(a, b); }
+ static inline VectorType cmpneq(const VectorType &a, const VectorType &b) { return _mm_andnot_si128(cmpeq(a, b), _mm_setallone_si128()); }
+
+#ifndef USE_INCORRECT_UNSIGNED_COMPARE
+ static inline VectorType INTRINSIC CONST cmplt(VectorType a, VectorType b) { return _mm_cmplt_epu16(a, b); }
+ static inline VectorType INTRINSIC CONST cmpgt(VectorType a, VectorType b) { return _mm_cmpgt_epu16(a, b); }
+#else
+ static inline VectorType INTRINSIC CONST cmplt(VectorType a, VectorType b) { return _mm_cmplt_epi16(a, b); }
+ static inline VectorType INTRINSIC CONST cmpgt(VectorType a, VectorType b) { return _mm_cmpgt_epi16(a, b); }
+#endif
+ static inline VectorType cmpnlt(const VectorType &a, const VectorType &b) { return _mm_andnot_si128(cmplt(a, b), _mm_setallone_si128()); }
+ static inline VectorType cmple (const VectorType &a, const VectorType &b) { return _mm_andnot_si128(cmpgt(a, b), _mm_setallone_si128()); }
+ static inline VectorType cmpnle(const VectorType &a, const VectorType &b) { return cmpgt(a, b); }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) { return a; }
+ };
+#undef OP1
+#undef OP
+#undef OP_
+#undef OPx
+#undef OPcmp
+
+template<> struct VectorHelper<char>
+{
+ typedef VectorTypeHelper<char>::Type VectorType;
+ typedef char EntryType;
+ typedef short ConcatType;
+};
+
+template<> struct VectorHelper<unsigned char>
+{
+ typedef VectorTypeHelper<unsigned char>::Type VectorType;
+ typedef unsigned char EntryType;
+ typedef unsigned short ConcatType;
+};
+
+} // namespace AVX
+} // namespace Vc
+
+#include "vectorhelper.tcc"
+
+#endif // AVX_VECTORHELPER_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "casts.h"
+#include <cstdlib>
+
+namespace Vc
+{
+namespace AVX
+{
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// float_v
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// loads
+template<> inline __m256 VectorHelper<__m256>::load(const float *m, AlignedFlag)
+{
+ return _mm256_load_ps(m);
+}
+template<> inline __m256 VectorHelper<__m256>::load(const float *m, UnalignedFlag)
+{
+ return _mm256_loadu_ps(m);
+}
+template<> inline __m256 VectorHelper<__m256>::load(const float *m, StreamingAndAlignedFlag)
+{
+ return avx_cast<__m256>(concat(_mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<float *>(m))),
+ _mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<float *>(&m[4])))));
+}
+template<> inline __m256
+ VC_WARN("AVX does not support streaming unaligned loads. Will use non-streaming unaligned load instead.")
+VectorHelper<__m256>::load(const float *m, StreamingAndUnalignedFlag)
+{
+ return _mm256_loadu_ps(m);
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// stores
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, AlignedFlag)
+{
+ _mm256_store_ps(mem, x);
+}
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, UnalignedFlag)
+{
+ _mm256_storeu_ps(mem, x);
+}
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, StreamingAndAlignedFlag)
+{
+ _mm256_stream_ps(mem, x);
+}
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(avx_cast<__m128i>(x), _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(_mm256_extractf128_si256(avx_cast<__m256i>(x), 1), _mm_setallone_si128(), reinterpret_cast<char *>(mem + 4));
+}
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, const VectorType m, AlignedFlag)
+{
+ _mm256_maskstore(mem, m, x);
+}
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, const VectorType m, UnalignedFlag)
+{
+ _mm256_maskstore(mem, m, x);
+}
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(avx_cast<__m128i>(x), avx_cast<__m128i>(m), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(_mm256_extractf128_si256(avx_cast<__m256i>(x), 1), _mm256_extractf128_si256(avx_cast<__m256i>(m), 1), reinterpret_cast<char *>(mem + 4));
+}
+inline void VectorHelper<__m256>::store(float *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(avx_cast<__m128i>(x), avx_cast<__m128i>(m), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(_mm256_extractf128_si256(avx_cast<__m256i>(x), 1), _mm256_extractf128_si256(avx_cast<__m256i>(m), 1), reinterpret_cast<char *>(mem + 4));
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// double_v
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// loads
+template<> inline __m256d VectorHelper<__m256d>::load(const double *m, AlignedFlag)
+{
+ return _mm256_load_pd(m);
+}
+template<> inline __m256d VectorHelper<__m256d>::load(const double *m, UnalignedFlag)
+{
+ return _mm256_loadu_pd(m);
+}
+template<> inline __m256d VectorHelper<__m256d>::load(const double *m, StreamingAndAlignedFlag)
+{
+ return avx_cast<__m256d>(concat(
+ _mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<double *>(m))),
+ _mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<double *>(&m[2])))));
+}
+template<> inline __m256d
+ VC_WARN("AVX does not support streaming unaligned loads. Will use non-streaming unaligned load instead.")
+VectorHelper<__m256d>::load(const double *m, StreamingAndUnalignedFlag)
+{
+ return _mm256_loadu_pd(m);
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// stores
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, AlignedFlag)
+{
+ _mm256_store_pd(mem, x);
+}
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, UnalignedFlag)
+{
+ _mm256_storeu_pd(mem, x);
+}
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, StreamingAndAlignedFlag)
+{
+ _mm256_stream_pd(mem, x);
+}
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(avx_cast<__m128i>(x), _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(avx_cast<__m128i>(_mm256_extractf128_pd(x, 1)), _mm_setallone_si128(), reinterpret_cast<char *>(mem + 2));
+}
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, const VectorType m, AlignedFlag)
+{
+ _mm256_maskstore(mem, m, x);
+}
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, const VectorType m, UnalignedFlag)
+{
+ _mm256_maskstore(mem, m, x);
+}
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(avx_cast<__m128i>(x), avx_cast<__m128i>(m), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(avx_cast<__m128i>(_mm256_extractf128_pd(x, 1)), avx_cast<__m128i>(_mm256_extractf128_pd(m, 1)), reinterpret_cast<char *>(mem + 2));
+}
+inline void VectorHelper<__m256d>::store(double *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(avx_cast<__m128i>(x), avx_cast<__m128i>(m), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(avx_cast<__m128i>(_mm256_extractf128_pd(x, 1)), avx_cast<__m128i>(_mm256_extractf128_pd(m, 1)), reinterpret_cast<char *>(mem + 2));
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// (u)int_v
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// loads
+template<typename T> inline __m256i VectorHelper<__m256i>::load(const T *m, AlignedFlag)
+{
+ return _mm256_load_si256(reinterpret_cast<const __m256i *>(m));
+}
+template<typename T> inline __m256i VectorHelper<__m256i>::load(const T *m, UnalignedFlag)
+{
+ return _mm256_loadu_si256(reinterpret_cast<const __m256i *>(m));
+}
+template<typename T> inline __m256i VectorHelper<__m256i>::load(const T *m, StreamingAndAlignedFlag)
+{
+ return concat(_mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<T *>(m))),
+ _mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<T *>(&m[4]))));
+}
+template<typename T> inline __m256i
+ VC_WARN("AVX does not support streaming unaligned loads. Will use non-streaming unaligned load instead.")
+VectorHelper<__m256i>::load(const T *m, StreamingAndUnalignedFlag)
+{
+ return _mm256_loadu_si256(reinterpret_cast<const __m256i *>(m));
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// stores
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, AlignedFlag)
+{
+ _mm256_store_si256(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, UnalignedFlag)
+{
+ _mm256_storeu_si256(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, StreamingAndAlignedFlag)
+{
+ _mm256_stream_si256(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(avx_cast<__m128i>(x), _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(_mm256_extractf128_si256(x, 1), _mm_setallone_si128(), reinterpret_cast<char *>(mem + 4));
+}
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, const VectorType m, AlignedFlag)
+{
+ _mm256_maskstore(mem, m, x);
+}
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, const VectorType m, UnalignedFlag)
+{
+ _mm256_maskstore(mem, m, x);
+}
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(lo128(x), lo128(m), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(hi128(x), hi128(m), reinterpret_cast<char *>(mem + 4));
+}
+template<typename T> inline void VectorHelper<__m256i>::store(T *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(lo128(x), lo128(m), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(hi128(x), hi128(m), reinterpret_cast<char *>(mem + 4));
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// (u)short_v
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// loads
+template<typename T> inline __m128i VectorHelper<__m128i>::load(const T *m, AlignedFlag)
+{
+ return _mm_load_si128(reinterpret_cast<const __m128i *>(m));
+}
+template<typename T> inline __m128i VectorHelper<__m128i>::load(const T *m, UnalignedFlag)
+{
+ return _mm_loadu_si128(reinterpret_cast<const __m128i *>(m));
+}
+template<typename T> inline __m128i VectorHelper<__m128i>::load(const T *m, StreamingAndAlignedFlag)
+{
+ return _mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<T *>(m)));
+}
+template<typename T> inline __m128i
+ VC_WARN("AVX does not support streaming unaligned loads. Will use non-streaming unaligned load instead.")
+VectorHelper<__m128i>::load(const T *m, StreamingAndUnalignedFlag)
+{
+ return _mm_loadu_si128(reinterpret_cast<const __m128i *>(m));
+}
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//// stores
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, AlignedFlag)
+{
+ _mm_store_si128(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, UnalignedFlag)
+{
+ _mm_storeu_si128(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, StreamingAndAlignedFlag)
+{
+ _mm_stream_si128(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(x, _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+}
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, const VectorType m, AlignedFlag align)
+{
+ store(mem, _mm_blendv_epi8(load(mem, align), x, m), align);
+}
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, const VectorType m, UnalignedFlag align)
+{
+ store(mem, _mm_blendv_epi8(load(mem, align), x, m), align);
+}
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(x, m, reinterpret_cast<char *>(mem));
+}
+template<typename T> inline void VectorHelper<__m128i>::store(T *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(x, m, reinterpret_cast<char *>(mem));
+}
+
+} // namespace AVX
+} // namespace Vc
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_AVX_WRITEMASKEDVECTOR_H
+#define VC_AVX_WRITEMASKEDVECTOR_H
+
+namespace Vc
+{
+namespace AVX
+{
+
+template<typename T>
+class WriteMaskedVector
+{
+ friend class Vector<T>;
+ typedef typename VectorTypeHelper<T>::Type VectorType;
+ typedef typename DetermineEntryType<T>::Type EntryType;
+ enum Constants { Size = sizeof(VectorType) / sizeof(EntryType) };
+ typedef typename Vc::AVX::Mask<Size, sizeof(VectorType)> Mask;
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(32)
+ //prefix
+ Vector<T> ALWAYS_INLINE_L &operator++() ALWAYS_INLINE_R;
+ Vector<T> ALWAYS_INLINE_L &operator--() ALWAYS_INLINE_R;
+ //postfix
+ Vector<T> ALWAYS_INLINE_L operator++(int) ALWAYS_INLINE_R;
+ Vector<T> ALWAYS_INLINE_L operator--(int) ALWAYS_INLINE_R;
+
+ Vector<T> ALWAYS_INLINE_L &operator+=(const Vector<T> &x) ALWAYS_INLINE_R;
+ Vector<T> ALWAYS_INLINE_L &operator-=(const Vector<T> &x) ALWAYS_INLINE_R;
+ Vector<T> ALWAYS_INLINE_L &operator*=(const Vector<T> &x) ALWAYS_INLINE_R;
+ Vector<T> ALWAYS_INLINE_L &operator/=(const Vector<T> &x) ALWAYS_INLINE_R;
+ Vector<T> ALWAYS_INLINE &operator+=(EntryType x) { return operator+=(Vector<T>(x)); }
+ Vector<T> ALWAYS_INLINE &operator-=(EntryType x) { return operator-=(Vector<T>(x)); }
+ Vector<T> ALWAYS_INLINE &operator*=(EntryType x) { return operator*=(Vector<T>(x)); }
+ Vector<T> ALWAYS_INLINE &operator/=(EntryType x) { return operator/=(Vector<T>(x)); }
+
+ Vector<T> ALWAYS_INLINE_L &operator=(const Vector<T> &x) ALWAYS_INLINE_R;
+ Vector<T> ALWAYS_INLINE &operator=(EntryType x) { return operator=(Vector<T>(x)); }
+
+ template<typename F> inline void INTRINSIC call(const F &f) const {
+ return vec->call(f, mask);
+ }
+ template<typename F> inline void INTRINSIC call(F &f) const {
+ return vec->call(f, mask);
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(const F &f) const {
+ return vec->apply(f, mask);
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(F &f) const {
+ return vec->apply(f, mask);
+ }
+ private:
+ inline WriteMaskedVector(Vector<T> *v, const Mask &k) : vec(v), mask(k) {}
+ Vector<T> *const vec;
+ Mask mask;
+};
+
+} // namespace AVX
+} // namespace Vc
+#include "writemaskedvector.tcc"
+#endif // VC_AVX_WRITEMASKEDVECTOR_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+namespace Vc
+{
+namespace AVX
+{
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE &WriteMaskedVector<T>::operator++()
+{
+ vec->data() = VectorHelper<T>::add(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return *vec;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE &WriteMaskedVector<T>::operator--() {
+ vec->data() = VectorHelper<T>::sub(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return *vec;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE WriteMaskedVector<T>::operator++(int) {
+ Vector<T> ret(*vec);
+ vec->data() = VectorHelper<T>::add(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return ret;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE WriteMaskedVector<T>::operator--(int) {
+ Vector<T> ret(*vec);
+ vec->data() = VectorHelper<T>::sub(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return ret;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE &WriteMaskedVector<T>::operator+=(const Vector<T> &x) {
+ vec->data() = VectorHelper<T>::add(vec->data(), VectorHelper<T>::notMaskedToZero(x.data(), mask.data()));
+ return *vec;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE &WriteMaskedVector<T>::operator-=(const Vector<T> &x) {
+ vec->data() = VectorHelper<T>::sub(vec->data(), VectorHelper<T>::notMaskedToZero(x.data(), mask.data()));
+ return *vec;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE &WriteMaskedVector<T>::operator*=(const Vector<T> &x) {
+ vec->assign(VectorHelper<T>::mul(vec->data(), x.data()), mask);
+ return *vec;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE &WriteMaskedVector<T>::operator/=(const Vector<T> &x) {
+ vec->assign(*vec / x, mask);
+ return *vec;
+}
+
+template<typename T>
+inline Vector<T> ALWAYS_INLINE &WriteMaskedVector<T>::operator=(const Vector<T> &x) {
+ vec->assign(x, mask);
+ return *vec;
+}
+
+} // namespace AVX
+} // namespace Vc
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_ALIASINGENTRYHELPER_H
+#define VC_COMMON_ALIASINGENTRYHELPER_H
+
+#include "macros.h"
+
+namespace Vc
+{
+namespace Common
+{
+
+template<class StorageType> class AliasingEntryHelper
+{
+ private:
+ typedef typename StorageType::EntryType T;
+#ifdef VC_ICC
+ StorageType *const m_storage;
+ const int m_index;
+ public:
+ AliasingEntryHelper(StorageType *d, int index) : m_storage(d), m_index(index) {}
+ AliasingEntryHelper(const AliasingEntryHelper &rhs) : m_storage(rhs.m_storage), m_index(rhs.m_index) {}
+ AliasingEntryHelper &operator=(const AliasingEntryHelper &rhs) {
+ m_storage->assign(m_index, rhs);
+ return *this;
+ }
+
+ AliasingEntryHelper &operator =(T x) { m_storage->assign(m_index, x); return *this; }
+ AliasingEntryHelper &operator +=(T x) { m_storage->assign(m_index, m_storage->m(m_index) + x); return *this; }
+ AliasingEntryHelper &operator -=(T x) { m_storage->assign(m_index, m_storage->m(m_index) - x); return *this; }
+ AliasingEntryHelper &operator /=(T x) { m_storage->assign(m_index, m_storage->m(m_index) / x); return *this; }
+ AliasingEntryHelper &operator *=(T x) { m_storage->assign(m_index, m_storage->m(m_index) * x); return *this; }
+ AliasingEntryHelper &operator |=(T x) { m_storage->assign(m_index, m_storage->m(m_index) | x); return *this; }
+ AliasingEntryHelper &operator &=(T x) { m_storage->assign(m_index, m_storage->m(m_index) & x); return *this; }
+ AliasingEntryHelper &operator ^=(T x) { m_storage->assign(m_index, m_storage->m(m_index) ^ x); return *this; }
+ AliasingEntryHelper &operator %=(T x) { m_storage->assign(m_index, m_storage->m(m_index) % x); return *this; }
+ AliasingEntryHelper &operator<<=(T x) { m_storage->assign(m_index, m_storage->m(m_index)<< x); return *this; }
+ AliasingEntryHelper &operator>>=(T x) { m_storage->assign(m_index, m_storage->m(m_index)>> x); return *this; }
+#define m_data m_storage->read(m_index)
+#else
+ typedef T A MAY_ALIAS;
+ A &m_data;
+ public:
+ template<typename T2>
+ AliasingEntryHelper(T2 &d) : m_data(reinterpret_cast<A &>(d)) {}
+
+ AliasingEntryHelper(A &d) : m_data(d) {}
+ AliasingEntryHelper &operator=(const AliasingEntryHelper &rhs) {
+ m_data = rhs.m_data;
+ return *this;
+ }
+
+ AliasingEntryHelper &operator =(T x) { m_data = x; return *this; }
+ AliasingEntryHelper &operator+=(T x) { m_data += x; return *this; }
+ AliasingEntryHelper &operator-=(T x) { m_data -= x; return *this; }
+ AliasingEntryHelper &operator/=(T x) { m_data /= x; return *this; }
+ AliasingEntryHelper &operator*=(T x) { m_data *= x; return *this; }
+ AliasingEntryHelper &operator|=(T x) { m_data |= x; return *this; }
+ AliasingEntryHelper &operator&=(T x) { m_data &= x; return *this; }
+ AliasingEntryHelper &operator^=(T x) { m_data ^= x; return *this; }
+ AliasingEntryHelper &operator%=(T x) { m_data %= x; return *this; }
+ AliasingEntryHelper &operator<<=(T x) { m_data <<= x; return *this; }
+ AliasingEntryHelper &operator>>=(T x) { m_data >>= x; return *this; }
+#endif
+
+ operator const T() const { return m_data; }
+
+ bool operator==(T x) const { return static_cast<T>(m_data) == x; }
+ bool operator!=(T x) const { return static_cast<T>(m_data) != x; }
+ bool operator<=(T x) const { return static_cast<T>(m_data) <= x; }
+ bool operator>=(T x) const { return static_cast<T>(m_data) >= x; }
+ bool operator< (T x) const { return static_cast<T>(m_data) < x; }
+ bool operator> (T x) const { return static_cast<T>(m_data) > x; }
+
+ T operator-() const { return -static_cast<T>(m_data); }
+ T operator~() const { return ~static_cast<T>(m_data); }
+ T operator+(T x) const { return static_cast<T>(m_data) + x; }
+ T operator-(T x) const { return static_cast<T>(m_data) - x; }
+ T operator/(T x) const { return static_cast<T>(m_data) / x; }
+ T operator*(T x) const { return static_cast<T>(m_data) * x; }
+ T operator|(T x) const { return static_cast<T>(m_data) | x; }
+ T operator&(T x) const { return static_cast<T>(m_data) & x; }
+ T operator^(T x) const { return static_cast<T>(m_data) ^ x; }
+ T operator%(T x) const { return static_cast<T>(m_data) % x; }
+ //T operator<<(T x) const { return static_cast<T>(m_data) << x; }
+ //T operator>>(T x) const { return static_cast<T>(m_data) >> x; }
+#ifdef m_data
+#undef m_data
+#endif
+};
+
+} // namespace Common
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_COMMON_ALIASINGENTRYHELPER_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_BITSCANINTRINSICS_H
+#define VC_COMMON_BITSCANINTRINSICS_H
+
+#if defined(VC_GCC) || defined(VC_CLANG)
+# if VC_GCC >= 0x40500
+ // GCC 4.5.0 introduced _bit_scan_forward / _bit_scan_reverse
+# include <x86intrin.h>
+# else
+ // GCC <= 4.4 and clang have x86intrin.h, but not the required functions
+# define _bit_scan_forward(x) __builtin_ctz(x)
+static inline int _Vc_bit_scan_reverse_asm(unsigned int x) {
+ int r;
+ __asm__("bsr %1,%0" : "=r"(r) : "X"(x));
+ return r;
+}
+# define _bit_scan_reverse(x) _Vc_bit_scan_reverse_asm(x)
+# endif
+#elif defined(VC_ICC)
+// for all I know ICC supports the _bit_scan_* intrinsics
+#elif defined(VC_OPEN64)
+// TODO
+#elif defined(VC_MSVC)
+#include "windows_fix_intrin.h"
+#pragma intrinsic(_BitScanForward)
+#pragma intrinsic(_BitScanReverse)
+static inline __forceinline unsigned long _bit_scan_forward(unsigned long x) {
+ unsigned long index;
+ _BitScanForward(&index, x);
+ return index;
+}
+static inline __forceinline unsigned long _bit_scan_reverse(unsigned long x) {
+ unsigned long index;
+ _BitScanReverse(&index, x);
+ return index;
+}
+#else
+// just assume the compiler can do it
+#endif
+
+
+#endif // VC_COMMON_BITSCANINTRINSICS_H
--- /dev/null
+/* This file is part of the Vc library. {{{
+
+ Copyright (C) 2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+}}}*/
+
+#ifndef VC_COMMON_CONST_H
+#define VC_COMMON_CONST_H
+
+#include "macros.h"
+
+namespace Vc
+{
+ template<typename T> struct Math;
+ template<> struct Math<float>
+ {
+ static _VC_CONSTEXPR float e() { return 2.7182818284590452353602874713526625f; }
+ static _VC_CONSTEXPR float log2e() { return 1.4426950408889634073599246810018921f; }
+ static _VC_CONSTEXPR float log10e() { return 0.4342944819032518276511289189166051f; }
+ static _VC_CONSTEXPR float ln2() { return 0.6931471805599453094172321214581766f; }
+ static _VC_CONSTEXPR float ln10() { return 2.3025850929940456840179914546843642f; }
+ static _VC_CONSTEXPR float pi() { return 3.1415926535897932384626433832795029f; }
+ static _VC_CONSTEXPR float pi_2() { return 1.5707963267948966192313216916397514f; }
+ static _VC_CONSTEXPR float pi_4() { return 0.7853981633974483096156608458198757f; }
+ static _VC_CONSTEXPR float _1_pi() { return 0.3183098861837906715377675267450287f; }
+ static _VC_CONSTEXPR float _2_pi() { return 0.6366197723675813430755350534900574f; }
+ static _VC_CONSTEXPR float _2_sqrtpi() { return 1.1283791670955125738961589031215452f; }
+ static _VC_CONSTEXPR float sqrt2() { return 1.4142135623730950488016887242096981f; }
+ static _VC_CONSTEXPR float sqrt1_2() { return 0.7071067811865475244008443621048490f; }
+ };
+ template<> struct Math<double>
+ {
+ static _VC_CONSTEXPR double e() { return 2.7182818284590452353602874713526625; }
+ static _VC_CONSTEXPR double log2e() { return 1.4426950408889634073599246810018921; }
+ static _VC_CONSTEXPR double log10e() { return 0.4342944819032518276511289189166051; }
+ static _VC_CONSTEXPR double ln2() { return 0.6931471805599453094172321214581766; }
+ static _VC_CONSTEXPR double ln10() { return 2.3025850929940456840179914546843642; }
+ static _VC_CONSTEXPR double pi() { return 3.1415926535897932384626433832795029; }
+ static _VC_CONSTEXPR double pi_2() { return 1.5707963267948966192313216916397514; }
+ static _VC_CONSTEXPR double pi_4() { return 0.7853981633974483096156608458198757; }
+ static _VC_CONSTEXPR double _1_pi() { return 0.3183098861837906715377675267450287; }
+ static _VC_CONSTEXPR double _2_pi() { return 0.6366197723675813430755350534900574; }
+ static _VC_CONSTEXPR double _2_sqrtpi() { return 1.1283791670955125738961589031215452; }
+ static _VC_CONSTEXPR double sqrt2() { return 1.4142135623730950488016887242096981; }
+ static _VC_CONSTEXPR double sqrt1_2() { return 0.7071067811865475244008443621048490; }
+ };
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_COMMON_CONST_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_DEINTERLEAVE_H
+#define VC_COMMON_DEINTERLEAVE_H
+
+namespace Vc
+{
+
+/**
+ * \ingroup Utilities
+ *
+ * Loads two vectors of values from an interleaved array.
+ *
+ * \param a, b The vectors to load the values from memory into.
+ * \param memory The memory location where to read the next 2 * V::Size values from
+ * \param align Either pass Vc::Aligned or Vc::Unaligned. It defaults to Vc::Aligned if nothing is
+ * specified.
+ *
+ * If you store your data as
+ * \code
+ * struct { float x, y; } m[1000];
+ * \endcode
+ * then the deinterleave function allows you to read \p Size concurrent x and y values like this:
+ * \code
+ * Vc::float_v x, y;
+ * Vc::deinterleave(&x, &y, &m[10], Vc::Unaligned);
+ * \endcode
+ * This code will load m[10], m[12], m[14], ... into \p x and m[11], m[13], m[15], ... into \p y.
+ *
+ * The deinterleave function supports the following type combinations:
+\verbatim
+ V \ M | float | double | ushort | short | uint | int
+=========|=======|========|========|=======|======|=====
+ float_v | X | | X | X | |
+---------|-------|--------|--------|-------|------|-----
+sfloat_v | X | | X | X | |
+---------|-------|--------|--------|-------|------|-----
+double_v | | X | | | |
+---------|-------|--------|--------|-------|------|-----
+ int_v | | | | X | | X
+---------|-------|--------|--------|-------|------|-----
+ uint_v | | | X | | X |
+---------|-------|--------|--------|-------|------|-----
+ short_v | | | | X | |
+---------|-------|--------|--------|-------|------|-----
+ushort_v | | | X | | |
+\endverbatim
+ */
+template<typename V, typename M, typename A> inline void deinterleave(V *a, V *b,
+ const M *memory, A align)
+{
+ Internal::Helper::deinterleave(*a, *b, memory, align);
+}
+
+// documented as default for align above
+template<typename V, typename M> inline void deinterleave(V *a, V *b,
+ const M *memory)
+{
+ Internal::Helper::deinterleave(*a, *b, memory, Aligned);
+}
+
+} // namespace Vc
+#endif // VC_COMMON_DEINTERLEAVE_H
--- /dev/null
+#ifndef COMMON_EXPONENTIAL_H
+#define COMMON_EXPONENTIAL_H
+/* This file is part of the Vc library. {{{
+
+ Copyright (C) 2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+ -------------------------------------------------------------------
+
+ The exp implementation is derived from Cephes, which carries the
+ following Copyright notice:
+
+ Cephes Math Library Release 2.2: June, 1992
+ Copyright 1984, 1987, 1989 by Stephen L. Moshier
+ Direct inquiries to 30 Frost Street, Cambridge, MA 02140
+
+}}}*/
+
+#ifndef VC_COMMON_EXPONENTIAL_H
+#define VC_COMMON_EXPONENTIAL_H
+
+#include "macros.h"
+namespace Vc
+{
+namespace Common
+{
+ using Vc::VC__USE_NAMESPACE::c_log;
+ using Vc::VC__USE_NAMESPACE::Vector;
+ using Vc::VC__USE_NAMESPACE::floor;
+ using Vc::VC__USE_NAMESPACE::ldexp;
+
+ static const float log2_e = 1.44269504088896341f;
+ static const float MAXLOGF = 88.72283905206835f;
+ static const float MINLOGF = -103.278929903431851103f; /* log(2^-149) */
+ static const float MAXNUMF = 3.4028234663852885981170418348451692544e38f;
+
+ template<typename T> struct TypenameForLdexp { typedef Vector<int> Type; };
+ template<> struct TypenameForLdexp<Vc::sfloat> { typedef Vector<short> Type; };
+
+ template<typename T> static inline Vector<T> exp(Vector<T> x) {
+ typedef Vector<T> V;
+ typedef typename V::Mask M;
+ typedef typename TypenameForLdexp<T>::Type I;
+ typedef Const<T> C;
+
+ const M overflow = x > MAXLOGF;
+ const M underflow = x < MINLOGF;
+
+ // log₂(eˣ) = x * log₂(e) * log₂(2)
+ // = log₂(2^(x * log₂(e)))
+ // => eˣ = 2^(x * log₂(e))
+ // => n = ⌊x * log₂(e) + ½⌋
+ // => y = x - n * ln(2) | recall that: ln(2) * log₂(e) == 1
+ // <=> eˣ = 2ⁿ * eʸ
+ V z = floor(C::log2_e() * x + 0.5f);
+ I n = static_cast<I>(z);
+ x -= z * C::ln2_large();
+ x -= z * C::ln2_small();
+
+ /* Theoretical peak relative error in [-0.5, +0.5] is 4.2e-9. */
+ z = ((((( 1.9875691500E-4f * x
+ + 1.3981999507E-3f) * x
+ + 8.3334519073E-3f) * x
+ + 4.1665795894E-2f) * x
+ + 1.6666665459E-1f) * x
+ + 5.0000001201E-1f) * (x * x)
+ + x
+ + 1.0f;
+
+ x = ldexp(z, n); // == z * 2ⁿ
+
+ x(overflow) = std::numeric_limits<typename V::EntryType>::infinity();
+ x.setZero(underflow);
+
+ return x;
+ }
+ static inline Vector<double> exp(Vector<double>::AsArg _x) {
+ Vector<double> x = _x;
+ typedef Vector<double> V;
+ typedef V::Mask M;
+ typedef Const<double> C;
+
+ const M overflow = x > Vc_buildDouble( 1, 0x0006232bdd7abcd2ull, 9); // max log
+ const M underflow = x < Vc_buildDouble(-1, 0x0006232bdd7abcd2ull, 9); // min log
+
+ V px = floor(C::log2_e() * x + 0.5);
+#ifdef VC_IMPL_SSE
+ Vector<int> n(px);
+ n.data() = Mem::permute<X0, X2, X1, X3>(n.data());
+#elif defined(VC_IMPL_AVX)
+ __m128i tmp = _mm256_cvttpd_epi32(px.data());
+ Vector<int> n = AVX::concat(_mm_unpacklo_epi32(tmp, tmp), _mm_unpackhi_epi32(tmp, tmp));
+#endif
+ x -= px * C::ln2_large(); //Vc_buildDouble(1, 0x00062e4000000000ull, -1); // ln2
+ x -= px * C::ln2_small(); //Vc_buildDouble(1, 0x0007f7d1cf79abcaull, -20); // ln2
+
+ const double P[] = {
+ Vc_buildDouble(1, 0x000089cdd5e44be8ull, -13),
+ Vc_buildDouble(1, 0x000f06d10cca2c7eull, -6),
+ Vc_buildDouble(1, 0x0000000000000000ull, 0)
+ };
+ const double Q[] = {
+ Vc_buildDouble(1, 0x00092eb6bc365fa0ull, -19),
+ Vc_buildDouble(1, 0x0004ae39b508b6c0ull, -9),
+ Vc_buildDouble(1, 0x000d17099887e074ull, -3),
+ Vc_buildDouble(1, 0x0000000000000000ull, 1)
+ };
+ const V x2 = x * x;
+ px = x * ((P[0] * x2 + P[1]) * x2 + P[2]);
+ x = px / ((((Q[0] * x2 + Q[1]) * x2 + Q[2]) * x2 + Q[3]) - px);
+ x = V::One() + 2.0 * x;
+
+ x = ldexp(x, n); // == x * 2ⁿ
+
+ x(overflow) = std::numeric_limits<double>::infinity();
+ x.setZero(underflow);
+
+ return x;
+ }
+} // namespace Common
+namespace VC__USE_NAMESPACE
+{
+ using Vc::Common::exp;
+} // namespace VC__USE_NAMESPACE
+} // namespace Vc
+#include "undomacros.h"
+
+#endif // VC_COMMON_EXPONENTIAL_H
+#endif // COMMON_EXPONENTIAL_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+/* The log implementations are based on code from Julien Pommier which carries the following
+ copyright information:
+ */
+/*
+ Inspired by Intel Approximate Math library, and based on the
+ corresponding algorithms of the cephes math library
+*/
+/* Copyright (C) 2007 Julien Pommier
+
+ This software is provided 'as-is', without any express or implied
+ warranty. In no event will the authors be held liable for any damages
+ arising from the use of this software.
+
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+
+ 1. The origin of this software must not be misrepresented; you must not
+ claim that you wrote the original software. If you use this software
+ in a product, an acknowledgment in the product documentation would be
+ appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+ misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
+
+ (this is the zlib license)
+*/
+
+#ifndef VC_COMMON_LOGARITHM_H
+#define VC_COMMON_LOGARITHM_H
+
+#include "macros.h"
+namespace Vc
+{
+namespace Common
+{
+#ifdef VC__USE_NAMESPACE
+using Vc::VC__USE_NAMESPACE::Const;
+using Vc::VC__USE_NAMESPACE::Vector;
+#endif
+enum LogarithmBase {
+ BaseE, Base10, Base2
+};
+
+template<LogarithmBase Base>
+struct LogImpl
+{
+ template<typename T> static inline ALWAYS_INLINE void log_series(Vector<T> &VC_RESTRICT x, typename Vector<T>::AsArg exponent) {
+ typedef Vector<T> V;
+ typedef Const<T> C;
+ // Taylor series around x = 2^exponent
+ // f(x) = ln(x) → exponent * ln(2) → C::ln2_small + C::ln2_large
+ // f'(x) = x⁻¹ → x → 1
+ // f''(x) = - x⁻² → -x² / 2 → C::_1_2()
+ // = 2!x⁻³ → x³ / 3 → C::P(8)
+ // = -3!x⁻⁴ → -x⁴ / 4 → C::P(7)
+ // = 4!x⁻⁵ → x⁵ / 5 → C::P(6)
+ // ...
+ // The high order coefficients are adjusted to reduce the error that occurs from ommission
+ // of higher order terms.
+ // P(0) is the smallest term and |x| < 1 ⇒ |xⁿ| > |xⁿ⁺¹|
+ // The order of additions must go from smallest to largest terms
+ const V x2 = x * x; // 0 → 4
+#ifdef VC_LOG_ILP
+ V y2 = (C::P(6) * /*4 → 8*/ x2 + /* 8 → 11*/ C::P(7) * /*1 → 5*/ x) + /*11 → 14*/ C::P(8);
+ V y0 = (C::P(0) * /*5 → 9*/ x2 + /* 9 → 12*/ C::P(1) * /*2 → 6*/ x) + /*12 → 15*/ C::P(2);
+ V y1 = (C::P(3) * /*6 → 10*/ x2 + /*10 → 13*/ C::P(4) * /*3 → 7*/ x) + /*13 → 16*/ C::P(5);
+ const V x3 = x2 * x; // 7 → 11
+ const V x6 = x3 * x3; // 11 → 15
+ const V x9 = x6 * x3; // 15 → 19
+ V y = (y0 * /*19 → 23*/ x9 + /*23 → 26*/ y1 * /*16 → 20*/ x6) + /*26 → 29*/ y2 * /*14 → 18*/ x3;
+#elif defined VC_LOG_ILP2
+ /*
+ * name start done
+ * movaps %xmm0, %xmm1 ; x 0 1
+ * movaps %xmm0, %xmm2 ; x 0 1
+ * mulps %xmm1, %xmm1 ; x2 1 5 *xmm1
+ * movaps <P8>, %xmm15 ; y8 1 2
+ * mulps %xmm1, %xmm2 ; x3 5 9 *xmm2
+ * movaps %xmm1, %xmm3 ; x2 5 6
+ * movaps %xmm1, %xmm4 ; x2 5 6
+ * mulps %xmm3, %xmm3 ; x4 6 10 *xmm3
+ * movaps %xmm2, %xmm5 ; x3 9 10
+ * movaps %xmm2, %xmm6 ; x3 9 10
+ * mulps %xmm2, %xmm4 ; x5 9 13 *xmm4
+ * movaps %xmm3, %xmm7 ; x4 10 11
+ * movaps %xmm3, %xmm8 ; x4 10 11
+ * movaps %xmm3, %xmm9 ; x4 10 11
+ * mulps %xmm5, %xmm5 ; x6 10 14 *xmm5
+ * mulps %xmm3, %xmm6 ; x7 11 15 *xmm6
+ * mulps %xmm7, %xmm7 ; x8 12 16 *xmm7
+ * movaps %xmm4, %xmm10 ; x5 13 14
+ * mulps %xmm4, %xmm8 ; x9 13 17 *xmm8
+ * mulps %xmm5, %xmm10 ; x11 14 18 *xmm10
+ * mulps %xmm5, %xmm9 ; x10 15 19 *xmm9
+ * mulps <P0>, %xmm10 ; y0 18 22
+ * mulps <P1>, %xmm9 ; y1 19 23
+ * mulps <P2>, %xmm8 ; y2 20 24
+ * mulps <P3>, %xmm7 ; y3 21 25
+ * addps %xmm10, %xmm9 ; y 23 26
+ * addps %xmm9, %xmm8 ; y 26 29
+ * addps %xmm8, %xmm7 ; y 29 32
+ */
+ const V x3 = x2 * x; // 4 → 8
+ const V x4 = x2 * x2; // 5 → 9
+ const V x5 = x2 * x3; // 8 → 12
+ const V x6 = x3 * x3; // 9 → 13
+ const V x7 = x4 * x3; //
+ const V x8 = x4 * x4;
+ const V x9 = x5 * x4;
+ const V x10 = x5 * x5;
+ const V x11 = x5 * x6; // 13 → 17
+ V y = C::P(0) * x11 + C::P(1) * x10 + C::P(2) * x9 + C::P(3) * x8 + C::P(4) * x7
+ + C::P(5) * x6 + C::P(6) * x5 + C::P(7) * x4 + C::P(8) * x3;
+#else
+ V y = C::P(0);
+ unrolled_loop16(i, 1, 9,
+ y = y * x + C::P(i);
+ );
+ y *= x * x2;
+#endif
+ switch (Base) {
+ case BaseE:
+ // ln(2) is split in two parts to increase precision (i.e. ln2_small + ln2_large = ln(2))
+ y += exponent * C::ln2_small();
+ y -= x2 * C::_1_2(); // [0, 0.25[
+ x += y;
+ x += exponent * C::ln2_large();
+ break;
+ case Base10:
+ y += exponent * C::ln2_small();
+ y -= x2 * C::_1_2(); // [0, 0.25[
+ x += y;
+ x += exponent * C::ln2_large();
+ x *= C::log10_e();
+ break;
+ case Base2:
+ {
+ const V x_ = x;
+ x *= C::log2_e();
+ y *= C::log2_e();
+ y -= x_ * x * C::_1_2(); // [0, 0.25[
+ x += y;
+ x += exponent;
+ break;
+ }
+ }
+ }
+
+ static inline ALWAYS_INLINE void log_series(Vector<double> &VC_RESTRICT x, Vector<double>::AsArg exponent) {
+ typedef Vector<double> V;
+ typedef Const<double> C;
+ const V x2 = x * x;
+ V y = C::P(0);
+ V y2 = C::Q(0) + x;
+ unrolled_loop16(i, 1, 5,
+ y = y * x + C::P(i);
+ y2 = y2 * x + C::Q(i);
+ );
+ y2 = x / y2;
+ y = y * x + C::P(5);
+ y = x2 * y * y2;
+ // TODO: refactor the following with the float implementation:
+ switch (Base) {
+ case BaseE:
+ // ln(2) is split in two parts to increase precision (i.e. ln2_small + ln2_large = ln(2))
+ y += exponent * C::ln2_small();
+ y -= x2 * C::_1_2(); // [0, 0.25[
+ x += y;
+ x += exponent * C::ln2_large();
+ break;
+ case Base10:
+ y += exponent * C::ln2_small();
+ y -= x2 * C::_1_2(); // [0, 0.25[
+ x += y;
+ x += exponent * C::ln2_large();
+ x *= C::log10_e();
+ break;
+ case Base2:
+ {
+ const V x_ = x;
+ x *= C::log2_e();
+ y *= C::log2_e();
+ y -= x_ * x * C::_1_2(); // [0, 0.25[
+ x += y;
+ x += exponent;
+ break;
+ }
+ }
+ }
+
+ template<typename T> static inline Vector<T> calc(Vector<T> x) {
+ typedef Vector<T> V;
+ typedef typename V::Mask M;
+ typedef Const<T> C;
+
+ const M invalidMask = x < V::Zero();
+ const M infinityMask = x == V::Zero();
+ const M denormal = x <= C::min();
+
+ x(denormal) *= V(Vc_buildDouble(1, 0, 54)); // 2²⁵
+ V exponent = x.exponent(); // = ⎣log₂(x)⎦
+ exponent(denormal) -= 54;
+
+ x.setZero(C::exponentMask()); // keep only the fractional part ⇒ x ∈ [1, 2[
+ x |= C::_1_2(); // and set the exponent to 2⁻¹ ⇒ x ∈ [½, 1[
+
+ // split calculation in two cases:
+ // A: x ∈ [½, √½[
+ // B: x ∈ [√½, 1[
+ // √½ defines the point where Δe(x) := log₂(x) - ⎣log₂(x)⎦ = ½, i.e.
+ // log₂(√½) - ⎣log₂(√½)⎦ = ½ * -1 - ⎣½ * -1⎦ = -½ + 1 = ½
+
+ const M smallX = x < C::_1_sqrt2();
+ x(smallX) += x; // => x ∈ [√½, 1[ ∪ [1.5, 1 + √½[
+ x -= V::One(); // => x ∈ [√½ - 1, 0[ ∪ [0.5, √½[
+ exponent(!smallX) += V::One();
+
+ log_series(x, exponent); // A: (ˣ⁄₂ᵉ - 1, e) B: (ˣ⁄₂ᵉ⁺¹ - 1, e + 1)
+
+ x.setQnan(invalidMask); // x < 0 → NaN
+ x(infinityMask) = C::neginf(); // x = 0 → -∞
+
+ return x;
+ }
+};
+
+template<typename T> static inline Vector<T> log(Vector<T> x) {
+ typedef typename Vector<T>::Mask M;
+ typedef Const<T> C;
+ return LogImpl<BaseE>::calc(x);
+}
+template<typename T> static inline Vector<T> log10(Vector<T> x) {
+ typedef typename Vector<T>::Mask M;
+ typedef Const<T> C;
+ return LogImpl<Base10>::calc(x);
+}
+template<typename T> static inline Vector<T> log2(Vector<T> x) {
+ typedef typename Vector<T>::Mask M;
+ typedef Const<T> C;
+ return LogImpl<Base2>::calc(x);
+}
+} // namespace Common
+#ifdef VC__USE_NAMESPACE
+namespace VC__USE_NAMESPACE
+{
+ using Vc::Common::log;
+ using Vc::Common::log10;
+ using Vc::Common::log2;
+} // namespace VC__USE_NAMESPACE
+#undef VC__USE_NAMESPACE
+#endif
+} // namespace Vc
+#include "undomacros.h"
+
+#endif // VC_COMMON_LOGARITHM_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_MACROS_H
+#define VC_COMMON_MACROS_H
+#undef VC_COMMON_UNDOMACROS_H
+
+#include <Vc/global.h>
+
+#ifdef VC_MSVC
+# define ALIGN(n) __declspec(align(n))
+# define STRUCT_ALIGN1(n) ALIGN(n)
+# define STRUCT_ALIGN2(n)
+# define ALIGNED_TYPEDEF(n, _type_, _newType_) typedef ALIGN(n) _type_ _newType_
+#else
+# define ALIGN(n) __attribute__((aligned(n)))
+# define STRUCT_ALIGN1(n)
+# define STRUCT_ALIGN2(n) ALIGN(n)
+# define ALIGNED_TYPEDEF(n, _type_, _newType_) typedef _type_ _newType_ ALIGN(n)
+#endif
+
+#define FREE_STORE_OPERATORS_ALIGNED(alignment) \
+ void *operator new(size_t size) { return _mm_malloc(size, alignment); } \
+ void *operator new(size_t, void *p) { return p; } \
+ void *operator new[](size_t size) { return _mm_malloc(size, alignment); } \
+ void operator delete(void *ptr, size_t) { _mm_free(ptr); } \
+ void operator delete[](void *ptr, size_t) { _mm_free(ptr); }
+
+#ifdef VC_CLANG
+# define INTRINSIC __attribute__((always_inline))
+# define INTRINSIC_L
+# define INTRINSIC_R INTRINSIC
+# define FLATTEN
+# define CONST __attribute__((const))
+# define CONST_L
+# define CONST_R CONST
+# define PURE __attribute__((pure))
+# define PURE_L
+# define PURE_R PURE
+# define MAY_ALIAS __attribute__((may_alias))
+# define ALWAYS_INLINE __attribute__((always_inline))
+# define ALWAYS_INLINE_L
+# define ALWAYS_INLINE_R ALWAYS_INLINE
+# define VC_IS_UNLIKELY(x) __builtin_expect(x, 0)
+# define VC_IS_LIKELY(x) __builtin_expect(x, 1)
+# define VC_RESTRICT __restrict__
+#elif defined(__GNUC__)
+# if defined(VC_OPEN64)
+# define INTRINSIC __attribute__((__flatten__, __always_inline__))
+# else
+# define INTRINSIC __attribute__((__flatten__, __always_inline__, __artificial__))
+# endif
+# define INTRINSIC_L
+# define INTRINSIC_R INTRINSIC
+# define FLATTEN __attribute__((__flatten__))
+# define CONST __attribute__((__const__))
+# define CONST_L
+# define CONST_R CONST
+# define PURE __attribute__((__pure__))
+# define PURE_L
+# define PURE_R PURE
+# define MAY_ALIAS __attribute__((__may_alias__))
+# define ALWAYS_INLINE __attribute__((__always_inline__))
+# define ALWAYS_INLINE_L
+# define ALWAYS_INLINE_R ALWAYS_INLINE
+# define VC_IS_UNLIKELY(x) __builtin_expect(x, 0)
+# define VC_IS_LIKELY(x) __builtin_expect(x, 1)
+# define VC_RESTRICT __restrict__
+#else
+# define FLATTEN
+# ifdef PURE
+# undef PURE
+# endif
+# define MAY_ALIAS
+# ifdef VC_MSVC
+# define ALWAYS_INLINE __forceinline
+# define ALWAYS_INLINE_L ALWAYS_INLINE
+# define ALWAYS_INLINE_R
+# define CONST __declspec(noalias)
+# define CONST_L CONST
+# define CONST_R
+# define PURE /*CONST*/
+# define PURE_L PURE
+# define PURE_R
+# define INTRINSIC __forceinline
+# define INTRINSIC_L INTRINSIC
+# define INTRINSIC_R
+# else
+# define ALWAYS_INLINE
+# define ALWAYS_INLINE_L
+# define ALWAYS_INLINE_R
+# define CONST
+# define CONST_L
+# define CONST_R
+# define PURE
+# define PURE_L
+# define PURE_R
+# define INTRINSIC
+# define INTRINSIC_L
+# define INTRINSIC_R
+# endif
+# define VC_IS_UNLIKELY(x) x
+# define VC_IS_LIKELY(x) x
+# define VC_RESTRICT __restrict
+#endif
+
+#if __cplusplus >= 201103 /*C++11*/
+#define _VC_CONSTEXPR constexpr
+#define _VC_CONSTEXPR_L _VC_CONSTEXPR
+#define _VC_CONSTEXPR_R
+#else
+#define _VC_CONSTEXPR inline INTRINSIC CONST
+#define _VC_CONSTEXPR_L inline INTRINSIC_L CONST_R
+#define _VC_CONSTEXPR_R INTRINSIC_R CONST_R
+#endif
+
+#if (defined(__GXX_EXPERIMENTAL_CXX0X__) && VC_GCC >= 0x40600) || __cplusplus >= 201103
+# define _VC_NOEXCEPT noexcept
+#else
+# define _VC_NOEXCEPT throw()
+#endif
+
+
+#ifdef VC_GCC
+# define VC_WARN_INLINE
+# define VC_WARN(msg) __attribute__((warning("\n\t" msg)))
+#else
+# define VC_WARN_INLINE inline
+# define VC_WARN(msg)
+#endif
+
+#define unrolled_loop16(_it_, _start_, _end_, _code_) \
+if (_start_ + 0 < _end_) { enum { _it_ = (_start_ + 0) < _end_ ? (_start_ + 0) : _start_ }; _code_ } \
+if (_start_ + 1 < _end_) { enum { _it_ = (_start_ + 1) < _end_ ? (_start_ + 1) : _start_ }; _code_ } \
+if (_start_ + 2 < _end_) { enum { _it_ = (_start_ + 2) < _end_ ? (_start_ + 2) : _start_ }; _code_ } \
+if (_start_ + 3 < _end_) { enum { _it_ = (_start_ + 3) < _end_ ? (_start_ + 3) : _start_ }; _code_ } \
+if (_start_ + 4 < _end_) { enum { _it_ = (_start_ + 4) < _end_ ? (_start_ + 4) : _start_ }; _code_ } \
+if (_start_ + 5 < _end_) { enum { _it_ = (_start_ + 5) < _end_ ? (_start_ + 5) : _start_ }; _code_ } \
+if (_start_ + 6 < _end_) { enum { _it_ = (_start_ + 6) < _end_ ? (_start_ + 6) : _start_ }; _code_ } \
+if (_start_ + 7 < _end_) { enum { _it_ = (_start_ + 7) < _end_ ? (_start_ + 7) : _start_ }; _code_ } \
+if (_start_ + 8 < _end_) { enum { _it_ = (_start_ + 8) < _end_ ? (_start_ + 8) : _start_ }; _code_ } \
+if (_start_ + 9 < _end_) { enum { _it_ = (_start_ + 9) < _end_ ? (_start_ + 9) : _start_ }; _code_ } \
+if (_start_ + 10 < _end_) { enum { _it_ = (_start_ + 10) < _end_ ? (_start_ + 10) : _start_ }; _code_ } \
+if (_start_ + 11 < _end_) { enum { _it_ = (_start_ + 11) < _end_ ? (_start_ + 11) : _start_ }; _code_ } \
+if (_start_ + 12 < _end_) { enum { _it_ = (_start_ + 12) < _end_ ? (_start_ + 12) : _start_ }; _code_ } \
+if (_start_ + 13 < _end_) { enum { _it_ = (_start_ + 13) < _end_ ? (_start_ + 13) : _start_ }; _code_ } \
+if (_start_ + 14 < _end_) { enum { _it_ = (_start_ + 14) < _end_ ? (_start_ + 14) : _start_ }; _code_ } \
+if (_start_ + 15 < _end_) { enum { _it_ = (_start_ + 15) < _end_ ? (_start_ + 15) : _start_ }; _code_ } \
+do {} while ( false )
+
+#define for_all_vector_entries(_it_, _code_) \
+ unrolled_loop16(_it_, 0, Size, _code_)
+
+#ifdef NDEBUG
+#define VC_ASSERT(x)
+#else
+#include <assert.h>
+#define VC_ASSERT(x) assert(x);
+#endif
+
+#ifdef VC_CLANG
+#define VC_HAS_BUILTIN(x) __has_builtin(x)
+#else
+#define VC_HAS_BUILTIN(x) 0
+#endif
+
+#ifndef VC_COMMON_MACROS_H_ONCE
+#define VC_COMMON_MACROS_H_ONCE
+
+#define _VC_CAT_HELPER(a, b, c, d) a##b##c##d
+#define _VC_CAT(a, b, c, d) _VC_CAT_HELPER(a, b, c, d)
+
+#if __cplusplus >= 201103 /*C++11*/ || VC_MSVC >= 160000000
+#define VC_STATIC_ASSERT_NC(cond, msg) \
+ static_assert(cond, #msg)
+#define VC_STATIC_ASSERT(cond, msg) VC_STATIC_ASSERT_NC(cond, msg)
+#else // C++98
+namespace Vc {
+ namespace {
+ template<bool> struct STATIC_ASSERT_FAILURE;
+ template<> struct STATIC_ASSERT_FAILURE<true> {};
+}}
+
+#define VC_STATIC_ASSERT_NC(cond, msg) \
+ typedef STATIC_ASSERT_FAILURE<cond> _VC_CAT(static_assert_failed_on_line_,__LINE__,_,msg); \
+ enum { \
+ _VC_CAT(static_assert_failed__on_line_,__LINE__,_,msg) = sizeof(_VC_CAT(static_assert_failed_on_line_,__LINE__,_,msg)) \
+ }
+#define VC_STATIC_ASSERT(cond, msg) VC_STATIC_ASSERT_NC(cond, msg)
+#endif // C++11/98
+
+ template<int e, int center> struct exponentToMultiplier { enum {
+ X = exponentToMultiplier<e - 1, center>::X * ((e - center < 31) ? 2 : 1),
+ Value = (X == 0 ? 1 : X)
+ }; };
+ template<int center> struct exponentToMultiplier<center,center> { enum { X = 1, Value = X }; };
+ template<int center> struct exponentToMultiplier< -1, center> { enum { X = 0 }; };
+ template<int center> struct exponentToMultiplier< -256, center> { enum { X = 0 }; };
+ template<int center> struct exponentToMultiplier< -512, center> { enum { X = 0 }; };
+ template<int center> struct exponentToMultiplier<-1024, center> { enum { X = 0 }; };
+
+ template<int e> struct exponentToDivisor { enum {
+ X = exponentToDivisor<e + 1>::X * 2,
+ Value = (X == 0 ? 1 : X)
+ }; };
+ template<> struct exponentToDivisor<0> { enum { X = 1, Value = X }; };
+ template<> struct exponentToDivisor<256> { enum { X = 0 }; };
+ template<> struct exponentToDivisor<512> { enum { X = 0 }; };
+ template<> struct exponentToDivisor<1024> { enum { X = 0 }; };
+#endif // VC_COMMON_MACROS_H_ONCE
+
+#define _CAT_IMPL(a, b) a##b
+#define CAT(a, b) _CAT_IMPL(a, b)
+
+#define Vc_buildDouble(sign, mantissa, exponent) \
+ ((static_cast<double>((mantissa & 0x000fffffffffffffull) | 0x0010000000000000ull) / 0x0010000000000000ull) \
+ * exponentToMultiplier<exponent, 0>::Value \
+ * exponentToMultiplier<exponent, 30>::Value \
+ * exponentToMultiplier<exponent, 60>::Value \
+ * exponentToMultiplier<exponent, 90>::Value \
+ / exponentToDivisor<exponent>::Value \
+ * static_cast<double>(sign))
+#define Vc_buildFloat(sign, mantissa, exponent) \
+ ((static_cast<float>((mantissa & 0x007fffffu) | 0x00800000) / 0x00800000) \
+ * exponentToMultiplier<exponent, 0>::Value \
+ * exponentToMultiplier<exponent, 30>::Value \
+ * exponentToMultiplier<exponent, 60>::Value \
+ * exponentToMultiplier<exponent, 90>::Value \
+ / exponentToDivisor<exponent>::Value \
+ * static_cast<float>(sign))
+
+#define _VC_APPLY_IMPL_1(macro, a, b, c, d, e) macro(a)
+#define _VC_APPLY_IMPL_2(macro, a, b, c, d, e) macro(a, b)
+#define _VC_APPLY_IMPL_3(macro, a, b, c, d, e) macro(a, b, c)
+#define _VC_APPLY_IMPL_4(macro, a, b, c, d, e) macro(a, b, c, d)
+#define _VC_APPLY_IMPL_5(macro, a, b, c, d, e) macro(a, b, c, d, e)
+
+#define VC_LIST_FLOAT_VECTOR_TYPES(size, macro, a, b, c, d) \
+ size(macro, double_v, a, b, c, d) \
+ size(macro, float_v, a, b, c, d) \
+ size(macro, sfloat_v, a, b, c, d)
+#define VC_LIST_INT_VECTOR_TYPES(size, macro, a, b, c, d) \
+ size(macro, int_v, a, b, c, d) \
+ size(macro, uint_v, a, b, c, d) \
+ size(macro, short_v, a, b, c, d) \
+ size(macro, ushort_v, a, b, c, d)
+#define VC_LIST_VECTOR_TYPES(size, macro, a, b, c, d) \
+ VC_LIST_FLOAT_VECTOR_TYPES(size, macro, a, b, c, d) \
+ VC_LIST_INT_VECTOR_TYPES(size, macro, a, b, c, d)
+#define VC_LIST_COMPARES(size, macro, a, b, c, d) \
+ size(macro, ==, a, b, c, d) \
+ size(macro, !=, a, b, c, d) \
+ size(macro, <=, a, b, c, d) \
+ size(macro, >=, a, b, c, d) \
+ size(macro, < , a, b, c, d) \
+ size(macro, > , a, b, c, d)
+#define VC_LIST_LOGICAL(size, macro, a, b, c, d) \
+ size(macro, &&, a, b, c, d) \
+ size(macro, ||, a, b, c, d)
+#define VC_LIST_BINARY(size, macro, a, b, c, d) \
+ size(macro, |, a, b, c, d) \
+ size(macro, &, a, b, c, d) \
+ size(macro, ^, a, b, c, d)
+#define VC_LIST_SHIFTS(size, macro, a, b, c, d) \
+ size(macro, <<, a, b, c, d) \
+ size(macro, >>, a, b, c, d)
+#define VC_LIST_ARITHMETICS(size, macro, a, b, c, d) \
+ size(macro, +, a, b, c, d) \
+ size(macro, -, a, b, c, d) \
+ size(macro, *, a, b, c, d) \
+ size(macro, /, a, b, c, d) \
+ size(macro, %, a, b, c, d)
+
+#define VC_APPLY_0(_list, macro) _list(_VC_APPLY_IMPL_1, macro, 0, 0, 0, 0)
+#define VC_APPLY_1(_list, macro, a) _list(_VC_APPLY_IMPL_2, macro, a, 0, 0, 0)
+#define VC_APPLY_2(_list, macro, a, b) _list(_VC_APPLY_IMPL_3, macro, a, b, 0, 0)
+#define VC_APPLY_3(_list, macro, a, b, c) _list(_VC_APPLY_IMPL_4, macro, a, b, c, 0)
+#define VC_APPLY_4(_list, macro, a, b, c, d) _list(_VC_APPLY_IMPL_5, macro, a, b, c, d)
+
+#define VC_ALL_COMPARES(macro) VC_APPLY_0(VC_LIST_COMPARES, macro)
+#define VC_ALL_LOGICAL(macro) VC_APPLY_0(VC_LIST_LOGICAL, macro)
+#define VC_ALL_BINARY(macro) VC_APPLY_0(VC_LIST_BINARY, macro)
+#define VC_ALL_SHIFTS(macro) VC_APPLY_0(VC_LIST_SHIFTS, macro)
+#define VC_ALL_ARITHMETICS(macro) VC_APPLY_0(VC_LIST_ARITHMETICS, macro)
+#define VC_ALL_FLOAT_VECTOR_TYPES(macro) VC_APPLY_0(VC_LIST_FLOAT_VECTOR_TYPES, macro)
+#define VC_ALL_VECTOR_TYPES(macro) VC_APPLY_0(VC_LIST_VECTOR_TYPES, macro)
+
+#define VC_EXACT_TYPE(_test, _reference, _type) \
+ typename EnableIf<IsEqualType<_test, _reference>::Value, _type>::Value
+
+#endif // VC_COMMON_MACROS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_MEMORY_H
+#define VC_COMMON_MEMORY_H
+
+#include "memorybase.h"
+#include <assert.h>
+#include <algorithm>
+#include <cstring>
+#include "memoryfwd.h"
+#include "macros.h"
+
+namespace Vc
+{
+
+/**
+ * Allocates memory on the Heap with alignment and padding.
+ *
+ * Memory that was allocated with this function must be released with Vc::free! Other methods might
+ * work but are not portable.
+ *
+ * \param n Specifies the number of scalar values the allocated memory must be able to store.
+ *
+ * \return Pointer to memory of the requested type and size, or 0 on error.
+ *
+ * \warning The standard malloc function specifies the number of Bytes to allocate whereas this
+ * function specifies the number of values, thus differing in a factor of sizeof(T)
+ *
+ * \see Vc::free
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+template<typename T, Vc::MallocAlignment A>
+inline ALWAYS_INLINE_L T *ALWAYS_INLINE_R malloc(size_t n)
+{
+ return static_cast<T *>(Internal::Helper::malloc<A>(n * sizeof(T)));
+}
+
+/**
+ * Frees memory that was allocated with Vc::malloc.
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+template<typename T>
+inline void ALWAYS_INLINE free(T *p)
+{
+ Internal::Helper::free(p);
+}
+
+template<typename V, size_t Size> struct _MemorySizeCalculation
+{
+ enum AlignmentCalculations {
+ Alignment = V::Size,
+ AlignmentMask = Alignment - 1,
+ MaskedSize = Size & AlignmentMask,
+ Padding = Alignment - MaskedSize,
+ PaddedSize = MaskedSize == 0 ? Size : Size + Padding
+ };
+};
+
+/**
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ *
+ * A helper class for fixed-size two-dimensional arrays.
+ *
+ * \param V The vector type you want to operate on. (e.g. float_v or uint_v)
+ * \param Size1 Number of rows
+ * \param Size2 Number of columns
+ */
+template<typename V, size_t Size1, size_t Size2> class Memory : public VectorAlignedBaseT<V>, public MemoryBase<V, Memory<V, Size1, Size2>, 2, Memory<V, Size2> >
+{
+ public:
+ typedef typename V::EntryType EntryType;
+ private:
+ typedef MemoryBase<V, Memory<V, Size1, Size2>, 2, Memory<V, Size2> > Base;
+ friend class MemoryBase<V, Memory<V, Size1, Size2>, 2, Memory<V, Size2> >;
+ friend class MemoryDimensionBase<V, Memory<V, Size1, Size2>, 2, Memory<V, Size2> >;
+ enum InternalConstants {
+ PaddedSize2 = _MemorySizeCalculation<V, Size2>::PaddedSize
+ };
+#if defined(VC_ICC) && defined(_WIN32)
+ __declspec(align(__alignof(VectorAlignedBaseT<V>)))
+#elif defined(VC_CLANG)
+ __attribute__((aligned(__alignof(VectorAlignedBaseT<V>))))
+#endif
+ EntryType m_mem[Size1][PaddedSize2];
+ public:
+ using Base::vector;
+ enum Constants {
+ RowCount = Size1,
+ VectorsCount = PaddedSize2 / V::Size
+ };
+
+ /**
+ * \return the number of rows in the array.
+ *
+ * \note This function can be eliminated by an optimizing compiler.
+ */
+ inline size_t rowsCount() const { return RowCount; }
+ /**
+ * \return the number of scalar entries in the whole array.
+ *
+ * \warning Do not use this function for scalar iteration over the array since there will be
+ * padding between rows if \c Size2 is not divisible by \c V::Size.
+ *
+ * \note This function can be optimized into a compile-time constant.
+ */
+ inline size_t entriesCount() const { return Size1 * Size2; }
+ /**
+ * \return the number of vectors in the whole array.
+ *
+ * \note This function can be optimized into a compile-time constant.
+ */
+ inline size_t vectorsCount() const { return VectorsCount * Size1; }
+
+ /**
+ * Copies the data from a different object.
+ *
+ * \param rhs The object to copy the data from.
+ *
+ * \return reference to the modified Memory object.
+ *
+ * \note Both objects must have the exact same vectorsCount().
+ */
+ template<typename Parent, typename RM>
+ inline Memory &operator=(const MemoryBase<V, Parent, 2, RM> &rhs) {
+ assert(vectorsCount() == rhs.vectorsCount());
+ std::memcpy(m_mem, rhs.m_mem, vectorsCount() * sizeof(V));
+ return *this;
+ }
+ /**
+ * Initialize all data with the given vector.
+ *
+ * \param v This vector will be used to initialize the memory.
+ *
+ * \return reference to the modified Memory object.
+ */
+ inline Memory &operator=(const V &v) {
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) = v;
+ }
+ return *this;
+ }
+ }
+#if defined(VC_ICC) && VC_ICC < 20120212 && !defined(_WIN32)
+ __attribute__((__aligned__(__alignof(VectorAlignedBaseT<V>))))
+#endif
+ ;
+
+ /**
+ * A helper class to simplify usage of correctly aligned and padded memory, allowing both vector and
+ * scalar access.
+ *
+ * Example:
+ * \code
+ Vc::Memory<int_v, 11> array;
+
+ // scalar access:
+ for (size_t i = 0; i < array.entriesCount(); ++i) {
+ int x = array[i]; // read
+ array[i] = x; // write
+ }
+ // more explicit alternative:
+ for (size_t i = 0; i < array.entriesCount(); ++i) {
+ int x = array.scalar(i); // read
+ array.scalar(i) = x; // write
+ }
+
+ // vector access:
+ for (size_t i = 0; i < array.vectorsCount(); ++i) {
+ int_v x = array.vector(i); // read
+ array.vector(i) = x; // write
+ }
+ * \endcode
+ * This code allocates a small array and implements three equivalent loops (that do nothing useful).
+ * The loops show how scalar and vector read/write access is best implemented.
+ *
+ * Since the size of 11 is not a multiple of int_v::Size (unless you use the
+ * scalar Vc implementation) the last write access of the vector loop would normally be out of
+ * bounds. But the Memory class automatically pads the memory such that the whole array can be
+ * accessed with correctly aligned memory addresses.
+ *
+ * \param V The vector type you want to operate on. (e.g. float_v or uint_v)
+ * \param Size The number of entries of the scalar base type the memory should hold. This
+ * is thus the same number as you would use for a normal C array (e.g. float mem[11] becomes
+ * Memory<float_v, 11> mem).
+ *
+ * \see Memory<V, 0u>
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+ template<typename V, size_t Size> class Memory<V, Size, 0u> : public VectorAlignedBaseT<V>, public MemoryBase<V, Memory<V, Size, 0u>, 1, void>
+ {
+ public:
+ typedef typename V::EntryType EntryType;
+ private:
+ typedef MemoryBase<V, Memory<V, Size, 0u>, 1, void> Base;
+ friend class MemoryBase<V, Memory<V, Size, 0u>, 1, void>;
+ friend class MemoryDimensionBase<V, Memory<V, Size, 0u>, 1, void>;
+ enum InternalConstants {
+ Alignment = V::Size,
+ AlignmentMask = Alignment - 1,
+ MaskedSize = Size & AlignmentMask,
+ Padding = Alignment - MaskedSize,
+ PaddedSize = MaskedSize == 0 ? Size : Size + Padding
+ };
+#if defined(__INTEL_COMPILER) && defined(_WIN32)
+ __declspec(align(__alignof(VectorAlignedBaseT<V>)))
+#elif defined(VC_CLANG)
+ __attribute__((aligned(__alignof(VectorAlignedBaseT<V>))))
+#endif
+ EntryType m_mem[PaddedSize];
+ public:
+ using Base::vector;
+ enum Constants {
+ EntriesCount = Size,
+ VectorsCount = PaddedSize / V::Size
+ };
+
+ /**
+ * \return the number of scalar entries in the whole array.
+ *
+ * \note This function can be optimized into a compile-time constant.
+ */
+ inline size_t entriesCount() const { return EntriesCount; }
+
+ /**
+ * \return the number of vectors in the whole array.
+ *
+ * \note This function can be optimized into a compile-time constant.
+ */
+ inline size_t vectorsCount() const { return VectorsCount; }
+
+ template<typename Parent, typename RM>
+ inline Memory<V> &operator=(const MemoryBase<V, Parent, 1, RM> &rhs) {
+ assert(vectorsCount() == rhs.vectorsCount());
+ std::memcpy(m_mem, rhs.m_mem, entriesCount() * sizeof(EntryType));
+ return *this;
+ }
+ inline Memory<V> &operator=(const EntryType *rhs) {
+ std::memcpy(m_mem, rhs, entriesCount() * sizeof(EntryType));
+ return *this;
+ }
+ inline Memory &operator=(const V &v) {
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) = v;
+ }
+ return *this;
+ }
+ }
+#if defined(VC_ICC) && VC_ICC < 20120212 && !defined(_WIN32)
+ __attribute__((__aligned__(__alignof(VectorAlignedBaseT<V>)) ))
+#endif
+ ;
+
+ /**
+ * A helper class that is very similar to Memory<V, Size> but with dynamically allocated memory and
+ * thus dynamic size.
+ *
+ * Example:
+ * \code
+ size_t size = 11;
+ Vc::Memory<int_v> array(size);
+
+ // scalar access:
+ for (size_t i = 0; i < array.entriesCount(); ++i) {
+ array[i] = i;
+ }
+
+ // vector access:
+ for (size_t i = 0; i < array.vectorsCount(); ++i) {
+ array.vector(i) = int_v::IndexesFromZero() + i * int_v::Size;
+ }
+ * \endcode
+ * This code allocates a small array with 11 scalar entries
+ * and implements two equivalent loops that initialize the memory.
+ * The scalar loop writes each individual int. The vectorized loop writes int_v::Size values to
+ * memory per iteration. Since the size of 11 is not a multiple of int_v::Size (unless you use the
+ * scalar Vc implementation) the last write access of the vector loop would normally be out of
+ * bounds. But the Memory class automatically pads the memory such that the whole array can be
+ * accessed with correctly aligned memory addresses.
+ * (Note: the scalar loop can be auto-vectorized, except for the last three assignments.)
+ *
+ * \note The internal data pointer is not declared with the \c __restrict__ keyword. Therefore
+ * modifying memory of V::EntryType will require the compiler to assume aliasing. If you want to use
+ * the \c __restrict__ keyword you need to use a standard pointer to memory and do the vector
+ * address calculation and loads and stores manually.
+ *
+ * \param V The vector type you want to operate on. (e.g. float_v or uint_v)
+ *
+ * \see Memory<V, Size>
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+ template<typename V> class Memory<V, 0u, 0u> : public MemoryBase<V, Memory<V, 0u, 0u>, 1, void>
+ {
+ public:
+ typedef typename V::EntryType EntryType;
+ private:
+ typedef MemoryBase<V, Memory<V>, 1, void> Base;
+ friend class MemoryBase<V, Memory<V>, 1, void>;
+ friend class MemoryDimensionBase<V, Memory<V>, 1, void>;
+ enum InternalConstants {
+ Alignment = V::Size,
+ AlignmentMask = Alignment - 1
+ };
+ size_t m_entriesCount;
+ size_t m_vectorsCount;
+ EntryType *m_mem;
+ size_t calcPaddedEntriesCount(size_t x)
+ {
+ size_t masked = x & AlignmentMask;
+ return (masked == 0 ? x : x + (Alignment - masked));
+ }
+ public:
+ using Base::vector;
+
+ /**
+ * Allocate enough memory to access \p size values of type \p V::EntryType.
+ *
+ * The allocated memory is aligned and padded correctly for fully vectorized access.
+ *
+ * \param size Determines how many scalar values will fit into the allocated memory.
+ */
+ inline Memory(size_t size)
+ : m_entriesCount(size),
+ m_vectorsCount(calcPaddedEntriesCount(m_entriesCount)),
+ m_mem(Vc::malloc<EntryType, Vc::AlignOnVector>(m_vectorsCount))
+ {
+ m_vectorsCount /= V::Size;
+ }
+
+ /**
+ * Copy the memory into a new memory area.
+ *
+ * The allocated memory is aligned and padded correctly for fully vectorized access.
+ *
+ * \param rhs The Memory object to copy from.
+ */
+ template<typename Parent, typename RM>
+ inline Memory(const MemoryBase<V, Parent, 1, RM> &rhs)
+ : m_entriesCount(rhs.entriesCount()),
+ m_vectorsCount(rhs.vectorsCount()),
+ m_mem(Vc::malloc<EntryType, Vc::AlignOnVector>(m_vectorsCount * V::Size))
+ {
+ std::memcpy(m_mem, rhs.m_mem, entriesCount() * sizeof(EntryType));
+ }
+
+ /**
+ * Overload of the above function.
+ *
+ * (Because C++ would otherwise not use the templated cctor and use a default-constructed cctor instead.)
+ *
+ * \param rhs The Memory object to copy from.
+ */
+ inline Memory(const Memory<V, 0u> &rhs)
+ : m_entriesCount(rhs.entriesCount()),
+ m_vectorsCount(rhs.vectorsCount()),
+ m_mem(Vc::malloc<EntryType, Vc::AlignOnVector>(m_vectorsCount * V::Size))
+ {
+ std::memcpy(m_mem, rhs.m_mem, entriesCount() * sizeof(EntryType));
+ }
+
+ /**
+ * Frees the memory which was allocated in the constructor.
+ */
+ inline ALWAYS_INLINE ~Memory()
+ {
+ Vc::free(m_mem);
+ }
+
+ /**
+ * Swap the contents and size information of two Memory objects.
+ *
+ * \param rhs The other Memory object to swap.
+ */
+ inline void swap(Memory &rhs) {
+ std::swap(m_mem, rhs.m_mem);
+ std::swap(m_entriesCount, rhs.m_entriesCount);
+ std::swap(m_vectorsCount, rhs.m_vectorsCount);
+ }
+
+ /**
+ * \return the number of scalar entries in the whole array.
+ */
+ inline size_t entriesCount() const { return m_entriesCount; }
+
+ /**
+ * \return the number of vectors in the whole array.
+ */
+ inline size_t vectorsCount() const { return m_vectorsCount; }
+
+ /**
+ * Overwrite all entries with the values stored in \p rhs.
+ *
+ * \param rhs The object to copy the data from.
+ *
+ * \return reference to the modified Memory object.
+ *
+ * \note this function requires the vectorsCount() of both Memory objects to be equal.
+ */
+ template<typename Parent, typename RM>
+ inline Memory<V> &operator=(const MemoryBase<V, Parent, 1, RM> &rhs) {
+ assert(vectorsCount() == rhs.vectorsCount());
+ std::memcpy(m_mem, rhs.m_mem, entriesCount() * sizeof(EntryType));
+ return *this;
+ }
+
+ /**
+ * Overwrite all entries with the values stored in the memory at \p rhs.
+ *
+ * \param rhs The array to copy the data from.
+ *
+ * \return reference to the modified Memory object.
+ *
+ * \note this function requires that there are entriesCount() many values accessible from \p rhs.
+ */
+ inline Memory<V> &operator=(const EntryType *rhs) {
+ std::memcpy(m_mem, rhs, entriesCount() * sizeof(EntryType));
+ return *this;
+ }
+};
+
+/**
+ * Prefetch the cacheline containing \p addr for a single read access.
+ *
+ * This prefetch completely bypasses the cache, not evicting any other data.
+ *
+ * \param addr The cacheline containing \p addr will be prefetched.
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+inline void ALWAYS_INLINE prefetchForOneRead(const void *addr)
+{
+ Internal::Helper::prefetchForOneRead(addr);
+}
+
+/**
+ * Prefetch the cacheline containing \p addr for modification.
+ *
+ * This prefetch evicts data from the cache. So use it only for data you really will use. When the
+ * target system supports it the cacheline will be marked as modified while prefetching, saving work
+ * later on.
+ *
+ * \param addr The cacheline containing \p addr will be prefetched.
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+inline void ALWAYS_INLINE prefetchForModify(const void *addr)
+{
+ Internal::Helper::prefetchForModify(addr);
+}
+
+/**
+ * Prefetch the cacheline containing \p addr to L1 cache.
+ *
+ * This prefetch evicts data from the cache. So use it only for data you really will use.
+ *
+ * \param addr The cacheline containing \p addr will be prefetched.
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+inline void ALWAYS_INLINE prefetchClose(const void *addr)
+{
+ Internal::Helper::prefetchClose(addr);
+}
+
+/**
+ * Prefetch the cacheline containing \p addr to L2 cache.
+ *
+ * This prefetch evicts data from the cache. So use it only for data you really will use.
+ *
+ * \param addr The cacheline containing \p addr will be prefetched.
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+inline void ALWAYS_INLINE prefetchMid(const void *addr)
+{
+ Internal::Helper::prefetchMid(addr);
+}
+
+/**
+ * Prefetch the cacheline containing \p addr to L3 cache.
+ *
+ * This prefetch evicts data from the cache. So use it only for data you really will use.
+ *
+ * \param addr The cacheline containing \p addr will be prefetched.
+ *
+ * \ingroup Utilities
+ * \headerfile memory.h <Vc/Memory>
+ */
+inline void ALWAYS_INLINE prefetchFar(const void *addr)
+{
+ Internal::Helper::prefetchFar(addr);
+}
+
+} // namespace Vc
+
+namespace std
+{
+ template<typename V> inline void swap(Vc::Memory<V> &a, Vc::Memory<V> &b) { a.swap(b); }
+} // namespace std
+
+#include "undomacros.h"
+
+#endif // VC_COMMON_MEMORY_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_MEMORYBASE_H
+#define VC_COMMON_MEMORYBASE_H
+
+#include <assert.h>
+#include "macros.h"
+
+namespace Vc
+{
+
+#if __cplusplus >= 201103 || defined(VC_MSVC)
+#define VC_DECLTYPE(T1, op, T2) decltype(T1() op T2())
+#else
+namespace
+{
+ struct one { char x; };
+ struct two { one x, y; };
+ template<typename T1, typename T2> struct DecltypeHelper
+ {
+ static one test(const T1 &) { return one(); }
+ static two test(const T2 &) { return two(); }
+ //static void test(...) {}
+ };
+ template<typename T1> struct DecltypeHelper<T1, T1>
+ {
+ static one test(const T1 &) { return one(); }
+ //static void test(...) {}
+ };
+ template<typename T1, typename T2, size_t ToSelect> struct Decltype { typedef T1 Value; };
+ template<typename T1, typename T2> struct Decltype<T1, T2, sizeof(one)> { typedef T1 Value; };
+ template<typename T1, typename T2> struct Decltype<T1, T2, sizeof(two)> { typedef T2 Value; };
+ static const void *SOME_PTR;
+} // anonymous namespace
+#define VC_DECLTYPE(T1, op, T2) typename Decltype<T1, T2, sizeof(DecltypeHelper<T1, T2>::test(*static_cast<const T1*>(SOME_PTR) op *static_cast<const T2*>(SOME_PTR)))>::Value
+#endif
+
+#define VC_MEM_OPERATOR_EQ(op) \
+ template<typename T> \
+ inline VectorPointerHelper<V, A> &operator op##=(const T &x) { \
+ const V result = V(m_ptr, Internal::FlagObject<A>::the()) op x; \
+ result.store(m_ptr, Internal::FlagObject<A>::the()); \
+ return *this; \
+ }
+
+/**
+ * Helper class for the Memory::vector(size_t) class of functions.
+ *
+ * You will never need to directly make use of this class. It is an implementation detail of the
+ * Memory API.
+ *
+ * \headerfile memorybase.h <Vc/Memory>
+ */
+template<typename V, typename A> class VectorPointerHelperConst
+{
+ typedef typename V::EntryType EntryType;
+ typedef typename V::Mask Mask;
+ public:
+ const EntryType *const m_ptr;
+
+ explicit VectorPointerHelperConst(const EntryType *ptr) : m_ptr(ptr) {}
+
+ /**
+ * Cast to \p V operator.
+ *
+ * This function allows to assign this object to any object of type \p V.
+ */
+ inline operator const V() const { return V(m_ptr, Internal::FlagObject<A>::the()); }
+};
+
+/**
+ * Helper class for the Memory::vector(size_t) class of functions.
+ *
+ * You will never need to directly make use of this class. It is an implementation detail of the
+ * Memory API.
+ *
+ * \headerfile memorybase.h <Vc/Memory>
+ */
+template<typename V, typename A> class VectorPointerHelper
+{
+ typedef typename V::EntryType EntryType;
+ typedef typename V::Mask Mask;
+ public:
+ EntryType *const m_ptr;
+
+ explicit VectorPointerHelper(EntryType *ptr) : m_ptr(ptr) {}
+
+ /**
+ * Cast to \p V operator.
+ *
+ * This function allows to assign this object to any object of type \p V.
+ */
+ inline operator const V() const { return V(m_ptr, Internal::FlagObject<A>::the()); }
+
+ template<typename T>
+ inline VectorPointerHelper &operator=(const T &x) {
+ V v;
+ v = x;
+ v.store(m_ptr, Internal::FlagObject<A>::the());
+ return *this;
+ }
+
+ VC_ALL_BINARY(VC_MEM_OPERATOR_EQ)
+ VC_ALL_ARITHMETICS(VC_MEM_OPERATOR_EQ)
+};
+#undef VC_MEM_OPERATOR_EQ
+
+#define VC_VPH_OPERATOR(op) \
+template<typename V1, typename A1, typename V2, typename A2> \
+VC_DECLTYPE(V1, op, V2) operator op(const VectorPointerHelper<V1, A1> &x, const VectorPointerHelper<V2, A2> &y) { \
+ return V1(x.m_ptr, Internal::FlagObject<A1>::the()) op V2(y.m_ptr, Internal::FlagObject<A2>::the()); \
+}
+VC_ALL_ARITHMETICS(VC_VPH_OPERATOR)
+VC_ALL_BINARY (VC_VPH_OPERATOR)
+VC_ALL_COMPARES (VC_VPH_OPERATOR)
+#undef VC_VPH_OPERATOR
+
+template<typename V, typename Parent, int Dimension, typename RowMemory> class MemoryDimensionBase;
+template<typename V, typename Parent, typename RowMemory> class MemoryDimensionBase<V, Parent, 1, RowMemory> // {{{1
+{
+ private:
+ Parent *p() { return static_cast<Parent *>(this); }
+ const Parent *p() const { return static_cast<const Parent *>(this); }
+ public:
+ /**
+ * The type of the scalar entries in the array.
+ */
+ typedef typename V::EntryType EntryType;
+
+ /**
+ * Returns a pointer to the start of the allocated memory.
+ */
+ inline EntryType *entries() { return &p()->m_mem[0]; }
+ /// Const overload of the above function.
+ inline const EntryType *entries() const { return &p()->m_mem[0]; }
+
+ /**
+ * Returns the \p i-th scalar value in the memory.
+ */
+ inline EntryType &scalar(size_t i) { return entries()[i]; }
+ /// Const overload of the above function.
+ inline const EntryType scalar(size_t i) const { return entries()[i]; }
+
+ /**
+ * Cast operator to the scalar type. This allows to use the object very much like a standard
+ * C array.
+ */
+ inline operator EntryType*() { return entries(); }
+ /// Const overload of the above function.
+ inline operator const EntryType*() const { return entries(); }
+
+ // omit operator[] because the EntryType* cast operator suffices, for dox it makes sense to
+ // show it, though because it helps API discoverability.
+#ifdef DOXYGEN
+ /**
+ * Returns the \p i-th scalar value in the memory.
+ */
+ inline EntryType &operator[](size_t i);
+ /// Const overload of the above function.
+ inline const EntryType &operator[](size_t i) const;
+#endif
+
+ /**
+ * Uses a vector gather to combine the entries at the indexes in \p i into the returned
+ * vector object.
+ *
+ * \param i An integer vector. It determines the entries to be gathered.
+ * \returns A vector object. Modification of this object will not modify the values in
+ * memory.
+ *
+ * \warning The API of this function might change in future versions of Vc to additionally
+ * support scatters.
+ */
+ template<typename IndexT> inline V operator[](Vector<IndexT> i) const
+ {
+ return V(entries(), i);
+ }
+};
+template<typename V, typename Parent, typename RowMemory> class MemoryDimensionBase<V, Parent, 2, RowMemory> // {{{1
+{
+ private:
+ Parent *p() { return static_cast<Parent *>(this); }
+ const Parent *p() const { return static_cast<const Parent *>(this); }
+ public:
+ /**
+ * The type of the scalar entries in the array.
+ */
+ typedef typename V::EntryType EntryType;
+
+ inline size_t rowCount() const { return Parent::RowCount; }
+
+ /**
+ * Returns a pointer to the start of the allocated memory.
+ */
+ inline EntryType *entries(size_t x = 0) { return &p()->m_mem[x][0]; }
+ /// Const overload of the above function.
+ inline const EntryType *entries(size_t x = 0) const { return &p()->m_mem[x][0]; }
+
+ /**
+ * Returns the \p i,j-th scalar value in the memory.
+ */
+ inline EntryType &scalar(size_t i, size_t j) { return entries(i)[j]; }
+ /// Const overload of the above function.
+ inline const EntryType scalar(size_t i, size_t j) const { return entries(i)[j]; }
+
+ /**
+ * Returns the \p i-th row in the memory.
+ */
+ inline RowMemory &operator[](size_t i) {
+ return *new(entries(i)) RowMemory;
+ }
+ /// Const overload of the above function.
+ inline const RowMemory &operator[](size_t i) const {
+ return *new(const_cast<EntryType *>(entries(i))) RowMemory;
+ }
+
+ /**
+ * \return the number of rows in the array.
+ *
+ * \note This function can be eliminated by an optimizing compiler.
+ */
+ inline size_t rowsCount() const { return p()->rowsCount(); }
+};
+
+//{{{1
+/**
+ * \headerfile memorybase.h <Vc/Memory>
+ *
+ * Common interface to all Memory classes, independent of allocation on the stack or heap.
+ *
+ * \param V The vector type you want to operate on. (e.g. float_v or uint_v)
+ * \param Parent This type is the complete type of the class that derives from MemoryBase.
+ * \param Dimension The number of dimensions the implementation provides.
+ * \param RowMemory Class to be used to work on a single row.
+ */
+template<typename V, typename Parent, int Dimension, typename RowMemory> class MemoryBase : public MemoryDimensionBase<V, Parent, Dimension, RowMemory> //{{{1
+{
+ private:
+ Parent *p() { return static_cast<Parent *>(this); }
+ const Parent *p() const { return static_cast<const Parent *>(this); }
+ public:
+ /**
+ * The type of the scalar entries in the array.
+ */
+ typedef typename V::EntryType EntryType;
+
+ /**
+ * \return the number of scalar entries in the array. This function is optimized away
+ * if a constant size array is used.
+ */
+ inline size_t entriesCount() const { return p()->entriesCount(); }
+ /**
+ * \return the number of vector entries that span the array. This function is optimized away
+ * if a constant size array is used.
+ */
+ inline size_t vectorsCount() const { return p()->vectorsCount(); }
+
+ using MemoryDimensionBase<V, Parent, Dimension, RowMemory>::entries;
+ using MemoryDimensionBase<V, Parent, Dimension, RowMemory>::scalar;
+
+ /**
+ * \param i Selects the offset, where the vector should be read.
+ *
+ * \return a smart object to wrap the \p i-th vector in the memory.
+ *
+ * The return value can be used as any other vector object. I.e. you can substitute
+ * something like
+ * \code
+ * float_v a = ..., b = ...;
+ * a += b;
+ * \endcode
+ * with
+ * \code
+ * mem.vector(i) += b;
+ * \endcode
+ *
+ * This function ensures that only \em aligned loads and stores are used. Thus it only allows to
+ * access memory at fixed strides. If access to known offsets from the aligned vectors is
+ * needed the vector(size_t, int) function can be used.
+ */
+ inline VectorPointerHelper<V, AlignedFlag> vector(size_t i) {
+ return VectorPointerHelper<V, AlignedFlag>(&entries()[i * V::Size]);
+ }
+ /** \brief Const overload of the above function
+ *
+ * \param i Selects the offset, where the vector should be read.
+ *
+ * \return a smart object to wrap the \p i-th vector in the memory.
+ */
+ inline const VectorPointerHelperConst<V, AlignedFlag> vector(size_t i) const {
+ return VectorPointerHelperConst<V, AlignedFlag>(&entries()[i * V::Size]);
+ }
+
+ /**
+ * \return a smart object to wrap the vector starting from the \p i-th scalar entry in the memory.
+ *
+ * Example:
+ * \code
+ * Memory<float_v, N> mem;
+ * mem.setZero();
+ * for (int i = 0; i < mem.entriesCount(); i += float_v::Size) {
+ * mem.vectorAt(i) += b;
+ * }
+ * \endcode
+ *
+ * \param i Specifies the scalar entry from where the vector will be loaded/stored. I.e. the
+ * values scalar(i), scalar(i + 1), ..., scalar(i + V::Size - 1) will be read/overwritten.
+ *
+ * \param align You must take care to determine whether an unaligned load/store is
+ * required. Per default an aligned load/store is used. If \p i is not a multiple of \c V::Size
+ * you must pass Vc::Unaligned here.
+ */
+#ifdef DOXYGEN
+ template<typename A> inline VectorPointerHelper<V, A> vectorAt(size_t i, A align = Vc::Aligned);
+ /** \brief Const overload of the above function
+ *
+ * \return a smart object to wrap the vector starting from the \p i-th scalar entry in the memory.
+ *
+ * \param i Specifies the scalar entry from where the vector will be loaded/stored. I.e. the
+ * values scalar(i), scalar(i + 1), ..., scalar(i + V::Size - 1) will be read/overwritten.
+ *
+ * \param align You must take care to determine whether an unaligned load/store is
+ * required. Per default an aligned load/store is used. If \p i is not a multiple of \c V::Size
+ * you must pass Vc::Unaligned here.
+ */
+ template<typename A> inline const VectorPointerHelperConst<V, A> vectorAt(size_t i, A align = Vc::Aligned) const;
+#else
+ template<typename A>
+ inline VectorPointerHelper<V, A> vectorAt(size_t i, A) {
+ return VectorPointerHelper<V, A>(&entries()[i]);
+ }
+ template<typename A>
+ inline const VectorPointerHelperConst<V, A> vectorAt(size_t i, A) const {
+ return VectorPointerHelperConst<V, A>(&entries()[i]);
+ }
+
+ inline VectorPointerHelper<V, AlignedFlag> vectorAt(size_t i) {
+ return VectorPointerHelper<V, AlignedFlag>(&entries()[i]);
+ }
+ inline const VectorPointerHelperConst<V, AlignedFlag> vectorAt(size_t i) const {
+ return VectorPointerHelperConst<V, AlignedFlag>(&entries()[i]);
+ }
+#endif
+
+ /**
+ * \return a smart object to wrap the \p i-th vector + \p shift in the memory.
+ *
+ * This function ensures that only \em unaligned loads and stores are used.
+ * It allows to access memory at any location aligned to the entry type.
+ *
+ * \param i Selects the memory location of the i-th vector. Thus if \p V::Size == 4 and
+ * \p i is set to 3 the base address for the load/store will be the 12th entry
+ * (same as \p &mem[12]).
+ * \param shift Shifts the base address determined by parameter \p i by \p shift many
+ * entries. Thus \p vector(3, 1) for \p V::Size == 4 will load/store the
+ * 13th - 16th entries (same as \p &mem[13]).
+ *
+ * \note Any shift value is allowed as long as you make sure it stays within bounds of the
+ * allocated memory. Shift values that are a multiple of \p V::Size will \em not result in
+ * aligned loads. You have to use the above vector(size_t) function for aligned loads
+ * instead.
+ *
+ * \note Thus a simple way to access vectors randomly is to set \p i to 0 and use \p shift as the
+ * parameter to select the memory address:
+ * \code
+ * // don't use:
+ * mem.vector(i / V::Size, i % V::Size) += 1;
+ * // instead use:
+ * mem.vector(0, i) += 1;
+ * \endcode
+ */
+ inline VectorPointerHelper<V, UnalignedFlag> vector(size_t i, int shift) {
+ return VectorPointerHelper<V, UnalignedFlag>(&entries()[i * V::Size + shift]);
+ }
+ /// Const overload of the above function.
+ inline const VectorPointerHelperConst<V, UnalignedFlag> vector(size_t i, int shift) const {
+ return VectorPointerHelperConst<V, UnalignedFlag>(&entries()[i * V::Size + shift]);
+ }
+
+ /**
+ * \return the first vector in the allocated memory.
+ *
+ * This function is simply a shorthand for vector(0).
+ */
+ inline VectorPointerHelper<V, AlignedFlag> firstVector() {
+ return VectorPointerHelper<V, AlignedFlag>(entries());
+ }
+ /// Const overload of the above function.
+ inline const VectorPointerHelperConst<V, AlignedFlag> firstVector() const {
+ return VectorPointerHelperConst<V, AlignedFlag>(entries());
+ }
+
+ /**
+ * \return the last vector in the allocated memory.
+ *
+ * This function is simply a shorthand for vector(vectorsCount() - 1).
+ */
+ inline VectorPointerHelper<V, AlignedFlag> lastVector() {
+ return VectorPointerHelper<V, AlignedFlag>(&entries()[vectorsCount() * V::Size - V::Size]);
+ }
+ /// Const overload of the above function.
+ inline const VectorPointerHelperConst<V, AlignedFlag> lastVector() const {
+ return VectorPointerHelperConst<V, AlignedFlag>(&entries()[vectorsCount() * V::Size - V::Size]);
+ }
+
+ inline V gather(const unsigned char *indexes) const { return V(entries(), indexes); }
+ inline V gather(const unsigned short *indexes) const { return V(entries(), indexes); }
+ inline V gather(const unsigned int *indexes) const { return V(entries(), indexes); }
+ inline V gather(const unsigned long *indexes) const { return V(entries(), indexes); }
+
+ inline void setZero() {
+ V zero(Vc::Zero);
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) = zero;
+ }
+ }
+
+ template<typename P2, typename RM>
+ inline Parent &operator+=(const MemoryBase<V, P2, Dimension, RM> &rhs) {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) += rhs.vector(i);
+ }
+ return static_cast<Parent &>(*this);
+ }
+ template<typename P2, typename RM>
+ inline Parent &operator-=(const MemoryBase<V, P2, Dimension, RM> &rhs) {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) -= rhs.vector(i);
+ }
+ return static_cast<Parent &>(*this);
+ }
+ template<typename P2, typename RM>
+ inline Parent &operator*=(const MemoryBase<V, P2, Dimension, RM> &rhs) {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) *= rhs.vector(i);
+ }
+ return static_cast<Parent &>(*this);
+ }
+ template<typename P2, typename RM>
+ inline Parent &operator/=(const MemoryBase<V, P2, Dimension, RM> &rhs) {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) /= rhs.vector(i);
+ }
+ return static_cast<Parent &>(*this);
+ }
+ inline Parent &operator+=(EntryType rhs) {
+ V v(rhs);
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) += v;
+ }
+ return static_cast<Parent &>(*this);
+ }
+ inline Parent &operator-=(EntryType rhs) {
+ V v(rhs);
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) -= v;
+ }
+ return static_cast<Parent &>(*this);
+ }
+ inline Parent &operator*=(EntryType rhs) {
+ V v(rhs);
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) *= v;
+ }
+ return static_cast<Parent &>(*this);
+ }
+ inline Parent &operator/=(EntryType rhs) {
+ V v(rhs);
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ vector(i) /= v;
+ }
+ return static_cast<Parent &>(*this);
+ }
+ template<typename P2, typename RM>
+ inline bool operator==(const MemoryBase<V, P2, Dimension, RM> &rhs) const {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ if (!(V(vector(i)) == V(rhs.vector(i))).isFull()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ template<typename P2, typename RM>
+ inline bool operator!=(const MemoryBase<V, P2, Dimension, RM> &rhs) const {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ if (!(V(vector(i)) == V(rhs.vector(i))).isEmpty()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ template<typename P2, typename RM>
+ inline bool operator<(const MemoryBase<V, P2, Dimension, RM> &rhs) const {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ if (!(V(vector(i)) < V(rhs.vector(i))).isFull()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ template<typename P2, typename RM>
+ inline bool operator<=(const MemoryBase<V, P2, Dimension, RM> &rhs) const {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ if (!(V(vector(i)) <= V(rhs.vector(i))).isFull()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ template<typename P2, typename RM>
+ inline bool operator>(const MemoryBase<V, P2, Dimension, RM> &rhs) const {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ if (!(V(vector(i)) > V(rhs.vector(i))).isFull()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ template<typename P2, typename RM>
+ inline bool operator>=(const MemoryBase<V, P2, Dimension, RM> &rhs) const {
+ assert(vectorsCount() == rhs.vectorsCount());
+ for (size_t i = 0; i < vectorsCount(); ++i) {
+ if (!(V(vector(i)) >= V(rhs.vector(i))).isFull()) {
+ return false;
+ }
+ }
+ return true;
+ }
+};
+
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_COMMON_MEMORYBASE_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_MEMORYFWD_H
+#define VC_COMMON_MEMORYFWD_H
+
+namespace Vc
+{
+ template<typename V, size_t Size1 = 0, size_t Size2 = 0> class Memory;
+} // namespace Vc
+
+#endif // VC_COMMON_MEMORYFWD_H
--- /dev/null
+#ifndef VC_ICC
+// ICC ICEs if the following type-traits are in the anonymous namespace
+namespace
+{
+#endif
+template<typename Cond, typename T> struct EnableIfNeitherIntegerNorVector : public EnableIf<!CanConvertToInt<Cond>::Value, T> {};
+template<typename Cond, typename T> struct EnableIfNeitherIntegerNorVector<Vector<Cond>, T>;
+
+template<typename T> struct IsVector { enum { Value = false }; };
+template<typename T> struct IsVector<Vector<T> > { enum { Value = true }; };
+
+template<typename T0, typename T1, typename V0, typename V1> struct IsTypeCombinationOf
+{
+ enum {
+ Value = IsVector<V0>::Value ? (IsVector<V1>::Value ? ( // Vec × Vec
+ ( IsEqualType<T0, V0>::Value && HasImplicitCast<T1, V1>::Value && !HasImplicitCast<T1, int>::Value) ||
+ (HasImplicitCast<T0, V0>::Value && IsEqualType<T1, V1>::Value && !HasImplicitCast<T0, int>::Value) ||
+ ( IsEqualType<T0, V1>::Value && HasImplicitCast<T1, V0>::Value && !HasImplicitCast<T1, int>::Value) ||
+ (HasImplicitCast<T0, V1>::Value && IsEqualType<T1, V0>::Value && !HasImplicitCast<T0, int>::Value)
+ ) : ( // Vec × Scalar
+ (HasImplicitCast<T0, V0>::Value && IsEqualType<T1, V1>::Value && !HasImplicitCast<T0, int>::Value) ||
+ ( IsEqualType<T0, V1>::Value && HasImplicitCast<T1, V0>::Value && !HasImplicitCast<T1, int>::Value)
+ )) : (IsVector<V1>::Value ? ( // Scalar × Vec
+ ( IsEqualType<T0, V0>::Value && HasImplicitCast<T1, V1>::Value && !HasImplicitCast<T1, int>::Value) ||
+ (HasImplicitCast<T0, V1>::Value && IsEqualType<T1, V0>::Value && !HasImplicitCast<T0, int>::Value)
+ ) : ( // Scalar × Scalar
+ ( IsEqualType<T0, V0>::Value && IsEqualType<T1, V1>::Value) ||
+ ( IsEqualType<T0, V1>::Value && IsEqualType<T1, V0>::Value)
+ ))
+ };
+};
+
+template<typename T0, typename T1, typename V> struct IsVectorOperands
+{
+ enum {
+ Value = (HasImplicitCast<T0, V>::Value && !HasImplicitCast<T0, int>::Value && !IsEqualType<T0, V>::Value && IsEqualType<T1, V>::Value)
+ || (HasImplicitCast<T1, V>::Value && !HasImplicitCast<T1, int>::Value && !IsEqualType<T1, V>::Value && IsEqualType<T0, V>::Value)
+ };
+};
+#ifndef VC_ICC
+}
+#endif
+
+// float-int arithmetic operators //{{{1
+// These operators must be very picky about the exact types they want to handle. Once (uncontrolled)
+// implicit type conversions get involved, ambiguous overloads will occur. E.g. a simple int × enum
+// will become ambiguous because it can convert both to a vector type, which then can execute the
+// operator. We can't argue that such code should not be used - it could break existing code, not
+// under control of the developer, just by putting the Vc header somewhere on top.
+//
+// The following type combinations are safe (always symmetric):
+// 1. Vector × Vector
+// 2. Vector × Scalar (int, float, enum value, ...)
+// 3. Some object that has a vector cast operator × Vector
+// 4. Some object that has a vector cast operator × Scalar
+//
+// Additionally there are restrictions on which types combine to what resulting type:
+// 1.a. float × double_v -> double_v
+// 1.b. any int × double_v -> double_v
+// 2.a. (u)int_v × float_v -> float_v
+// 2.b. (u)int_v × float -> float_v
+// 2.c. any int × float_v -> float_v
+// 3.a. (u)short_v × sfloat_v -> sfloat_v
+// 3.b. (u)short_v × float -> sfloat_v
+// 3.c. short × sfloat_v -> sfloat_v
+// 4.a. int_v × uint_v -> uint_v
+// 4.b. any int × uint_v -> uint_v
+// 4.c. unsigned int × int_v -> uint_v
+// 4.d. signed int × int_v -> int_v
+// 5. shorts like ints
+
+#define VC_OPERATOR_FORWARD_(ret, op) \
+template<typename T0, typename T1> static inline typename EnableIf< \
+ IsVectorOperands<T0, T1, double_v>::Value || \
+ ((IsEqualType<T0, float>::Value || IsLikeInteger<T0>::Value) && HasImplicitCast<T1, double_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ ((IsEqualType<T1, float>::Value || IsLikeInteger<T1>::Value) && HasImplicitCast<T0, double_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ false, double_##ret>::Value operator op(const T0 &x, const T1 &y) { return double_v(x) op double_v(y); } \
+\
+template<typename T0, typename T1> static inline typename EnableIf< \
+ IsVectorOperands<T0, T1, float_v>::Value || \
+ IsTypeCombinationOf<T0, T1, int_v, float_v>::Value || \
+ IsTypeCombinationOf<T0, T1, uint_v, float_v>::Value || \
+ IsTypeCombinationOf<T0, T1, int_v, float>::Value || \
+ IsTypeCombinationOf<T0, T1, uint_v, float>::Value || \
+ (IsLikeInteger<T0>::Value && HasImplicitCast<T1, float_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsLikeInteger<T1>::Value && HasImplicitCast<T0, float_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ false, float_##ret>::Value operator op(const T0 &x, const T1 &y) { return float_v(x) op float_v(y); } \
+\
+template<typename T0, typename T1> static inline typename EnableIf< \
+ IsVectorOperands<T0, T1, sfloat_v>::Value || \
+ IsTypeCombinationOf<T0, T1, short_v, sfloat_v>::Value || \
+ IsTypeCombinationOf<T0, T1, ushort_v, sfloat_v>::Value || \
+ IsTypeCombinationOf<T0, T1, short_v, float>::Value || \
+ IsTypeCombinationOf<T0, T1, ushort_v, float>::Value || \
+ (IsLikeInteger<T0>::Value && HasImplicitCast<T1, sfloat_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsLikeInteger<T1>::Value && HasImplicitCast<T0, sfloat_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ false, sfloat_##ret>::Value operator op(const T0 &x, const T1 &y) { return sfloat_v(x) op sfloat_v(y); } \
+\
+template<typename T0, typename T1> static inline typename EnableIf< \
+ IsVectorOperands<T0, T1, uint_v>::Value || \
+ IsTypeCombinationOf<T0, T1, int_v, uint_v>::Value || \
+ (IsUnsignedInteger<T0>::Value && HasImplicitCast<T1, int_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsUnsignedInteger<T1>::Value && HasImplicitCast<T0, int_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ (IsLikeInteger<T0>::Value && !IsEqualType<T0, unsigned int>::Value && HasImplicitCast<T1, uint_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsLikeInteger<T1>::Value && !IsEqualType<T1, unsigned int>::Value && HasImplicitCast<T0, uint_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ false, uint_##ret>::Value operator op(const T0 &x, const T1 &y) { return uint_v(x) op uint_v(y); } \
+template<typename T0, typename T1> static inline typename EnableIf< \
+ IsVectorOperands<T0, T1, int_v>::Value || \
+ (IsLikeSignedInteger<T0>::Value && !IsEqualType<T0, int>::Value && HasImplicitCast<T1, int_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsLikeSignedInteger<T1>::Value && !IsEqualType<T1, int>::Value && HasImplicitCast<T0, int_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ false, int_##ret>::Value operator op(const T0 &x, const T1 &y) { return int_v(x) op int_v(y); } \
+\
+template<typename T0, typename T1> static inline typename EnableIf< \
+ IsVectorOperands<T0, T1, ushort_v>::Value || \
+ IsTypeCombinationOf<T0, T1, short_v, ushort_v>::Value || \
+ (IsUnsignedInteger<T0>::Value && HasImplicitCast<T1, short_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsUnsignedInteger<T1>::Value && HasImplicitCast<T0, short_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ (IsLikeInteger<T0>::Value && !IsEqualType<T0, unsigned short>::Value && HasImplicitCast<T1, ushort_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsLikeInteger<T1>::Value && !IsEqualType<T1, unsigned short>::Value && HasImplicitCast<T0, ushort_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ false, ushort_##ret>::Value operator op(const T0 &x, const T1 &y) { return ushort_v(x) op ushort_v(y); } \
+template<typename T0, typename T1> static inline typename EnableIf< \
+ IsVectorOperands<T0, T1, short_v>::Value || \
+ (IsLikeSignedInteger<T0>::Value && !IsEqualType<T0, short>::Value && HasImplicitCast<T1, short_v>::Value && !HasImplicitCast<T1, int>::Value) || \
+ (IsLikeSignedInteger<T1>::Value && !IsEqualType<T1, short>::Value && HasImplicitCast<T0, short_v>::Value && !HasImplicitCast<T0, int>::Value) || \
+ false, short_##ret>::Value operator op(const T0 &x, const T1 &y) { return short_v(x) op short_v(y); }
+
+
+// break incorrect combinations
+#define VC_OPERATOR_INTENTIONAL_ERROR_1(V, op) \
+template<typename T> static inline typename EnableIfNeitherIntegerNorVector<T, Vc::Error::invalid_operands_of_types<V, T> >::Value operator op(const V &, const T &) { return Vc::Error::invalid_operands_of_types<V, T>(); } \
+template<typename T> static inline typename EnableIfNeitherIntegerNorVector<T, Vc::Error::invalid_operands_of_types<T, V> >::Value operator op(const T &, const V &) { return Vc::Error::invalid_operands_of_types<T, V>(); }
+
+#define VC_OPERATOR_INTENTIONAL_ERROR_2(V1, V2, op) \
+static inline Vc::Error::invalid_operands_of_types<V1, V2> operator op(V1::AsArg, V2::AsArg) { return Vc::Error::invalid_operands_of_types<V1, V2>(); } \
+static inline Vc::Error::invalid_operands_of_types<V2, V1> operator op(V2::AsArg, V1::AsArg) { return Vc::Error::invalid_operands_of_types<V2, V1>(); }
+
+#define VC_OPERATOR_INTENTIONAL_ERROR_3(V, _T, op) \
+template<typename T> static inline typename EnableIf<IsEqualType<T, _T>::Value, Vc::Error::invalid_operands_of_types<V, T> >::Value operator op(const V &, const T &) { return Vc::Error::invalid_operands_of_types<V, T>(); } \
+template<typename T> static inline typename EnableIf<IsEqualType<T, _T>::Value, Vc::Error::invalid_operands_of_types<T, V> >::Value operator op(const T &, const V &) { return Vc::Error::invalid_operands_of_types<T, V>(); }
+
+//#define VC_EXTRA_CHECKING
+#ifdef VC_EXTRA_CHECKING
+#define VC_OPERATOR_INTENTIONAL_ERROR(op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(double_v, sfloat_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(double_v, float_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(double_v, int_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(double_v, uint_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(double_v, short_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(double_v, ushort_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2( int_v, short_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2( uint_v, short_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2( int_v, ushort_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2( uint_v, ushort_v, op) \
+ VC_APPLY_1(VC_LIST_VECTOR_TYPES, VC_OPERATOR_INTENTIONAL_ERROR_1, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2( float_v, short_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2( float_v, ushort_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(sfloat_v, float_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(sfloat_v, int_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_2(sfloat_v, uint_v, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_3( float_v, double, op) \
+ VC_OPERATOR_INTENTIONAL_ERROR_3(sfloat_v, double, op)
+#else
+#define VC_OPERATOR_INTENTIONAL_ERROR(op)
+#endif
+
+#define VC_OPERATOR_FORWARD_COMMUTATIVE(ret, op, op2) \
+template<typename T> static inline VC_EXACT_TYPE(T, double, double_##ret) operator op(T x, double_v::AsArg y) { return y op2 x; } \
+template<typename T> static inline VC_EXACT_TYPE(T, float, sfloat_##ret) operator op(T x, sfloat_v::AsArg y) { return y op2 x; } \
+template<typename T> static inline VC_EXACT_TYPE(T, float, float_##ret) operator op(T x, float_v::AsArg y) { return y op2 x; } \
+template<typename T> static inline VC_EXACT_TYPE(T, int, int_##ret) operator op(T x, int_v::AsArg y) { return y op2 x; } \
+template<typename T> static inline VC_EXACT_TYPE(T, unsigned int, uint_##ret) operator op(T x, uint_v::AsArg y) { return y op2 x; } \
+template<typename T> static inline VC_EXACT_TYPE(T, short, short_##ret) operator op(T x, short_v::AsArg y) { return y op2 x; } \
+template<typename T> static inline VC_EXACT_TYPE(T, unsigned short, ushort_##ret) operator op(T x, ushort_v::AsArg y) { return y op2 x; } \
+VC_OPERATOR_FORWARD_(ret, op) \
+VC_OPERATOR_INTENTIONAL_ERROR(op)
+
+#define VC_OPERATOR_FORWARD(ret, op) \
+template<typename T> static inline VC_EXACT_TYPE(T, double, double_##ret) operator op(T x, double_v::AsArg y) { return double_v(x) op y; } \
+template<typename T> static inline VC_EXACT_TYPE(T, float, sfloat_##ret) operator op(T x, sfloat_v::AsArg y) { return sfloat_v(x) op y; } \
+template<typename T> static inline VC_EXACT_TYPE(T, float, float_##ret) operator op(T x, float_v::AsArg y) { return float_v(x) op y; } \
+template<typename T> static inline VC_EXACT_TYPE(T, int, int_##ret) operator op(T x, int_v::AsArg y) { return int_v(x) op y; } \
+template<typename T> static inline VC_EXACT_TYPE(T, unsigned int, uint_##ret) operator op(T x, uint_v::AsArg y) { return uint_v(x) op y; } \
+template<typename T> static inline VC_EXACT_TYPE(T, short, short_##ret) operator op(T x, short_v::AsArg y) { return short_v(x) op y; } \
+template<typename T> static inline VC_EXACT_TYPE(T, unsigned short, ushort_##ret) operator op(T x, ushort_v::AsArg y) { return ushort_v(x) op y; } \
+VC_OPERATOR_FORWARD_(ret, op) \
+VC_OPERATOR_INTENTIONAL_ERROR(op)
+
+VC_OPERATOR_FORWARD_COMMUTATIVE(v, *, *)
+VC_OPERATOR_FORWARD(v, /)
+VC_OPERATOR_FORWARD_COMMUTATIVE(v, +, +)
+VC_OPERATOR_FORWARD(v, -)
+VC_OPERATOR_FORWARD_COMMUTATIVE(v, |, |)
+VC_OPERATOR_FORWARD_COMMUTATIVE(v, &, &)
+VC_OPERATOR_FORWARD_COMMUTATIVE(v, ^, ^)
+VC_OPERATOR_FORWARD_COMMUTATIVE(m, <, >)
+VC_OPERATOR_FORWARD_COMMUTATIVE(m, >, <)
+VC_OPERATOR_FORWARD_COMMUTATIVE(m, <=, >=)
+VC_OPERATOR_FORWARD_COMMUTATIVE(m, >=, <=)
+VC_OPERATOR_FORWARD_COMMUTATIVE(m, ==, ==)
+VC_OPERATOR_FORWARD_COMMUTATIVE(m, !=, !=)
+
+#undef VC_OPERATOR_FORWARD_
+#undef VC_OPERATOR_INTENTIONAL_ERROR_1
+#undef VC_OPERATOR_INTENTIONAL_ERROR_2
+#undef VC_OPERATOR_INTENTIONAL_ERROR
+#undef VC_OPERATOR_FORWARD_COMMUTATIVE
+#undef VC_OPERATOR_FORWARD
+
+// }}}1
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_STORAGE_H
+#define VC_COMMON_STORAGE_H
+
+#include "aliasingentryhelper.h"
+#include "macros.h"
+
+namespace Vc
+{
+namespace Common
+{
+
+template<typename _VectorType, typename _EntryType> class VectorMemoryUnion
+{
+ public:
+ typedef _VectorType VectorType;
+ typedef _EntryType EntryType;
+ typedef EntryType AliasingEntryType MAY_ALIAS;
+ inline VectorMemoryUnion() {}
+#if defined VC_ICC || defined VC_MSVC
+ inline VectorMemoryUnion(const VectorType &x) { data.v = x; }
+ inline VectorMemoryUnion &operator=(const VectorType &x) {
+ data.v = x; return *this;
+ }
+
+ VectorType &v() { return data.v; }
+ const VectorType &v() const { return data.v; }
+
+#if defined VC_ICC
+ AliasingEntryHelper<VectorMemoryUnion<VectorType, EntryType> > m(int index) {
+ return AliasingEntryHelper<VectorMemoryUnion<VectorType, EntryType> >(this, index);
+ }
+ void assign(int index, EntryType x) {
+ data.m[index] = x;
+ }
+ EntryType read(int index) const {
+ return data.m[index];
+ }
+#else
+ EntryType &m(int index) {
+ return data.m[index];
+ }
+#endif
+
+ EntryType m(int index) const {
+ return data.m[index];
+ }
+
+ private:
+ union VectorScalarUnion {
+ VectorType v;
+ EntryType m[sizeof(VectorType)/sizeof(EntryType)];
+ } data;
+#else
+ inline VectorMemoryUnion(VectorType x) : data(x) {}
+ inline VectorMemoryUnion &operator=(VectorType x) {
+ data = x; return *this;
+ }
+
+ inline VectorType &v() { return data; }
+ inline const VectorType &v() const { return data; }
+
+ inline AliasingEntryType &m(int index) {
+ return reinterpret_cast<AliasingEntryType *>(&data)[index];
+ }
+
+ inline EntryType m(int index) const {
+ return reinterpret_cast<const AliasingEntryType *>(&data)[index];
+ }
+
+ private:
+ VectorType data;
+#endif
+};
+
+#if VC_GCC == 0x40700 || (VC_GCC >= 0x40600 && VC_GCC <= 0x40603)
+// workaround bug 52736 in GCC
+template<typename T, typename V> static inline T &vectorMemoryUnionAliasedMember(V *data, int index) {
+ if (__builtin_constant_p(index) && index == 0) {
+ T *ret;
+ asm("mov %1,%0" : "=r"(ret) : "r"(data));
+ return *ret;
+ } else {
+ return reinterpret_cast<T *>(data)[index];
+ }
+}
+template<> inline VectorMemoryUnion<__m128d, double>::AliasingEntryType &VectorMemoryUnion<__m128d, double>::m(int index) {
+ return vectorMemoryUnionAliasedMember<AliasingEntryType>(&data, index);
+}
+template<> inline VectorMemoryUnion<__m128i, long long>::AliasingEntryType &VectorMemoryUnion<__m128i, long long>::m(int index) {
+ return vectorMemoryUnionAliasedMember<AliasingEntryType>(&data, index);
+}
+template<> inline VectorMemoryUnion<__m128i, unsigned long long>::AliasingEntryType &VectorMemoryUnion<__m128i, unsigned long long>::m(int index) {
+ return vectorMemoryUnionAliasedMember<AliasingEntryType>(&data, index);
+}
+#endif
+
+} // namespace Common
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_COMMON_STORAGE_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_SUPPORT_H
+#define VC_COMMON_SUPPORT_H
+
+#ifndef VC_GLOBAL_H
+#error "Vc/global.h must be included first!"
+#endif
+
+namespace Vc
+{
+
+/**
+ * \ingroup Utilities
+ *
+ * Tests whether the given implementation is supported by the system the code is executing on.
+ *
+ * \return \c true if the OS and hardware support execution of instructions defined by \p impl.
+ * \return \c false otherwise
+ *
+ * \param impl The SIMD target to test for.
+ */
+bool isImplementationSupported(Vc::Implementation impl);
+
+#ifndef VC_COMPILE_LIB
+/**
+ * \ingroup Utilities
+ *
+ * Tests that the CPU and Operating System support the vector unit which was compiled for. This
+ * function should be called before any other Vc functionality is used. It checks whether the program
+ * will work. If this function returns \c false then the program should exit with a useful error
+ * message before the OS has to kill it because of an invalid instruction exception.
+ *
+ * If the program continues and makes use of any vector features not supported by
+ * hard- or software then the program will crash.
+ *
+ * Example:
+ * \code
+ * int main()
+ * {
+ * if (!Vc::currentImplementationSupported()) {
+ * std::cerr << "CPU or OS requirements not met for the compiled in vector unit!\n";
+ * exit -1;
+ * }
+ * ...
+ * }
+ * \endcode
+ *
+ * \return \c true if the OS and hardware support execution of the currently selected SIMD
+ * instructions.
+ * \return \c false otherwise
+ */
+#ifdef VC_GCC
+ __attribute__((target("no-sse2,no-avx")))
+#endif
+bool currentImplementationSupported()
+{
+ return isImplementationSupported(
+#ifdef VC_USE_VEX_CODING
+ // everything will use VEX coding, so the system has to support AVX even if VC_IMPL_AVX
+ // is not set
+ AVXImpl
+#else
+ VC_IMPL
+#endif
+#ifdef VC_IMPL_XOP
+ ) && isImplementationSupported(XopImpl
+#endif
+#ifdef VC_IMPL_FMA4
+ ) && isImplementationSupported(Fma4Impl
+#endif
+ );
+}
+#endif // VC_COMPILE_LIB
+
+} // namespace Vc
+
+#endif // VC_COMMON_SUPPORT_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_TRIGONOMETRIC_H
+#define VC_COMMON_TRIGONOMETRIC_H
+
+#include "const.h"
+#include "macros.h"
+
+namespace Vc
+{
+namespace Common
+{
+#ifdef VC__USE_NAMESPACE
+ using Vc::VC__USE_NAMESPACE::Const;
+ using Vc::VC__USE_NAMESPACE::Vector;
+#endif
+ namespace {
+ template<typename T> static inline ALWAYS_INLINE CONST Vector<T> _foldMinusPiToPi(const Vector<T> &x) {
+ typedef Const<T> C;
+ // put the input in the range [-π, π]
+ // 'f(x) = 2π * round(x/2π)' is the offset:
+ // ⇒ f(x) = 0 ∀ x ∈ ]-π, π[ ; f(x) = 2π ∀ x ∈ [π, 3π[
+ return x - round(x * C::_1_2pi()) * C::_2pi();
+ }
+ }
+
+ template<typename T> static inline Vector<T> sin(const Vector<T> &_x) {
+ typedef Vector<T> V;
+ typedef Const<T> C;
+
+ // x - x**3/3! + x**5/5! - x**7/7! + x**9/9! - x**11/11! for [-½π, ½π]
+
+ V x = _foldMinusPiToPi(_x); // [-π, π[
+ // fold the left and right fourths in to reduce the range to [-½π, ½π]
+ x(x > C::_pi_2()) = C::_pi() - x;
+ x(x < -C::_pi_2()) = -C::_pi() - x;
+
+ const V &x2 = x * x;
+ return x * (V::One() - x2 * (C::_1_3fac() - x2 * (C::_1_5fac() - x2 * (C::_1_7fac() - x2 * C::_1_9fac()))));
+ }
+ template<typename T> static inline Vector<T> cos(const Vector<T> &_x) {
+ typedef Vector<T> V;
+ typedef Const<T> C;
+
+ V x = _foldMinusPiToPi(_x) + C::_pi_2(); // [-½π, ¾π[
+ x(x > C::_pi_2()) = C::_pi() - x; // [-½π, ½π]
+
+ const V &x2 = x * x;
+ return x * (V::One() - x2 * (C::_1_3fac() - x2 * (C::_1_5fac() - x2 * (C::_1_7fac() - x2 * C::_1_9fac()))));
+ }
+ template<typename T> static inline void sincos(const Vector<T> &_x, Vector<T> *_sin, Vector<T> *_cos) {
+ typedef Vector<T> V;
+ typedef Const<T> C;
+ // I did a short test how the results would look if I make use of 1=s²+c². There seems to be
+ // no easy way to keep the results in an acceptable precision.
+
+ V sin_x = _foldMinusPiToPi(_x); // [-π, π]
+ V cos_x = sin_x + C::_pi_2(); // [-½π, ¾π]
+ cos_x(cos_x > C::_pi_2()) = C::_pi() - cos_x; // [-½π, ½π]
+
+ // fold the left and right fourths in to reduce the range to [-½π, ½π]
+ sin_x(sin_x > C::_pi_2()) = C::_pi() - sin_x;
+ sin_x(sin_x < -C::_pi_2()) = -C::_pi() - sin_x;
+
+ const V &sin_x2 = sin_x * sin_x;
+ const V &cos_x2 = cos_x * cos_x;
+
+ *_sin = sin_x * (V::One() - sin_x2 * (C::_1_3fac() - sin_x2 * (C::_1_5fac() - sin_x2 * (C::_1_7fac() - sin_x2 * C::_1_9fac()))));
+ *_cos = cos_x * (V::One() - cos_x2 * (C::_1_3fac() - cos_x2 * (C::_1_5fac() - cos_x2 * (C::_1_7fac() - cos_x2 * C::_1_9fac()))));
+ }
+ template<typename _T> static inline Vector<_T> asin (const Vector<_T> &_x) {
+ typedef Vector<_T> V;
+ typedef typename V::EntryType T;
+ typedef typename V::Mask M;
+
+ const V pi_2(Math<T>::pi_2());
+ const M &negative = _x < V::Zero();
+
+ const V &a = abs(_x);
+ //const M &outOfRange = a > V::One();
+ const M &small = a < V(T(1.e-4));
+ const M >_0_5 = a > V(T(0.5));
+ V x = a;
+ V z = a * a;
+ z(gt_0_5) = (V::One() - a) * V(T(0.5));
+ x(gt_0_5) = sqrt(z);
+ z = ((((T(4.2163199048e-2) * z
+ + T(2.4181311049e-2)) * z
+ + T(4.5470025998e-2)) * z
+ + T(7.4953002686e-2)) * z
+ + T(1.6666752422e-1)) * z * x
+ + x;
+ z(gt_0_5) = pi_2 - (z + z);
+ z(small) = a;
+ z(negative) = -z;
+ //z(outOfRange) = nan;
+
+ return z;
+ }
+ template<typename _T> static inline Vector<_T> atan (const Vector<_T> &_x) {
+ typedef Vector<_T> V;
+ typedef typename V::EntryType T;
+ typedef typename V::Mask M;
+ V x = abs(_x);
+ const V pi_2(Math<T>::pi_2());
+ const V pi_4(Math<T>::pi_4());
+ const M >_tan_3pi_8 = x > V(T(2.414213562373095));
+ const M >_tan_pi_8 = x > V(T(0.4142135623730950)) && !gt_tan_3pi_8;
+ const V minusOne(-1);
+ V y = V::Zero();
+ y(gt_tan_3pi_8) = pi_2;
+ y(gt_tan_pi_8) = pi_4;
+ x(gt_tan_3pi_8) = minusOne / x;
+ x(gt_tan_pi_8) = (x - V::One()) / (x + V::One());
+ const V &x2 = x * x;
+ y += (((T(8.05374449538e-2) * x2
+ - T(1.38776856032E-1)) * x2
+ + T(1.99777106478E-1)) * x2
+ - T(3.33329491539E-1)) * x2 * x
+ + x;
+ y(_x < V::Zero()) = -y;
+ return y;
+ }
+ template<typename _T> static inline Vector<_T> atan2(const Vector<_T> &y, const Vector<_T> &x) {
+ typedef Vector<_T> V;
+ typedef typename V::EntryType T;
+ typedef typename V::Mask M;
+ const V pi(Math<T>::pi());
+ const V pi_2(Math<T>::pi_2());
+
+ const M &xZero = x == V::Zero();
+ const M &yZero = y == V::Zero();
+ const M &xNeg = x < V::Zero();
+ const M &yNeg = y < V::Zero();
+
+ const V &absX = abs(x);
+ const V &absY = abs(y);
+
+ V a = absY / absX;
+ const V pi_4(Math<T>::pi_4());
+ const M >_tan_3pi_8 = a > V(T(2.414213562373095));
+ const M >_tan_pi_8 = a > V(T(0.4142135623730950)) && !gt_tan_3pi_8;
+ const V minusOne(-1);
+ V b = V::Zero();
+ b(gt_tan_3pi_8) = pi_2;
+ b(gt_tan_pi_8) = pi_4;
+ a(gt_tan_3pi_8) = minusOne / a;
+ a(gt_tan_pi_8) = (absY - absX) / (absY + absX);
+ const V &a2 = a * a;
+ b += (((T(8.05374449538e-2) * a2
+ - T(1.38776856032E-1)) * a2
+ + T(1.99777106478E-1)) * a2
+ - T(3.33329491539E-1)) * a2 * a
+ + a;
+ b(xNeg ^ yNeg) = -b;
+
+ b(xNeg && !yNeg) += pi;
+ b(xNeg && yNeg) -= pi;
+ //b(xZero) = pi_2;
+ b.setZero(xZero && yZero);
+ b(xZero && yNeg) = -pi_2;
+ //b(yZero && xNeg) = pi;
+ return b;
+ }
+} // namespace Common
+#ifdef VC__USE_NAMESPACE
+namespace VC__USE_NAMESPACE
+{
+ using Vc::Common::sin;
+ using Vc::Common::cos;
+ using Vc::Common::sincos;
+ using Vc::Common::asin;
+ using Vc::Common::atan;
+ using Vc::Common::atan2;
+} // namespace VC__USE_NAMESPACE
+#undef VC__USE_NAMESPACE
+#endif
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_COMMON_TRIGONOMETRIC_H
--- /dev/null
+/* This file is part of the Vc library. {{{
+
+ Copyright (C) 2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+}}}*/
+
+#ifndef VC_COMMON_TYPES_H
+#define VC_COMMON_TYPES_H
+
+namespace Vc
+{
+
+// helper type to implement sfloat_v (Vector<Vc::sfloat>)
+struct sfloat {};
+
+template<typename T> struct DetermineEntryType { typedef T Type; };
+template<> struct DetermineEntryType<sfloat> { typedef float Type; };
+
+template<typename T> struct NegateTypeHelper { typedef T Type; };
+template<> struct NegateTypeHelper<unsigned char > { typedef char Type; };
+template<> struct NegateTypeHelper<unsigned short> { typedef short Type; };
+template<> struct NegateTypeHelper<unsigned int > { typedef int Type; };
+
+namespace VectorSpecialInitializerZero { enum ZEnum { Zero = 0 }; }
+namespace VectorSpecialInitializerOne { enum OEnum { One = 1 }; }
+namespace VectorSpecialInitializerIndexesFromZero { enum IEnum { IndexesFromZero }; }
+
+template<typename V, size_t Size1, size_t Size2> class Memory;
+#ifdef VC_MSVC
+# if defined(VC_IMPL_Scalar)
+namespace Scalar {
+ template<typename T> class Vector;
+ template<unsigned int VectorSize> class Mask;
+}
+#define _Vector Vc::Scalar::Vector
+# elif defined(VC_IMPL_SSE)
+namespace SSE {
+ template<typename T> class Vector;
+ template<unsigned int VectorSize> class Mask;
+ class Float8Mask;
+}
+#define _Vector Vc::SSE::Vector
+# elif defined(VC_IMPL_AVX)
+namespace AVX {
+ template<typename T> class Vector;
+ template<unsigned int VectorSize, size_t RegisterWidth> class Mask;
+}
+#define _Vector Vc::AVX::Vector
+# else
+# error "Sorry, MSVC is a nasty compiler and needs extra care. Please help."
+# endif
+#endif
+namespace
+{
+ template<bool Test, typename T = void> struct EnableIf { typedef T Value; };
+ template<typename T> struct EnableIf<false, T> {};
+
+ template<typename T> struct IsSignedInteger { enum { Value = 0 }; };
+ template<> struct IsSignedInteger<signed char> { enum { Value = 1 }; };
+ template<> struct IsSignedInteger<short> { enum { Value = 1 }; };
+ template<> struct IsSignedInteger<int> { enum { Value = 1 }; };
+ template<> struct IsSignedInteger<long> { enum { Value = 1 }; };
+ template<> struct IsSignedInteger<long long> { enum { Value = 1 }; };
+
+ template<typename T> struct IsUnsignedInteger { enum { Value = 0 }; };
+ template<> struct IsUnsignedInteger<unsigned char> { enum { Value = 1 }; };
+ template<> struct IsUnsignedInteger<unsigned short> { enum { Value = 1 }; };
+ template<> struct IsUnsignedInteger<unsigned int> { enum { Value = 1 }; };
+ template<> struct IsUnsignedInteger<unsigned long> { enum { Value = 1 }; };
+ template<> struct IsUnsignedInteger<unsigned long long> { enum { Value = 1 }; };
+
+ template<typename T> struct IsInteger { enum { Value = IsSignedInteger<T>::Value | IsUnsignedInteger<T>::Value }; };
+
+ template<typename T> struct IsReal { enum { Value = 0 }; };
+ template<> struct IsReal<float> { enum { Value = 1 }; };
+ template<> struct IsReal<double> { enum { Value = 1 }; };
+
+ template<typename T, typename U> struct IsEqualType { enum { Value = 0 }; };
+ template<typename T> struct IsEqualType<T, T> { enum { Value = 1 }; };
+
+ template<typename T, typename List0, typename List1 = void, typename List2 = void, typename List3 = void, typename List4 = void, typename List5 = void, typename List6 = void>
+ struct IsInTypelist { enum { Value = false }; };
+ template<typename T, typename List1, typename List2, typename List3, typename List4, typename List5, typename List6> struct IsInTypelist<T, T, List1, List2, List3, List4, List5, List6> { enum { Value = true }; };
+ template<typename T, typename List0, typename List2, typename List3, typename List4, typename List5, typename List6> struct IsInTypelist<T, List0, T, List2, List3, List4, List5, List6> { enum { Value = true }; };
+ template<typename T, typename List0, typename List1, typename List3, typename List4, typename List5, typename List6> struct IsInTypelist<T, List0, List1, T, List3, List4, List5, List6> { enum { Value = true }; };
+ template<typename T, typename List0, typename List1, typename List2, typename List4, typename List5, typename List6> struct IsInTypelist<T, List0, List1, List2, T, List4, List5, List6> { enum { Value = true }; };
+ template<typename T, typename List0, typename List1, typename List2, typename List3, typename List5, typename List6> struct IsInTypelist<T, List0, List1, List2, List3, T, List5, List6> { enum { Value = true }; };
+ template<typename T, typename List0, typename List1, typename List2, typename List3, typename List4, typename List6> struct IsInTypelist<T, List0, List1, List2, List3, List4, T, List6> { enum { Value = true }; };
+ template<typename T, typename List0, typename List1, typename List2, typename List3, typename List4, typename List5> struct IsInTypelist<T, List0, List1, List2, List3, List4, List5, T> { enum { Value = true }; };
+
+ template<typename Arg0, typename Arg1, typename T0, typename T1> struct IsCombinationOf { enum { Value = false }; };
+ template<typename Arg0, typename Arg1> struct IsCombinationOf<Arg0, Arg1, Arg0, Arg1> { enum { Value = true }; };
+ template<typename Arg0, typename Arg1> struct IsCombinationOf<Arg0, Arg1, Arg1, Arg0> { enum { Value = true }; };
+
+ namespace
+ {
+ struct yes { char x; };
+ struct no { yes x, y; };
+ } // anonymous namespace
+
+ template<typename From, typename To> struct HasImplicitCast
+ {
+#ifdef VC_MSVC
+ // MSVC can't compile this code if we pass a type that has large alignment restrictions by
+ // value
+ // clang OTOH warns about this code if we pass a null-reference, thus we ifdef the const-ref
+ // for MSVC only
+ static yes test(const To &) { return yes(); }
+#else
+ static yes test( To) { return yes(); }
+#endif
+ static no test(...) { return no(); }
+ enum {
+ Value = !!(sizeof(test(*static_cast<From *>(0))) == sizeof(yes))
+ };
+ };
+
+#ifdef VC_MSVC
+ // MSVC is such a broken compiler :'(
+ // HasImplicitCast breaks if From has an __declspec(align(#)) modifier and has no implicit cast
+ // to To. That's because it'll call test(...) as test(From) and not test(const From &).
+ // This results in C2718. And MSVC is too stupid to see that it should just shut up and
+ // everybody would be happy.
+ //
+ // Because the HasImplicitCast specializations can only be implemented after the Vector class
+ // was declared we have to write some nasty hacks.
+ template<typename T1, typename T2> struct HasImplicitCast<_Vector<T1>, T2> { enum { Value = false }; };
+#if defined(VC_IMPL_Scalar)
+ template<unsigned int VS, typename T2> struct HasImplicitCast<Vc::Scalar::Mask<VS>, T2> { enum { Value = false }; };
+ template<unsigned int VS> struct HasImplicitCast<Vc::Scalar::Mask<VS>, Vc::Scalar::Mask<VS> > { enum { Value = true }; };
+#elif defined(VC_IMPL_SSE)
+ template<unsigned int VS, typename T2> struct HasImplicitCast<Vc::SSE::Mask<VS>, T2> { enum { Value = false }; };
+ template<unsigned int VS> struct HasImplicitCast<Vc::SSE::Mask<VS>, Vc::SSE::Mask<VS> > { enum { Value = true }; };
+ template<typename T2> struct HasImplicitCast<Vc::SSE::Float8Mask, T2> { enum { Value = false }; };
+ template<> struct HasImplicitCast<Vc::SSE::Float8Mask, Vc::SSE::Float8Mask> { enum { Value = true }; };
+#elif defined(VC_IMPL_AVX)
+ template<unsigned int VectorSize, size_t RegisterWidth, typename T2> struct HasImplicitCast<Vc::AVX::Mask<VectorSize, RegisterWidth>, T2> { enum { Value = false }; };
+ template<unsigned int VectorSize, size_t RegisterWidth> struct HasImplicitCast<Vc::AVX::Mask<VectorSize, RegisterWidth>, Vc::AVX::Mask<VectorSize, RegisterWidth> > { enum { Value = true }; };
+#endif
+ template<typename T> struct HasImplicitCast<_Vector<T>, _Vector<T> > { enum { Value = true }; };
+ //template<> struct HasImplicitCast<_Vector< int>, _Vector< unsigned int>> { enum { Value = true }; };
+ //template<> struct HasImplicitCast<_Vector< unsigned int>, _Vector< int>> { enum { Value = true }; };
+ //template<> struct HasImplicitCast<_Vector< short>, _Vector<unsigned short>> { enum { Value = true }; };
+ //template<> struct HasImplicitCast<_Vector<unsigned short>, _Vector< short>> { enum { Value = true }; };
+ template<typename V, size_t Size1, size_t Size2, typename T2> struct HasImplicitCast<Vc::Memory<V, Size1, Size2>, T2> { enum { Value = false }; };
+ template<typename V, size_t Size1, size_t Size2> struct HasImplicitCast<Vc::Memory<V, Size1, Size2>, Vc::Memory<V, Size1, Size2> > { enum { Value = true }; };
+#undef _Vector
+#endif
+
+ template<typename T> struct CanConvertToInt : public HasImplicitCast<T, int> {};
+ template<> struct CanConvertToInt<bool> { enum { Value = 0 }; };
+ //template<> struct CanConvertToInt<float> { enum { Value = 0 }; };
+ //template<> struct CanConvertToInt<double> { enum { Value = 0 }; };
+
+ enum TestEnum {};
+ VC_STATIC_ASSERT(CanConvertToInt<int>::Value == 1, CanConvertToInt_is_broken);
+ VC_STATIC_ASSERT(CanConvertToInt<unsigned char>::Value == 1, CanConvertToInt_is_broken);
+ VC_STATIC_ASSERT(CanConvertToInt<bool>::Value == 0, CanConvertToInt_is_broken);
+ VC_STATIC_ASSERT(CanConvertToInt<float>::Value == 1, CanConvertToInt_is_broken);
+ VC_STATIC_ASSERT(CanConvertToInt<double>::Value == 1, CanConvertToInt_is_broken);
+ VC_STATIC_ASSERT(CanConvertToInt<float*>::Value == 0, CanConvertToInt_is_broken);
+ VC_STATIC_ASSERT(CanConvertToInt<TestEnum>::Value == 1, CanConvertToInt_is_broken);
+
+ typedef HasImplicitCast<TestEnum, short> HasImplicitCastTest0;
+ typedef HasImplicitCast<int *, void *> HasImplicitCastTest1;
+ typedef HasImplicitCast<int *, const void *> HasImplicitCastTest2;
+ typedef HasImplicitCast<const int *, const void *> HasImplicitCastTest3;
+ typedef HasImplicitCast<const int *, int *> HasImplicitCastTest4;
+
+ VC_STATIC_ASSERT(HasImplicitCastTest0::Value == true, HasImplicitCast0_is_broken);
+ VC_STATIC_ASSERT(HasImplicitCastTest1::Value == true, HasImplicitCast1_is_broken);
+ VC_STATIC_ASSERT(HasImplicitCastTest2::Value == true, HasImplicitCast2_is_broken);
+ VC_STATIC_ASSERT(HasImplicitCastTest3::Value == true, HasImplicitCast3_is_broken);
+ VC_STATIC_ASSERT(HasImplicitCastTest4::Value == false, HasImplicitCast4_is_broken);
+
+ template<typename T> struct IsLikeInteger { enum { Value = !IsReal<T>::Value && CanConvertToInt<T>::Value }; };
+ template<typename T> struct IsLikeSignedInteger { enum { Value = IsLikeInteger<T>::Value && !IsUnsignedInteger<T>::Value }; };
+} // anonymous namespace
+
+} // namespace Vc
+
+#endif // VC_COMMON_TYPES_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_UNDOMACROS_H
+#define VC_COMMON_UNDOMACROS_H
+#undef VC_COMMON_MACROS_H
+
+#undef INTRINSIC
+#undef INTRINSIC_L
+#undef INTRINSIC_R
+#undef CONST
+#undef CONST_L
+#undef CONST_R
+#undef PURE
+#undef PURE_L
+#undef PURE_R
+#undef MAY_ALIAS
+#undef ALWAYS_INLINE
+#undef ALWAYS_INLINE_L
+#undef ALWAYS_INLINE_R
+#undef _VC_CONSTEXPR
+#undef _VC_CONSTEXPR_L
+#undef _VC_CONSTEXPR_R
+#undef _VC_NOEXCEPT
+
+#undef ALIGN
+#undef STRUCT_ALIGN1
+#undef STRUCT_ALIGN2
+#undef ALIGNED_TYPEDEF
+#undef _CAT_IMPL
+#undef CAT
+#undef unrolled_loop16
+#undef for_all_vector_entries
+#undef FREE_STORE_OPERATORS_ALIGNED
+
+#undef VC_WARN_INLINE
+#undef VC_WARN
+
+#undef VC_HAS_BUILTIN
+
+#undef Vc_buildDouble
+#undef Vc_buildFloat
+
+#undef _VC_APPLY_IMPL_1
+#undef _VC_APPLY_IMPL_2
+#undef _VC_APPLY_IMPL_3
+#undef _VC_APPLY_IMPL_4
+#undef _VC_APPLY_IMPL_5
+
+#undef VC_LIST_FLOAT_VECTOR_TYPES
+#undef VC_LIST_INT_VECTOR_TYPES
+#undef VC_LIST_VECTOR_TYPES
+#undef VC_LIST_COMPARES
+#undef VC_LIST_LOGICAL
+#undef VC_LIST_BINARY
+#undef VC_LIST_SHIFTS
+#undef VC_LIST_ARITHMETICS
+
+#undef VC_APPLY_0
+#undef VC_APPLY_1
+#undef VC_APPLY_2
+#undef VC_APPLY_3
+#undef VC_APPLY_4
+
+#undef VC_ALL_COMPARES
+#undef VC_ALL_LOGICAL
+#undef VC_ALL_BINARY
+#undef VC_ALL_SHIFTS
+#undef VC_ALL_ARITHMETICS
+#undef VC_ALL_FLOAT_VECTOR_TYPES
+#undef VC_ALL_VECTOR_TYPES
+
+#undef VC_EXACT_TYPE
+
+#endif // VC_COMMON_UNDOMACROS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_COMMON_WINDOWS_FIX_INTRIN_H
+#define VC_COMMON_WINDOWS_FIX_INTRIN_H
+
+#if defined(_MSC_VER) && !defined(__midl)
+// MSVC sucks. If you include intrin.h you get all SSE and AVX intrinsics
+// declared. This is a problem because we need to implement the intrinsics
+// that are not supported in hardware ourselves.
+// Something always includes intrin.h even if you don't
+// do it explicitly. Therefore we try to be the first to include it
+// but with __midl defined, in which case it is basically empty.
+#ifdef __INTRIN_H_
+#error "intrin.h was already included, polluting the namespace. Please fix your code to include the Vc headers before anything that includes intrin.h. (Vc will declare the relevant intrinsics as they are required by some system headers.)"
+#endif
+#define __midl
+#include <intrin.h>
+#undef __midl
+#include <crtdefs.h>
+#include <setjmp.h>
+#include <stddef.h>
+extern "C" {
+
+#ifdef _WIN64
+_CRTIMP double ceil(_In_ double);
+__int64 _InterlockedDecrement64(__int64 volatile *);
+__int64 _InterlockedExchange64(__int64 volatile *, __int64);
+void * _InterlockedExchangePointer(void * volatile *, void *);
+__int64 _InterlockedExchangeAdd64(__int64 volatile *, __int64);
+void *_InterlockedCompareExchangePointer (void * volatile *, void *, void *);
+__int64 _InterlockedIncrement64(__int64 volatile *);
+int __cdecl _setjmpex(jmp_buf);
+void __faststorefence(void);
+__int64 __mulh(__int64,__int64);
+unsigned __int64 __umulh(unsigned __int64,unsigned __int64);
+unsigned __int64 __readcr0(void);
+unsigned __int64 __readcr2(void);
+unsigned __int64 __readcr3(void);
+unsigned __int64 __readcr4(void);
+unsigned __int64 __readcr8(void);
+void __writecr0(unsigned __int64);
+void __writecr3(unsigned __int64);
+void __writecr4(unsigned __int64);
+void __writecr8(unsigned __int64);
+unsigned __int64 __readdr(unsigned int);
+void __writedr(unsigned int, unsigned __int64);
+unsigned __int64 __readeflags(void);
+void __writeeflags(unsigned __int64);
+void __movsq(unsigned long long *, unsigned long long const *, size_t);
+unsigned char __readgsbyte(unsigned long Offset);
+unsigned short __readgsword(unsigned long Offset);
+unsigned long __readgsdword(unsigned long Offset);
+unsigned __int64 __readgsqword(unsigned long Offset);
+void __writegsbyte(unsigned long Offset, unsigned char Data);
+void __writegsword(unsigned long Offset, unsigned short Data);
+void __writegsdword(unsigned long Offset, unsigned long Data);
+void __writegsqword(unsigned long Offset, unsigned __int64 Data);
+void __addgsbyte(unsigned long Offset, unsigned char Data);
+void __addgsword(unsigned long Offset, unsigned short Data);
+void __addgsdword(unsigned long Offset, unsigned long Data);
+void __addgsqword(unsigned long Offset, unsigned __int64 Data);
+void __incgsbyte(unsigned long Offset);
+void __incgsword(unsigned long Offset);
+void __incgsdword(unsigned long Offset);
+void __incgsqword(unsigned long Offset);
+unsigned char __vmx_vmclear(unsigned __int64*);
+unsigned char __vmx_vmlaunch(void);
+unsigned char __vmx_vmptrld(unsigned __int64*);
+unsigned char __vmx_vmread(size_t, size_t*);
+unsigned char __vmx_vmresume(void);
+unsigned char __vmx_vmwrite(size_t, size_t);
+unsigned char __vmx_on(unsigned __int64*);
+void __stosq(unsigned __int64 *, unsigned __int64, size_t);
+unsigned char _interlockedbittestandset64(__int64 volatile *a, __int64 b);
+unsigned char _interlockedbittestandreset64(__int64 volatile *a, __int64 b);
+short _InterlockedCompareExchange16_np(short volatile *Destination, short Exchange, short Comparand);
+long _InterlockedCompareExchange_np (long volatile *, long, long);
+__int64 _InterlockedCompareExchange64_np(__int64 volatile *, __int64, __int64);
+void *_InterlockedCompareExchangePointer_np (void * volatile *, void *, void *);
+unsigned char _InterlockedCompareExchange128(__int64 volatile *, __int64, __int64, __int64 *);
+unsigned char _InterlockedCompareExchange128_np(__int64 volatile *, __int64, __int64, __int64 *);
+long _InterlockedAnd_np(long volatile *, long);
+char _InterlockedAnd8_np(char volatile *, char);
+short _InterlockedAnd16_np(short volatile *, short);
+__int64 _InterlockedAnd64_np(__int64 volatile *, __int64);
+long _InterlockedOr_np(long volatile *, long);
+char _InterlockedOr8_np(char volatile *, char);
+short _InterlockedOr16_np(short volatile *, short);
+__int64 _InterlockedOr64_np(__int64 volatile *, __int64);
+long _InterlockedXor_np(long volatile *, long);
+char _InterlockedXor8_np(char volatile *, char);
+short _InterlockedXor16_np(short volatile *, short);
+__int64 _InterlockedXor64_np(__int64 volatile *, __int64);
+unsigned __int64 __lzcnt64(unsigned __int64);
+unsigned __int64 __popcnt64(unsigned __int64);
+__int64 _InterlockedOr64(__int64 volatile *, __int64);
+__int64 _InterlockedXor64(__int64 volatile *, __int64);
+__int64 _InterlockedAnd64(__int64 volatile *, __int64);
+unsigned char _bittest64(__int64 const *a, __int64 b);
+unsigned char _bittestandset64(__int64 *a, __int64 b);
+unsigned char _bittestandreset64(__int64 *a, __int64 b);
+unsigned char _bittestandcomplement64(__int64 *a, __int64 b);
+unsigned char _BitScanForward64(unsigned long* Index, unsigned __int64 Mask);
+unsigned char _BitScanReverse64(unsigned long* Index, unsigned __int64 Mask);
+unsigned __int64 __shiftleft128(unsigned __int64 LowPart, unsigned __int64 HighPart, unsigned char Shift);
+unsigned __int64 __shiftright128(unsigned __int64 LowPart, unsigned __int64 HighPart, unsigned char Shift);
+unsigned __int64 _umul128(unsigned __int64 multiplier, unsigned __int64 multiplicand, unsigned __int64 *highproduct);
+__int64 _mul128(__int64 multiplier, __int64 multiplicand, __int64 *highproduct);
+#endif
+
+long _InterlockedOr(long volatile *, long);
+char _InterlockedOr8(char volatile *, char);
+short _InterlockedOr16(short volatile *, short);
+long _InterlockedXor(long volatile *, long);
+char _InterlockedXor8(char volatile *, char);
+short _InterlockedXor16(short volatile *, short);
+long _InterlockedAnd(long volatile *, long);
+char _InterlockedAnd8(char volatile *, char);
+short _InterlockedAnd16(short volatile *, short);
+unsigned char _bittest(long const *a, long b);
+unsigned char _bittestandset(long *a, long b);
+unsigned char _bittestandreset(long *a, long b);
+unsigned char _bittestandcomplement(long *a, long b);
+unsigned char _BitScanForward(unsigned long* Index, unsigned long Mask);
+unsigned char _BitScanReverse(unsigned long* Index, unsigned long Mask);
+_CRTIMP wchar_t * __cdecl wcscat( _Pre_cap_for_(_Source) _Prepost_z_ wchar_t *, _In_z_ const wchar_t * _Source);
+_Check_return_ _CRTIMP int __cdecl wcscmp(_In_z_ const wchar_t *,_In_z_ const wchar_t *);
+_CRTIMP wchar_t * __cdecl wcscpy(_Pre_cap_for_(_Source) _Post_z_ wchar_t *, _In_z_ const wchar_t * _Source);
+_Check_return_ _CRTIMP size_t __cdecl wcslen(_In_z_ const wchar_t *);
+#pragma warning(suppress: 4985)
+_CRTIMP wchar_t * __cdecl _wcsset(_Inout_z_ wchar_t *, wchar_t);
+void _ReadBarrier(void);
+unsigned char _rotr8(unsigned char value, unsigned char shift);
+unsigned short _rotr16(unsigned short value, unsigned char shift);
+unsigned char _rotl8(unsigned char value, unsigned char shift);
+unsigned short _rotl16(unsigned short value, unsigned char shift);
+short _InterlockedIncrement16(short volatile *Addend);
+short _InterlockedDecrement16(short volatile *Addend);
+short _InterlockedCompareExchange16(short volatile *Destination, short Exchange, short Comparand);
+void __nvreg_save_fence(void);
+void __nvreg_restore_fence(void);
+
+#ifdef _M_IX86
+unsigned long __readcr0(void);
+unsigned long __readcr2(void);
+unsigned long __readcr3(void);
+unsigned long __readcr4(void);
+unsigned long __readcr8(void);
+void __writecr0(unsigned);
+void __writecr3(unsigned);
+void __writecr4(unsigned);
+void __writecr8(unsigned);
+unsigned __readdr(unsigned int);
+void __writedr(unsigned int, unsigned);
+unsigned __readeflags(void);
+void __writeeflags(unsigned);
+void __addfsbyte(unsigned long Offset, unsigned char Data);
+void __addfsword(unsigned long Offset, unsigned short Data);
+void __addfsdword(unsigned long Offset, unsigned long Data);
+void __incfsbyte(unsigned long Offset);
+void __incfsword(unsigned long Offset);
+void __incfsdword(unsigned long Offset);
+unsigned char __readfsbyte(unsigned long Offset);
+unsigned short __readfsword(unsigned long Offset);
+unsigned long __readfsdword(unsigned long Offset);
+unsigned __int64 __readfsqword(unsigned long Offset);
+void __writefsbyte(unsigned long Offset, unsigned char Data);
+void __writefsword(unsigned long Offset, unsigned short Data);
+void __writefsdword(unsigned long Offset, unsigned long Data);
+void __writefsqword(unsigned long Offset, unsigned __int64 Data);
+long _InterlockedAddLargeStatistic(__int64 volatile *, long);
+#endif
+
+_Ret_bytecap_(_Size) void * __cdecl _alloca(size_t _Size);
+int __cdecl abs(_In_ int);
+_Check_return_ unsigned short __cdecl _byteswap_ushort(_In_ unsigned short value);
+_Check_return_ unsigned long __cdecl _byteswap_ulong(_In_ unsigned long value);
+_Check_return_ unsigned __int64 __cdecl _byteswap_uint64(_In_ unsigned __int64 value);
+void __cdecl __debugbreak(void);
+void __cdecl _disable(void);
+__int64 __emul(int,int);
+unsigned __int64 __emulu(unsigned int,unsigned int);
+void __cdecl _enable(void);
+long __cdecl _InterlockedDecrement(long volatile *);
+long _InterlockedExchange(long volatile *, long);
+short _InterlockedExchange16(short volatile *, short);
+char _InterlockedExchange8(char volatile *, char);
+long _InterlockedExchangeAdd(long volatile *, long);
+short _InterlockedExchangeAdd16(short volatile *, short);
+char _InterlockedExchangeAdd8(char volatile *, char);
+long _InterlockedCompareExchange (long volatile *, long, long);
+__int64 _InterlockedCompareExchange64(__int64 volatile *, __int64, __int64);
+long __cdecl _InterlockedIncrement(long volatile *);
+int __cdecl _inp(unsigned short);
+int __cdecl inp(unsigned short);
+unsigned long __cdecl _inpd(unsigned short);
+unsigned long __cdecl inpd(unsigned short);
+unsigned short __cdecl _inpw(unsigned short);
+unsigned short __cdecl inpw(unsigned short);
+long __cdecl labs(_In_ long);
+_Check_return_ unsigned long __cdecl _lrotl(_In_ unsigned long,_In_ int);
+_Check_return_ unsigned long __cdecl _lrotr(_In_ unsigned long,_In_ int);
+unsigned __int64 __ll_lshift(unsigned __int64,int);
+__int64 __ll_rshift(__int64,int);
+_Check_return_ int __cdecl memcmp(_In_opt_bytecount_(_Size) const void *,_In_opt_bytecount_(_Size) const void *,_In_ size_t _Size);
+void * __cdecl memcpy(_Out_opt_bytecapcount_(_Size) void *,_In_opt_bytecount_(_Size) const void *,_In_ size_t _Size);
+void * __cdecl memset(_Out_opt_bytecapcount_(_Size) void *,_In_ int,_In_ size_t _Size);
+int __cdecl _outp(unsigned short,int);
+int __cdecl outp(unsigned short,int);
+unsigned long __cdecl _outpd(unsigned short,unsigned long);
+unsigned long __cdecl outpd(unsigned short,unsigned long);
+unsigned short __cdecl _outpw(unsigned short,unsigned short);
+unsigned short __cdecl outpw(unsigned short,unsigned short);
+void * _ReturnAddress(void);
+_Check_return_ unsigned int __cdecl _rotl(_In_ unsigned int,_In_ int);
+_Check_return_ unsigned int __cdecl _rotr(_In_ unsigned int,_In_ int);
+int __cdecl _setjmp(jmp_buf);
+_Check_return_ int __cdecl strcmp(_In_z_ const char *,_In_z_ const char *);
+_Check_return_ size_t __cdecl strlen(_In_z_ const char *);
+char * __cdecl strset(_Inout_z_ char *,_In_ int);
+unsigned __int64 __ull_rshift(unsigned __int64,int);
+void * _AddressOfReturnAddress(void);
+
+void _WriteBarrier(void);
+void _ReadWriteBarrier(void);
+void __wbinvd(void);
+void __invlpg(void*);
+unsigned __int64 __readmsr(unsigned long);
+void __writemsr(unsigned long, unsigned __int64);
+unsigned __int64 __rdtsc(void);
+void __movsb(unsigned char *, unsigned char const *, size_t);
+void __movsw(unsigned short *, unsigned short const *, size_t);
+void __movsd(unsigned long *, unsigned long const *, size_t);
+unsigned char __inbyte(unsigned short Port);
+unsigned short __inword(unsigned short Port);
+unsigned long __indword(unsigned short Port);
+void __outbyte(unsigned short Port, unsigned char Data);
+void __outword(unsigned short Port, unsigned short Data);
+void __outdword(unsigned short Port, unsigned long Data);
+void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count);
+void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count);
+void __indwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count);
+void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned long Count);
+void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned long Count);
+void __outdwordstring(unsigned short Port, unsigned long *Buffer, unsigned long Count);
+unsigned int __getcallerseflags();
+void __vmx_vmptrst(unsigned __int64 *);
+void __vmx_off(void);
+void __svm_clgi(void);
+void __svm_invlpga(void*, int);
+void __svm_skinit(int);
+void __svm_stgi(void);
+void __svm_vmload(size_t);
+void __svm_vmrun(size_t);
+void __svm_vmsave(size_t);
+void __halt(void);
+void __sidt(void*);
+void __lidt(void*);
+void __ud2(void);
+void __nop(void);
+void __stosb(unsigned char *, unsigned char, size_t);
+void __stosw(unsigned short *, unsigned short, size_t);
+void __stosd(unsigned long *, unsigned long, size_t);
+unsigned char _interlockedbittestandset(long volatile *a, long b);
+unsigned char _interlockedbittestandreset(long volatile *a, long b);
+void __cpuid(int a[4], int b);
+void __cpuidex(int a[4], int b, int c);
+unsigned __int64 __readpmc(unsigned long a);
+unsigned long __segmentlimit(unsigned long a);
+_Check_return_ unsigned __int64 __cdecl _rotl64(_In_ unsigned __int64,_In_ int);
+_Check_return_ unsigned __int64 __cdecl _rotr64(_In_ unsigned __int64,_In_ int);
+__int64 __cdecl _abs64(__int64);
+void __int2c(void);
+char _InterlockedCompareExchange8(char volatile *Destination, char Exchange, char Comparand);
+unsigned short __lzcnt16(unsigned short);
+unsigned int __lzcnt(unsigned int);
+unsigned short __popcnt16(unsigned short);
+unsigned int __popcnt(unsigned int);
+unsigned __int64 __rdtscp(unsigned int*);
+}
+#endif
+
+#endif // VC_COMMON_WINDOWS_FIX_INTRIN_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef CPUID_H
+#define CPUID_H
+
+#include <iostream>
+
+namespace Vc
+{
+class CpuId
+{
+ typedef unsigned char uchar;
+ typedef unsigned short ushort;
+ typedef unsigned int uint;
+
+ public:
+ enum ProcessorType {
+ OriginalOemProcessor = 0,
+ IntelOverDriveProcessor = 1,
+ DualProcessor = 2,
+ IntelReserved = 3
+ };
+
+ static void init();
+ static inline ushort cacheLineSize() { return static_cast<ushort>(s_cacheLineSize) * 8u; }
+ static inline ProcessorType processorType() { return s_processorType; }
+ static inline uint processorFamily() { return s_processorFamily; }
+ static inline uint processorModel() { return s_processorModel; }
+ static inline uint logicalProcessors() { return s_logicalProcessors; }
+ static inline bool isAmd () { return s_ecx0 == 0x444D4163; }
+ static inline bool isIntel () { return s_ecx0 == 0x6C65746E; }
+ static inline bool hasSse3 () { return s_processorFeaturesC & (1 << 0); }
+ static inline bool hasVmx () { return (s_processorFeaturesC & (1 << 5)) != 0; }
+ static inline bool hasSmx () { return (s_processorFeaturesC & (1 << 6)) != 0; }
+ static inline bool hasEst () { return (s_processorFeaturesC & (1 << 7)) != 0; }
+ static inline bool hasTm2 () { return (s_processorFeaturesC & (1 << 8)) != 0; }
+ static inline bool hasSsse3() { return (s_processorFeaturesC & (1 << 9)) != 0; }
+ static inline bool hasPdcm () { return (s_processorFeaturesC & (1 << 15)) != 0; }
+ static inline bool hasSse41() { return (s_processorFeaturesC & (1 << 19)) != 0; }
+ static inline bool hasSse42() { return (s_processorFeaturesC & (1 << 20)) != 0; }
+ static inline bool hasAes () { return (s_processorFeaturesC & (1 << 25)) != 0; }
+ static inline bool hasOsxsave() { return (s_processorFeaturesC & (1 << 27)) != 0; }
+ static inline bool hasAvx () { return (s_processorFeaturesC & (1 << 28)) != 0; }
+ static inline bool hasFpu () { return (s_processorFeaturesD & (1 << 0)) != 0; }
+ static inline bool hasVme () { return (s_processorFeaturesD & (1 << 1)) != 0; }
+ static inline bool hasDe () { return (s_processorFeaturesD & (1 << 2)) != 0; }
+ static inline bool hasPse () { return (s_processorFeaturesD & (1 << 3)) != 0; }
+ static inline bool hasTsc () { return (s_processorFeaturesD & (1 << 4)) != 0; }
+ static inline bool hasMsr () { return (s_processorFeaturesD & (1 << 5)) != 0; }
+ static inline bool hasPae () { return (s_processorFeaturesD & (1 << 6)) != 0; }
+ static inline bool hasMtrr () { return (s_processorFeaturesD & (1 << 12)) != 0; }
+ static inline bool hasCmov () { return (s_processorFeaturesD & (1 << 15)) != 0; }
+ static inline bool hasClfsh() { return (s_processorFeaturesD & (1 << 19)) != 0; }
+ static inline bool hasAcpi () { return (s_processorFeaturesD & (1 << 22)) != 0; }
+ static inline bool hasMmx () { return (s_processorFeaturesD & (1 << 23)) != 0; }
+ static inline bool hasSse () { return (s_processorFeaturesD & (1 << 25)) != 0; }
+ static inline bool hasSse2 () { return (s_processorFeaturesD & (1 << 26)) != 0; }
+ static inline bool hasHtt () { return (s_processorFeaturesD & (1 << 28)) != 0; }
+ static inline bool hasSse4a() { return (s_processorFeatures8C & (1 << 6)) != 0; }
+ static inline bool hasMisAlignSse() { return (s_processorFeatures8C & (1 << 7)) != 0; }
+ static inline bool hasAmdPrefetch() { return (s_processorFeatures8C & (1 << 8)) != 0; }
+ static inline bool hasXop () { return (s_processorFeatures8C & (1 << 11)) != 0; }
+ static inline bool hasFma4 () { return (s_processorFeatures8C & (1 << 16)) != 0; }
+ static inline bool hasRdtscp() { return (s_processorFeatures8D & (1 << 27)) != 0; }
+ static inline bool has3DNow() { return (s_processorFeatures8D & (1u << 31)) != 0; }
+ static inline bool has3DNowExt() { return (s_processorFeatures8D & (1 << 30)) != 0; }
+ static inline uint L1Instruction() { return s_L1Instruction; }
+ static inline uint L1Data() { return s_L1Data; }
+ static inline uint L2Data() { return s_L2Data; }
+ static inline uint L3Data() { return s_L3Data; }
+ static inline ushort L1InstructionLineSize() { return s_L1InstructionLineSize; }
+ static inline ushort L1DataLineSize() { return s_L1DataLineSize; }
+ static inline ushort L2DataLineSize() { return s_L2DataLineSize; }
+ static inline ushort L3DataLineSize() { return s_L3DataLineSize; }
+ static inline uint L1Associativity() { return s_L1Associativity; }
+ static inline uint L2Associativity() { return s_L2Associativity; }
+ static inline uint L3Associativity() { return s_L3Associativity; }
+ static inline ushort prefetch() { return s_prefetch; }
+
+ private:
+ static void interpret(uchar byte, bool *checkLeaf4);
+
+ static uint s_ecx0;
+ static uint s_logicalProcessors;
+ static uint s_processorFeaturesC;
+ static uint s_processorFeaturesD;
+ static uint s_processorFeatures8C;
+ static uint s_processorFeatures8D;
+ static uint s_L1Instruction;
+ static uint s_L1Data;
+ static uint s_L2Data;
+ static uint s_L3Data;
+ static ushort s_L1InstructionLineSize;
+ static ushort s_L1DataLineSize;
+ static ushort s_L2DataLineSize;
+ static ushort s_L3DataLineSize;
+ static uint s_L1Associativity;
+ static uint s_L2Associativity;
+ static uint s_L3Associativity;
+ static ushort s_prefetch;
+ static uchar s_brandIndex;
+ static uchar s_cacheLineSize;
+ static uchar s_processorModel;
+ static uchar s_processorFamily;
+ static ProcessorType s_processorType;
+ static bool s_noL2orL3;
+};
+} // namespace Vc
+
+#endif // CPUID_H
--- /dev/null
+#ifdef __GNUC__
+#warning "Use of the Vc/double_v header is deprecated. The header file will be removed in a future version of Vc."
+#endif
--- /dev/null
+#ifdef __GNUC__
+#warning "Use of the Vc/float_v header is deprecated. The header file will be removed in a future version of Vc."
+#endif
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_GLOBAL_H
+#define VC_GLOBAL_H
+
+// Compiler defines
+#ifdef __INTEL_COMPILER
+#define VC_ICC __INTEL_COMPILER_BUILD_DATE
+#elif defined(__OPENCC__)
+#define VC_OPEN64 1
+#elif defined(__clang__)
+#define VC_CLANG (__clang_major__ * 0x10000 + __clang_minor__ * 0x100 + __clang_patchlevel__)
+#elif defined(__GNUC__)
+#define VC_GCC (__GNUC__ * 0x10000 + __GNUC_MINOR__ * 0x100 + __GNUC_PATCHLEVEL__)
+#elif defined(_MSC_VER)
+#define VC_MSVC _MSC_FULL_VER
+#else
+#define VC_UNSUPPORTED_COMPILER 1
+#endif
+
+// Features/Quirks defines
+#if defined VC_MSVC && defined _WIN32
+// the Win32 ABI can't handle function parameters with alignment >= 16
+#define VC_PASSING_VECTOR_BY_VALUE_IS_BROKEN 1
+#endif
+#if defined(__GNUC__) && !defined(VC_NO_INLINE_ASM)
+#define VC_GNU_ASM 1
+#endif
+#if defined(VC_GCC) && (VC_GCC <= 0x40405 || (VC_GCC >= 0x40500 && VC_GCC <= 0x40502)) && !(VC_GCC == 0x40502 && defined(__GNUC_UBUNTU_VERSION__) && __GNUC_UBUNTU_VERSION__ == 0xb0408)
+// GCC 4.6.0 / 4.5.3 / 4.4.6 switched to the interface as defined by ICC
+// (Ubuntu 11.04 ships a GCC 4.5.2 with the new interface)
+#define VC_MM256_MASKSTORE_WRONG_MASK_TYPE 1
+#endif
+#if defined(VC_GCC) && VC_GCC >= 0x40300
+#define VC_HAVE_ATTRIBUTE_ERROR 1
+#define VC_HAVE_ATTRIBUTE_WARNING 1
+#endif
+
+#define SSE 9875294
+#define SSE2 9875295
+#define SSE3 9875296
+#define SSSE3 9875297
+#define SSE4_1 9875298
+#define Scalar 9875299
+#define SSE4_2 9875301
+#define SSE4a 9875302
+#define AVX 9875303
+
+#ifdef _M_IX86_FP
+# if _M_IX86_FP >= 1
+# ifndef __SSE__
+# define __SSE__ 1
+# endif
+# endif
+# if _M_IX86_FP >= 2
+# ifndef __SSE2__
+# define __SSE2__ 1
+# endif
+# endif
+#endif
+
+#ifndef VC_IMPL
+
+# if defined(__AVX__)
+# define VC_IMPL_AVX 1
+# else
+# if defined(__SSE4a__)
+# define VC_IMPL_SSE 1
+# define VC_IMPL_SSE4a 1
+# endif
+# if defined(__SSE4_2__)
+# define VC_IMPL_SSE 1
+# define VC_IMPL_SSE4_2 1
+# endif
+# if defined(__SSE4_1__)
+# define VC_IMPL_SSE 1
+# define VC_IMPL_SSE4_1 1
+# endif
+# if defined(__SSE3__)
+# define VC_IMPL_SSE 1
+# define VC_IMPL_SSE3 1
+# endif
+# if defined(__SSSE3__)
+# define VC_IMPL_SSE 1
+# define VC_IMPL_SSSE3 1
+# endif
+# if defined(__SSE2__)
+# define VC_IMPL_SSE 1
+# define VC_IMPL_SSE2 1
+# endif
+
+# if defined(VC_IMPL_SSE)
+ // nothing
+# else
+# define VC_IMPL_Scalar 1
+# endif
+# endif
+
+#else // VC_IMPL
+
+# if VC_IMPL == AVX // AVX supersedes SSE
+# define VC_IMPL_AVX 1
+# elif VC_IMPL == Scalar
+# define VC_IMPL_Scalar 1
+# elif VC_IMPL == SSE4a
+# define VC_IMPL_SSE4a 1
+# define VC_IMPL_SSE3 1
+# define VC_IMPL_SSE2 1
+# define VC_IMPL_SSE 1
+# elif VC_IMPL == SSE4_2
+# define VC_IMPL_SSE4_2 1
+# define VC_IMPL_SSE4_1 1
+# define VC_IMPL_SSSE3 1
+# define VC_IMPL_SSE3 1
+# define VC_IMPL_SSE2 1
+# define VC_IMPL_SSE 1
+# elif VC_IMPL == SSE4_1
+# define VC_IMPL_SSE4_1 1
+# define VC_IMPL_SSSE3 1
+# define VC_IMPL_SSE3 1
+# define VC_IMPL_SSE2 1
+# define VC_IMPL_SSE 1
+# elif VC_IMPL == SSSE3
+# define VC_IMPL_SSSE3 1
+# define VC_IMPL_SSE3 1
+# define VC_IMPL_SSE2 1
+# define VC_IMPL_SSE 1
+# elif VC_IMPL == SSE3
+# define VC_IMPL_SSE3 1
+# define VC_IMPL_SSE2 1
+# define VC_IMPL_SSE 1
+# elif VC_IMPL == SSE2
+# define VC_IMPL_SSE2 1
+# define VC_IMPL_SSE 1
+# elif VC_IMPL == SSE
+# define VC_IMPL_SSE 1
+# if defined(__SSE4a__)
+# define VC_IMPL_SSE4a 1
+# endif
+# if defined(__SSE4_2__)
+# define VC_IMPL_SSE4_2 1
+# endif
+# if defined(__SSE4_1__)
+# define VC_IMPL_SSE4_1 1
+# endif
+# if defined(__SSE3__)
+# define VC_IMPL_SSE3 1
+# endif
+# if defined(__SSSE3__)
+# define VC_IMPL_SSSE3 1
+# endif
+# if defined(__SSE2__)
+# define VC_IMPL_SSE2 1
+# endif
+# endif
+# undef VC_IMPL
+
+#endif // VC_IMPL
+
+// If AVX is enabled in the compiler it will use VEX coding for the SIMD instructions.
+#ifdef __AVX__
+# define VC_USE_VEX_CODING 1
+#endif
+
+// There are no explicit switches for FMA4/XOP in Vc yet, so enable it when the compiler
+// says it's active
+#ifdef __FMA4__
+# define VC_IMPL_FMA4 1
+#endif
+#ifdef __XOP__
+# define VC_IMPL_XOP 1
+#endif
+
+#if defined(VC_GCC) && VC_GCC < 0x40300 && !defined(VC_IMPL_Scalar)
+# ifndef VC_DONT_WARN_OLD_GCC
+# warning "GCC < 4.3 does not have full support for SSE2 intrinsics. Using scalar types/operations only. Define VC_DONT_WARN_OLD_GCC to silence this warning."
+# endif
+# undef VC_IMPL_SSE
+# undef VC_IMPL_SSE2
+# undef VC_IMPL_SSE3
+# undef VC_IMPL_SSE4a
+# undef VC_IMPL_SSE4_1
+# undef VC_IMPL_SSE4_2
+# undef VC_IMPL_SSSE3
+# undef VC_IMPL_AVX
+# undef VC_IMPL_FMA4
+# undef VC_IMPL_XOP
+# undef VC_USE_VEX_CODING
+# define VC_IMPL_Scalar 1
+#endif
+
+# if !defined(VC_IMPL_Scalar) && !defined(VC_IMPL_SSE) && !defined(VC_IMPL_AVX)
+# error "No suitable Vc implementation was selected! Probably VC_IMPL was set to an invalid value."
+# elif defined(VC_IMPL_SSE) && !defined(VC_IMPL_SSE2)
+# error "SSE requested but no SSE2 support. Vc needs at least SSE2!"
+# endif
+
+#undef SSE
+#undef SSE2
+#undef SSE3
+#undef SSSE3
+#undef SSE4_1
+#undef SSE4_2
+#undef SSE4a
+#undef AVX
+#undef Scalar
+
+#ifndef DOXYGEN
+namespace Vc {
+enum AlignedFlag {
+ Aligned = 0
+};
+enum UnalignedFlag {
+ Unaligned = 1
+};
+enum StreamingAndAlignedFlag { // implies Aligned
+ Streaming = 2
+};
+enum StreamingAndUnalignedFlag {
+ StreamingAndUnaligned = 3
+};
+#endif
+
+/**
+ * \ingroup Utilities
+ *
+ * Enum that specifies the alignment and padding restrictions to use for memory allocation with
+ * Vc::malloc.
+ */
+enum MallocAlignment {
+ /**
+ * Align on boundary of vector sizes (e.g. 16 Bytes on SSE platforms) and pad to allow
+ * vector access to the end. Thus the allocated memory contains a multiple of
+ * VectorAlignment bytes.
+ */
+ AlignOnVector,
+ /**
+ * Align on boundary of cache line sizes (e.g. 64 Bytes on x86) and pad to allow
+ * full cache line access to the end. Thus the allocated memory contains a multiple of
+ * 64 bytes.
+ */
+ AlignOnCacheline,
+ /**
+ * Align on boundary of page sizes (e.g. 4096 Bytes on x86) and pad to allow
+ * full page access to the end. Thus the allocated memory contains a multiple of
+ * 4096 bytes.
+ */
+ AlignOnPage
+};
+
+static inline StreamingAndUnalignedFlag operator|(UnalignedFlag, StreamingAndAlignedFlag) { return StreamingAndUnaligned; }
+static inline StreamingAndUnalignedFlag operator|(StreamingAndAlignedFlag, UnalignedFlag) { return StreamingAndUnaligned; }
+static inline StreamingAndUnalignedFlag operator&(UnalignedFlag, StreamingAndAlignedFlag) { return StreamingAndUnaligned; }
+static inline StreamingAndUnalignedFlag operator&(StreamingAndAlignedFlag, UnalignedFlag) { return StreamingAndUnaligned; }
+
+static inline StreamingAndAlignedFlag operator|(AlignedFlag, StreamingAndAlignedFlag) { return Streaming; }
+static inline StreamingAndAlignedFlag operator|(StreamingAndAlignedFlag, AlignedFlag) { return Streaming; }
+static inline StreamingAndAlignedFlag operator&(AlignedFlag, StreamingAndAlignedFlag) { return Streaming; }
+static inline StreamingAndAlignedFlag operator&(StreamingAndAlignedFlag, AlignedFlag) { return Streaming; }
+
+/**
+ * \ingroup Utilities
+ *
+ * Enum to identify a certain SIMD instruction set.
+ *
+ * You can use \ref VC_IMPL for the currently active implementation.
+ */
+enum Implementation {
+ /// uses only built-in types
+ ScalarImpl,
+ /// x86 SSE + SSE2
+ SSE2Impl,
+ /// x86 SSE + SSE2 + SSE3
+ SSE3Impl,
+ /// x86 SSE + SSE2 + SSE3 + SSSE3
+ SSSE3Impl,
+ /// x86 SSE + SSE2 + SSE3 + SSSE3 + SSE4.1
+ SSE41Impl,
+ /// x86 SSE + SSE2 + SSE3 + SSSE3 + SSE4.1 + SSE4.2
+ SSE42Impl,
+ /// x86 (AMD only) SSE + SSE2 + SSE3 + SSE4a
+ SSE4aImpl,
+ /// x86 AVX
+ AVXImpl,
+ /// x86 (AMD only) XOP
+ Fma4Impl,
+ /// x86 (AMD only) FMA4
+ XopImpl
+};
+
+#ifdef DOXYGEN
+/**
+ * \ingroup Utilities
+ *
+ * This macro is set to the value of Vc::Implementation that the current translation unit is
+ * compiled with.
+ */
+#define VC_IMPL
+#elif VC_IMPL_Scalar
+#define VC_IMPL ::Vc::ScalarImpl
+#elif VC_IMPL_AVX
+#define VC_IMPL ::Vc::AVXImpl
+#elif VC_IMPL_SSE4a
+#define VC_IMPL ::Vc::SSE4aImpl
+#elif VC_IMPL_SSE4_2
+#define VC_IMPL ::Vc::SSE42Impl
+#elif VC_IMPL_SSE4_1
+#define VC_IMPL ::Vc::SSE41Impl
+#elif VC_IMPL_SSSE3
+#define VC_IMPL ::Vc::SSSE3Impl
+#elif VC_IMPL_SSE3
+#define VC_IMPL ::Vc::SSE3Impl
+#elif VC_IMPL_SSE2
+#define VC_IMPL ::Vc::SSE2Impl
+#endif
+
+namespace Internal {
+ template<Implementation Impl> struct HelperImpl;
+ typedef HelperImpl<VC_IMPL> Helper;
+
+ template<typename A> struct FlagObject;
+ template<> struct FlagObject<AlignedFlag> { static inline AlignedFlag the() { return Aligned; } };
+ template<> struct FlagObject<UnalignedFlag> { static inline UnalignedFlag the() { return Unaligned; } };
+ template<> struct FlagObject<StreamingAndAlignedFlag> { static inline StreamingAndAlignedFlag the() { return Streaming; } };
+ template<> struct FlagObject<StreamingAndUnalignedFlag> { static inline StreamingAndUnalignedFlag the() { return StreamingAndUnaligned; } };
+} // namespace Internal
+
+namespace Warnings
+{
+ void _operator_bracket_warning()
+#if VC_HAVE_ATTRIBUTE_WARNING
+ __attribute__((warning("\n\tUse of Vc::Vector::operator[] to modify scalar entries is known to miscompile with GCC 4.3.x.\n\tPlease upgrade to a more recent GCC or avoid operator[] altogether.\n\t(This warning adds an unnecessary function call to operator[] which should work around the problem at a little extra cost.)")))
+#endif
+ ;
+} // namespace Warnings
+
+namespace Error
+{
+ template<typename L, typename R> struct invalid_operands_of_types {};
+} // namespace Error
+
+} // namespace Vc
+
+#include "version.h"
+
+#endif // VC_GLOBAL_H
--- /dev/null
+#ifdef __GNUC__
+#warning "Use of the Vc/int_v header is deprecated. The header file will be removed in a future version of Vc."
+#endif
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#if VC_IMPL_Scalar
+# define VECTOR_NAMESPACE Vc::Scalar
+#elif VC_IMPL_AVX
+# define VECTOR_NAMESPACE Vc::AVX
+#elif VC_IMPL_SSE
+# define VECTOR_NAMESPACE Vc::SSE
+#endif
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef INCLUDE_VC_LIMITS
+#define INCLUDE_VC_LIMITS
+
+#include <limits>
+#include "vector.h"
+#include "common/macros.h"
+#include <limits>
+
+namespace std
+{
+template<typename T> struct numeric_limits<Vc::Vector<T> > : public numeric_limits<typename Vc::Vector<T>::EntryType>
+{
+private:
+ typedef numeric_limits<typename Vc::Vector<T>::EntryType> _Base;
+public:
+ static inline INTRINSIC CONST Vc::Vector<T> max() { return Vc::Vector<T>(_Base::max()); }
+ static inline INTRINSIC CONST Vc::Vector<T> min() { return Vc::Vector<T>(_Base::min()); }
+ static inline INTRINSIC CONST Vc::Vector<T> lowest() { return Vc::Vector<T>(_Base::lowest()); }
+ static inline INTRINSIC CONST Vc::Vector<T> epsilon() { return Vc::Vector<T>(_Base::epsilon()); }
+ static inline INTRINSIC CONST Vc::Vector<T> round_error() { return Vc::Vector<T>(_Base::round_error()); }
+ static inline INTRINSIC CONST Vc::Vector<T> infinity() { return Vc::Vector<T>(_Base::infinity()); }
+ static inline INTRINSIC CONST Vc::Vector<T> quiet_NaN() { return Vc::Vector<T>(_Base::quiet_NaN()); }
+ static inline INTRINSIC CONST Vc::Vector<T> signaling_NaN() { return Vc::Vector<T>(_Base::signaling_NaN()); }
+ static inline INTRINSIC CONST Vc::Vector<T> denorm_min() { return Vc::Vector<T>(_Base::denorm_min()); }
+};
+} // namespace std
+
+#include "common/undomacros.h"
+#if VC_IMPL_Scalar
+# include "scalar/limits.h"
+#elif VC_IMPL_AVX
+# include "avx/limits.h"
+#elif VC_IMPL_SSE
+# include "sse/limits.h"
+#endif
+
+#endif // INCLUDE_VC_LIMITS
+
+// vim: ft=cpp
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_DEINTERLEAVE_H
+#define VC_SCALAR_DEINTERLEAVE_H
+
+#include "macros.h"
+
+namespace Vc
+{
+namespace Internal
+{
+
+template<> struct HelperImpl<Vc::ScalarImpl>
+{
+ template<typename V, typename M, typename A>
+ static inline void ALWAYS_INLINE deinterleave(V &a, V &b, const M *mem, A)
+ {
+ a = mem[0];
+ b = mem[1];
+ }
+
+ static inline void prefetchForOneRead(const void *) {}
+ static inline void prefetchForModify(const void *) {}
+ static inline void prefetchClose(const void *) {}
+ static inline void prefetchMid(const void *) {}
+ static inline void prefetchFar(const void *) {}
+
+ template<Vc::MallocAlignment A>
+ static inline ALWAYS_INLINE_L void *malloc(size_t n) ALWAYS_INLINE_R;
+ static inline ALWAYS_INLINE_L void free(void *p) ALWAYS_INLINE_R;
+};
+
+} // namespace Scalar
+} // namespace Vc
+
+#include "helperimpl.tcc"
+#include "undomacros.h"
+
+#endif // VC_SCALAR_DEINTERLEAVE_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_HELPERIMPL_TCC
+#define VC_SCALAR_HELPERIMPL_TCC
+
+#include <cstdlib>
+#if defined _WIN32 || defined _WIN64
+#include <malloc.h>
+#endif
+
+namespace Vc
+{
+namespace Internal
+{
+
+template<size_t X>
+static inline size_t nextMultipleOf(size_t value)
+{
+ const size_t offset = value % X;
+ if ( offset > 0 ) {
+ return value + X - offset;
+ }
+ return value;
+}
+
+template<Vc::MallocAlignment A>
+inline void *HelperImpl<ScalarImpl>::malloc(size_t n)
+{
+ void *ptr = 0;
+ switch (A) {
+ case Vc::AlignOnVector:
+ return std::malloc(n);
+ case Vc::AlignOnCacheline:
+ // TODO: hardcoding 64 is not such a great idea
+#if defined _WIN32 || defined _WIN64
+ ptr = _aligned_malloc(nextMultipleOf<64>(n), 64);
+ return ptr;
+#else
+ if (0 == posix_memalign(&ptr, 64, nextMultipleOf<64>(n))) {
+ return ptr;
+ }
+ break;
+#endif
+ case Vc::AlignOnPage:
+ // TODO: hardcoding 4096 is not such a great idea
+#if defined _WIN32 || defined _WIN64
+ ptr = _aligned_malloc(nextMultipleOf<4096>(n), 4096);
+ return ptr;
+#else
+ if (0 == posix_memalign(&ptr, 4096, nextMultipleOf<4096>(n))) {
+ return ptr;
+ }
+ break;
+#endif
+ }
+ return 0;
+}
+
+inline void HelperImpl<ScalarImpl>::free(void *p)
+{
+ std::free(p);
+}
+
+} // namespace Internal
+} // namespace Vc
+
+#endif // VC_SCALAR_HELPERIMPL_TCC
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2010 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_LIMITS_H
+#define VC_SCALAR_LIMITS_H
+
+
+#endif // VC_SCALAR_LIMITS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2010 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "../common/macros.h"
+
+#ifndef VC_SCALAR_MACROS_H
+#define VC_SCALAR_MACROS_H
+
+#endif // VC_SCALAR_MACROS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_MASK_H
+#define VC_SCALAR_MASK_H
+
+#include "types.h"
+
+namespace Vc
+{
+namespace Scalar
+{
+template<unsigned int VectorSize = 1> class Mask
+{
+ public:
+ inline Mask() {}
+ inline explicit Mask(bool b) : m(b) {}
+ inline explicit Mask(VectorSpecialInitializerZero::ZEnum) : m(false) {}
+ inline explicit Mask(VectorSpecialInitializerOne::OEnum) : m(true) {}
+ inline Mask(const Mask<VectorSize> *a) : m(a[0].m) {}
+
+ inline Mask &operator=(const Mask &rhs) { m = rhs.m; return *this; }
+ inline Mask &operator=(bool rhs) { m = rhs; return *this; }
+
+ inline void expand(Mask *x) { x[0].m = m; }
+
+ inline bool operator==(const Mask &rhs) const { return Mask(m == rhs.m); }
+ inline bool operator!=(const Mask &rhs) const { return Mask(m != rhs.m); }
+
+ inline Mask operator&&(const Mask &rhs) const { return Mask(m && rhs.m); }
+ inline Mask operator& (const Mask &rhs) const { return Mask(m && rhs.m); }
+ inline Mask operator||(const Mask &rhs) const { return Mask(m || rhs.m); }
+ inline Mask operator| (const Mask &rhs) const { return Mask(m || rhs.m); }
+ inline Mask operator^ (const Mask &rhs) const { return Mask(m ^ rhs.m); }
+ inline Mask operator!() const { return Mask(!m); }
+
+ inline Mask &operator&=(const Mask &rhs) { m &= rhs.m; return *this; }
+ inline Mask &operator|=(const Mask &rhs) { m |= rhs.m; return *this; }
+
+ inline bool isFull () const { return m; }
+ inline bool isEmpty() const { return !m; }
+ inline bool isMix () const { return false; }
+
+ inline bool data () const { return m; }
+ inline bool dataI() const { return m; }
+ inline bool dataD() const { return m; }
+
+#ifndef VC_NO_AUTOMATIC_BOOL_FROM_MASK
+ inline operator bool() const { return isFull(); }
+#endif
+
+ template<unsigned int OtherSize>
+ inline Mask cast() const { return *this; }
+
+ inline bool operator[](int) const { return m; }
+
+ inline int count() const { return m ? 1 : 0; }
+
+ /**
+ * Returns the index of the first one in the mask.
+ *
+ * The return value is undefined if the mask is empty.
+ */
+ int firstOne() const { return 0; }
+
+ private:
+ bool m;
+};
+
+struct ForeachHelper
+{
+ bool first;
+ inline ForeachHelper(bool mask) : first(mask) {}
+ inline void next() { first = false; }
+};
+
+#define Vc_foreach_bit(_it_, _mask_) \
+ for (Vc::Scalar::ForeachHelper _Vc_foreach_bit_helper(_mask_); _Vc_foreach_bit_helper.first; ) \
+ for (_it_ = 0; _Vc_foreach_bit_helper.first; _Vc_foreach_bit_helper.next())
+
+} // namespace Scalar
+} // namespace Vc
+
+#endif // VC_SCALAR_MASK_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_MATH_H
+#define VC_SCALAR_MATH_H
+
+#include "../common/const.h"
+#include "macros.h"
+
+namespace Vc
+{
+namespace Scalar
+{
+
+#define VC_MINMAX(V) \
+static inline V min(const V &x, const V &y) { return V(std::min(x.data(), y.data())); } \
+static inline V max(const V &x, const V &y) { return V(std::max(x.data(), y.data())); }
+VC_ALL_VECTOR_TYPES(VC_MINMAX)
+#undef VC_MINMAX
+
+template<typename T> static inline Vector<T> sqrt (const Vector<T> &x)
+{
+ return Vector<T>(std::sqrt(x.data()));
+}
+
+template<typename T> static inline Vector<T> rsqrt(const Vector<T> &x)
+{
+ const typename Vector<T>::EntryType one = 1; return Vector<T>(one / std::sqrt(x.data()));
+}
+
+template<typename T> static inline Vector<T> abs (const Vector<T> &x)
+{
+ return Vector<T>(std::abs(x.data()));
+}
+
+template<typename T> static inline void sincos(const Vector<T> &x, Vector<T> *sin, Vector<T> *cos)
+{
+#if (defined(VC_CLANG) && VC_HAS_BUILTIN(__builtin_sincosf)) || (!defined(VC_CLANG) && defined(__GNUC__))
+ __builtin_sincosf(x.data(), &sin->data(), &cos->data());
+#elif VC_MSVC
+ sin->data() = std::sin(x.data());
+ cos->data() = std::cos(x.data());
+#else
+ sincosf(x.data(), &sin->data(), &cos->data());
+#endif
+}
+
+template<> inline void sincos(const Vector<double> &x, Vector<double> *sin, Vector<double> *cos)
+{
+#if (defined(VC_CLANG) && VC_HAS_BUILTIN(__builtin_sincos)) || (!defined(VC_CLANG) && defined(__GNUC__))
+ __builtin_sincos(x.data(), &sin->data(), &cos->data());
+#elif VC_MSVC
+ sin->data() = std::sin(x.data());
+ cos->data() = std::cos(x.data());
+#else
+ ::sincos(x.data(), &sin->data(), &cos->data());
+#endif
+}
+
+template<typename T> static inline Vector<T> sin (const Vector<T> &x)
+{
+ return Vector<T>(std::sin(x.data()));
+}
+
+template<typename T> static inline Vector<T> asin (const Vector<T> &x)
+{
+ return Vector<T>(std::asin(x.data()));
+}
+
+template<typename T> static inline Vector<T> cos (const Vector<T> &x)
+{
+ return Vector<T>(std::cos(x.data()));
+}
+
+template<typename T> static inline Vector<T> log (const Vector<T> &x)
+{
+ return Vector<T>(std::log(x.data()));
+}
+
+template<typename T> static inline Vector<T> log10(const Vector<T> &x)
+{
+ return Vector<T>(std::log10(x.data()));
+}
+
+#if _XOPEN_SOURCE >= 600 || _ISOC99_SOURCE || _POSIX_C_SOURCE >= 200112L
+static inline double_v log2(double_v::AsArg x) { return double_v(::log2 (x.data())); }
+static inline sfloat_v log2(sfloat_v::AsArg x) { return sfloat_v(::log2f(x.data())); }
+static inline float_v log2( float_v::AsArg x) { return float_v(::log2f(x.data())); }
+#else
+#define VC_LOG2(V) \
+static inline V log2(const V &x) \
+{ \
+ return V(std::log(x.data()) / Math<V::EntryType>::ln2()); \
+}
+VC_ALL_FLOAT_VECTOR_TYPES(VC_LOG2)
+#undef VC_LOG2
+#endif
+
+template<typename T> static inline Vector<T> exp (const Vector<T> &x)
+{
+ return Vector<T>(std::exp(x.data()));
+}
+
+template<typename T> static inline Vector<T> atan (const Vector<T> &x)
+{
+ return Vector<T>(std::atan( x.data() ));
+}
+
+template<typename T> static inline Vector<T> atan2(const Vector<T> &x, const Vector<T> &y)
+{
+ return Vector<T>(std::atan2( x.data(), y.data() ));
+}
+
+template<typename T> static inline Vector<T> floor(const Vector<T> &x)
+{
+ return Vector<T>(std::floor(x.data()));
+}
+
+template<typename T> static inline Vector<T> ceil(const Vector<T> &x)
+{
+ return Vector<T>(std::ceil(x.data()));
+}
+
+template<typename T> static inline Vector<T> round(const Vector<T> &x)
+{
+ return x;
+}
+
+namespace
+{
+ template<typename T> bool _realIsEvenHalf(T x) {
+ const T two = 2;
+ const T half = 0.5;
+ const T f = std::floor(x * half) * two;
+ return (x - f) == half;
+ }
+} // namespace
+template<> inline Vector<float> round(const Vector<float> &x)
+{
+ return float_v(std::floor(x.data() + 0.5f) - (_realIsEvenHalf(x.data()) ? 1.f : 0.f));
+}
+
+template<> inline Vector<sfloat> round(const Vector<sfloat> &x)
+{
+ return sfloat_v(std::floor(x.data() + 0.5f) - (_realIsEvenHalf(x.data()) ? 1.f : 0.f));
+}
+
+template<> inline Vector<double> round(const Vector<double> &x)
+{
+ return double_v(std::floor(x.data() + 0.5 ) - (_realIsEvenHalf(x.data()) ? 1. : 0. ));
+}
+
+template<typename T> static inline Vector<T> reciprocal(const Vector<T> &x)
+{
+ const typename Vector<T>::EntryType one = 1; return Vector<T>(one / x.data());
+}
+
+#ifdef isfinite
+#undef isfinite
+#endif
+#ifdef isnan
+#undef isnan
+#endif
+template<typename T> static inline typename Vector<T>::Mask isfinite(const Vector<T> &x)
+{
+ return typename Vector<T>::Mask(
+#ifdef _MSC_VER
+ !!_finite(x.data())
+#elif defined(__INTEL_COMPILER)
+ ::isfinite(x.data())
+#else
+ std::isfinite(x.data())
+#endif
+ );
+}
+
+template<typename T> static inline typename Vector<T>::Mask isnan(const Vector<T> &x)
+{
+ return typename Vector<T>::Mask(
+#ifdef _MSC_VER
+ !!_isnan(x.data())
+#elif defined(__INTEL_COMPILER)
+ ::isnan(x.data())
+#else
+ std::isnan(x.data())
+#endif
+ );
+}
+
+inline Vector<float> frexp(Vector<float> x, Vector<int> *e) {
+ return float_v(::frexpf(x.data(), &e->data()));
+}
+inline Vector<double> frexp(Vector<double> x, Vector<int> *e) {
+ return double_v(::frexp(x.data(), &e->data()));
+}
+inline sfloat_v frexp(sfloat_v x, short_v *e) {
+ int ee;
+ const float r = ::frexpf(x.data(), &ee);
+ e->data() = ee;
+ return sfloat_v(r);
+}
+
+inline Vector<float> ldexp(Vector<float> x, Vector<int> e) {
+ return float_v(::ldexpf(x.data(), e.data()));
+}
+inline Vector<double> ldexp(Vector<double> x, Vector<int> e) {
+ return double_v(::ldexp(x.data(), e.data()));
+}
+inline sfloat_v ldexp(sfloat_v x, short_v e) {
+ return sfloat_v(::ldexpf(x.data(), e.data()));
+}
+
+} // namespace Scalar
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_SCALAR_MATH_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_TYPES_H
+#define VC_SCALAR_TYPES_H
+
+#define VC_DOUBLE_V_SIZE 1
+#define VC_FLOAT_V_SIZE 1
+#define VC_SFLOAT_V_SIZE 1
+#define VC_INT_V_SIZE 1
+#define VC_UINT_V_SIZE 1
+#define VC_SHORT_V_SIZE 1
+#define VC_USHORT_V_SIZE 1
+
+#include "../common/types.h"
+
+namespace Vc
+{
+ namespace Scalar
+ {
+ template<typename V = float> class VectorAlignedBaseT {};
+ template<typename T> class Vector;
+ } // namespace Scalar
+} // namespace Vc
+
+#endif // VC_SCALAR_TYPES_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2010 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_UNDOMACROS_H
+#define VC_SCALAR_UNDOMACROS_H
+
+#endif // VC_SCALAR_UNDOMACROS_H
+
+#include "../common/undomacros.h"
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef SCALAR_VECTOR_H
+#define SCALAR_VECTOR_H
+
+#include <assert.h>
+#include <algorithm>
+#include <cmath>
+
+#ifdef _MSC_VER
+#include <float.h>
+#endif
+
+#include "../common/memoryfwd.h"
+#include "macros.h"
+#include "types.h"
+#include "mask.h"
+#include "writemaskedvector.h"
+
+namespace Vc
+{
+namespace Scalar
+{
+ enum VectorAlignmentEnum { VectorAlignment = 4 };
+
+template<typename T>
+class Vector
+{
+ friend class WriteMaskedVector<T>;
+ public:
+ typedef typename DetermineEntryType<T>::Type EntryType;
+ protected:
+ EntryType m_data;
+ public:
+ typedef Vc::Memory<Vector<T>, 1> Memory;
+ typedef Vector<unsigned int> IndexType;
+ typedef Scalar::Mask<1u> Mask;
+ typedef Vector<T> AsArg;
+
+ EntryType &data() { return m_data; }
+ EntryType data() const { return m_data; }
+
+ enum Constants { Size = 1 };
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // uninitialized
+ inline Vector() {}
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // constants
+ inline Vector(VectorSpecialInitializerZero::ZEnum) : m_data(0) {}
+ inline Vector(VectorSpecialInitializerOne::OEnum) : m_data(1) {}
+ inline Vector(VectorSpecialInitializerIndexesFromZero::IEnum) : m_data(0) {}
+ static inline Vector Zero() { Vector r; r.m_data = 0; return r; }
+ static inline Vector One() { Vector r; r.m_data = 1; return r; }
+ static inline Vector IndexesFromZero() { return Zero(); }
+ static inline INTRINSIC_L Vector Random() INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // static_cast / copy ctor
+ template<typename OtherT> explicit inline Vector(const Vector<OtherT> &x) : m_data(static_cast<EntryType>(x.data())) {}
+
+ // implicit cast
+ template<typename OtherT> inline Vector &operator=(const Vector<OtherT> &x);
+
+ // copy assignment
+ inline Vector &operator=(Vector v) { m_data = v.data(); return *this; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // broadcast
+ explicit inline Vector(EntryType x) : m_data(x) {}
+ template<typename TT> inline Vector(TT x, VC_EXACT_TYPE(TT, EntryType, void *) = 0) : m_data(x) {}
+ inline Vector &operator=(EntryType a) { m_data = a; return *this; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // load ctors
+ explicit inline Vector(const EntryType *x) : m_data(x[0]) {}
+ template<typename A> inline Vector(const EntryType *x, A) : m_data(x[0]) {}
+ template<typename Other> explicit inline Vector(const Other *x) : m_data(x[0]) {}
+ template<typename Other, typename A> inline Vector(const Other *x, A) : m_data(x[0]) {}
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // expand 1 float_v to 2 double_v XXX rationale? remove it for release? XXX
+ template<typename OtherT> inline void expand(Vector<OtherT> *x) const { x->data() = static_cast<OtherT>(m_data); }
+ template<typename OtherT> explicit inline Vector(const Vector<OtherT> *a) : m_data(static_cast<EntryType>(a->data())) {}
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // zeroing
+ inline void setZero() { m_data = 0; }
+ inline void setZero(Mask k) { if (k) m_data = 0; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // load member functions
+ template<typename Other> inline void load(const Other *mem) { m_data = mem[0]; }
+ template<typename Other, typename A> inline void load(const Other *mem, A) { m_data = mem[0]; }
+ template<typename Other> inline void load(const Other *mem, Mask m) { if (m.data()) m_data = mem[0]; }
+
+ inline void load(const EntryType *mem) { m_data = mem[0]; }
+ template<typename A> inline void load(const EntryType *mem, A) { m_data = mem[0]; }
+ inline void load(const EntryType *mem, Mask m) { if (m.data()) m_data = mem[0]; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // stores
+ inline void store(EntryType *mem) const { mem[0] = m_data; }
+ inline void store(EntryType *mem, Mask m) const { if (m.data()) mem[0] = m_data; }
+ template<typename A> inline void store(EntryType *mem, A) const { store(mem); }
+ template<typename A> inline void store(EntryType *mem, Mask m, A) const { store(mem, m); }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // swizzles
+ inline INTRINSIC const Vector<T> &abcd() const { return *this; }
+ inline INTRINSIC const Vector<T> cdab() const { return *this; }
+ inline INTRINSIC const Vector<T> badc() const { return *this; }
+ inline INTRINSIC const Vector<T> aaaa() const { return *this; }
+ inline INTRINSIC const Vector<T> bbbb() const { return *this; }
+ inline INTRINSIC const Vector<T> cccc() const { return *this; }
+ inline INTRINSIC const Vector<T> dddd() const { return *this; }
+ inline INTRINSIC const Vector<T> bcad() const { return *this; }
+ inline INTRINSIC const Vector<T> bcda() const { return *this; }
+ inline INTRINSIC const Vector<T> dabc() const { return *this; }
+ inline INTRINSIC const Vector<T> acbd() const { return *this; }
+ inline INTRINSIC const Vector<T> dbca() const { return *this; }
+ inline INTRINSIC const Vector<T> dcba() const { return *this; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // gathers
+ template<typename IndexT> inline Vector(const EntryType *array, const IndexT *indexes) : m_data(array[indexes[0]]) {}
+ template<typename IndexT> inline Vector(const EntryType *array, Vector<IndexT> indexes) : m_data(array[indexes[0]]) {}
+ template<typename IndexT> inline Vector(const EntryType *array, IndexT indexes, Mask m) : m_data(m.data() ? array[indexes[0]] : 0) {}
+ template<typename S1, typename IT> inline Vector(const S1 *array, const EntryType S1::* member1, IT indexes, Mask mask = Mask(true))
+ : m_data(mask.data() ? (&array[indexes[0]])->*(member1) : 0) {}
+ template<typename S1, typename S2, typename IT> inline Vector(const S1 *array, const S2 S1::* member1,
+ const EntryType S2::* member2, IT indexes, Mask mask = Mask(true))
+ : m_data(mask.data() ? array[indexes[0]].*(member1).*(member2) : 0) {}
+ template<typename S1, typename IT1, typename IT2> inline Vector(const S1 *array, const EntryType *const S1::* ptrMember1,
+ IT1 outerIndex, IT2 innerIndex, Mask mask = Mask(true))
+ : m_data(mask.data() ? (array[outerIndex[0]].*(ptrMember1))[innerIndex[0]] : 0) {}
+
+ template<typename IT> inline void gather(const EntryType *array, IT indexes, Mask mask = Mask(true))
+ { if (mask.data()) m_data = array[indexes[0]]; }
+ template<typename S1, typename IT> inline void gather(const S1 *array, const EntryType S1::* member1, IT indexes, Mask mask = Mask(true))
+ { if (mask.data()) m_data = (&array[indexes[0]])->*(member1); }
+ template<typename S1, typename S2, typename IT> inline void gather(const S1 *array, const S2 S1::* member1,
+ const EntryType S2::* member2, IT indexes, Mask mask = Mask(true))
+ { if (mask.data()) m_data = array[indexes[0]].*(member1).*(member2); }
+ template<typename S1, typename IT1, typename IT2> inline void gather(const S1 *array, const EntryType *const S1::* ptrMember1,
+ IT1 outerIndex, IT2 innerIndex, Mask mask = Mask(true))
+ { if (mask.data()) m_data = (array[outerIndex[0]].*(ptrMember1))[innerIndex[0]]; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // scatters
+ inline void scatter(EntryType *array, const Vector<unsigned int> &indexes, Mask m = Mask(true)) const { if (m.data()) array[indexes[0]] = m_data; }
+ template<typename S1> inline void scatter(S1 *array, EntryType S1::* member, const Vector<unsigned int> &indexes, Mask m = Mask(true)) const {
+ if (m.data()) array[indexes[0]].*(member) = m_data;
+ }
+ template<typename S1, typename S2> inline void scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2,
+ const Vector<unsigned int> &indexes, Mask m = Mask(true)) const {
+ if (m.data()) array[indexes[0]].*(member1).*(member2) = m_data;
+ }
+
+ inline void scatter(EntryType *array, const Vector<unsigned short> &indexes, Mask m = Mask(true)) const { if (m.data()) array[indexes[0]] = m_data; }
+ template<typename S1> inline void scatter(S1 *array, EntryType S1::* member, const Vector<unsigned short> &indexes, Mask m = Mask(true)) const {
+ if (m.data()) array[indexes[0]].*(member) = m_data;
+ }
+ template<typename S1, typename S2> inline void scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2,
+ const Vector<unsigned short> &indexes, Mask m = Mask(true)) const {
+ if (m.data()) array[indexes[0]].*(member1).*(member2) = m_data;
+ }
+
+ //prefix
+ inline Vector &operator++() { ++m_data; return *this; }
+ //postfix
+ inline Vector operator++(int) { return m_data++; }
+
+ inline EntryType &operator[](int index) {
+ assert(index == 0); if(index) {}
+ return m_data;
+ }
+
+ inline EntryType operator[](int index) const {
+ assert(index == 0); if(index) {}
+ return m_data;
+ }
+
+ inline Vector operator~() const { return Vector(~m_data); }
+ inline Vector<typename NegateTypeHelper<T>::Type> operator-() const { return Vector<typename NegateTypeHelper<T>::Type>(-m_data); }
+
+#define OPshift(symbol) \
+ inline Vector &operator symbol##=(const Vector<T> &x) { m_data symbol##= x.m_data; return *this; } \
+ inline Vector &operator symbol##=(EntryType x) { return operator symbol##=(Vector(x)); } \
+ inline Vector operator symbol(const Vector<T> &x) const { return Vector<T>(m_data symbol x.m_data); }
+#define OPshift_int(symbol) \
+ inline Vector operator symbol(int x) const { return Vector(m_data symbol x); }
+#define OP(symbol) \
+ OPshift(symbol) \
+ template<typename TT> inline VC_EXACT_TYPE(TT, EntryType, Vector) operator symbol(TT x) const { return operator symbol(Vector(x)); }
+#define OPcmp(symbol) \
+ inline Mask operator symbol(const Vector<T> &x) const { return Mask(m_data symbol x.m_data); } \
+ template<typename TT> inline VC_EXACT_TYPE(TT, EntryType, Mask) operator symbol(TT x) const { return Mask(m_data symbol x); }
+
+ VC_ALL_ARITHMETICS(OP)
+ VC_ALL_BINARY(OP)
+ VC_ALL_SHIFTS(OPshift)
+ VC_ALL_SHIFTS(OPshift_int)
+ VC_ALL_COMPARES(OPcmp)
+#undef OP
+#undef OPcmp
+#undef OPshift
+#undef OPshift_int
+
+ inline void multiplyAndAdd(const Vector<T> &factor, const Vector<T> &summand) {
+ m_data *= factor;
+ m_data += summand;
+ }
+
+ inline Vector<T> multiplyAndAdd(const Vector<T> &factor, const Vector<T> &summand) const {
+ return Vector<T>( m_data * factor.m_data + summand.m_data );
+ }
+
+ inline void assign(const Vector<T> &v, const Mask &m) {
+ if (m.data()) m_data = v.m_data;
+ }
+
+ template<typename V2> inline V2 staticCast() const { return V2(static_cast<typename V2::EntryType>(m_data)); }
+ template<typename V2> inline V2 reinterpretCast() const {
+ typedef typename V2::EntryType AliasT2 MAY_ALIAS;
+ return V2(*reinterpret_cast<const AliasT2 *>(&m_data));
+ }
+
+ inline WriteMaskedVector<T> operator()(Mask m) { return WriteMaskedVector<T>(this, m); }
+
+ inline bool pack(Mask &m1, Vector<T> &v2, Mask &m2) {
+ if (!m1.data() && m2.data()) {
+ m_data = v2.m_data;
+ m1 = true;
+ m2 = false;
+ return true;
+ }
+ return m1;
+ }
+
+ inline EntryType min() const { return m_data; }
+ inline EntryType max() const { return m_data; }
+ inline EntryType product() const { return m_data; }
+ inline EntryType sum() const { return m_data; }
+ inline EntryType min(Mask) const { return m_data; }
+ inline EntryType max(Mask) const { return m_data; }
+ inline EntryType product(Mask) const { return m_data; }
+ inline EntryType sum(Mask m) const { if (m) return m_data; return static_cast<EntryType>(0); }
+
+ Vector sorted() const { return *this; }
+
+ template<typename F> void callWithValuesSorted(F &f) {
+ f(m_data);
+ }
+
+ template<typename F> inline void INTRINSIC call(const F &f) const {
+ f(m_data);
+ }
+ template<typename F> inline void INTRINSIC call(F &f) const {
+ f(m_data);
+ }
+
+ template<typename F> inline void INTRINSIC call(const F &f, Mask mask) const {
+ if (mask) {
+ f(m_data);
+ }
+ }
+ template<typename F> inline void INTRINSIC call(F &f, Mask mask) const {
+ if (mask) {
+ f(m_data);
+ }
+ }
+
+ template<typename F> inline Vector INTRINSIC apply(const F &f) const {
+ return Vector(f(m_data));
+ }
+ template<typename F> inline Vector INTRINSIC apply(F &f) const {
+ return Vector(f(m_data));
+ }
+
+ template<typename F> inline Vector INTRINSIC apply(const F &f, Mask mask) const {
+ if (mask) {
+ return Vector(f(m_data));
+ } else {
+ return *this;
+ }
+ }
+ template<typename F> inline Vector INTRINSIC apply(F &f, Mask mask) const {
+ if (mask) {
+ return Vector(f(m_data));
+ } else {
+ return *this;
+ }
+ }
+
+ template<typename IndexT> inline void INTRINSIC fill(EntryType (&f)(IndexT)) {
+ m_data = f(0);
+ }
+ inline void INTRINSIC fill(EntryType (&f)()) {
+ m_data = f();
+ }
+
+ inline INTRINSIC_L Vector copySign(Vector reference) const INTRINSIC_R;
+ inline INTRINSIC_L Vector exponent() const INTRINSIC_R;
+};
+
+typedef Vector<double> double_v;
+typedef Vector<float> float_v;
+typedef Vector<sfloat> sfloat_v;
+typedef Vector<int> int_v;
+typedef Vector<unsigned int> uint_v;
+typedef Vector<short> short_v;
+typedef Vector<unsigned short> ushort_v;
+typedef double_v::Mask double_m;
+typedef float_v::Mask float_m;
+typedef sfloat_v::Mask sfloat_m;
+typedef int_v::Mask int_m;
+typedef uint_v::Mask uint_m;
+typedef short_v::Mask short_m;
+typedef ushort_v::Mask ushort_m;
+
+template<typename T> class SwizzledVector : public Vector<T> {};
+
+#ifdef _MSC_VER
+ template<typename T> static inline void forceToRegisters(const Vector<T> &) {
+ }
+#else
+ template<typename T> static inline void forceToRegisters(const Vector<T> &x01) {
+ __asm__ __volatile__(""::"r"(x01.data()));
+ }
+ template<> inline void forceToRegisters(const Vector<float> &x01) {
+ __asm__ __volatile__(""::"x"(x01.data()));
+ }
+ template<> inline void forceToRegisters(const Vector<double> &x01) {
+ __asm__ __volatile__(""::"x"(x01.data()));
+ }
+#endif
+ template<typename T1, typename T2> static inline void forceToRegisters(
+ const Vector<T1> &x01, const Vector<T2> &x02) {
+ forceToRegisters(x01);
+ forceToRegisters(x02);
+ }
+ template<typename T1, typename T2, typename T3> static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &, const Vector<T3> &) {}
+ template<typename T1, typename T2, typename T3, typename T4> static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9, typename T10>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &, const Vector<T10> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9, typename T10, typename T11>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &, const Vector<T10> &,
+ const Vector<T11> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9, typename T10, typename T11, typename T12>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &, const Vector<T10> &,
+ const Vector<T11> &, const Vector<T12> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9, typename T10, typename T11, typename T12, typename T13>
+ static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &, const Vector<T10> &,
+ const Vector<T11> &, const Vector<T12> &,
+ const Vector<T13> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9, typename T10, typename T11, typename T12, typename T13,
+ typename T14> static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &, const Vector<T10> &,
+ const Vector<T11> &, const Vector<T12> &,
+ const Vector<T13> &, const Vector<T14> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9, typename T10, typename T11, typename T12, typename T13,
+ typename T14, typename T15> static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &, const Vector<T10> &,
+ const Vector<T11> &, const Vector<T12> &,
+ const Vector<T13> &, const Vector<T14> &,
+ const Vector<T15> &) {}
+ template<typename T1, typename T2, typename T3, typename T4, typename T5, typename T6,
+ typename T7, typename T8, typename T9, typename T10, typename T11, typename T12, typename T13,
+ typename T14, typename T15, typename T16> static inline void forceToRegisters(
+ const Vector<T1> &, const Vector<T2> &,
+ const Vector<T3> &, const Vector<T4> &,
+ const Vector<T5> &, const Vector<T6> &,
+ const Vector<T7> &, const Vector<T8> &,
+ const Vector<T9> &, const Vector<T10> &,
+ const Vector<T11> &, const Vector<T12> &,
+ const Vector<T13> &, const Vector<T14> &,
+ const Vector<T15> &, const Vector<T16> &) {}
+
+} // namespace Scalar
+} // namespace Vc
+
+#include "vector.tcc"
+#include "math.h"
+#include "undomacros.h"
+
+#endif // SCALAR_VECTOR_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+namespace Vc
+{
+ALIGN(64) extern unsigned int RandomState[16];
+
+namespace Scalar
+{
+
+// conversion/casts {{{1
+template<> template<> inline INTRINSIC short_v &Vector<short>::operator=(const ushort_v &x) {
+ data() = static_cast<short>(x.data()); return *this;
+}
+template<> template<> inline INTRINSIC ushort_v &Vector<unsigned short>::operator=(const short_v &x) {
+ data() = static_cast<unsigned short>(x.data()); return *this;
+}
+template<> template<> inline INTRINSIC int_v &Vector<int>::operator=(const uint_v &x) {
+ data() = static_cast<int>(x.data()); return *this;
+}
+template<> template<> inline INTRINSIC uint_v &Vector<unsigned int>::operator=(const int_v &x) {
+ data() = static_cast<unsigned int>(x.data()); return *this;
+}
+
+// copySign ///////////////////////////////////////////////////////////////////////// {{{1
+template<> inline Vector<float> INTRINSIC Vector<float>::copySign(Vector<float> reference) const
+{
+ union {
+ float f;
+ unsigned int i;
+ } value, sign;
+ value.f = data();
+ sign.f = reference.data();
+ value.i = (sign.i & 0x80000000u) | (value.i & 0x7fffffffu);
+ return float_v(value.f);
+}
+template<> inline sfloat_v INTRINSIC Vector<sfloat>::copySign(sfloat_v reference) const
+{
+ return sfloat_v(float_v(m_data).copySign(float_v(reference.data())).data());
+}
+template<> inline Vector<double> INTRINSIC Vector<double>::copySign(Vector<double> reference) const
+{
+ union {
+ double f;
+ unsigned long long i;
+ } value, sign;
+ value.f = data();
+ sign.f = reference.data();
+ value.i = (sign.i & 0x8000000000000000ull) | (value.i & 0x7fffffffffffffffull);
+ return double_v(value.f);
+} // }}}1
+// bitwise operators {{{1
+#define VC_CAST_OPERATOR_FORWARD(op, IntT, VecT) \
+template<> inline VecT &VecT::operator op##=(const VecT &x) { \
+ typedef IntT uinta MAY_ALIAS; \
+ uinta *left = reinterpret_cast<uinta *>(&m_data); \
+ const uinta *right = reinterpret_cast<const uinta *>(&x.m_data); \
+ *left op##= *right; \
+ return *this; \
+} \
+template<> inline VecT VecT::operator op(const VecT &x) const { \
+ VecT ret = *this; \
+ return VecT(ret op##= x); \
+}
+#define VC_CAST_OPERATOR_FORWARD_FLOAT(op) VC_CAST_OPERATOR_FORWARD(op, unsigned int, Vector<float>)
+#define VC_CAST_OPERATOR_FORWARD_SFLOAT(op) VC_CAST_OPERATOR_FORWARD(op, unsigned int, Vector<sfloat>)
+#define VC_CAST_OPERATOR_FORWARD_DOUBLE(op) VC_CAST_OPERATOR_FORWARD(op, unsigned long, Vector<double>)
+VC_ALL_BINARY(VC_CAST_OPERATOR_FORWARD_FLOAT)
+VC_ALL_BINARY(VC_CAST_OPERATOR_FORWARD_SFLOAT)
+VC_ALL_BINARY(VC_CAST_OPERATOR_FORWARD_DOUBLE)
+#undef VC_CAST_OPERATOR_FORWARD
+#undef VC_CAST_OPERATOR_FORWARD_FLOAT
+#undef VC_CAST_OPERATOR_FORWARD_SFLOAT
+#undef VC_CAST_OPERATOR_FORWARD_DOUBLE
+// }}}1
+// operators {{{1
+#include "../common/operators.h"
+// }}}1
+// exponent {{{1
+template<> inline Vector<float> INTRINSIC Vector<float>::exponent() const
+{
+ VC_ASSERT(m_data > 0.f);
+ union { float f; int i; } value;
+ value.f = m_data;
+ return float_v(static_cast<float>((value.i >> 23) - 0x7f));
+}
+template<> inline sfloat_v INTRINSIC Vector<sfloat>::exponent() const
+{
+ return sfloat_v(float_v(m_data).exponent().data());
+}
+template<> inline Vector<double> INTRINSIC Vector<double>::exponent() const
+{
+ VC_ASSERT(m_data > 0.);
+ union { double f; long long i; } value;
+ value.f = m_data;
+ return double_v(static_cast<double>((value.i >> 52) - 0x3ff));
+}
+// }}}1
+// Random {{{1
+static inline ALWAYS_INLINE void _doRandomStep(Vector<unsigned int> &state0,
+ Vector<unsigned int> &state1)
+{
+ state0.load(&Vc::RandomState[0]);
+ state1.load(&Vc::RandomState[uint_v::Size]);
+ (state1 * 0xdeece66du + 11).store(&Vc::RandomState[uint_v::Size]);
+ uint_v((state0 * 0xdeece66du + 11).data() ^ (state1.data() >> 16)).store(&Vc::RandomState[0]);
+}
+
+template<typename T> inline INTRINSIC Vector<T> Vector<T>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ return Vector<T>(static_cast<EntryType>(state0.data()));
+}
+template<> inline INTRINSIC Vector<float> Vector<float>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ union { unsigned int i; float f; } x;
+ x.i = (state0.data() & 0x0fffffffu) | 0x3f800000u;
+ return float_v(x.f - 1.f);
+}
+template<> inline INTRINSIC sfloat_v Vector<sfloat>::Random()
+{
+ return sfloat_v(Vector<float>::Random().data());
+}
+template<> inline INTRINSIC Vector<double> Vector<double>::Random()
+{
+ typedef unsigned long long uint64 MAY_ALIAS;
+ uint64 state0 = *reinterpret_cast<const uint64 *>(&Vc::RandomState[8]);
+ state0 = (state0 * 0x5deece66dull + 11) & 0x000fffffffffffffull;
+ *reinterpret_cast<uint64 *>(&Vc::RandomState[8]) = state0;
+ union { unsigned long long i; double f; } x;
+ x.i = state0 | 0x3ff0000000000000ull;
+ return double_v(x.f - 1.);
+}
+// }}}1
+} // namespace Scalar
+} // namespace Vc
+// vim: foldmethod=marker
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SCALAR_WRITEMASKEDVECTOR_H
+#define VC_SCALAR_WRITEMASKEDVECTOR_H
+
+namespace Vc
+{
+namespace Scalar
+{
+
+template<typename T> class WriteMaskedVector
+{
+ friend class Vector<T>;
+ typedef typename Vector<T>::Mask Mask;
+ typedef typename Vector<T>::EntryType EntryType;
+ public:
+ //prefix
+ inline Vector<T> &operator++() { if (mask) ++vec->m_data; return *vec; }
+ inline Vector<T> &operator--() { if (mask) --vec->m_data; return *vec; }
+ //postfix
+ inline Vector<T> operator++(int) { if (mask) vec->m_data++; return *vec; }
+ inline Vector<T> operator--(int) { if (mask) vec->m_data--; return *vec; }
+
+ inline Vector<T> &operator+=(Vector<T> x) { if (mask) vec->m_data += x.m_data; return *vec; }
+ inline Vector<T> &operator-=(Vector<T> x) { if (mask) vec->m_data -= x.m_data; return *vec; }
+ inline Vector<T> &operator*=(Vector<T> x) { if (mask) vec->m_data *= x.m_data; return *vec; }
+ inline Vector<T> &operator/=(Vector<T> x) { if (mask) vec->m_data /= x.m_data; return *vec; }
+
+ inline Vector<T> &operator=(Vector<T> x) {
+ vec->assign(x, mask);
+ return *vec;
+ }
+
+ inline Vector<T> &operator+=(EntryType x) { if (mask) vec->m_data += x; return *vec; }
+ inline Vector<T> &operator-=(EntryType x) { if (mask) vec->m_data -= x; return *vec; }
+ inline Vector<T> &operator*=(EntryType x) { if (mask) vec->m_data *= x; return *vec; }
+ inline Vector<T> &operator/=(EntryType x) { if (mask) vec->m_data /= x; return *vec; }
+
+ inline Vector<T> &operator=(EntryType x) {
+ vec->assign(Vector<T>(x), mask);
+ return *vec;
+ }
+
+ template<typename F> inline void call(const F &f) const {
+ vec->call(f, mask);
+ }
+ template<typename F> inline void call(F &f) const {
+ vec->call(f, mask);
+ }
+ template<typename F> inline Vector<T> apply(const F &f) const {
+ if (mask) {
+ return Vector<T>(f(vec->m_data));
+ } else {
+ return *vec;
+ }
+ }
+ template<typename F> inline Vector<T> apply(F &f) const {
+ if (mask) {
+ return Vector<T>(f(vec->m_data));
+ } else {
+ return *vec;
+ }
+ }
+ private:
+ WriteMaskedVector(Vector<T> *v, Mask k) : vec(v), mask(k) {}
+ Vector<T> *const vec;
+ Mask mask;
+};
+
+} // namespace Scalar
+} // namespace Vc
+#endif // VC_SCALAR_WRITEMASKEDVECTOR_H
--- /dev/null
+#ifdef __GNUC__
+#warning "Use of the Vc/sfloat_v header is deprecated. The header file will be removed in a future version of Vc."
+#endif
--- /dev/null
+#ifdef __GNUC__
+#warning "Use of the Vc/short_v header is deprecated. The header file will be removed in a future version of Vc."
+#endif
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef SSE_CASTS_H
+#define SSE_CASTS_H
+
+#include "intrinsics.h"
+#include "types.h"
+
+namespace Vc
+{
+namespace SSE
+{
+ template<typename To, typename From> static inline To CONST mm128_reinterpret_cast(From v) { return v; }
+ template<> inline _M128I CONST mm128_reinterpret_cast<_M128I, _M128 >(_M128 v) { return _mm_castps_si128(v); }
+ template<> inline _M128I CONST mm128_reinterpret_cast<_M128I, _M128D>(_M128D v) { return _mm_castpd_si128(v); }
+ template<> inline _M128 CONST mm128_reinterpret_cast<_M128 , _M128D>(_M128D v) { return _mm_castpd_ps(v); }
+ template<> inline _M128 CONST mm128_reinterpret_cast<_M128 , _M128I>(_M128I v) { return _mm_castsi128_ps(v); }
+ template<> inline _M128D CONST mm128_reinterpret_cast<_M128D, _M128I>(_M128I v) { return _mm_castsi128_pd(v); }
+ template<> inline _M128D CONST mm128_reinterpret_cast<_M128D, _M128 >(_M128 v) { return _mm_castps_pd(v); }
+
+ template<typename From, typename To> struct StaticCastHelper {};
+ template<> struct StaticCastHelper<float , int > { static _M128I cast(const _M128 &v) { return _mm_cvttps_epi32(v); } };
+ template<> struct StaticCastHelper<double , int > { static _M128I cast(const _M128D &v) { return _mm_cvttpd_epi32(v); } };
+ template<> struct StaticCastHelper<int , int > { static _M128I cast(const _M128I &v) { return v; } };
+ template<> struct StaticCastHelper<unsigned int, int > { static _M128I cast(const _M128I &v) { return v; } };
+ template<> struct StaticCastHelper<float , unsigned int> { static _M128I cast(const _M128 &v) {
+ return _mm_castps_si128(_mm_blendv_ps(
+ _mm_castsi128_ps(_mm_cvttps_epi32(v)),
+ _mm_castsi128_ps(_mm_add_epi32(_mm_cvttps_epi32(_mm_sub_ps(v, _mm_set1_ps(1u << 31))), _mm_set1_epi32(1 << 31))),
+ _mm_cmpge_ps(v, _mm_set1_ps(1u << 31))
+ ));
+
+ } };
+ template<> struct StaticCastHelper<double , unsigned int> { static _M128I cast(const _M128D &v) { return _mm_cvttpd_epi32(v); } };
+ template<> struct StaticCastHelper<int , unsigned int> { static _M128I cast(const _M128I &v) { return v; } };
+ template<> struct StaticCastHelper<unsigned int, unsigned int> { static _M128I cast(const _M128I &v) { return v; } };
+ template<> struct StaticCastHelper<float , float > { static _M128 cast(const _M128 &v) { return v; } };
+ template<> struct StaticCastHelper<double , float > { static _M128 cast(const _M128D &v) { return _mm_cvtpd_ps(v); } };
+ template<> struct StaticCastHelper<int , float > { static _M128 cast(const _M128I &v) { return _mm_cvtepi32_ps(v); } };
+ template<> struct StaticCastHelper<unsigned int, float > { static _M128 cast(const _M128I &v) {
+ return _mm_blendv_ps(
+ _mm_cvtepi32_ps(v),
+ _mm_add_ps(_mm_cvtepi32_ps(_mm_sub_epi32(v, _mm_set1_epi32(1 << 31))), _mm_set1_ps(1u << 31)),
+ _mm_castsi128_ps(_mm_cmplt_epi32(v, _mm_setzero_si128()))
+ );
+ } };
+ template<> struct StaticCastHelper<float , double > { static _M128D cast(const _M128 &v) { return _mm_cvtps_pd(v); } };
+ template<> struct StaticCastHelper<double , double > { static _M128D cast(const _M128D &v) { return v; } };
+ template<> struct StaticCastHelper<int , double > { static _M128D cast(const _M128I &v) { return _mm_cvtepi32_pd(v); } };
+ template<> struct StaticCastHelper<unsigned int, double > { static _M128D cast(const _M128I &v) { return _mm_cvtepi32_pd(v); } };
+
+ template<> struct StaticCastHelper<unsigned short, float8 > { static M256 cast(const _M128I &v) {
+ return M256::create(_mm_cvtepi32_ps(_mm_unpacklo_epi16(v, _mm_setzero_si128())),
+ _mm_cvtepi32_ps(_mm_unpackhi_epi16(v, _mm_setzero_si128())));
+ } };
+ template<> struct StaticCastHelper<short , float8 > { static M256 cast(const _M128I &v) {
+ const _M128I neg = _mm_cmplt_epi16(v, _mm_setzero_si128());
+ return M256::create(_mm_cvtepi32_ps(_mm_unpacklo_epi16(v, neg)),
+ _mm_cvtepi32_ps(_mm_unpackhi_epi16(v, neg)));
+ } };
+ template<> struct StaticCastHelper<float8 , short > { static _M128I cast(const M256 &v) { return _mm_packs_epi32(_mm_cvttps_epi32(v[0]), _mm_cvttps_epi32(v[1])); } };
+#ifdef VC_IMPL_SSE4_1
+ template<> struct StaticCastHelper<float8 , unsigned short> { static _M128I cast(const M256 &v) { return _mm_packus_epi32(_mm_cvttps_epi32(v[0]), _mm_cvttps_epi32(v[1])); } };
+#else
+ template<> struct StaticCastHelper<float8 , unsigned short> { static _M128I cast(const M256 &v) {
+ return _mm_add_epi16(_mm_set1_epi16(-32768),
+ _mm_packs_epi32(
+ _mm_add_epi32(_mm_set1_epi32(-32768), _mm_cvttps_epi32(v[0])),
+ _mm_add_epi32(_mm_set1_epi32(-32768), _mm_cvttps_epi32(v[1]))
+ )
+ );
+ } };
+#endif
+
+ template<> struct StaticCastHelper<float , short > { static _M128I cast(const _M128 &v) { return _mm_packs_epi32(_mm_cvttps_epi32(v), _mm_setzero_si128()); } };
+ template<> struct StaticCastHelper<short , short > { static _M128I cast(const _M128I &v) { return v; } };
+ template<> struct StaticCastHelper<unsigned short, short > { static _M128I cast(const _M128I &v) { return v; } };
+ template<> struct StaticCastHelper<float , unsigned short> { static _M128I cast(const _M128 &v) { return _mm_packs_epi32(_mm_cvttps_epi32(v), _mm_setzero_si128()); } };
+ template<> struct StaticCastHelper<short , unsigned short> { static _M128I cast(const _M128I &v) { return v; } };
+ template<> struct StaticCastHelper<unsigned short, unsigned short> { static _M128I cast(const _M128I &v) { return v; } };
+} // namespace SSE
+} // namespace Vc
+
+#endif // SSE_CASTS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_CONST_H
+#define VC_SSE_CONST_H
+
+#include "const_data.h"
+#include "vector.h"
+#include "macros.h"
+
+namespace Vc
+{
+namespace SSE
+{
+ template<typename T> class Vector;
+
+ template<typename T> struct Const
+ {
+ typedef Vector<T> V;
+ typedef typename V::Mask M;
+
+ static inline ALWAYS_INLINE_L CONST_L V _1_2pi() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[0 * V::Size]); }
+ static inline ALWAYS_INLINE_L CONST_L V _2pi() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[1 * V::Size]); }
+ static inline ALWAYS_INLINE_L CONST_L V _pi_2() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[2 * V::Size]); }
+ static inline ALWAYS_INLINE_L CONST_L V _pi() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[3 * V::Size]); }
+ static inline ALWAYS_INLINE_L CONST_L V _1_3fac() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[4 * V::Size]); }
+ static inline ALWAYS_INLINE_L CONST_L V _1_5fac() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[5 * V::Size]); }
+ static inline ALWAYS_INLINE_L CONST_L V _1_7fac() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[6 * V::Size]); }
+ static inline ALWAYS_INLINE_L CONST_L V _1_9fac() ALWAYS_INLINE_R CONST_R { return V(&c_sin<T>::data[7 * V::Size]); }
+
+ static inline ALWAYS_INLINE_L CONST_L M exponentMask() ALWAYS_INLINE_R CONST_R { return M(V(c_log<T>::d(1)).data()); }
+ static inline ALWAYS_INLINE_L CONST_L V _1_2() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(18)); }
+ static inline ALWAYS_INLINE_L CONST_L V _1_sqrt2() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(15)); }
+ static inline ALWAYS_INLINE_L CONST_L V P(int i) ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(2 + i)); }
+ static inline ALWAYS_INLINE_L CONST_L V Q(int i) ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(8 + i)); }
+ static inline ALWAYS_INLINE_L CONST_L V min() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(14)); }
+ static inline ALWAYS_INLINE_L CONST_L V ln2_small() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(17)); }
+ static inline ALWAYS_INLINE_L CONST_L V ln2_large() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(16)); }
+ static inline ALWAYS_INLINE_L CONST_L V neginf() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(13)); }
+ static inline ALWAYS_INLINE_L CONST_L V log10_e() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(19)); }
+ static inline ALWAYS_INLINE_L CONST_L V log2_e() ALWAYS_INLINE_R CONST_R { return V(c_log<T>::d(20)); }
+ };
+#define VC_FLOAT8_CONST_IMPL(name) \
+ template<> inline ALWAYS_INLINE CONST Vector<float8> Const<float8>::name() { \
+ return M256::dup(Const<float>::name().data()); \
+ }
+ VC_FLOAT8_CONST_IMPL(_1_2pi)
+ VC_FLOAT8_CONST_IMPL(_2pi)
+ VC_FLOAT8_CONST_IMPL(_pi_2)
+ VC_FLOAT8_CONST_IMPL(_pi)
+ VC_FLOAT8_CONST_IMPL(_1_3fac)
+ VC_FLOAT8_CONST_IMPL(_1_5fac)
+ VC_FLOAT8_CONST_IMPL(_1_7fac)
+ VC_FLOAT8_CONST_IMPL(_1_9fac)
+ VC_FLOAT8_CONST_IMPL(_1_2)
+ VC_FLOAT8_CONST_IMPL(_1_sqrt2)
+ VC_FLOAT8_CONST_IMPL(min)
+ VC_FLOAT8_CONST_IMPL(ln2_small)
+ VC_FLOAT8_CONST_IMPL(ln2_large)
+ VC_FLOAT8_CONST_IMPL(neginf)
+ VC_FLOAT8_CONST_IMPL(log10_e)
+ VC_FLOAT8_CONST_IMPL(log2_e)
+ template<> inline ALWAYS_INLINE CONST Vector<float8> Const<float8>::P(int i) {
+ return M256::dup(Const<float>::P(i).data());
+ }
+ template<> inline ALWAYS_INLINE CONST Vector<float8> Const<float8>::Q(int i) {
+ return M256::dup(Const<float>::Q(i).data());
+ }
+ template<> inline ALWAYS_INLINE CONST Vector<float8>::Mask Const<float8>::exponentMask() {
+ return M256::dup(Const<float>::exponentMask().data());
+ }
+#undef VC_FLOAT8_CONST_IMPL
+} // namespace SSE
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_SSE_CONST_H
--- /dev/null
+/* This file is part of the Vc library. {{{
+
+ Copyright (C) 2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+}}}*/
+
+#ifndef VC_SSE_CONST_DATA_H
+#define VC_SSE_CONST_DATA_H
+
+#include "macros.h"
+
+namespace Vc
+{
+namespace SSE
+{
+
+ALIGN(16) extern const unsigned int _IndexesFromZero4[4];
+ALIGN(16) extern const unsigned short _IndexesFromZero8[8];
+ALIGN(16) extern const unsigned char _IndexesFromZero16[16];
+
+struct c_general
+{
+ ALIGN(64) static const unsigned int allone[4];
+ ALIGN(16) static const unsigned short one16[8];
+ ALIGN(16) static const unsigned int one32[4];
+ ALIGN(16) static const float oneFloat[4];
+ ALIGN(16) static const double oneDouble[2];
+ ALIGN(16) static const int absMaskFloat[4];
+ ALIGN(16) static const long long absMaskDouble[2];
+ ALIGN(16) static const unsigned int signMaskFloat[4];
+ ALIGN(16) static const unsigned long long signMaskDouble[2];
+ ALIGN(16) static const short minShort[8];
+ ALIGN(16) static const unsigned long long frexpMask[2];
+};
+
+template<typename T> struct c_sin
+{
+ ALIGN(64) static const T data[];
+};
+
+template<typename T> struct c_log
+{
+ enum VectorSize { Size = 16 / sizeof(T) };
+ static inline ALWAYS_INLINE CONST const float *d(int i) { return reinterpret_cast<const float *>(&data[i * Size]); }
+ ALIGN(64) static const unsigned int data[];
+};
+
+template<> struct c_log<double>
+{
+ enum VectorSize { Size = 16 / sizeof(double) };
+ static inline ALWAYS_INLINE CONST const double *d(int i) { return reinterpret_cast<const double *>(&data[i * Size]); }
+ ALIGN(64) static const unsigned long long data[];
+};
+
+} // namespace SSE
+} // namespace Vc
+
+#include "undomacros.h"
+
+#endif // VC_SSE_CONST_DATA_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_DEBUG_H
+#define VC_SSE_DEBUG_H
+
+#ifndef NDEBUG
+#include "types.h"
+#include <iostream>
+#include <iomanip>
+#endif
+
+namespace Vc
+{
+namespace SSE
+{
+
+#ifdef NDEBUG
+class DebugStream
+{
+ public:
+ DebugStream(const char *, const char *, int) {}
+ template<typename T> inline DebugStream &operator<<(const T &) { return *this; }
+};
+#else
+class DebugStream
+{
+ private:
+ template<typename T, typename V> static void printVector(V _x)
+ {
+ enum { Size = sizeof(V) / sizeof(T) };
+ union { V v; T m[Size]; } x = { _x };
+ std::cerr << '[' << std::setprecision(24) << x.m[0];
+ for (int i = 1; i < Size; ++i) {
+ std::cerr << ", " << std::setprecision(24) << x.m[i];
+ }
+ std::cerr << ']';
+ }
+ public:
+ DebugStream(const char *func, const char *file, int line)
+ {
+ std::cerr << "\033[1;40;33mDEBUG: " << file << ':' << line << ' ' << func << ' ';
+ }
+
+ template<typename T> DebugStream &operator<<(const T &x) { std::cerr << x; return *this; }
+
+ DebugStream &operator<<(__m128 x) {
+ printVector<float, __m128>(x);
+ return *this;
+ }
+ DebugStream &operator<<(__m128d x) {
+ printVector<double, __m128d>(x);
+ return *this;
+ }
+ DebugStream &operator<<(__m128i x) {
+ printVector<unsigned int, __m128i>(x);
+ return *this;
+ }
+
+ ~DebugStream()
+ {
+ std::cerr << "\033[0m" << std::endl;
+ }
+};
+#endif
+
+#define VC_DEBUG ::Vc::SSE::DebugStream(__PRETTY_FUNCTION__, __FILE__, __LINE__)
+
+} // namespace SSE
+} // namespace Vc
+
+#endif // VC_SSE_DEBUG_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+namespace Vc
+{
+namespace SSE
+{
+
+inline void deinterleave(Vector<float> &a, Vector<float> &b)
+{
+ const _M128 tmp0 = _mm_unpacklo_ps(a.data(), b.data());
+ const _M128 tmp1 = _mm_unpackhi_ps(a.data(), b.data());
+ a.data() = _mm_unpacklo_ps(tmp0, tmp1);
+ b.data() = _mm_unpackhi_ps(tmp0, tmp1);
+}
+
+inline void deinterleave(Vector<float> &a, Vector<float> &b, Vector<short>::AsArg tmp)
+{
+ a.data() = _mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(tmp.data(), 16), 16));
+ b.data() = _mm_cvtepi32_ps(_mm_srai_epi32(tmp.data(), 16));
+}
+
+inline void deinterleave(Vector<float> &a, Vector<float> &b, Vector<unsigned short>::AsArg tmp)
+{
+ a.data() = _mm_cvtepi32_ps(_mm_srli_epi32(_mm_slli_epi32(tmp.data(), 16), 16));
+ b.data() = _mm_cvtepi32_ps(_mm_srli_epi32(tmp.data(), 16));
+}
+
+inline void deinterleave(Vector<float8> &a, Vector<float8> &b)
+{
+ _M128 tmp0 = _mm_unpacklo_ps(a.data()[0], a.data()[1]);
+ _M128 tmp1 = _mm_unpackhi_ps(a.data()[0], a.data()[1]);
+ _M128 tmp2 = _mm_unpacklo_ps(b.data()[0], b.data()[1]);
+ _M128 tmp3 = _mm_unpackhi_ps(b.data()[0], b.data()[1]);
+ a.data()[0] = _mm_unpacklo_ps(tmp0, tmp1);
+ b.data()[0] = _mm_unpackhi_ps(tmp0, tmp1);
+ a.data()[1] = _mm_unpacklo_ps(tmp2, tmp3);
+ b.data()[1] = _mm_unpackhi_ps(tmp2, tmp3);
+}
+
+inline void deinterleave(Vector<float8> &a, Vector<float8> &b, Vector<short>::AsArg tmp0, Vector<short>::AsArg tmp1)
+{
+ a.data()[0] = _mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(tmp0.data(), 16), 16));
+ b.data()[0] = _mm_cvtepi32_ps(_mm_srai_epi32(tmp0.data(), 16));
+ a.data()[1] = _mm_cvtepi32_ps(_mm_srai_epi32(_mm_slli_epi32(tmp1.data(), 16), 16));
+ b.data()[1] = _mm_cvtepi32_ps(_mm_srai_epi32(tmp1.data(), 16));
+}
+
+inline void deinterleave(Vector<float8> &a, Vector<float8> &b, Vector<unsigned short>::AsArg tmp0, Vector<unsigned short>::AsArg tmp1)
+{
+ a.data()[0] = _mm_cvtepi32_ps(_mm_srli_epi32(_mm_slli_epi32(tmp0.data(), 16), 16));
+ b.data()[0] = _mm_cvtepi32_ps(_mm_srli_epi32(tmp0.data(), 16));
+ a.data()[1] = _mm_cvtepi32_ps(_mm_srli_epi32(_mm_slli_epi32(tmp1.data(), 16), 16));
+ b.data()[1] = _mm_cvtepi32_ps(_mm_srli_epi32(tmp1.data(), 16));
+}
+
+inline void deinterleave(Vector<double> &a, Vector<double> &b)
+{
+ _M128D tmp = _mm_unpacklo_pd(a.data(), b.data());
+ b.data() = _mm_unpackhi_pd(a.data(), b.data());
+ a.data() = tmp;
+}
+
+inline void deinterleave(Vector<int> &a, Vector<int> &b)
+{
+ const _M128I tmp0 = _mm_unpacklo_epi32(a.data(), b.data());
+ const _M128I tmp1 = _mm_unpackhi_epi32(a.data(), b.data());
+ a.data() = _mm_unpacklo_epi32(tmp0, tmp1);
+ b.data() = _mm_unpackhi_epi32(tmp0, tmp1);
+}
+
+inline void deinterleave(Vector<unsigned int> &a, Vector<unsigned int> &b)
+{
+ const _M128I tmp0 = _mm_unpacklo_epi32(a.data(), b.data());
+ const _M128I tmp1 = _mm_unpackhi_epi32(a.data(), b.data());
+ a.data() = _mm_unpacklo_epi32(tmp0, tmp1);
+ b.data() = _mm_unpackhi_epi32(tmp0, tmp1);
+}
+
+inline void deinterleave(Vector<short> &a, Vector<short> &b)
+{
+ _M128I tmp0 = _mm_unpacklo_epi16(a.data(), b.data()); // a0 a4 b0 b4 a1 a5 b1 b5
+ _M128I tmp1 = _mm_unpackhi_epi16(a.data(), b.data()); // a2 a6 b2 b6 a3 a7 b3 b7
+ _M128I tmp2 = _mm_unpacklo_epi16(tmp0, tmp1); // a0 a2 a4 a6 b0 b2 b4 b6
+ _M128I tmp3 = _mm_unpackhi_epi16(tmp0, tmp1); // a1 a3 a5 a7 b1 b3 b5 b7
+ a.data() = _mm_unpacklo_epi16(tmp2, tmp3);
+ b.data() = _mm_unpackhi_epi16(tmp2, tmp3);
+}
+
+inline void deinterleave(Vector<unsigned short> &a, Vector<unsigned short> &b)
+{
+ _M128I tmp0 = _mm_unpacklo_epi16(a.data(), b.data()); // a0 a4 b0 b4 a1 a5 b1 b5
+ _M128I tmp1 = _mm_unpackhi_epi16(a.data(), b.data()); // a2 a6 b2 b6 a3 a7 b3 b7
+ _M128I tmp2 = _mm_unpacklo_epi16(tmp0, tmp1); // a0 a2 a4 a6 b0 b2 b4 b6
+ _M128I tmp3 = _mm_unpackhi_epi16(tmp0, tmp1); // a1 a3 a5 a7 b1 b3 b5 b7
+ a.data() = _mm_unpacklo_epi16(tmp2, tmp3);
+ b.data() = _mm_unpackhi_epi16(tmp2, tmp3);
+}
+
+inline void deinterleave(Vector<int> &a, Vector<int> &b, Vector<short>::AsArg tmp)
+{
+ a.data() = _mm_srai_epi32(_mm_slli_epi32(tmp.data(), 16), 16);
+ b.data() = _mm_srai_epi32(tmp.data(), 16);
+}
+
+inline void deinterleave(Vector<unsigned int> &a, Vector<unsigned int> &b, Vector<unsigned short>::AsArg tmp)
+{
+ a.data() = _mm_srli_epi32(_mm_slli_epi32(tmp.data(), 16), 16);
+ b.data() = _mm_srli_epi32(tmp.data(), 16);
+}
+
+} // namespace SSE
+
+
+namespace Internal
+{
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ float_v &a, float_v &b, const float *m, A align)
+{
+ a.load(m, align);
+ b.load(m + float_v::Size, align);
+ Vc::SSE::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ float_v &a, float_v &b, const short *m, A align)
+{
+ short_v tmp(m, align);
+ Vc::SSE::deinterleave(a, b, tmp);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ float_v &a, float_v &b, const unsigned short *m, A align)
+{
+ ushort_v tmp(m, align);
+ Vc::SSE::deinterleave(a, b, tmp);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ sfloat_v &a, sfloat_v &b, const float *m, A align)
+{
+ a.load(m, align);
+ b.load(m + sfloat_v::Size, align);
+ Vc::SSE::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ sfloat_v &a, sfloat_v &b, const short *m, A align)
+{
+ short_v tmp0(m, align);
+ short_v tmp1(m + short_v::Size, align);
+ Vc::SSE::deinterleave(a, b, tmp0, tmp1);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ sfloat_v &a, sfloat_v &b, const unsigned short *m, A align)
+{
+ ushort_v tmp0(m, align);
+ ushort_v tmp1(m + short_v::Size, align);
+ Vc::SSE::deinterleave(a, b, tmp0, tmp1);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ double_v &a, double_v &b, const double *m, A align)
+{
+ a.load(m, align);
+ b.load(m + double_v::Size, align);
+ Vc::SSE::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ int_v &a, int_v &b, const int *m, A align)
+{
+ a.load(m, align);
+ b.load(m + int_v::Size, align);
+ Vc::SSE::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ int_v &a, int_v &b, const short *m, A align)
+{
+ short_v tmp(m, align);
+ Vc::SSE::deinterleave(a, b, tmp);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ uint_v &a, uint_v &b, const unsigned int *m, A align)
+{
+ a.load(m, align);
+ b.load(m + uint_v::Size, align);
+ Vc::SSE::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ uint_v &a, uint_v &b, const unsigned short *m, A align)
+{
+ ushort_v tmp(m, align);
+ Vc::SSE::deinterleave(a, b, tmp);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ short_v &a, short_v &b, const short *m, A align)
+{
+ a.load(m, align);
+ b.load(m + short_v::Size, align);
+ Vc::SSE::deinterleave(a, b);
+}
+
+template<typename A> inline void HelperImpl<Vc::SSE2Impl>::deinterleave(
+ ushort_v &a, ushort_v &b, const unsigned short *m, A align)
+{
+ a.load(m, align);
+ b.load(m + ushort_v::Size, align);
+ Vc::SSE::deinterleave(a, b);
+}
+
+} // namespace Internal
+} // namespace Vc
--- /dev/null
+#ifdef VC_GNU_ASM
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x1.data()));
+}
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x1.data()));
+}
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x2.data()), "x"(x1.data()));
+}
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T6> &x6, const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x6.data()), "x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T6> &x6, Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x6.data()), "+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T7> &x7, const Vector<T6> &x6, const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x7.data()), "x"(x6.data()), "x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T7> &x7, Vector<T6> &x6, Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x7.data()), "+x"(x6.data()), "+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T8> &x8, const Vector<T7> &x7, const Vector<T6> &x6, const Vector<T5> &x5, const Vector<T4> &x4, const Vector<T3> &x3, const Vector<T2> &x2, const Vector<T1> &x1) {
+ __asm__ __volatile__(""::"x"(x8.data()), "x"(x7.data()), "x"(x6.data()), "x"(x5.data()), "x"(x4.data()), "x"(x3.data()), "x"(x2.data()), "x"(x1.data()));
+}
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T8> &x8, Vector<T7> &x7, Vector<T6> &x6, Vector<T5> &x5, Vector<T4> &x4, Vector<T3> &x3, Vector<T2> &x2, Vector<T1> &x1) {
+ __asm__ __volatile__("":"+x"(x8.data()), "+x"(x7.data()), "+x"(x6.data()), "+x"(x5.data()), "+x"(x4.data()), "+x"(x3.data()), "+x"(x2.data()), "+x"(x1.data()));
+}
+#elif defined(VC_MSVC)
+#pragma optimize("g", off)
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T6> &/*x6*/, const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T6> &/*x6*/, Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T7> &/*x7*/, const Vector<T6> &/*x6*/, const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T7> &/*x7*/, Vector<T6> &/*x6*/, Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#pragma optimize("g", off)
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegisters(const Vector<T8> &/*x8*/, const Vector<T7> &/*x7*/, const Vector<T6> &/*x6*/, const Vector<T5> &/*x5*/, const Vector<T4> &/*x4*/, const Vector<T3> &/*x3*/, const Vector<T2> &/*x2*/, const Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", off)
+template<typename T8, typename T7, typename T6, typename T5, typename T4, typename T3, typename T2, typename T1>
+static inline void ALWAYS_INLINE forceToRegistersDirty(Vector<T8> &/*x8*/, Vector<T7> &/*x7*/, Vector<T6> &/*x6*/, Vector<T5> &/*x5*/, Vector<T4> &/*x4*/, Vector<T3> &/*x3*/, Vector<T2> &/*x2*/, Vector<T1> &/*x1*/) {
+}
+#pragma optimize("g", on)
+#else
+#error "forceToRegisters unsupported on this compiler"
+#endif
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_DEINTERLEAVE_H
+#define VC_SSE_DEINTERLEAVE_H
+
+#include "macros.h"
+
+namespace Vc
+{
+namespace Internal
+{
+
+template<> struct HelperImpl<Vc::SSE2Impl>
+{
+ typedef SSE::Vector<float> float_v;
+ typedef SSE::Vector<SSE::float8> sfloat_v;
+ typedef SSE::Vector<double> double_v;
+ typedef SSE::Vector<int> int_v;
+ typedef SSE::Vector<unsigned int> uint_v;
+ typedef SSE::Vector<short> short_v;
+ typedef SSE::Vector<unsigned short> ushort_v;
+
+ template<typename A> static void deinterleave(float_v &, float_v &, const float *, A);
+ template<typename A> static void deinterleave(float_v &, float_v &, const short *, A);
+ template<typename A> static void deinterleave(float_v &, float_v &, const unsigned short *, A);
+
+ template<typename A> static void deinterleave(sfloat_v &, sfloat_v &, const float *, A);
+ template<typename A> static void deinterleave(sfloat_v &, sfloat_v &, const short *, A);
+ template<typename A> static void deinterleave(sfloat_v &, sfloat_v &, const unsigned short *, A);
+
+ template<typename A> static void deinterleave(double_v &, double_v &, const double *, A);
+
+ template<typename A> static void deinterleave(int_v &, int_v &, const int *, A);
+ template<typename A> static void deinterleave(int_v &, int_v &, const short *, A);
+
+ template<typename A> static void deinterleave(uint_v &, uint_v &, const unsigned int *, A);
+ template<typename A> static void deinterleave(uint_v &, uint_v &, const unsigned short *, A);
+
+ template<typename A> static void deinterleave(short_v &, short_v &, const short *, A);
+
+ template<typename A> static void deinterleave(ushort_v &, ushort_v &, const unsigned short *, A);
+
+ static inline ALWAYS_INLINE_L void prefetchForOneRead(const void *addr) ALWAYS_INLINE_R;
+ static inline ALWAYS_INLINE_L void prefetchForModify(const void *addr) ALWAYS_INLINE_R;
+ static inline ALWAYS_INLINE_L void prefetchClose(const void *addr) ALWAYS_INLINE_R;
+ static inline ALWAYS_INLINE_L void prefetchMid(const void *addr) ALWAYS_INLINE_R;
+ static inline ALWAYS_INLINE_L void prefetchFar(const void *addr) ALWAYS_INLINE_R;
+
+ template<Vc::MallocAlignment A>
+ static inline ALWAYS_INLINE_L void *malloc(size_t n) ALWAYS_INLINE_R;
+ static inline ALWAYS_INLINE_L void free(void *p) ALWAYS_INLINE_R;
+};
+
+template<> struct HelperImpl<SSE3Impl> : public HelperImpl<SSE2Impl> {};
+template<> struct HelperImpl<SSSE3Impl> : public HelperImpl<SSE3Impl> {};
+template<> struct HelperImpl<SSE41Impl> : public HelperImpl<SSSE3Impl> {};
+template<> struct HelperImpl<SSE42Impl> : public HelperImpl<SSE41Impl> {};
+template<> struct HelperImpl<SSE4aImpl> : public HelperImpl<SSE3Impl> {};
+
+
+} // namespace Internal
+} // namespace Vc
+
+#include "undomacros.h"
+#include "deinterleave.tcc"
+#include "prefetches.tcc"
+#include "helperimpl.tcc"
+
+#endif // VC_SSE_DEINTERLEAVE_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_HELPERIMPL_TCC
+#define VC_SSE_HELPERIMPL_TCC
+
+#include "intrinsics.h"
+#include <cstdio>
+
+namespace Vc
+{
+namespace Internal
+{
+
+template<size_t X>
+static inline size_t nextMultipleOf(size_t value)
+{
+ const size_t offset = value % X;
+ if ( offset > 0 ) {
+ return value + X - offset;
+ }
+ return value;
+}
+
+template<Vc::MallocAlignment A>
+inline void *HelperImpl<SSE2Impl>::malloc(size_t n)
+{
+ switch (A) {
+ case Vc::AlignOnVector:
+ return _mm_malloc(nextMultipleOf<Vc::SSE::VectorAlignment>(n), Vc::SSE::VectorAlignment);
+ case Vc::AlignOnCacheline:
+ // TODO: hardcoding 64 is not such a great idea
+ return _mm_malloc(nextMultipleOf<64>(n), 64);
+ case Vc::AlignOnPage:
+ // TODO: hardcoding 4096 is not such a great idea
+ return _mm_malloc(nextMultipleOf<4096>(n), 4096);
+ default:
+#ifndef NDEBUG
+ abort();
+#endif
+ return _mm_malloc(n, 8);
+ }
+}
+
+inline void HelperImpl<SSE2Impl>::free(void *p)
+{
+ _mm_free(p);
+}
+
+} // namespace Internal
+} // namespace Vc
+
+#endif // VC_SSE_HELPERIMPL_TCC
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef SSE_INTRINSICS_H
+#define SSE_INTRINSICS_H
+
+#include "../common/windows_fix_intrin.h"
+
+// MMX
+#include <mmintrin.h>
+// SSE
+#include <xmmintrin.h>
+// SSE2
+#include <emmintrin.h>
+
+#if defined(__GNUC__) && !defined(VC_IMPL_SSE2)
+#error "SSE Vector class needs at least SSE2"
+#endif
+
+#include "const_data.h"
+#include "macros.h"
+#include <cstdlib>
+
+#ifdef __3dNOW__
+#include <mm3dnow.h>
+#endif
+
+namespace Vc
+{
+namespace SSE
+{
+ enum VectorAlignmentEnum { VectorAlignment = 16 };
+
+#if defined(VC_GCC) && VC_GCC < 0x40600 && !defined(VC_DONT_FIX_SSE_SHIFT)
+ static inline __m128i CONST _mm_sll_epi16(__m128i a, __m128i count) { __asm__("psllw %1,%0" : "+x"(a) : "x"(count)); return a; }
+ static inline __m128i CONST _mm_sll_epi32(__m128i a, __m128i count) { __asm__("pslld %1,%0" : "+x"(a) : "x"(count)); return a; }
+ static inline __m128i CONST _mm_sll_epi64(__m128i a, __m128i count) { __asm__("psllq %1,%0" : "+x"(a) : "x"(count)); return a; }
+ static inline __m128i CONST _mm_srl_epi16(__m128i a, __m128i count) { __asm__("psrlw %1,%0" : "+x"(a) : "x"(count)); return a; }
+ static inline __m128i CONST _mm_srl_epi32(__m128i a, __m128i count) { __asm__("psrld %1,%0" : "+x"(a) : "x"(count)); return a; }
+ static inline __m128i CONST _mm_srl_epi64(__m128i a, __m128i count) { __asm__("psrlq %1,%0" : "+x"(a) : "x"(count)); return a; }
+#endif
+
+#if defined(VC_GNU_ASM) && !defined(NVALGRIND)
+ static inline __m128i CONST _mm_setallone() { __m128i r; __asm__("pcmpeqb %0,%0":"=x"(r)); return r; }
+#else
+ static inline __m128i CONST _mm_setallone() { __m128i r = _mm_setzero_si128(); return _mm_cmpeq_epi8(r, r); }
+#endif
+ static inline __m128i CONST _mm_setallone_si128() { return _mm_setallone(); }
+ static inline __m128d CONST _mm_setallone_pd() { return _mm_castsi128_pd(_mm_setallone()); }
+ static inline __m128 CONST _mm_setallone_ps() { return _mm_castsi128_ps(_mm_setallone()); }
+
+ static inline __m128i CONST _mm_setone_epi8 () { return _mm_set1_epi8(1); }
+ static inline __m128i CONST _mm_setone_epu8 () { return _mm_setone_epi8(); }
+ static inline __m128i CONST _mm_setone_epi16() { return _mm_load_si128(reinterpret_cast<const __m128i *>(c_general::one16)); }
+ static inline __m128i CONST _mm_setone_epu16() { return _mm_setone_epi16(); }
+ static inline __m128i CONST _mm_setone_epi32() { return _mm_load_si128(reinterpret_cast<const __m128i *>(c_general::one32)); }
+ static inline __m128i CONST _mm_setone_epu32() { return _mm_setone_epi32(); }
+
+ static inline __m128 CONST _mm_setone_ps() { return _mm_load_ps(c_general::oneFloat); }
+ static inline __m128d CONST _mm_setone_pd() { return _mm_load_pd(c_general::oneDouble); }
+
+ static inline __m128d CONST _mm_setabsmask_pd() { return _mm_load_pd(reinterpret_cast<const double *>(c_general::absMaskDouble)); }
+ static inline __m128 CONST _mm_setabsmask_ps() { return _mm_load_ps(reinterpret_cast<const float *>(c_general::absMaskFloat)); }
+ static inline __m128d CONST _mm_setsignmask_pd(){ return _mm_load_pd(reinterpret_cast<const double *>(c_general::signMaskDouble)); }
+ static inline __m128 CONST _mm_setsignmask_ps(){ return _mm_load_ps(reinterpret_cast<const float *>(c_general::signMaskFloat)); }
+
+ //X static inline __m128i CONST _mm_setmin_epi8 () { return _mm_slli_epi8 (_mm_setallone_si128(), 7); }
+ static inline __m128i CONST _mm_setmin_epi16() { return _mm_load_si128(reinterpret_cast<const __m128i *>(c_general::minShort)); }
+ static inline __m128i CONST _mm_setmin_epi32() { return _mm_load_si128(reinterpret_cast<const __m128i *>(c_general::signMaskFloat)); }
+
+ //X static inline __m128i CONST _mm_cmplt_epu8 (__m128i a, __m128i b) { return _mm_cmplt_epi8 (
+ //X _mm_xor_si128(a, _mm_setmin_epi8 ()), _mm_xor_si128(b, _mm_setmin_epi8 ())); }
+ //X static inline __m128i CONST _mm_cmpgt_epu8 (__m128i a, __m128i b) { return _mm_cmpgt_epi8 (
+ //X _mm_xor_si128(a, _mm_setmin_epi8 ()), _mm_xor_si128(b, _mm_setmin_epi8 ())); }
+ static inline __m128i CONST _mm_cmplt_epu16(__m128i a, __m128i b) { return _mm_cmplt_epi16(
+ _mm_xor_si128(a, _mm_setmin_epi16()), _mm_xor_si128(b, _mm_setmin_epi16())); }
+ static inline __m128i CONST _mm_cmpgt_epu16(__m128i a, __m128i b) { return _mm_cmpgt_epi16(
+ _mm_xor_si128(a, _mm_setmin_epi16()), _mm_xor_si128(b, _mm_setmin_epi16())); }
+ static inline __m128i CONST _mm_cmplt_epu32(__m128i a, __m128i b) { return _mm_cmplt_epi32(
+ _mm_xor_si128(a, _mm_setmin_epi32()), _mm_xor_si128(b, _mm_setmin_epi32())); }
+ static inline __m128i CONST _mm_cmpgt_epu32(__m128i a, __m128i b) { return _mm_cmpgt_epi32(
+ _mm_xor_si128(a, _mm_setmin_epi32()), _mm_xor_si128(b, _mm_setmin_epi32())); }
+} // namespace SSE
+} // namespace Vc
+
+// SSE3
+#ifdef VC_IMPL_SSE3
+#include <pmmintrin.h>
+#elif defined _PMMINTRIN_H_INCLUDED
+#error "SSE3 was disabled but something includes <pmmintrin.h>. Please fix your code."
+#endif
+// SSSE3
+#ifdef VC_IMPL_SSSE3
+#include <tmmintrin.h>
+namespace Vc
+{
+namespace SSE
+{
+
+ // not overriding _mm_set1_epi8 because this one should only be used for non-constants
+ static inline __m128i CONST set1_epi8(int a) {
+#if defined(VC_GCC) && VC_GCC < 0x40500
+ return _mm_shuffle_epi8(_mm_cvtsi32_si128(a), _mm_setzero_si128());
+#else
+ // GCC 4.5 nows about the pshufb improvement
+ return _mm_set1_epi8(a);
+#endif
+ }
+
+} // namespace SSE
+} // namespace Vc
+#elif defined _TMMINTRIN_H_INCLUDED
+#error "SSSE3 was disabled but something includes <tmmintrin.h>. Please fix your code."
+#else
+namespace Vc
+{
+namespace SSE
+{
+ static inline __m128i CONST _mm_abs_epi8 (__m128i a) {
+ __m128i negative = _mm_cmplt_epi8 (a, _mm_setzero_si128());
+ return _mm_add_epi8 (_mm_xor_si128(a, negative), _mm_and_si128(negative, _mm_setone_epi8()));
+ }
+ // positive value:
+ // negative == 0
+ // a unchanged after xor
+ // 0 >> 31 -> 0
+ // a + 0 -> a
+ // negative value:
+ // negative == -1
+ // a xor -1 -> -a - 1
+ // -1 >> 31 -> 1
+ // -a - 1 + 1 -> -a
+ static inline __m128i CONST _mm_abs_epi16(__m128i a) {
+ __m128i negative = _mm_cmplt_epi16(a, _mm_setzero_si128());
+ return _mm_add_epi16(_mm_xor_si128(a, negative), _mm_srli_epi16(negative, 15));
+ }
+ static inline __m128i CONST _mm_abs_epi32(__m128i a) {
+ __m128i negative = _mm_cmplt_epi32(a, _mm_setzero_si128());
+ return _mm_add_epi32(_mm_xor_si128(a, negative), _mm_srli_epi32(negative, 31));
+ }
+ static inline __m128i CONST set1_epi8(int a) {
+ return _mm_set1_epi8(a);
+ }
+ static inline __m128i CONST _mm_alignr_epi8(__m128i a, __m128i b, const int s) {
+ switch (s) {
+ case 0: return b;
+ case 1: return _mm_or_si128(_mm_slli_si128(a, 15), _mm_srli_si128(b, 1));
+ case 2: return _mm_or_si128(_mm_slli_si128(a, 14), _mm_srli_si128(b, 2));
+ case 3: return _mm_or_si128(_mm_slli_si128(a, 13), _mm_srli_si128(b, 3));
+ case 4: return _mm_or_si128(_mm_slli_si128(a, 12), _mm_srli_si128(b, 4));
+ case 5: return _mm_or_si128(_mm_slli_si128(a, 11), _mm_srli_si128(b, 5));
+ case 6: return _mm_or_si128(_mm_slli_si128(a, 10), _mm_srli_si128(b, 6));
+ case 7: return _mm_or_si128(_mm_slli_si128(a, 9), _mm_srli_si128(b, 7));
+ case 8: return _mm_or_si128(_mm_slli_si128(a, 8), _mm_srli_si128(b, 8));
+ case 9: return _mm_or_si128(_mm_slli_si128(a, 7), _mm_srli_si128(b, 9));
+ case 10: return _mm_or_si128(_mm_slli_si128(a, 6), _mm_srli_si128(b, 10));
+ case 11: return _mm_or_si128(_mm_slli_si128(a, 5), _mm_srli_si128(b, 11));
+ case 12: return _mm_or_si128(_mm_slli_si128(a, 4), _mm_srli_si128(b, 12));
+ case 13: return _mm_or_si128(_mm_slli_si128(a, 3), _mm_srli_si128(b, 13));
+ case 14: return _mm_or_si128(_mm_slli_si128(a, 2), _mm_srli_si128(b, 14));
+ case 15: return _mm_or_si128(_mm_slli_si128(a, 1), _mm_srli_si128(b, 15));
+ case 16: return a;
+ case 17: return _mm_srli_si128(a, 1);
+ case 18: return _mm_srli_si128(a, 2);
+ case 19: return _mm_srli_si128(a, 3);
+ case 20: return _mm_srli_si128(a, 4);
+ case 21: return _mm_srli_si128(a, 5);
+ case 22: return _mm_srli_si128(a, 6);
+ case 23: return _mm_srli_si128(a, 7);
+ case 24: return _mm_srli_si128(a, 8);
+ case 25: return _mm_srli_si128(a, 9);
+ case 26: return _mm_srli_si128(a, 10);
+ case 27: return _mm_srli_si128(a, 11);
+ case 28: return _mm_srli_si128(a, 12);
+ case 29: return _mm_srli_si128(a, 13);
+ case 30: return _mm_srli_si128(a, 14);
+ case 31: return _mm_srli_si128(a, 15);
+ }
+ return _mm_setzero_si128();
+ }
+
+} // namespace SSE
+} // namespace Vc
+
+#endif
+
+// SSE4.1
+#ifdef VC_IMPL_SSE4_1
+#include <smmintrin.h>
+#else
+#ifdef _SMMINTRIN_H_INCLUDED
+#error "SSE4.1 was disabled but something includes <smmintrin.h>. Please fix your code."
+#endif
+namespace Vc
+{
+namespace SSE
+{
+ static inline __m128d INTRINSIC _mm_blendv_pd(__m128d a, __m128d b, __m128d c) {
+ return _mm_or_pd(_mm_andnot_pd(c, a), _mm_and_pd(c, b));
+ }
+ static inline __m128 INTRINSIC _mm_blendv_ps(__m128 a, __m128 b, __m128 c) {
+ return _mm_or_ps(_mm_andnot_ps(c, a), _mm_and_ps(c, b));
+ }
+ static inline __m128i INTRINSIC _mm_blendv_epi8(__m128i a, __m128i b, __m128i c) {
+ return _mm_or_si128(_mm_andnot_si128(c, a), _mm_and_si128(c, b));
+ }
+
+ // only use the following blend functions with immediates as mask and, of course, compiling
+ // with optimization
+ static inline __m128d INTRINSIC _mm_blend_pd(__m128d a, __m128d b, const int mask) {
+ switch (mask) {
+ case 0x0:
+ return a;
+ case 0x1:
+ return _mm_shuffle_pd(b, a, 2);
+ case 0x2:
+ return _mm_shuffle_pd(a, b, 2);
+ case 0x3:
+ return b;
+ default:
+ abort();
+ }
+ }
+ static inline __m128 INTRINSIC _mm_blend_ps(__m128 a, __m128 b, const int mask) {
+ __m128i c;
+ switch (mask) {
+ case 0x0:
+ return a;
+ case 0x1:
+ c = _mm_srli_si128(_mm_setallone_si128(), 12);
+ break;
+ case 0x2:
+ c = _mm_slli_si128(_mm_srli_si128(_mm_setallone_si128(), 12), 4);
+ break;
+ case 0x3:
+ c = _mm_srli_si128(_mm_setallone_si128(), 8);
+ break;
+ case 0x4:
+ c = _mm_slli_si128(_mm_srli_si128(_mm_setallone_si128(), 12), 8);
+ break;
+ case 0x5:
+ c = _mm_set_epi32(0, -1, 0, -1);
+ break;
+ case 0x6:
+ c = _mm_slli_si128(_mm_srli_si128(_mm_setallone_si128(), 8), 4);
+ break;
+ case 0x7:
+ c = _mm_srli_si128(_mm_setallone_si128(), 4);
+ break;
+ case 0x8:
+ c = _mm_slli_si128(_mm_setallone_si128(), 12);
+ break;
+ case 0x9:
+ c = _mm_set_epi32(-1, 0, 0, -1);
+ break;
+ case 0xa:
+ c = _mm_set_epi32(-1, 0, -1, 0);
+ break;
+ case 0xb:
+ c = _mm_set_epi32(-1, 0, -1, -1);
+ break;
+ case 0xc:
+ c = _mm_slli_si128(_mm_setallone_si128(), 8);
+ break;
+ case 0xd:
+ c = _mm_set_epi32(-1, -1, 0, -1);
+ break;
+ case 0xe:
+ c = _mm_slli_si128(_mm_setallone_si128(), 4);
+ break;
+ case 0xf:
+ return b;
+ default: // may not happen
+ abort();
+ c = _mm_setzero_si128();
+ break;
+ }
+ __m128 _c = _mm_castsi128_ps(c);
+ return _mm_or_ps(_mm_andnot_ps(_c, a), _mm_and_ps(_c, b));
+ }
+ static inline __m128i INTRINSIC _mm_blend_epi16(__m128i a, __m128i b, const int mask) {
+ __m128i c;
+ switch (mask) {
+ case 0x00:
+ return a;
+ case 0x01:
+ c = _mm_srli_si128(_mm_setallone_si128(), 14);
+ break;
+ case 0x03:
+ c = _mm_srli_si128(_mm_setallone_si128(), 12);
+ break;
+ case 0x07:
+ c = _mm_srli_si128(_mm_setallone_si128(), 10);
+ break;
+ case 0x0f:
+ return _mm_unpackhi_epi64(_mm_slli_si128(b, 8), a);
+ case 0x1f:
+ c = _mm_srli_si128(_mm_setallone_si128(), 6);
+ break;
+ case 0x3f:
+ c = _mm_srli_si128(_mm_setallone_si128(), 4);
+ break;
+ case 0x7f:
+ c = _mm_srli_si128(_mm_setallone_si128(), 2);
+ break;
+ case 0x80:
+ c = _mm_slli_si128(_mm_setallone_si128(), 14);
+ break;
+ case 0xc0:
+ c = _mm_slli_si128(_mm_setallone_si128(), 12);
+ break;
+ case 0xe0:
+ c = _mm_slli_si128(_mm_setallone_si128(), 10);
+ break;
+ case 0xf0:
+ c = _mm_slli_si128(_mm_setallone_si128(), 8);
+ break;
+ case 0xf8:
+ c = _mm_slli_si128(_mm_setallone_si128(), 6);
+ break;
+ case 0xfc:
+ c = _mm_slli_si128(_mm_setallone_si128(), 4);
+ break;
+ case 0xfe:
+ c = _mm_slli_si128(_mm_setallone_si128(), 2);
+ break;
+ case 0xff:
+ return b;
+ case 0xcc:
+ return _mm_unpacklo_epi32(_mm_shuffle_epi32(a, _MM_SHUFFLE(2, 0, 2, 0)), _mm_shuffle_epi32(b, _MM_SHUFFLE(3, 1, 3, 1)));
+ case 0x33:
+ return _mm_unpacklo_epi32(_mm_shuffle_epi32(b, _MM_SHUFFLE(2, 0, 2, 0)), _mm_shuffle_epi32(a, _MM_SHUFFLE(3, 1, 3, 1)));
+ default:
+ const __m128i shift = _mm_set_epi16(0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, -0x7fff);
+ c = _mm_srai_epi16(_mm_mullo_epi16(_mm_set1_epi16(mask), shift), 15);
+ break;
+ }
+ return _mm_or_si128(_mm_andnot_si128(c, a), _mm_and_si128(c, b));
+ }
+
+ static inline __m128i CONST _mm_max_epi8 (__m128i a, __m128i b) {
+ return _mm_blendv_epi8(b, a, _mm_cmpgt_epi8 (a, b));
+ }
+ static inline __m128i CONST _mm_max_epi32(__m128i a, __m128i b) {
+ return _mm_blendv_epi8(b, a, _mm_cmpgt_epi32(a, b));
+ }
+//X static inline __m128i CONST _mm_max_epu8 (__m128i a, __m128i b) {
+//X return _mm_blendv_epi8(b, a, _mm_cmpgt_epu8 (a, b));
+//X }
+ static inline __m128i CONST _mm_max_epu16(__m128i a, __m128i b) {
+ return _mm_blendv_epi8(b, a, _mm_cmpgt_epu16(a, b));
+ }
+ static inline __m128i CONST _mm_max_epu32(__m128i a, __m128i b) {
+ return _mm_blendv_epi8(b, a, _mm_cmpgt_epu32(a, b));
+ }
+//X static inline __m128i CONST _mm_min_epu8 (__m128i a, __m128i b) {
+//X return _mm_blendv_epi8(a, b, _mm_cmpgt_epu8 (a, b));
+//X }
+ static inline __m128i CONST _mm_min_epu16(__m128i a, __m128i b) {
+ return _mm_blendv_epi8(a, b, _mm_cmpgt_epu16(a, b));
+ }
+ static inline __m128i CONST _mm_min_epu32(__m128i a, __m128i b) {
+ return _mm_blendv_epi8(a, b, _mm_cmpgt_epu32(a, b));
+ }
+ static inline __m128i CONST _mm_min_epi8 (__m128i a, __m128i b) {
+ return _mm_blendv_epi8(a, b, _mm_cmpgt_epi8 (a, b));
+ }
+ static inline __m128i CONST _mm_min_epi32(__m128i a, __m128i b) {
+ return _mm_blendv_epi8(a, b, _mm_cmpgt_epi32(a, b));
+ }
+ static inline __m128i INTRINSIC _mm_cvtepu8_epi16(__m128i epu8) {
+ return _mm_unpacklo_epi8(epu8, _mm_setzero_si128());
+ }
+ static inline __m128i INTRINSIC _mm_cvtepi8_epi16(__m128i epi8) {
+ return _mm_unpacklo_epi8(epi8, _mm_cmplt_epi8(epi8, _mm_setzero_si128()));
+ }
+ static inline __m128i INTRINSIC _mm_cvtepu16_epi32(__m128i epu16) {
+ return _mm_unpacklo_epi16(epu16, _mm_setzero_si128());
+ }
+ static inline __m128i INTRINSIC _mm_cvtepi16_epi32(__m128i epu16) {
+ return _mm_unpacklo_epi16(epu16, _mm_cmplt_epi16(epu16, _mm_setzero_si128()));
+ }
+ static inline __m128i INTRINSIC _mm_cvtepu8_epi32(__m128i epu8) {
+ return _mm_cvtepu16_epi32(_mm_cvtepu8_epi16(epu8));
+ }
+ static inline __m128i INTRINSIC _mm_cvtepi8_epi32(__m128i epi8) {
+ const __m128i neg = _mm_cmplt_epi8(epi8, _mm_setzero_si128());
+ const __m128i epi16 = _mm_unpacklo_epi8(epi8, neg);
+ return _mm_unpacklo_epi16(epi16, _mm_unpacklo_epi8(neg, neg));
+ }
+ static inline __m128i INTRINSIC _mm_stream_load_si128(__m128i *mem) {
+ return _mm_load_si128(mem);
+ }
+
+} // namespace SSE
+} // namespace Vc
+#endif
+
+// SSE4.2
+#ifdef VC_IMPL_SSE4_2
+#include <nmmintrin.h>
+#elif defined _NMMINTRIN_H_INCLUDED
+#error "SSE4.2 was disabled but something includes <nmmintrin.h>. Please fix your code."
+#endif
+
+namespace Vc
+{
+namespace SSE
+{
+ static inline float INTRINSIC extract_float_imm(const __m128 v, const size_t i) {
+ float f;
+ switch (i) {
+ case 0:
+ f = _mm_cvtss_f32(v);
+ break;
+#if defined VC_IMPL_SSE4_1 && !defined VC_MSVC
+ default:
+#ifdef VC_GCC
+ f = __builtin_ia32_vec_ext_v4sf(static_cast<__v4sf>(v), (i));
+#else
+ // MSVC fails to compile this because it can't optimize i to an immediate
+ _MM_EXTRACT_FLOAT(f, v, i);
+#endif
+ break;
+#else
+ case 1:
+ f = _mm_cvtss_f32(_mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(v), 4)));
+ break;
+ case 2:
+ f = _mm_cvtss_f32(_mm_movehl_ps(v, v));
+ break;
+ case 3:
+ f = _mm_cvtss_f32(_mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(v), 12)));
+ break;
+#endif
+ }
+ return f;
+ }
+ static inline double INTRINSIC extract_double_imm(const __m128d v, const size_t i) {
+ if (i == 0) {
+ return _mm_cvtsd_f64(v);
+ }
+ return _mm_cvtsd_f64(_mm_castps_pd(_mm_movehl_ps(_mm_castpd_ps(v), _mm_castpd_ps(v))));
+ }
+ static inline float INTRINSIC extract_float(const __m128 v, const size_t i) {
+#ifdef VC_GCC
+ if (__builtin_constant_p(i)) {
+ return extract_float_imm(v, i);
+//X if (index <= 1) {
+//X unsigned long long tmp = _mm_cvtsi128_si64(_mm_castps_si128(v));
+//X if (index == 0) tmp &= 0xFFFFFFFFull;
+//X if (index == 1) tmp >>= 32;
+//X return Common::AliasingEntryHelper<EntryType>(tmp);
+//X }
+ } else {
+ typedef float float4[4] MAY_ALIAS;
+ const float4 &data = reinterpret_cast<const float4 &>(v);
+ return data[i];
+ }
+#else
+ union { __m128 v; float m[4]; } u;
+ u.v = v;
+ return u.m[i];
+#endif
+ }
+
+ static inline __m128 INTRINSIC _mm_stream_load(const float *mem) {
+#ifdef VC_IMPL_SSE4_1
+ return _mm_castsi128_ps(_mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<float *>(mem))));
+#else
+ return _mm_load_ps(mem);
+#endif
+ }
+ static inline __m128d INTRINSIC _mm_stream_load(const double *mem) {
+#ifdef VC_IMPL_SSE4_1
+ return _mm_castsi128_pd(_mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<double *>(mem))));
+#else
+ return _mm_load_pd(mem);
+#endif
+ }
+ static inline __m128i INTRINSIC _mm_stream_load(const int *mem) {
+#ifdef VC_IMPL_SSE4_1
+ return _mm_stream_load_si128(reinterpret_cast<__m128i *>(const_cast<int *>(mem)));
+#else
+ return _mm_load_si128(reinterpret_cast<const __m128i *>(mem));
+#endif
+ }
+ static inline __m128i INTRINSIC _mm_stream_load(const unsigned int *mem) {
+ return _mm_stream_load(reinterpret_cast<const int *>(mem));
+ }
+ static inline __m128i INTRINSIC _mm_stream_load(const short *mem) {
+ return _mm_stream_load(reinterpret_cast<const int *>(mem));
+ }
+ static inline __m128i INTRINSIC _mm_stream_load(const unsigned short *mem) {
+ return _mm_stream_load(reinterpret_cast<const int *>(mem));
+ }
+ static inline __m128i INTRINSIC _mm_stream_load(const signed char *mem) {
+ return _mm_stream_load(reinterpret_cast<const int *>(mem));
+ }
+ static inline __m128i INTRINSIC _mm_stream_load(const unsigned char *mem) {
+ return _mm_stream_load(reinterpret_cast<const int *>(mem));
+ }
+} // namespace SSE
+} // namespace Vc
+
+#include "shuffle.h"
+
+#endif // SSE_INTRINSICS_H
--- /dev/null
+/* This file is part of the Vc library. {{{
+
+ Copyright (C) 2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+}}}*/
+
+#ifndef VC_SSE_ITERATORS_H
+#define VC_SSE_ITERATORS_H
+
+namespace Vc
+{
+namespace Iterators
+{
+
+template<typename T> class VectorRef : public Vector<T>
+{
+private:
+ typedef Vector<T> Base;
+public:
+ typedef typename Base::EntryType EntryType;
+private:
+ EntryType *VC_RESTRICT const m_memoryLocation;
+
+public:
+ VectorRef(EntryType *m)
+ : Base(),
+ m_memoryLocation(m)
+ {
+ Base::load(m_memoryLocation);
+ }
+
+ VectorRef(VectorRef &&a) = default;
+
+ ~VectorRef()
+ {
+ Base::store(m_memoryLocation);
+ }
+private:
+ // disable copies
+ VectorRef &operator=(const VectorRef &) = delete;
+ VectorRef(const VectorRef &) = delete;
+};
+
+template<typename V> class ConstRangeIterator
+{
+protected:
+ typedef typename V::EntryType EntryType;
+ size_t m_offset;
+
+public:
+ ConstRangeIterator(size_t o)
+ : m_offset(o)
+ {
+ }
+
+ const Vector<V> operator()(const EntryType *mem) const
+ {
+ return Vector<V>(mem);
+ }
+
+ bool operator!=(const ConstRangeIterator &rhs) const
+ {
+ return m_offset != rhs.m_offset;
+ }
+
+ ConstRangeIterator &operator++()
+ {
+ ++m_offset;
+ return *this;
+ }
+ ConstRangeIterator operator++(int) {
+ ConstRangeIterator ret(*this);
+ ++m_offset;
+ return ret;
+ }
+ ConstRangeIterator &operator--()
+ {
+ --m_offset;
+ return *this;
+ }
+ ConstRangeIterator operator--(int) {
+ ConstRangeIterator ret(*this);
+ --m_offset;
+ return ret;
+ }
+ ConstRangeIterator &operator*() {
+ return *this;
+ }
+};
+
+template<typename V> class RangeIterator : public ConstRangeIterator<V>
+{
+ typedef typename ConstRangeIterator<V>::EntryType EntryType;
+public:
+ RangeIterator(size_t o) : ConstRangeIterator<V>(o) {}
+
+ VectorRef<V> operator()(EntryType *mem) const
+ {
+ return VectorRef<V>(mem);
+ }
+ RangeIterator &operator*() {
+ return *this;
+ }
+};
+
+template<typename V> class RangeIteration
+{
+private:
+ size_t m_begin;
+ size_t m_end;
+
+public:
+ RangeIteration(size_t b, size_t e)
+ : m_begin(b),
+ m_end(e)
+ {
+ }
+
+ RangeIterator<V> begin() { return RangeIterator<V>(m_begin); }
+ RangeIterator<V> end() { return RangeIterator<V>(m_end); }
+ RangeIterator<V> begin() const { return ConstRangeIterator<V>(m_begin); }
+ RangeIterator<V> end() const { return ConstRangeIterator<V>(m_end); }
+};
+} // namespace Iterators
+} // namespace Vc
+
+namespace std
+{
+ template<typename V>
+ Vc::Iterators::RangeIterator<V> begin(Vc::Iterators::RangeIteration<V> &range)
+ {
+ return range.begin();
+ }
+ template<typename V>
+ Vc::Iterators::RangeIterator<V> end(Vc::Iterators::RangeIteration<V> &range)
+ {
+ return range.end();
+ }
+} // namespace std
+
+#endif // VC_SSE_ITERATORS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_LIMITS_H
+#define VC_SSE_LIMITS_H
+
+#include "intrinsics.h"
+#include "types.h"
+#include "macros.h"
+
+namespace std
+{
+template<> struct numeric_limits<Vc::SSE::ushort_v> : public numeric_limits<unsigned short>
+{
+ static inline INTRINSIC CONST Vc::SSE::ushort_v max() _VC_NOEXCEPT { return Vc::SSE::_mm_setallone_si128(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v min() _VC_NOEXCEPT { return Vc::SSE::ushort_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v lowest() _VC_NOEXCEPT { return min(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v epsilon() _VC_NOEXCEPT { return Vc::SSE::ushort_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v round_error() _VC_NOEXCEPT { return Vc::SSE::ushort_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v infinity() _VC_NOEXCEPT { return Vc::SSE::ushort_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v quiet_NaN() _VC_NOEXCEPT { return Vc::SSE::ushort_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v signaling_NaN() _VC_NOEXCEPT { return Vc::SSE::ushort_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::ushort_v denorm_min() _VC_NOEXCEPT { return Vc::SSE::ushort_v::Zero(); }
+};
+template<> struct numeric_limits<Vc::SSE::short_v> : public numeric_limits<short>
+{
+ static inline INTRINSIC CONST Vc::SSE::short_v max() _VC_NOEXCEPT { return _mm_srli_epi16(Vc::SSE::_mm_setallone_si128(), 1); }
+ static inline INTRINSIC CONST Vc::SSE::short_v min() _VC_NOEXCEPT { return Vc::SSE::_mm_setmin_epi16(); }
+ static inline INTRINSIC CONST Vc::SSE::short_v lowest() _VC_NOEXCEPT { return min(); }
+ static inline INTRINSIC CONST Vc::SSE::short_v epsilon() _VC_NOEXCEPT { return Vc::SSE::short_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::short_v round_error() _VC_NOEXCEPT { return Vc::SSE::short_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::short_v infinity() _VC_NOEXCEPT { return Vc::SSE::short_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::short_v quiet_NaN() _VC_NOEXCEPT { return Vc::SSE::short_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::short_v signaling_NaN() _VC_NOEXCEPT { return Vc::SSE::short_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::short_v denorm_min() _VC_NOEXCEPT { return Vc::SSE::short_v::Zero(); }
+};
+template<> struct numeric_limits<Vc::SSE::uint_v> : public numeric_limits<unsigned int>
+{
+ static inline INTRINSIC CONST Vc::SSE::uint_v max() _VC_NOEXCEPT { return Vc::SSE::_mm_setallone_si128(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v min() _VC_NOEXCEPT { return Vc::SSE::uint_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v lowest() _VC_NOEXCEPT { return min(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v epsilon() _VC_NOEXCEPT { return Vc::SSE::uint_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v round_error() _VC_NOEXCEPT { return Vc::SSE::uint_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v infinity() _VC_NOEXCEPT { return Vc::SSE::uint_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v quiet_NaN() _VC_NOEXCEPT { return Vc::SSE::uint_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v signaling_NaN() _VC_NOEXCEPT { return Vc::SSE::uint_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::uint_v denorm_min() _VC_NOEXCEPT { return Vc::SSE::uint_v::Zero(); }
+};
+template<> struct numeric_limits<Vc::SSE::int_v> : public numeric_limits<int>
+{
+ static inline INTRINSIC CONST Vc::SSE::int_v max() _VC_NOEXCEPT { return _mm_srli_epi32(Vc::SSE::_mm_setallone_si128(), 1); }
+ static inline INTRINSIC CONST Vc::SSE::int_v min() _VC_NOEXCEPT { return Vc::SSE::_mm_setmin_epi32(); }
+ static inline INTRINSIC CONST Vc::SSE::int_v lowest() _VC_NOEXCEPT { return min(); }
+ static inline INTRINSIC CONST Vc::SSE::int_v epsilon() _VC_NOEXCEPT { return Vc::SSE::int_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::int_v round_error() _VC_NOEXCEPT { return Vc::SSE::int_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::int_v infinity() _VC_NOEXCEPT { return Vc::SSE::int_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::int_v quiet_NaN() _VC_NOEXCEPT { return Vc::SSE::int_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::int_v signaling_NaN() _VC_NOEXCEPT { return Vc::SSE::int_v::Zero(); }
+ static inline INTRINSIC CONST Vc::SSE::int_v denorm_min() _VC_NOEXCEPT { return Vc::SSE::int_v::Zero(); }
+};
+} // namespace std
+
+#include "undomacros.h"
+
+#endif // VC_SSE_LIMITS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2010 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "../common/macros.h"
+
+#ifndef VC_SSE_MACROS_H
+#define VC_SSE_MACROS_H
+#undef VC_SSE_UNDOMACROS_H
+
+#ifndef _M128
+# define _M128 __m128
+#endif
+
+#ifndef _M128I
+# define _M128I __m128i
+#endif
+
+#ifndef _M128D
+# define _M128D __m128d
+#endif
+
+#define STORE_VECTOR(type, name, vec) \
+ union { __m128i p; type v[16 / sizeof(type)]; } CAT(u, __LINE__); \
+ _mm_store_si128(&CAT(u, __LINE__).p, vec); \
+ const type *const name = &CAT(u, __LINE__).v[0]
+
+#if defined(VC_IMPL_SSE4_1) && !defined(VC_DISABLE_PTEST)
+#define VC_USE_PTEST
+#endif
+
+#endif // VC_SSE_MACROS_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef SSE_MASK_H
+#define SSE_MASK_H
+
+#include "intrinsics.h"
+
+namespace Vc
+{
+namespace SSE
+{
+
+template<unsigned int Size1> struct MaskHelper;
+template<> struct MaskHelper<2> {
+ static inline bool cmpeq (_M128 k1, _M128 k2) { return _mm_movemask_pd(_mm_castps_pd(k1)) == _mm_movemask_pd(_mm_castps_pd(k2)); }
+ static inline bool cmpneq(_M128 k1, _M128 k2) { return _mm_movemask_pd(_mm_castps_pd(k1)) != _mm_movemask_pd(_mm_castps_pd(k2)); }
+};
+template<> struct MaskHelper<4> {
+ static inline bool cmpeq (_M128 k1, _M128 k2) { return _mm_movemask_ps(k1) == _mm_movemask_ps(k2); }
+ static inline bool cmpneq(_M128 k1, _M128 k2) { return _mm_movemask_ps(k1) != _mm_movemask_ps(k2); }
+};
+template<> struct MaskHelper<8> {
+ static inline bool cmpeq (_M128 k1, _M128 k2) { return _mm_movemask_epi8(_mm_castps_si128(k1)) == _mm_movemask_epi8(_mm_castps_si128(k2)); }
+ static inline bool cmpneq(_M128 k1, _M128 k2) { return _mm_movemask_epi8(_mm_castps_si128(k1)) != _mm_movemask_epi8(_mm_castps_si128(k2)); }
+};
+
+class Float8Mask;
+template<unsigned int VectorSize> class Mask
+{
+ friend class Mask<2u>;
+ friend class Mask<4u>;
+ friend class Mask<8u>;
+ friend class Mask<16u>;
+ friend class Float8Mask;
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(16)
+
+ // abstracts the way Masks are passed to functions, it can easily be changed to const ref here
+ // Also Float8Mask requires const ref on MSVC 32bit.
+#if defined VC_MSVC && defined _WIN32
+ typedef const Mask<VectorSize> &Argument;
+#else
+ typedef Mask<VectorSize> Argument;
+#endif
+
+ inline Mask() {}
+ inline Mask(const __m128 &x) : k(x) {}
+ inline Mask(const __m128d &x) : k(_mm_castpd_ps(x)) {}
+ inline Mask(const __m128i &x) : k(_mm_castsi128_ps(x)) {}
+ inline explicit Mask(VectorSpecialInitializerZero::ZEnum) : k(_mm_setzero_ps()) {}
+ inline explicit Mask(VectorSpecialInitializerOne::OEnum) : k(_mm_setallone_ps()) {}
+ inline explicit Mask(bool b) : k(b ? _mm_setallone_ps() : _mm_setzero_ps()) {}
+ inline Mask(const Mask &rhs) : k(rhs.k) {}
+ inline Mask(const Mask<VectorSize / 2> *a)
+ : k(_mm_castsi128_ps(_mm_packs_epi16(a[0].dataI(), a[1].dataI()))) {}
+ inline explicit Mask(const Float8Mask &m);
+
+ template<unsigned int OtherSize> explicit Mask(const Mask<OtherSize> &x);
+//X {
+//X _M128I tmp = x.dataI();
+//X if (OtherSize < VectorSize) {
+//X tmp = _mm_packs_epi16(tmp, _mm_setzero_si128());
+//X if (VectorSize / OtherSize >= 4u) { tmp = _mm_packs_epi16(tmp, _mm_setzero_si128()); }
+//X if (VectorSize / OtherSize >= 8u) { tmp = _mm_packs_epi16(tmp, _mm_setzero_si128()); }
+//X } else if (OtherSize > VectorSize) {
+//X tmp = _mm_unpacklo_epi8(tmp, tmp);
+//X if (OtherSize / VectorSize >= 4u) { tmp = _mm_unpacklo_epi8(tmp, tmp); }
+//X if (OtherSize / VectorSize >= 8u) { tmp = _mm_unpacklo_epi8(tmp, tmp); }
+//X }
+//X k = _mm_castsi128_ps(tmp);
+//X }
+
+ void expand(Mask<VectorSize / 2> *x) const;
+
+ inline bool operator==(const Mask &rhs) const { return MaskHelper<VectorSize>::cmpeq (k, rhs.k); }
+ inline bool operator!=(const Mask &rhs) const { return MaskHelper<VectorSize>::cmpneq(k, rhs.k); }
+
+ inline Mask operator&&(const Mask &rhs) const { return _mm_and_ps(k, rhs.k); }
+ inline Mask operator& (const Mask &rhs) const { return _mm_and_ps(k, rhs.k); }
+ inline Mask operator||(const Mask &rhs) const { return _mm_or_ps (k, rhs.k); }
+ inline Mask operator| (const Mask &rhs) const { return _mm_or_ps (k, rhs.k); }
+ inline Mask operator^ (const Mask &rhs) const { return _mm_xor_ps(k, rhs.k); }
+ inline Mask operator!() const { return _mm_andnot_si128(dataI(), _mm_setallone_si128()); }
+
+ inline Mask &operator&=(const Mask &rhs) { k = _mm_and_ps(k, rhs.k); return *this; }
+ inline Mask &operator|=(const Mask &rhs) { k = _mm_or_ps (k, rhs.k); return *this; }
+
+ inline bool isFull () const { return
+#ifdef VC_USE_PTEST
+ _mm_testc_si128(dataI(), _mm_setallone_si128()); // return 1 if (0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff) == (~0 & k)
+#else
+ _mm_movemask_epi8(dataI()) == 0xffff;
+#endif
+ }
+ inline bool isEmpty() const { return
+#ifdef VC_USE_PTEST
+ _mm_testz_si128(dataI(), dataI()); // return 1 if (0, 0, 0, 0) == (k & k)
+#else
+ _mm_movemask_epi8(dataI()) == 0x0000;
+#endif
+ }
+ inline bool isMix() const {
+#ifdef VC_USE_PTEST
+ return _mm_test_mix_ones_zeros(dataI(), _mm_setallone_si128());
+#else
+ const int tmp = _mm_movemask_epi8(dataI());
+ return tmp != 0 && (tmp ^ 0xffff) != 0;
+#endif
+ }
+
+#ifndef VC_NO_AUTOMATIC_BOOL_FROM_MASK
+ inline operator bool() const { return isFull(); }
+#endif
+
+ inline int CONST_L shiftMask() const CONST_R;
+
+ int CONST_L toInt() const CONST_R;
+
+ inline _M128 data () const { return k; }
+ inline _M128I dataI() const { return _mm_castps_si128(k); }
+ inline _M128D dataD() const { return _mm_castps_pd(k); }
+
+ template<unsigned int OtherSize> inline Mask<OtherSize> cast() const { return Mask<OtherSize>(k); }
+
+ bool operator[](int index) const;
+
+ int count() const;
+
+ /**
+ * Returns the index of the first one in the mask.
+ *
+ * The return value is undefined if the mask is empty.
+ */
+ int firstOne() const;
+
+ private:
+ _M128 k;
+};
+
+struct ForeachHelper
+{
+ _long mask;
+ bool brk;
+ inline ForeachHelper(_long _mask) : mask(_mask), brk(false) {}
+ inline bool outer() const { return mask != 0; }
+ inline bool inner() { return (brk = !brk); }
+ inline _long next() {
+#ifdef VC_GNU_ASM
+ const _long bit = __builtin_ctzl(mask);
+ __asm__("btr %1,%0" : "+r"(mask) : "r"(bit));
+#elif defined(_WIN64)
+ unsigned long bit;
+ _BitScanForward64(&bit, mask);
+ _bittestandreset64(&mask, bit);
+#elif defined(_WIN32)
+ unsigned long bit;
+ _BitScanForward(&bit, mask);
+ _bittestandreset(&mask, bit);
+#else
+#error "Not implemented yet. Please contact vc-devel@compeng.uni-frankfurt.de"
+#endif
+ return bit;
+ }
+};
+
+#define Vc_foreach_bit(_it_, _mask_) \
+ for (Vc::SSE::ForeachHelper _Vc_foreach_bit_helper((_mask_).toInt()); _Vc_foreach_bit_helper.outer(); ) \
+ for (_it_ = _Vc_foreach_bit_helper.next(); _Vc_foreach_bit_helper.inner(); )
+
+template<unsigned int Size> inline int Mask<Size>::shiftMask() const
+{
+ return _mm_movemask_epi8(dataI());
+}
+
+template<> template<> inline Mask<2>::Mask(const Mask<4> &x) {
+ k = _mm_unpacklo_ps(x.data(), x.data());
+}
+template<> template<> inline Mask<2>::Mask(const Mask<8> &x) {
+ _M128I tmp = _mm_unpacklo_epi16(x.dataI(), x.dataI());
+ k = _mm_castsi128_ps(_mm_unpacklo_epi32(tmp, tmp));
+}
+template<> template<> inline Mask<2>::Mask(const Mask<16> &x) {
+ _M128I tmp = _mm_unpacklo_epi8(x.dataI(), x.dataI());
+ tmp = _mm_unpacklo_epi16(tmp, tmp);
+ k = _mm_castsi128_ps(_mm_unpacklo_epi32(tmp, tmp));
+}
+template<> template<> inline Mask<4>::Mask(const Mask<2> &x) {
+ k = _mm_castsi128_ps(_mm_packs_epi16(x.dataI(), _mm_setzero_si128()));
+}
+template<> template<> inline Mask<4>::Mask(const Mask<8> &x) {
+ k = _mm_castsi128_ps(_mm_unpacklo_epi16(x.dataI(), x.dataI()));
+}
+template<> template<> inline Mask<4>::Mask(const Mask<16> &x) {
+ _M128I tmp = _mm_unpacklo_epi8(x.dataI(), x.dataI());
+ k = _mm_castsi128_ps(_mm_unpacklo_epi16(tmp, tmp));
+}
+template<> template<> inline Mask<8>::Mask(const Mask<2> &x) {
+ _M128I tmp = _mm_packs_epi16(x.dataI(), x.dataI());
+ k = _mm_castsi128_ps(_mm_packs_epi16(tmp, tmp));
+}
+template<> template<> inline Mask<8>::Mask(const Mask<4> &x) {
+ k = _mm_castsi128_ps(_mm_packs_epi16(x.dataI(), x.dataI()));
+}
+template<> template<> inline Mask<8>::Mask(const Mask<16> &x) {
+ k = _mm_castsi128_ps(_mm_unpacklo_epi8(x.dataI(), x.dataI()));
+}
+
+template<> inline void Mask< 4>::expand(Mask<2> *x) const {
+ x[0].k = _mm_unpacklo_ps(data(), data());
+ x[1].k = _mm_unpackhi_ps(data(), data());
+}
+template<> inline void Mask< 8>::expand(Mask<4> *x) const {
+ x[0].k = _mm_castsi128_ps(_mm_unpacklo_epi16(dataI(), dataI()));
+ x[1].k = _mm_castsi128_ps(_mm_unpackhi_epi16(dataI(), dataI()));
+}
+template<> inline void Mask<16>::expand(Mask<8> *x) const {
+ x[0].k = _mm_castsi128_ps(_mm_unpacklo_epi8 (dataI(), dataI()));
+ x[1].k = _mm_castsi128_ps(_mm_unpackhi_epi8 (dataI(), dataI()));
+}
+
+template<> inline int Mask< 2>::toInt() const { return _mm_movemask_pd(dataD()); }
+template<> inline int Mask< 4>::toInt() const { return _mm_movemask_ps(data ()); }
+template<> inline int Mask< 8>::toInt() const { return _mm_movemask_epi8(_mm_packs_epi16(dataI(), _mm_setzero_si128())); }
+template<> inline int Mask<16>::toInt() const { return _mm_movemask_epi8(dataI()); }
+
+template<> inline bool Mask< 2>::operator[](int index) const { return toInt() & (1 << index); }
+template<> inline bool Mask< 4>::operator[](int index) const { return toInt() & (1 << index); }
+template<> inline bool Mask< 8>::operator[](int index) const { return shiftMask() & (1 << 2 * index); }
+template<> inline bool Mask<16>::operator[](int index) const { return toInt() & (1 << index); }
+
+template<> inline int Mask<2>::count() const
+{
+ int mask = _mm_movemask_pd(dataD());
+ return (mask & 1) + (mask >> 1);
+}
+
+template<> inline int Mask<4>::count() const
+{
+#ifdef VC_IMPL_SSE4_2
+ return _mm_popcnt_u32(_mm_movemask_ps(data()));
+//X tmp = (tmp & 5) + ((tmp >> 1) & 5);
+//X return (tmp & 3) + ((tmp >> 2) & 3);
+#else
+ _M128I x = _mm_srli_epi32(dataI(), 31);
+ x = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(0, 1, 2, 3)));
+ x = _mm_add_epi32(x, _mm_shufflelo_epi16(x, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(x);
+#endif
+}
+
+template<> inline int Mask<8>::count() const
+{
+#ifdef VC_IMPL_SSE4_2
+ return _mm_popcnt_u32(_mm_movemask_epi8(dataI())) / 2;
+#else
+//X int tmp = _mm_movemask_epi8(dataI());
+//X tmp = (tmp & 0x1111) + ((tmp >> 2) & 0x1111);
+//X tmp = (tmp & 0x0303) + ((tmp >> 4) & 0x0303);
+//X return (tmp & 0x000f) + ((tmp >> 8) & 0x000f);
+ _M128I x = _mm_srli_epi16(dataI(), 15);
+ x = _mm_add_epi16(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(0, 1, 2, 3)));
+ x = _mm_add_epi16(x, _mm_shufflelo_epi16(x, _MM_SHUFFLE(0, 1, 2, 3)));
+ x = _mm_add_epi16(x, _mm_shufflelo_epi16(x, _MM_SHUFFLE(2, 3, 0, 1)));
+ return _mm_extract_epi16(x, 0);
+#endif
+}
+
+template<> inline int Mask<16>::count() const
+{
+ int tmp = _mm_movemask_epi8(dataI());
+#ifdef VC_IMPL_SSE4_2
+ return _mm_popcnt_u32(tmp);
+#else
+ tmp = (tmp & 0x5555) + ((tmp >> 1) & 0x5555);
+ tmp = (tmp & 0x3333) + ((tmp >> 2) & 0x3333);
+ tmp = (tmp & 0x0f0f) + ((tmp >> 4) & 0x0f0f);
+ return (tmp & 0x00ff) + ((tmp >> 8) & 0x00ff);
+#endif
+}
+
+
+class Float8Mask
+{
+ enum Constants {
+ PartialSize = 4,
+ VectorSize = 8
+ };
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(16)
+
+ // abstracts the way Masks are passed to functions, it can easily be changed to const ref here
+ // Also Float8Mask requires const ref on MSVC 32bit.
+#if defined VC_MSVC && defined _WIN32
+ typedef const Float8Mask & Argument;
+#else
+ typedef Float8Mask Argument;
+#endif
+
+ inline Float8Mask() {}
+ inline Float8Mask(const M256 &x) : k(x) {}
+ inline explicit Float8Mask(VectorSpecialInitializerZero::ZEnum) {
+ k[0] = _mm_setzero_ps();
+ k[1] = _mm_setzero_ps();
+ }
+ inline explicit Float8Mask(VectorSpecialInitializerOne::OEnum) {
+ k[0] = _mm_setallone_ps();
+ k[1] = _mm_setallone_ps();
+ }
+ inline explicit Float8Mask(bool b) {
+ const __m128 tmp = b ? _mm_setallone_ps() : _mm_setzero_ps();
+ k[0] = tmp;
+ k[1] = tmp;
+ }
+ inline Float8Mask(const Mask<VectorSize> &a) {
+ k[0] = _mm_castsi128_ps(_mm_unpacklo_epi16(a.dataI(), a.dataI()));
+ k[1] = _mm_castsi128_ps(_mm_unpackhi_epi16(a.dataI(), a.dataI()));
+ }
+
+ inline bool operator==(const Float8Mask &rhs) const {
+ return MaskHelper<PartialSize>::cmpeq (k[0], rhs.k[0])
+ && MaskHelper<PartialSize>::cmpeq (k[1], rhs.k[1]);
+ }
+ inline bool operator!=(const Float8Mask &rhs) const {
+ return MaskHelper<PartialSize>::cmpneq(k[0], rhs.k[0])
+ && MaskHelper<PartialSize>::cmpneq(k[1], rhs.k[1]);
+ }
+
+ inline Float8Mask operator&&(const Float8Mask &rhs) const {
+ Float8Mask r;
+ r.k[0] = _mm_and_ps(k[0], rhs.k[0]);
+ r.k[1] = _mm_and_ps(k[1], rhs.k[1]);
+ return r;
+ }
+ inline Float8Mask operator& (const Float8Mask &rhs) const {
+ Float8Mask r;
+ r.k[0] = _mm_and_ps(k[0], rhs.k[0]);
+ r.k[1] = _mm_and_ps(k[1], rhs.k[1]);
+ return r;
+ }
+ inline Float8Mask operator||(const Float8Mask &rhs) const {
+ Float8Mask r;
+ r.k[0] = _mm_or_ps(k[0], rhs.k[0]);
+ r.k[1] = _mm_or_ps(k[1], rhs.k[1]);
+ return r;
+ }
+ inline Float8Mask operator| (const Float8Mask &rhs) const {
+ Float8Mask r;
+ r.k[0] = _mm_or_ps(k[0], rhs.k[0]);
+ r.k[1] = _mm_or_ps(k[1], rhs.k[1]);
+ return r;
+ }
+ inline Float8Mask operator^ (const Float8Mask &rhs) const {
+ Float8Mask r;
+ r.k[0] = _mm_xor_ps(k[0], rhs.k[0]);
+ r.k[1] = _mm_xor_ps(k[1], rhs.k[1]);
+ return r;
+ }
+ inline Float8Mask operator!() const {
+ Float8Mask r;
+ r.k[0] = _mm_andnot_ps(k[0], _mm_setallone_ps());
+ r.k[1] = _mm_andnot_ps(k[1], _mm_setallone_ps());
+ return r;
+ }
+ inline Float8Mask &operator&=(const Float8Mask &rhs) {
+ k[0] = _mm_and_ps(k[0], rhs.k[0]);
+ k[1] = _mm_and_ps(k[1], rhs.k[1]);
+ return *this;
+ }
+ inline Float8Mask &operator|=(const Float8Mask &rhs) {
+ k[0] = _mm_or_ps (k[0], rhs.k[0]);
+ k[1] = _mm_or_ps (k[1], rhs.k[1]);
+ return *this;
+ }
+
+ inline bool isFull () const {
+ const _M128 tmp = _mm_and_ps(k[0], k[1]);
+#ifdef VC_USE_PTEST
+ return _mm_testc_si128(_mm_castps_si128(tmp), _mm_setallone_si128());
+#else
+ return _mm_movemask_ps(tmp) == 0xf;
+ //_mm_movemask_ps(k[0]) == 0xf &&
+ //_mm_movemask_ps(k[1]) == 0xf;
+#endif
+ }
+ inline bool isEmpty() const {
+ const _M128 tmp = _mm_or_ps(k[0], k[1]);
+#ifdef VC_USE_PTEST
+ return _mm_testz_si128(_mm_castps_si128(tmp), _mm_castps_si128(tmp));
+#else
+ return _mm_movemask_ps(tmp) == 0x0;
+ //_mm_movemask_ps(k[0]) == 0x0 &&
+ //_mm_movemask_ps(k[1]) == 0x0;
+#endif
+ }
+ inline bool isMix() const {
+#ifdef VC_USE_PTEST
+ return _mm_test_mix_ones_zeros(_mm_castps_si128(k[0]), _mm_castps_si128(k[0])) &&
+ _mm_test_mix_ones_zeros(_mm_castps_si128(k[1]), _mm_castps_si128(k[1]));
+#else
+ const int tmp = _mm_movemask_ps(k[0]) + _mm_movemask_ps(k[1]);
+ return tmp > 0x0 && tmp < (0xf + 0xf);
+#endif
+ }
+
+#ifndef VC_NO_AUTOMATIC_BOOL_FROM_MASK
+ inline operator bool() const { return isFull(); }
+#endif
+
+ inline int shiftMask() const {
+ return (_mm_movemask_ps(k[1]) << 4) + _mm_movemask_ps(k[0]);
+ }
+ inline int toInt() const { return (_mm_movemask_ps(k[1]) << 4) + _mm_movemask_ps(k[0]); }
+
+ inline const M256 &data () const { return k; }
+
+ inline bool operator[](int index) const {
+ return (toInt() & (1 << index)) != 0;
+ }
+
+ inline int count() const {
+#ifdef VC_IMPL_SSE4_2
+ return _mm_popcnt_u32(toInt());
+#else
+//X int tmp1 = _mm_movemask_ps(k[0]);
+//X int tmp2 = _mm_movemask_ps(k[1]);
+//X tmp1 = (tmp1 & 5) + ((tmp1 >> 1) & 5);
+//X tmp2 = (tmp2 & 5) + ((tmp2 >> 1) & 5);
+//X return (tmp1 & 3) + (tmp2 & 3) + ((tmp1 >> 2) & 3) + ((tmp2 >> 2) & 3);
+ _M128I x = _mm_add_epi32(_mm_srli_epi32(_mm_castps_si128(k[0]), 31),
+ _mm_srli_epi32(_mm_castps_si128(k[1]), 31));
+ x = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(0, 1, 2, 3)));
+ x = _mm_add_epi32(x, _mm_shufflelo_epi16(x, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(x);
+#endif
+ }
+
+ int firstOne() const;
+
+ private:
+ M256 k;
+};
+
+template<unsigned int Size> inline int Mask<Size>::firstOne() const
+{
+ const int mask = toInt();
+#ifdef _MSC_VER
+ unsigned long bit;
+ _BitScanForward(&bit, mask);
+#else
+ int bit;
+ __asm__("bsf %1,%0" : "=&r"(bit) : "r"(mask));
+#endif
+ return bit;
+}
+inline int Float8Mask::firstOne() const
+{
+ const int mask = toInt();
+#ifdef _MSC_VER
+ unsigned long bit;
+ _BitScanForward(&bit, mask);
+#else
+ int bit;
+ __asm__("bsf %1,%0" : "=&r"(bit) : "r"(mask));
+#endif
+ return bit;
+}
+
+template<unsigned int VectorSize>
+inline Mask<VectorSize>::Mask(const Float8Mask &m)
+ : k(_mm_castsi128_ps(_mm_packs_epi32(_mm_castps_si128(m.data()[0]), _mm_castps_si128(m.data()[1])))) {}
+
+class Float8GatherMask
+{
+ public:
+ Float8GatherMask(const Mask<8u> &k) : mask(k.toInt()) {}
+ Float8GatherMask(const Float8Mask &k) : mask(k.toInt()) {}
+ int toInt() const { return mask; }
+ private:
+ const int mask;
+};
+
+/**
+ * Loop over all set bits in the mask. The iterator variable will be set to the position of the set
+ * bits. A mask of e.g. 00011010 would result in the loop being called with the iterator being set to
+ * 1, 3, and 4.
+ *
+ * This allows you to write:
+ * \code
+ * float_v a = ...;
+ * foreach_bit(int i, a < 0.f) {
+ * std::cout << a[i] << "\n";
+ * }
+ * \endcode
+ * The example prints all the values in \p a that are negative, and only those.
+ *
+ * \param it The iterator variable. For example "int i".
+ * \param mask The mask to iterate over. You can also just write a vector operation that returns a
+ * mask.
+ */
+//X #define foreach_bit(it, mask)
+//X for (int _sse_vector_foreach_inner = 1, ForeachScope _sse_vector_foreach_scope(mask.toInt()), int it = _sse_vector_foreach_scope.bit(); _sse_vector_foreach_inner; --_sse_vector_foreach_inner)
+//X for (int _sse_vector_foreach_mask = (mask).toInt(), int _sse_vector_foreach_it = _sse_bitscan(mask.toInt());
+//X _sse_vector_foreach_it > 0;
+//X _sse_vector_foreach_it = _sse_bitscan_initialized(_sse_vector_foreach_it, mask.data()))
+//X for (int _sse_vector_foreach_inner = 1, it = _sse_vector_foreach_it; _sse_vector_foreach_inner; --_sse_vector_foreach_inner)
+
+} // namespace SSE
+} // namespace Vc
+
+#endif // SSE_MASK_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_MATH_H
+#define VC_SSE_MATH_H
+
+#include "const.h"
+
+namespace Vc
+{
+namespace SSE
+{
+ /**
+ * splits \p v into exponent and mantissa, the sign is kept with the mantissa
+ *
+ * The return value will be in the range [0.5, 1.0[
+ * The \p e value will be an integer defining the power-of-two exponent
+ */
+ inline double_v frexp(const double_v &v, int_v *e) {
+ const __m128i exponentBits = Const<double>::exponentMask().dataI();
+ const __m128i exponentPart = _mm_and_si128(_mm_castpd_si128(v.data()), exponentBits);
+ *e = _mm_sub_epi32(_mm_srli_epi64(exponentPart, 52), _mm_set1_epi32(0x3fe));
+ const __m128d exponentMaximized = _mm_or_pd(v.data(), _mm_castsi128_pd(exponentBits));
+ double_v ret = _mm_and_pd(exponentMaximized, _mm_load_pd(reinterpret_cast<const double *>(&c_general::frexpMask[0])));
+ double_m zeroMask = v == double_v::Zero();
+ ret(isnan(v) || !isfinite(v) || zeroMask) = v;
+ e->setZero(zeroMask.data());
+ return ret;
+ }
+ inline float_v frexp(const float_v &v, int_v *e) {
+ const __m128i exponentBits = Const<float>::exponentMask().dataI();
+ const __m128i exponentPart = _mm_and_si128(_mm_castps_si128(v.data()), exponentBits);
+ *e = _mm_sub_epi32(_mm_srli_epi32(exponentPart, 23), _mm_set1_epi32(0x7e));
+ const __m128 exponentMaximized = _mm_or_ps(v.data(), _mm_castsi128_ps(exponentBits));
+ float_v ret = _mm_and_ps(exponentMaximized, _mm_castsi128_ps(_mm_set1_epi32(0xbf7fffffu)));
+ ret(isnan(v) || !isfinite(v) || v == float_v::Zero()) = v;
+ e->setZero(v == float_v::Zero());
+ return ret;
+ }
+ inline sfloat_v frexp(const sfloat_v &v, short_v *e) {
+ const __m128i exponentBits = Const<float>::exponentMask().dataI();
+ const __m128i exponentPart0 = _mm_and_si128(_mm_castps_si128(v.data()[0]), exponentBits);
+ const __m128i exponentPart1 = _mm_and_si128(_mm_castps_si128(v.data()[1]), exponentBits);
+ *e = _mm_sub_epi16(_mm_packs_epi32(_mm_srli_epi32(exponentPart0, 23), _mm_srli_epi32(exponentPart1, 23)),
+ _mm_set1_epi16(0x7e));
+ const __m128 exponentMaximized0 = _mm_or_ps(v.data()[0], _mm_castsi128_ps(exponentBits));
+ const __m128 exponentMaximized1 = _mm_or_ps(v.data()[1], _mm_castsi128_ps(exponentBits));
+ sfloat_v ret = M256::create(
+ _mm_and_ps(exponentMaximized0, _mm_castsi128_ps(_mm_set1_epi32(0xbf7fffffu))),
+ _mm_and_ps(exponentMaximized1, _mm_castsi128_ps(_mm_set1_epi32(0xbf7fffffu)))
+ );
+ sfloat_m zeroMask = v == sfloat_v::Zero();
+ ret(isnan(v) || !isfinite(v) || zeroMask) = v;
+ e->setZero(static_cast<short_m>(zeroMask));
+ return ret;
+ }
+
+ /* -> x * 2^e
+ * x == NaN -> NaN
+ * x == (-)inf -> (-)inf
+ */
+ inline double_v ldexp(double_v::AsArg v, int_v::AsArg _e) {
+ int_v e = _e;
+ e.setZero((v == double_v::Zero()).dataI());
+ const __m128i exponentBits = _mm_slli_epi64(e.data(), 52);
+ return _mm_castsi128_pd(_mm_add_epi64(_mm_castpd_si128(v.data()), exponentBits));
+ }
+ inline float_v ldexp(float_v::AsArg v, int_v::AsArg _e) {
+ int_v e = _e;
+ e.setZero(static_cast<int_m>(v == float_v::Zero()));
+ return (v.reinterpretCast<int_v>() + (e << 23)).reinterpretCast<float_v>();
+ }
+ inline sfloat_v ldexp(sfloat_v::AsArg v, short_v::AsArg _e) {
+ short_v e = _e;
+ e.setZero(static_cast<short_m>(v == sfloat_v::Zero()));
+ e <<= (23 - 16);
+ const __m128i exponentBits0 = _mm_unpacklo_epi16(_mm_setzero_si128(), e.data());
+ const __m128i exponentBits1 = _mm_unpackhi_epi16(_mm_setzero_si128(), e.data());
+ return M256::create(_mm_castsi128_ps(_mm_add_epi32(_mm_castps_si128(v.data()[0]), exponentBits0)),
+ _mm_castsi128_ps(_mm_add_epi32(_mm_castps_si128(v.data()[1]), exponentBits1)));
+ }
+
+#ifdef VC_IMPL_SSE4_1
+ inline double_v floor(double_v::AsArg v) { return _mm_floor_pd(v.data()); }
+ inline float_v floor(float_v::AsArg v) { return _mm_floor_ps(v.data()); }
+ inline sfloat_v floor(sfloat_v::AsArg v) { return M256::create(_mm_floor_ps(v.data()[0]),
+ _mm_floor_ps(v.data()[1])); }
+ inline double_v ceil(double_v::AsArg v) { return _mm_ceil_pd(v.data()); }
+ inline float_v ceil(float_v::AsArg v) { return _mm_ceil_ps(v.data()); }
+ inline sfloat_v ceil(sfloat_v::AsArg v) { return M256::create(_mm_ceil_ps(v.data()[0]),
+ _mm_ceil_ps(v.data()[1])); }
+#else
+ static inline void floor_shift(float_v &v, float_v::AsArg e)
+ {
+ int_v x = _mm_setallone_si128();
+ x <<= 23;
+ x >>= static_cast<int_v>(e);
+ v &= x.reinterpretCast<float_v>();
+ }
+
+ static inline void floor_shift(sfloat_v &v, sfloat_v::AsArg e)
+ {
+ int_v x = _mm_setallone_si128();
+ x <<= 23;
+ int_v y = x;
+ x >>= _mm_cvttps_epi32(e.data()[0]);
+ y >>= _mm_cvttps_epi32(e.data()[1]);
+ v.data()[0] = _mm_and_ps(v.data()[0], _mm_castsi128_ps(x.data()));
+ v.data()[1] = _mm_and_ps(v.data()[1], _mm_castsi128_ps(y.data()));
+ }
+
+ static inline void floor_shift(double_v &v, double_v::AsArg e)
+ {
+ const long long initialMask = 0xfff0000000000000ull;
+ const uint_v shifts = static_cast<uint_v>(e);
+ union d_ll {
+ long long ll;
+ double d;
+ };
+ d_ll mask0 = { initialMask >> shifts[0] };
+ d_ll mask1 = { initialMask >> shifts[1] };
+ v &= double_v(_mm_setr_pd(mask0.d, mask1.d));
+ }
+
+ template<typename T>
+ inline Vector<T> floor(Vector<T> _v) {
+ typedef Vector<T> V;
+ typedef typename V::Mask M;
+
+ V v = _v;
+ V e = abs(v).exponent();
+ const M negativeExponent = e < 0;
+ e.setZero(negativeExponent);
+ const M negativeInput = v < V::Zero();
+
+ floor_shift(v, e);
+
+ v.setZero(negativeExponent);
+ v(negativeInput && _v != v) -= V::One();
+ return v;
+ }
+
+ template<typename T>
+ inline Vector<T> ceil(Vector<T> _v) {
+ typedef Vector<T> V;
+ typedef typename V::Mask M;
+
+ V v = _v;
+ V e = abs(v).exponent();
+ const M negativeExponent = e < 0;
+ e.setZero(negativeExponent);
+ const M positiveInput = v > V::Zero();
+
+ floor_shift(v, e);
+
+ v.setZero(negativeExponent);
+ v(positiveInput && _v != v) += V::One();
+ return v;
+ }
+#endif
+} // namespace SSE
+} // namespace Vc
+
+#define VC__USE_NAMESPACE SSE
+#include "../common/trigonometric.h"
+#define VC__USE_NAMESPACE SSE
+#include "../common/logarithm.h"
+#define VC__USE_NAMESPACE SSE
+#include "../common/exponential.h"
+#undef VC__USE_NAMESPACE
+
+#endif // VC_SSE_MATH_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_PREFETCHES_TCC
+#define VC_SSE_PREFETCHES_TCC
+
+namespace Vc
+{
+namespace Internal
+{
+
+inline void HelperImpl<Vc::SSE2Impl>::prefetchForOneRead(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_NTA);
+}
+inline void HelperImpl<Vc::SSE2Impl>::prefetchClose(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T0);
+}
+inline void HelperImpl<Vc::SSE2Impl>::prefetchMid(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T1);
+}
+inline void HelperImpl<Vc::SSE2Impl>::prefetchFar(const void *addr)
+{
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T2);
+}
+inline void HelperImpl<Vc::SSE2Impl>::prefetchForModify(const void *addr)
+{
+#ifdef __3dNOW__
+ _m_prefetchw(const_cast<void *>(addr));
+#else
+ _mm_prefetch(static_cast<char *>(const_cast<void *>(addr)), _MM_HINT_T0);
+#endif
+}
+
+} // namespace Internal
+} // namespace Vc
+
+#endif // VC_SSE_PREFETCHES_TCC
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_SHUFFLE_H
+#define VC_SSE_SHUFFLE_H
+
+namespace Vc
+{
+ enum VecPos {
+ X0, X1, X2, X3, X4, X5, X6, X7,
+ Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7
+ };
+
+ namespace Mem
+ {
+ // shuffle<X1, X2, Y0, Y2>([x0 x1 x2 x3], [y0 y1 y2 y3]) = [x1 x2 y0 y2]
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m128 ALWAYS_INLINE CONST shuffle(__m128 x, __m128 y) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= Y0 && Dst3 >= Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= Y3 && Dst3 <= Y3, Incorrect_Range);
+ return _mm_shuffle_ps(x, y, Dst0 + Dst1 * 4 + (Dst2 - Y0) * 16 + (Dst3 - Y0) * 64);
+ }
+
+ // shuffle<X1, Y0>([x0 x1], [y0 y1]) = [x1 y0]
+ template<VecPos Dst0, VecPos Dst1> static inline __m128d ALWAYS_INLINE CONST shuffle(__m128d x, __m128d y) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X1 && Dst1 <= Y1, Incorrect_Range);
+ return _mm_shuffle_pd(x, y, Dst0 + (Dst1 - Y0) * 2);
+ }
+
+ // blend<X0, Y1>([x0 x1], [y0, y1]) = [x0 y1]
+ template<VecPos Dst0, VecPos Dst1> static inline __m128d ALWAYS_INLINE CONST blend(__m128d x, __m128d y) {
+ VC_STATIC_ASSERT(Dst0 == X0 || Dst0 == Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst1 == X1 || Dst1 == Y1, Incorrect_Range);
+#if !defined(VC_IMPL_SSE4_1) && !defined(VC_IMPL_AVX)
+ using Vc::SSE::_mm_blend_pd;
+#endif
+ return _mm_blend_pd(x, y, (Dst0 / Y0) + (Dst1 / Y0) * 2);
+ }
+
+ // blend<X0, Y1>([x0 x1], [y0, y1]) = [x0 y1]
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m128 ALWAYS_INLINE CONST blend(__m128 x, __m128 y) {
+ VC_STATIC_ASSERT(Dst0 == X0 || Dst0 == Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst1 == X1 || Dst1 == Y1, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst2 == X2 || Dst2 == Y2, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst3 == X3 || Dst3 == Y3, Incorrect_Range);
+#if !defined(VC_IMPL_SSE4_1) && !defined(VC_IMPL_AVX)
+ using Vc::SSE::_mm_blend_ps;
+#endif
+ return _mm_blend_ps(x, y,
+ (Dst0 / Y0) * 1 + (Dst1 / Y1) * 2 +
+ (Dst2 / Y2) * 4 + (Dst3 / Y3) * 8);
+ }
+
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3, VecPos Dst4, VecPos Dst5, VecPos Dst6, VecPos Dst7>
+ static inline __m128i ALWAYS_INLINE CONST blend(__m128i x, __m128i y) {
+ VC_STATIC_ASSERT(Dst0 == X0 || Dst0 == Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst1 == X1 || Dst1 == Y1, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst2 == X2 || Dst2 == Y2, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst3 == X3 || Dst3 == Y3, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst4 == X4 || Dst4 == Y4, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst5 == X5 || Dst5 == Y5, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst6 == X6 || Dst6 == Y6, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst7 == X7 || Dst7 == Y7, Incorrect_Range);
+#if !defined(VC_IMPL_SSE4_1) && !defined(VC_IMPL_AVX)
+ using Vc::SSE::_mm_blend_epi16;
+#endif
+ return _mm_blend_epi16(x, y,
+ (Dst0 / Y0) * 1 + (Dst1 / Y1) * 2 +
+ (Dst2 / Y2) * 4 + (Dst3 / Y3) * 8 +
+ (Dst4 / Y4) * 16 + (Dst5 / Y5) * 32 +
+ (Dst6 / Y6) * 64 + (Dst7 / Y7) *128
+ );
+ }
+
+ // permute<X1, X2, Y0, Y2>([x0 x1 x2 x3], [y0 y1 y2 y3]) = [x1 x2 y0 y2]
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m128 ALWAYS_INLINE CONST permute(__m128 x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm_shuffle_ps(x, x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m128i ALWAYS_INLINE CONST permute(__m128i x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm_shuffle_epi32(x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m128i ALWAYS_INLINE CONST permuteLo(__m128i x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm_shufflelo_epi16(x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3> static inline __m128i ALWAYS_INLINE CONST permuteHi(__m128i x) {
+ VC_STATIC_ASSERT(Dst0 >= X4 && Dst1 >= X4 && Dst2 >= X4 && Dst3 >= X4, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X7 && Dst1 <= X7 && Dst2 <= X7 && Dst3 <= X7, Incorrect_Range);
+ return _mm_shufflehi_epi16(x, (Dst0 - X4) + (Dst1 - X4) * 4 + (Dst2 - X4) * 16 + (Dst3 - X4) * 64);
+ }
+
+ template<VecPos Dst0, VecPos Dst1, VecPos Dst2, VecPos Dst3, VecPos Dst4, VecPos Dst5, VecPos Dst6, VecPos Dst7>
+ static inline __m128i ALWAYS_INLINE CONST permute(__m128i x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst4 >= X4 && Dst5 >= X4 && Dst6 >= X4 && Dst7 >= X4, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst4 <= X7 && Dst5 <= X7 && Dst6 <= X7 && Dst7 <= X7, Incorrect_Range);
+ if (Dst0 != X0 || Dst1 != X1 || Dst2 != X2 || Dst3 != X3) {
+ x = _mm_shufflelo_epi16(x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+ if (Dst4 != X4 || Dst5 != X5 || Dst6 != X6 || Dst7 != X7) {
+ x = _mm_shufflehi_epi16(x, (Dst4 - X4) + (Dst5 - X4) * 4 + (Dst6 - X4) * 16 + (Dst7 - X4) * 64);
+ }
+ return x;
+ }
+ } // namespace Mem
+ // The shuffles and permutes above use memory ordering. The ones below use register ordering:
+ namespace Reg
+ {
+ // shuffle<Y2, Y0, X2, X1>([x3 x2 x1 x0], [y3 y2 y1 y0]) = [y2 y0 x2 x1]
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m128 ALWAYS_INLINE CONST shuffle(__m128 x, __m128 y) {
+ return Mem::shuffle<Dst0, Dst1, Dst2, Dst3>(x, y);
+ }
+
+ // shuffle<Y0, X1>([x1 x0], [y1 y0]) = [y0 x1]
+ template<VecPos Dst1, VecPos Dst0> static inline __m128d ALWAYS_INLINE CONST shuffle(__m128d x, __m128d y) {
+ return Mem::shuffle<Dst0, Dst1>(x, y);
+ }
+
+ // shuffle<X3, X0, X2, X1>([x3 x2 x1 x0]) = [x3 x0 x2 x1]
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m128i ALWAYS_INLINE CONST permute(__m128i x) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= X0 && Dst3 >= X0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= X3 && Dst3 <= X3, Incorrect_Range);
+ return _mm_shuffle_epi32(x, Dst0 + Dst1 * 4 + Dst2 * 16 + Dst3 * 64);
+ }
+
+ // shuffle<Y2, Y0, X2, X1>([x3 x2 x1 x0], [y3 y2 y1 y0]) = [y2 y0 x2 x1]
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m128i ALWAYS_INLINE CONST shuffle(__m128i x, __m128i y) {
+ VC_STATIC_ASSERT(Dst0 >= X0 && Dst1 >= X0 && Dst2 >= Y0 && Dst3 >= Y0, Incorrect_Range);
+ VC_STATIC_ASSERT(Dst0 <= X3 && Dst1 <= X3 && Dst2 <= Y3 && Dst3 <= Y3, Incorrect_Range);
+ return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(x), _mm_castsi128_ps(y), Dst0 + Dst1 * 4 + (Dst2 - Y0) * 16 + (Dst3 - Y0) * 64));
+ }
+
+ // blend<Y1, X0>([x1 x0], [y1, y0]) = [x1 y0]
+ template<VecPos Dst1, VecPos Dst0> static inline __m128d ALWAYS_INLINE CONST blend(__m128d x, __m128d y) {
+ return Mem::blend<Dst0, Dst1>(x, y);
+ }
+
+ template<VecPos Dst3, VecPos Dst2, VecPos Dst1, VecPos Dst0> static inline __m128 ALWAYS_INLINE CONST blend(__m128 x, __m128 y) {
+ return Mem::blend<Dst0, Dst1, Dst2, Dst3>(x, y);
+ }
+ } // namespace Reg
+} // namespace Vc
+
+#endif // VC_SSE_SHUFFLE_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef SSE_TYPES_H
+#define SSE_TYPES_H
+
+#include "intrinsics.h"
+#include "../common/storage.h"
+#include "macros.h"
+
+#define VC_DOUBLE_V_SIZE 2
+#define VC_FLOAT_V_SIZE 4
+#define VC_SFLOAT_V_SIZE 8
+#define VC_INT_V_SIZE 4
+#define VC_UINT_V_SIZE 4
+#define VC_SHORT_V_SIZE 8
+#define VC_USHORT_V_SIZE 8
+
+#include "../common/types.h"
+
+namespace Vc
+{
+namespace SSE
+{
+ template<typename T> class Vector;
+ template<typename T> class WriteMaskedVector;
+
+ // define our own long because on Windows64 long == int while on Linux long == max. register width
+ // since we want to have a type that depends on 32 vs. 64 bit we need to do some special casing on Windows
+#ifdef _WIN64
+ typedef __int64 _long;
+ typedef unsigned __int64 _ulong;
+#else
+ typedef long _long;
+ typedef unsigned long _ulong;
+#endif
+
+
+ class Float8Mask;
+ class Float8GatherMask;
+ template<unsigned int VectorSize> class Mask;
+
+ /*
+ * Hack to create a vector object with 8 floats
+ */
+ typedef Vc::sfloat float8;
+
+ class M256 {
+ public:
+ //inline M256() {}
+ //inline M256(_M128 a, _M128 b) { d[0] = a; d[1] = b; }
+ static inline M256 dup(_M128 a) { M256 r; r.d[0] = a; r.d[1] = a; return r; }
+ static inline M256 create(_M128 a, _M128 b) { M256 r; r.d[0] = a; r.d[1] = b; return r; }
+ inline _M128 &operator[](int i) { return d[i]; }
+ inline const _M128 &operator[](int i) const { return d[i]; }
+ private:
+ _M128 d[2];
+ };
+
+ template<typename T> struct ParameterHelper {
+ typedef T ByValue;
+ typedef T & Reference;
+ typedef const T & ConstRef;
+ };
+#if defined VC_MSVC && !defined _WIN64
+ // The calling convention on WIN32 can't guarantee alignment.
+ // An exception are the first three arguments, which may be passed in a register.
+ template<> struct ParameterHelper<M256> {
+ typedef const M256 & ByValue;
+ typedef M256 & Reference;
+ typedef const M256 & ConstRef;
+ };
+#endif
+
+ template<typename T> struct VectorHelper {};
+
+ template<unsigned int Size> struct IndexTypeHelper;
+ template<> struct IndexTypeHelper<2u> { typedef unsigned int Type; };
+ template<> struct IndexTypeHelper<4u> { typedef unsigned int Type; };
+ template<> struct IndexTypeHelper<8u> { typedef unsigned short Type; };
+ template<> struct IndexTypeHelper<16u>{ typedef unsigned char Type; };
+
+ template<typename T> struct CtorTypeHelper { typedef T Type; };
+ template<> struct CtorTypeHelper<short> { typedef int Type; };
+ template<> struct CtorTypeHelper<unsigned short> { typedef unsigned int Type; };
+ template<> struct CtorTypeHelper<float> { typedef double Type; };
+
+ template<typename T> struct ExpandTypeHelper { typedef T Type; };
+ template<> struct ExpandTypeHelper<short> { typedef int Type; };
+ template<> struct ExpandTypeHelper<unsigned short> { typedef unsigned int Type; };
+ template<> struct ExpandTypeHelper<float> { typedef double Type; };
+
+ template<typename T> struct VectorTypeHelper { typedef __m128i Type; };
+ template<> struct VectorTypeHelper<double> { typedef __m128d Type; };
+ template<> struct VectorTypeHelper< float> { typedef __m128 Type; };
+ template<> struct VectorTypeHelper<sfloat> { typedef M256 Type; };
+
+ template<typename T, unsigned int Size> struct DetermineMask { typedef Mask<Size> Type; };
+ template<> struct DetermineMask<sfloat, 8> { typedef Float8Mask Type; };
+
+ template<typename T> struct DetermineGatherMask { typedef T Type; };
+ template<> struct DetermineGatherMask<Float8Mask> { typedef Float8GatherMask Type; };
+
+ template<typename T> struct VectorTraits
+ {
+ typedef typename VectorTypeHelper<T>::Type VectorType;
+ typedef typename DetermineEntryType<T>::Type EntryType;
+ enum Constants {
+ Size = sizeof(VectorType) / sizeof(EntryType),
+ HasVectorDivision = !IsInteger<T>::Value
+ };
+ typedef typename DetermineMask<T, Size>::Type MaskType;
+ typedef typename DetermineGatherMask<MaskType>::Type GatherMaskType;
+ typedef Vector<typename IndexTypeHelper<Size>::Type> IndexType;
+ typedef Common::VectorMemoryUnion<VectorType, EntryType> StorageType;
+ };
+
+ template<typename T> struct VectorHelperSize;
+
+ template<typename V = Vector<float> >
+ class STRUCT_ALIGN1(16) VectorAlignedBaseT
+ {
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(16)
+ } STRUCT_ALIGN2(16);
+
+} // namespace SSE
+} // namespace Vc
+
+#endif // SSE_TYPES_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2010 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_SSE_UNDOMACROS_H
+#define VC_SSE_UNDOMACROS_H
+#undef VC_SSE_MACROS_H
+
+#undef STORE_VECTOR
+
+#ifdef VC_USE_PTEST
+#undef VC_USE_PTEST
+#endif
+
+#endif // VC_SSE_UNDOMACROS_H
+
+#include "../common/undomacros.h"
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef SSE_VECTOR_H
+#define SSE_VECTOR_H
+
+#include "intrinsics.h"
+#include "types.h"
+#include "vectorhelper.h"
+#include "mask.h"
+#include "../common/aliasingentryhelper.h"
+#include "../common/memoryfwd.h"
+#include <algorithm>
+#include <cmath>
+
+#include "macros.h"
+
+#ifdef isfinite
+#undef isfinite
+#endif
+#ifdef isnan
+#undef isnan
+#endif
+
+namespace Vc
+{
+namespace SSE
+{
+template<typename T>
+class WriteMaskedVector
+{
+ friend class Vector<T>;
+ typedef typename VectorTraits<T>::MaskType Mask;
+ typedef typename Vector<T>::EntryType EntryType;
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(16)
+ //prefix
+ inline INTRINSIC Vector<T> &operator++() {
+ vec->data() = VectorHelper<T>::add(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return *vec;
+ }
+ inline INTRINSIC Vector<T> &operator--() {
+ vec->data() = VectorHelper<T>::sub(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return *vec;
+ }
+ //postfix
+ inline INTRINSIC Vector<T> operator++(int) {
+ Vector<T> ret(*vec);
+ vec->data() = VectorHelper<T>::add(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return ret;
+ }
+ inline INTRINSIC Vector<T> operator--(int) {
+ Vector<T> ret(*vec);
+ vec->data() = VectorHelper<T>::sub(vec->data(),
+ VectorHelper<T>::notMaskedToZero(VectorHelper<T>::one(), mask.data())
+ );
+ return ret;
+ }
+
+ inline INTRINSIC Vector<T> &operator+=(const Vector<T> &x) {
+ vec->data() = VectorHelper<T>::add(vec->data(), VectorHelper<T>::notMaskedToZero(x.data(), mask.data()));
+ return *vec;
+ }
+ inline INTRINSIC Vector<T> &operator-=(const Vector<T> &x) {
+ vec->data() = VectorHelper<T>::sub(vec->data(), VectorHelper<T>::notMaskedToZero(x.data(), mask.data()));
+ return *vec;
+ }
+ inline INTRINSIC Vector<T> &operator*=(const Vector<T> &x) {
+ vec->data() = VectorHelper<T>::mul(vec->data(), x.data(), mask.data());
+ return *vec;
+ }
+ inline INTRINSIC CONST Vector<T> &operator/=(const Vector<T> &x);
+
+ inline INTRINSIC Vector<T> &operator+=(EntryType x) {
+ return operator+=(Vector<T>(x));
+ }
+ inline INTRINSIC Vector<T> &operator-=(EntryType x) {
+ return operator-=(Vector<T>(x));
+ }
+ inline INTRINSIC Vector<T> &operator*=(EntryType x) {
+ return operator*=(Vector<T>(x));
+ }
+ inline INTRINSIC Vector<T> &operator/=(EntryType x) {
+ return operator/=(Vector<T>(x));
+ }
+
+ inline INTRINSIC Vector<T> &operator=(const Vector<T> &x) {
+ vec->assign(x, mask);
+ return *vec;
+ }
+
+ inline INTRINSIC Vector<T> &operator=(EntryType x) {
+ vec->assign(Vector<T>(x), mask);
+ return *vec;
+ }
+
+ template<typename F> inline void INTRINSIC call(const F &f) const {
+ return vec->call(f, mask);
+ }
+ template<typename F> inline void INTRINSIC call(F &f) const {
+ return vec->call(f, mask);
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(const F &f) const {
+ return vec->apply(f, mask);
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(F &f) const {
+ return vec->apply(f, mask);
+ }
+
+ private:
+ WriteMaskedVector(Vector<T> *v, const Mask &k) : vec(v), mask(k) {}
+ Vector<T> *const vec;
+ Mask mask;
+};
+
+template<typename T> class Vector
+{
+ friend class WriteMaskedVector<T>;
+ protected:
+ typedef typename VectorTraits<T>::StorageType StorageType;
+ StorageType d;
+ typedef typename VectorTraits<T>::GatherMaskType GatherMask;
+ typedef VectorHelper<typename VectorTraits<T>::VectorType> HV;
+ typedef VectorHelper<T> HT;
+ public:
+ FREE_STORE_OPERATORS_ALIGNED(16)
+
+ enum Constants { Size = VectorTraits<T>::Size };
+ typedef typename VectorTraits<T>::VectorType VectorType;
+ typedef typename VectorTraits<T>::EntryType EntryType;
+ typedef typename VectorTraits<T>::IndexType IndexType;
+ typedef typename VectorTraits<T>::MaskType Mask;
+ typedef typename Mask::Argument MaskArg;
+ typedef Vc::Memory<Vector<T>, Size> Memory;
+#ifdef VC_PASSING_VECTOR_BY_VALUE_IS_BROKEN
+ typedef const Vector<T> &AsArg;
+#else
+ typedef const Vector<T> AsArg;
+#endif
+
+ typedef T _T;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // uninitialized
+ inline Vector() {}
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // constants
+ explicit inline INTRINSIC_L Vector(VectorSpecialInitializerZero::ZEnum) INTRINSIC_R;
+ explicit inline INTRINSIC_L Vector(VectorSpecialInitializerOne::OEnum) INTRINSIC_R;
+ explicit inline INTRINSIC_L Vector(VectorSpecialInitializerIndexesFromZero::IEnum) INTRINSIC_R;
+ static inline INTRINSIC_L Vector Zero() INTRINSIC_R;
+ static inline INTRINSIC_L Vector One() INTRINSIC_R;
+ static inline INTRINSIC_L Vector IndexesFromZero() INTRINSIC_R;
+ static inline INTRINSIC_L Vector Random() INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // internal: required to enable returning objects of VectorType
+ inline Vector(const VectorType &x) : d(x) {}
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // static_cast / copy ctor
+ template<typename OtherT> explicit inline INTRINSIC_L Vector(const Vector<OtherT> &x) INTRINSIC_R;
+
+ // implicit cast
+ template<typename OtherT> inline INTRINSIC_L Vector &operator=(const Vector<OtherT> &x) INTRINSIC_R;
+
+ // copy assignment
+ inline Vector &operator=(AsArg v) { d.v() = v.d.v(); return *this; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // broadcast
+ explicit Vector(EntryType a);
+ template<typename TT> inline INTRINSIC Vector(TT x, VC_EXACT_TYPE(TT, EntryType, void *) = 0) : d(HT::set(x)) {}
+ static inline Vector INTRINSIC broadcast4(const EntryType *x) { return Vector<T>(x); }
+ inline Vector &operator=(EntryType a) { d.v() = HT::set(a); return *this; }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // load ctors
+ explicit inline INTRINSIC_L
+ Vector(const EntryType *x) INTRINSIC_R;
+ template<typename Alignment> inline INTRINSIC_L
+ Vector(const EntryType *x, Alignment align) INTRINSIC_R;
+ template<typename OtherT> explicit inline INTRINSIC_L
+ Vector(const OtherT *x) INTRINSIC_R;
+ template<typename OtherT, typename Alignment> inline INTRINSIC_L
+ Vector(const OtherT *x, Alignment align) INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // load member functions
+ inline INTRINSIC_L
+ void load(const EntryType *mem) INTRINSIC_R;
+ template<typename Alignment> inline INTRINSIC_L
+ void load(const EntryType *mem, Alignment align) INTRINSIC_R;
+ template<typename OtherT> inline INTRINSIC_L
+ void load(const OtherT *mem) INTRINSIC_R;
+ template<typename OtherT, typename Alignment> inline INTRINSIC_L
+ void load(const OtherT *mem, Alignment align) INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // expand 1 float_v to 2 double_v XXX rationale? remove it for release? XXX
+ explicit inline INTRINSIC_L Vector(const Vector<typename CtorTypeHelper<T>::Type> *a) INTRINSIC_R;
+ void expand(Vector<typename ExpandTypeHelper<T>::Type> *x) const;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // zeroing
+ inline void INTRINSIC_L setZero() INTRINSIC_R;
+ inline void INTRINSIC_L setZero(const Mask &k) INTRINSIC_R;
+
+ inline void INTRINSIC_L setQnan() INTRINSIC_R;
+ inline void INTRINSIC_L setQnan(typename Mask::Argument k) INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // stores
+ inline void INTRINSIC_L store(EntryType *mem) const INTRINSIC_R;
+ inline void INTRINSIC_L store(EntryType *mem, const Mask &mask) const INTRINSIC_R;
+ template<typename A> inline void INTRINSIC_L store(EntryType *mem, A align) const INTRINSIC_R;
+ template<typename A> inline void INTRINSIC_L store(EntryType *mem, const Mask &mask, A align) const INTRINSIC_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // swizzles
+ inline const Vector<T> INTRINSIC_L CONST_L &abcd() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L cdab() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L badc() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L aaaa() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L bbbb() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L cccc() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dddd() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L bcad() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L bcda() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dabc() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L acbd() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dbca() const INTRINSIC_R CONST_R;
+ inline const Vector<T> INTRINSIC_L CONST_L dcba() const INTRINSIC_R CONST_R;
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // gathers
+ template<typename IndexT> Vector(const EntryType *mem, const IndexT *indexes);
+ template<typename IndexT> Vector(const EntryType *mem, const Vector<IndexT> indexes);
+ template<typename IndexT> Vector(const EntryType *mem, const IndexT *indexes, MaskArg mask);
+ template<typename IndexT> Vector(const EntryType *mem, const Vector<IndexT> indexes, MaskArg mask);
+ template<typename S1, typename IT> Vector(const S1 *array, const EntryType S1::* member1, const IT indexes);
+ template<typename S1, typename IT> Vector(const S1 *array, const EntryType S1::* member1, const IT indexes, MaskArg mask);
+ template<typename S1, typename S2, typename IT> Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes);
+ template<typename S1, typename S2, typename IT> Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes, MaskArg mask);
+ template<typename S1, typename IT1, typename IT2> Vector(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes);
+ template<typename S1, typename IT1, typename IT2> Vector(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes, MaskArg mask);
+ template<typename Index> void gather(const EntryType *mem, const Index indexes);
+ template<typename Index> void gather(const EntryType *mem, const Index indexes, MaskArg mask);
+#ifdef VC_USE_SET_GATHERS
+ template<typename IT> void gather(const EntryType *mem, Vector<IT> indexes, MaskArg mask);
+#endif
+ template<typename S1, typename IT> void gather(const S1 *array, const EntryType S1::* member1, const IT indexes);
+ template<typename S1, typename IT> void gather(const S1 *array, const EntryType S1::* member1, const IT indexes, MaskArg mask);
+ template<typename S1, typename S2, typename IT> void gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes);
+ template<typename S1, typename S2, typename IT> void gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes, MaskArg mask);
+ template<typename S1, typename IT1, typename IT2> void gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes);
+ template<typename S1, typename IT1, typename IT2> void gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes, MaskArg mask);
+
+ ///////////////////////////////////////////////////////////////////////////////////////////
+ // scatters
+ template<typename Index> void scatter(EntryType *mem, const Index indexes) const;
+ template<typename Index> void scatter(EntryType *mem, const Index indexes, MaskArg mask) const;
+ template<typename S1, typename IT> void scatter(S1 *array, EntryType S1::* member1, const IT indexes) const;
+ template<typename S1, typename IT> void scatter(S1 *array, EntryType S1::* member1, const IT indexes, MaskArg mask) const;
+ template<typename S1, typename S2, typename IT> void scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, const IT indexes) const;
+ template<typename S1, typename S2, typename IT> void scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, const IT indexes, MaskArg mask) const;
+ template<typename S1, typename IT1, typename IT2> void scatter(S1 *array, EntryType *S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes) const;
+ template<typename S1, typename IT1, typename IT2> void scatter(S1 *array, EntryType *S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes, MaskArg mask) const;
+
+ //prefix
+ inline Vector INTRINSIC &operator++() { data() = VectorHelper<T>::add(data(), VectorHelper<T>::one()); return *this; }
+ //postfix
+ inline Vector INTRINSIC operator++(int) { const Vector<T> r = *this; data() = VectorHelper<T>::add(data(), VectorHelper<T>::one()); return r; }
+
+ inline Common::AliasingEntryHelper<StorageType> INTRINSIC operator[](size_t index) {
+#if defined(VC_GCC) && VC_GCC >= 0x40300 && VC_GCC < 0x40400
+ ::Vc::Warnings::_operator_bracket_warning();
+#endif
+ return d.m(index);
+ }
+ inline EntryType INTRINSIC_L operator[](size_t index) const PURE INTRINSIC_R;
+
+ inline Vector PURE INTRINSIC operator~() const { return VectorHelper<VectorType>::andnot_(data(), VectorHelper<VectorType>::allone()); }
+ inline Vector<typename NegateTypeHelper<T>::Type> operator-() const;
+
+#define OP(symbol, fun) \
+ inline Vector INTRINSIC &operator symbol##=(const Vector<T> &x) { data() = VectorHelper<T>::fun(data(), x.data()); return *this; } \
+ inline Vector INTRINSIC &operator symbol##=(EntryType x) { return operator symbol##=(Vector<T>(x)); } \
+ inline Vector PURE INTRINSIC operator symbol(const Vector<T> &x) const { return HT::fun(data(), x.data()); } \
+ template<typename TT> inline VC_EXACT_TYPE(TT, EntryType, Vector) PURE INTRINSIC operator symbol(TT x) const { return operator symbol(Vector(x)); }
+
+ OP(+, add)
+ OP(-, sub)
+ OP(*, mul)
+#undef OP
+
+ inline INTRINSIC_L Vector &operator<<=(AsArg shift) INTRINSIC_R;
+ inline INTRINSIC_L Vector operator<< (AsArg shift) const INTRINSIC_R;
+ inline INTRINSIC_L Vector &operator<<=( int shift) INTRINSIC_R;
+ inline INTRINSIC_L Vector operator<< ( int shift) const INTRINSIC_R;
+ inline INTRINSIC_L Vector &operator>>=(AsArg shift) INTRINSIC_R;
+ inline INTRINSIC_L Vector operator>> (AsArg shift) const INTRINSIC_R;
+ inline INTRINSIC_L Vector &operator>>=( int shift) INTRINSIC_R;
+ inline INTRINSIC_L Vector operator>> ( int shift) const INTRINSIC_R;
+
+ inline INTRINSIC_L Vector &operator/=(const Vector<T> &x) INTRINSIC_R;
+ inline INTRINSIC_L Vector operator/ (const Vector<T> &x) const PURE INTRINSIC_R;
+ inline INTRINSIC_L Vector &operator/=(EntryType x) INTRINSIC_R;
+ template<typename TT> inline INTRINSIC_L VC_EXACT_TYPE(TT, typename DetermineEntryType<T>::Type, Vector<T>) operator/(TT x) const PURE INTRINSIC_R;
+
+#define OP(symbol, fun) \
+ inline Vector INTRINSIC_L &operator symbol##=(const Vector<T> &x) INTRINSIC_R; \
+ inline Vector INTRINSIC_L operator symbol(const Vector<T> &x) const PURE INTRINSIC_R; \
+ inline Vector INTRINSIC &operator symbol##=(EntryType x) { return operator symbol##=(Vector(x)); } \
+ template<typename TT> inline VC_EXACT_TYPE(TT, EntryType, Vector) PURE INTRINSIC operator symbol(TT x) const { return operator symbol(Vector(x)); }
+ OP(|, or_)
+ OP(&, and_)
+ OP(^, xor_)
+#undef OP
+#define OPcmp(symbol, fun) \
+ inline Mask PURE INTRINSIC operator symbol(const Vector<T> &x) const { return VectorHelper<T>::fun(data(), x.data()); } \
+ template<typename TT> inline VC_EXACT_TYPE(TT, EntryType, Mask) PURE INTRINSIC operator symbol(TT x) const { return operator symbol(Vector(x)); }
+
+ OPcmp(==, cmpeq)
+ OPcmp(!=, cmpneq)
+ OPcmp(>=, cmpnlt)
+ OPcmp(>, cmpnle)
+ OPcmp(<, cmplt)
+ OPcmp(<=, cmple)
+#undef OPcmp
+
+ inline void multiplyAndAdd(const Vector<T> &factor, const Vector<T> &summand) {
+ VectorHelper<T>::multiplyAndAdd(data(), factor, summand);
+ }
+
+ inline void assign( const Vector<T> &v, const Mask &mask ) {
+ const VectorType k = mm128_reinterpret_cast<VectorType>(mask.data());
+ data() = VectorHelper<VectorType>::blend(data(), v.data(), k);
+ }
+
+ template<typename V2> inline V2 staticCast() const { return StaticCastHelper<T, typename V2::_T>::cast(data()); }
+ template<typename V2> inline V2 reinterpretCast() const { return mm128_reinterpret_cast<typename V2::VectorType>(data()); }
+
+ inline WriteMaskedVector<T> INTRINSIC operator()(const Mask &k) { return WriteMaskedVector<T>(this, k); }
+
+ /**
+ * \return \p true This vector was completely filled. m2 might be 0 or != 0. You still have
+ * to test this.
+ * \p false This vector was not completely filled. m2 is all 0.
+ */
+ //inline bool pack(Mask &m1, Vector<T> &v2, Mask &m2) {
+ //return VectorHelper<T>::pack(data(), m1.data, v2.data(), m2.data);
+ //}
+
+ inline VectorType &data() { return d.v(); }
+ inline const VectorType &data() const { return d.v(); }
+
+ inline EntryType INTRINSIC min() const { return VectorHelper<T>::min(data()); }
+ inline EntryType INTRINSIC max() const { return VectorHelper<T>::max(data()); }
+ inline EntryType INTRINSIC product() const { return VectorHelper<T>::mul(data()); }
+ inline EntryType INTRINSIC sum() const { return VectorHelper<T>::add(data()); }
+ inline INTRINSIC_L EntryType min(MaskArg m) const INTRINSIC_R;
+ inline INTRINSIC_L EntryType max(MaskArg m) const INTRINSIC_R;
+ inline INTRINSIC_L EntryType product(MaskArg m) const INTRINSIC_R;
+ inline INTRINSIC_L EntryType sum(MaskArg m) const INTRINSIC_R;
+
+ inline Vector sorted() const { return SortHelper<VectorType, Size>::sort(data()); }
+
+ template<typename F> void callWithValuesSorted(F &f) {
+ EntryType value = d.m(0);
+ f(value);
+ for (int i = 1; i < Size; ++i) {
+ if (d.m(i) != value) {
+ value = d.m(i);
+ f(value);
+ }
+ }
+ }
+
+ template<typename F> inline void INTRINSIC call(const F &f) const {
+ for_all_vector_entries(i,
+ f(EntryType(d.m(i)));
+ );
+ }
+ template<typename F> inline void INTRINSIC call(F &f) const {
+ for_all_vector_entries(i,
+ f(EntryType(d.m(i)));
+ );
+ }
+
+ template<typename F> inline void INTRINSIC call(const F &f, const Mask &mask) const {
+ Vc_foreach_bit(size_t i, mask) {
+ f(EntryType(d.m(i)));
+ }
+ }
+ template<typename F> inline void INTRINSIC call(F &f, const Mask &mask) const {
+ Vc_foreach_bit(size_t i, mask) {
+ f(EntryType(d.m(i)));
+ }
+ }
+
+ template<typename F> inline Vector<T> INTRINSIC apply(const F &f) const {
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = f(EntryType(d.m(i)));
+ );
+ return r;
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(F &f) const {
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = f(EntryType(d.m(i)));
+ );
+ return r;
+ }
+
+ template<typename F> inline Vector<T> INTRINSIC apply(const F &f, const Mask &mask) const {
+ Vector<T> r(*this);
+ Vc_foreach_bit (size_t i, mask) {
+ r.d.m(i) = f(EntryType(r.d.m(i)));
+ }
+ return r;
+ }
+ template<typename F> inline Vector<T> INTRINSIC apply(F &f, const Mask &mask) const {
+ Vector<T> r(*this);
+ Vc_foreach_bit (size_t i, mask) {
+ r.d.m(i) = f(EntryType(r.d.m(i)));
+ }
+ return r;
+ }
+
+ template<typename IndexT> inline void INTRINSIC fill(EntryType (&f)(IndexT)) {
+ for_all_vector_entries(i,
+ d.m(i) = f(i);
+ );
+ }
+ inline void INTRINSIC fill(EntryType (&f)()) {
+ for_all_vector_entries(i,
+ d.m(i) = f();
+ );
+ }
+
+ inline INTRINSIC_L Vector copySign(typename Vector::AsArg reference) const INTRINSIC_R;
+ inline INTRINSIC_L Vector exponent() const INTRINSIC_R;
+};
+
+typedef Vector<double> double_v;
+typedef Vector<float> float_v;
+typedef Vector<float8> sfloat_v;
+typedef Vector<int> int_v;
+typedef Vector<unsigned int> uint_v;
+typedef Vector<short> short_v;
+typedef Vector<unsigned short> ushort_v;
+typedef double_v::Mask double_m;
+typedef float_v::Mask float_m;
+typedef sfloat_v::Mask sfloat_m;
+typedef int_v::Mask int_m;
+typedef uint_v::Mask uint_m;
+typedef short_v::Mask short_m;
+typedef ushort_v::Mask ushort_m;
+
+template<> inline Vector<float8> Vector<float8>::broadcast4(const float *x) {
+ const _M128 &v = VectorHelper<_M128>::load(x, Aligned);
+ return Vector<float8>(M256::create(v, v));
+}
+
+template<typename T> class SwizzledVector : public Vector<T> {};
+
+static inline int_v min(const int_v &x, const int_v &y) { return _mm_min_epi32(x.data(), y.data()); }
+static inline uint_v min(const uint_v &x, const uint_v &y) { return _mm_min_epu32(x.data(), y.data()); }
+static inline short_v min(const short_v &x, const short_v &y) { return _mm_min_epi16(x.data(), y.data()); }
+static inline ushort_v min(const ushort_v &x, const ushort_v &y) { return _mm_min_epu16(x.data(), y.data()); }
+static inline float_v min(const float_v &x, const float_v &y) { return _mm_min_ps(x.data(), y.data()); }
+static inline double_v min(const double_v &x, const double_v &y) { return _mm_min_pd(x.data(), y.data()); }
+static inline int_v max(const int_v &x, const int_v &y) { return _mm_max_epi32(x.data(), y.data()); }
+static inline uint_v max(const uint_v &x, const uint_v &y) { return _mm_max_epu32(x.data(), y.data()); }
+static inline short_v max(const short_v &x, const short_v &y) { return _mm_max_epi16(x.data(), y.data()); }
+static inline ushort_v max(const ushort_v &x, const ushort_v &y) { return _mm_max_epu16(x.data(), y.data()); }
+static inline float_v max(const float_v &x, const float_v &y) { return _mm_max_ps(x.data(), y.data()); }
+static inline double_v max(const double_v &x, const double_v &y) { return _mm_max_pd(x.data(), y.data()); }
+
+static inline sfloat_v min(const sfloat_v &x, const sfloat_v &y) {
+ return M256::create(_mm_min_ps(x.data()[0], y.data()[0]), _mm_min_ps(x.data()[1], y.data()[1]));
+}
+static inline sfloat_v max(const sfloat_v &x, const sfloat_v &y) {
+ return M256::create(_mm_max_ps(x.data()[0], y.data()[0]), _mm_max_ps(x.data()[1], y.data()[1]));
+}
+
+ template<typename T> static inline Vector<T> sqrt (const Vector<T> &x) { return VectorHelper<T>::sqrt(x.data()); }
+ template<typename T> static inline Vector<T> rsqrt(const Vector<T> &x) { return VectorHelper<T>::rsqrt(x.data()); }
+ template<typename T> static inline Vector<T> abs (const Vector<T> &x) { return VectorHelper<T>::abs(x.data()); }
+ template<typename T> static inline Vector<T> reciprocal(const Vector<T> &x) { return VectorHelper<T>::reciprocal(x.data()); }
+ template<typename T> static inline Vector<T> round(const Vector<T> &x) { return VectorHelper<T>::round(x.data()); }
+
+ template<typename T> static inline typename Vector<T>::Mask isfinite(const Vector<T> &x) { return VectorHelper<T>::isFinite(x.data()); }
+ template<typename T> static inline typename Vector<T>::Mask isnan(const Vector<T> &x) { return VectorHelper<T>::isNaN(x.data()); }
+
+#include "forceToRegisters.tcc"
+#ifdef VC_GNU_ASM
+template<>
+inline void ALWAYS_INLINE forceToRegisters(const Vector<float8> &x1) {
+ __asm__ __volatile__(""::"x"(x1.data()[0]), "x"(x1.data()[1]));
+}
+#elif defined(VC_MSVC)
+#pragma optimize("g", off)
+template<>
+inline void ALWAYS_INLINE forceToRegisters(const Vector<float8> &/*x1*/) {
+}
+#endif
+} // namespace SSE
+} // namespace Vc
+
+#include "undomacros.h"
+#include "vector.tcc"
+#include "math.h"
+#endif // SSE_VECTOR_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "limits.h"
+#include "../common/bitscanintrinsics.h"
+#include "macros.h"
+
+namespace Vc
+{
+ALIGN(64) extern unsigned int RandomState[16];
+
+namespace SSE
+{
+
+template<typename T, int Size> static inline const T *_IndexesFromZero() {
+ if (Size == 4) {
+ return reinterpret_cast<const T *>(_IndexesFromZero4);
+ } else if (Size == 8) {
+ return reinterpret_cast<const T *>(_IndexesFromZero8);
+ } else if (Size == 16) {
+ return reinterpret_cast<const T *>(_IndexesFromZero16);
+ }
+ return 0;
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// constants {{{1
+template<typename T> inline Vector<T>::Vector(VectorSpecialInitializerZero::ZEnum)
+ : d(VectorHelper<VectorType>::zero())
+{
+}
+
+template<typename T> inline Vector<T>::Vector(VectorSpecialInitializerOne::OEnum)
+ : d(VectorHelper<T>::one())
+{
+}
+
+template<typename T> inline Vector<T>::Vector(VectorSpecialInitializerIndexesFromZero::IEnum)
+ : d(VectorHelper<VectorType>::load(_IndexesFromZero<EntryType, Size>(), Aligned))
+{
+}
+
+template<typename T> inline Vector<T> Vector<T>::Zero()
+{
+ return VectorHelper<VectorType>::zero();
+}
+
+template<typename T> inline Vector<T> Vector<T>::One()
+{
+ return VectorHelper<T>::one();
+}
+
+template<typename T> inline Vector<T> Vector<T>::IndexesFromZero()
+{
+ return VectorHelper<VectorType>::load(_IndexesFromZero<EntryType, Size>(), Aligned);
+}
+
+// conversion/casts {{{1
+template<typename T> template<typename OtherT> inline INTRINSIC Vector<T>::Vector(const Vector<OtherT> &x)
+ : d(StaticCastHelper<OtherT, T>::cast(x.data()))
+{
+}
+
+template<> template<> inline INTRINSIC short_v &Vector<short>::operator=(const ushort_v &x) {
+ data() = StaticCastHelper<unsigned short, short>::cast(x.data()); return *this;
+}
+template<> template<> inline INTRINSIC ushort_v &Vector<unsigned short>::operator=(const short_v &x) {
+ data() = StaticCastHelper<short, unsigned short>::cast(x.data()); return *this;
+}
+template<> template<> inline INTRINSIC int_v &Vector<int>::operator=(const uint_v &x) {
+ data() = StaticCastHelper<unsigned int, int>::cast(x.data()); return *this;
+}
+template<> template<> inline INTRINSIC uint_v &Vector<unsigned int>::operator=(const int_v &x) {
+ data() = StaticCastHelper<int, unsigned int>::cast(x.data()); return *this;
+}
+
+// broadcasts {{{1
+template<typename T> inline Vector<T>::Vector(EntryType a)
+ : d(VectorHelper<T>::set(a))
+{
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// load ctors {{{1
+template<typename T> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *x) { load(x); }
+template<typename T> template<typename A> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *x, A a) { load(x, a); }
+template<typename T> template<typename OtherT> inline ALWAYS_INLINE Vector<T>::Vector(const OtherT *x) { load(x); }
+template<typename T> template<typename OtherT, typename A> inline ALWAYS_INLINE Vector<T>::Vector(const OtherT *x, A a) { load(x, a); }
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// load member functions {{{1
+template<typename T> inline void INTRINSIC Vector<T>::load(const EntryType *mem)
+{
+ load(mem, Aligned);
+}
+
+template<typename T> template<typename A> inline void INTRINSIC Vector<T>::load(const EntryType *mem, A align)
+{
+ d.v() = VectorHelper<VectorType>::load(mem, align);
+}
+
+template<typename T> template<typename OtherT> inline void INTRINSIC Vector<T>::load(const OtherT *mem)
+{
+ load(mem, Aligned);
+}
+
+// float8: simply use the float implementation twice {{{2
+template<> template<typename OtherT, typename A> inline void INTRINSIC Vector<float8>::load(const OtherT *x, A a)
+{
+ d.v() = M256::create(
+ Vector<float>(&x[0], a).data(),
+ Vector<float>(&x[4], a).data()
+ );
+}
+
+// LoadHelper {{{2
+template<typename DstT, typename SrcT, typename Flags> struct LoadHelper;
+
+// float {{{2
+template<typename Flags> struct LoadHelper<float, double, Flags> {
+ static inline __m128 load(const double *mem, Flags f)
+ {
+ return _mm_movelh_ps(_mm_cvtpd_ps(VectorHelper<__m128d>::load(&mem[0], f)),
+ _mm_cvtpd_ps(VectorHelper<__m128d>::load(&mem[2], f)));
+ }
+};
+template<typename Flags> struct LoadHelper<float, unsigned int, Flags> {
+ static inline __m128 load(const unsigned int *mem, Flags f)
+ {
+ return StaticCastHelper<unsigned int, float>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, int, Flags> {
+ static inline __m128 load(const int *mem, Flags f)
+ {
+ return StaticCastHelper<int, float>::cast(VectorHelper<__m128i>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, unsigned short, Flags> {
+ static inline __m128 load(const unsigned short *mem, Flags f)
+ {
+ return _mm_cvtepi32_ps(LoadHelper<int, unsigned short, Flags>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, short, Flags> {
+ static inline __m128 load(const short *mem, Flags f)
+ {
+ return _mm_cvtepi32_ps(LoadHelper<int, short, Flags>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, unsigned char, Flags> {
+ static inline __m128 load(const unsigned char *mem, Flags f)
+ {
+ return _mm_cvtepi32_ps(LoadHelper<int, unsigned char, Flags>::load(mem, f));
+ }
+};
+template<typename Flags> struct LoadHelper<float, signed char, Flags> {
+ static inline __m128 load(const signed char *mem, Flags f)
+ {
+ return _mm_cvtepi32_ps(LoadHelper<int, signed char, Flags>::load(mem, f));
+ }
+};
+
+// int {{{2
+template<typename Flags> struct LoadHelper<int, unsigned int, Flags> {
+ static inline __m128i load(const unsigned int *mem, Flags f)
+ {
+ return VectorHelper<__m128i>::load(mem, f);
+ }
+};
+// no difference between streaming and alignment, because the
+// 32/64 bit loads are not available as streaming loads, and can always be unaligned
+template<typename Flags> struct LoadHelper<int, unsigned short, Flags> {
+ static inline __m128i load(const unsigned short *mem, Flags)
+ {
+ return _mm_cvtepu16_epi32( _mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem)));
+ }
+};
+template<typename Flags> struct LoadHelper<int, short, Flags> {
+ static inline __m128i load(const short *mem, Flags)
+ {
+ return _mm_cvtepi16_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem)));
+ }
+};
+template<typename Flags> struct LoadHelper<int, unsigned char, Flags> {
+ static inline __m128i load(const unsigned char *mem, Flags)
+ {
+ return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(*reinterpret_cast<const int *>(mem)));
+ }
+};
+template<typename Flags> struct LoadHelper<int, signed char, Flags> {
+ static inline __m128i load(const signed char *mem, Flags)
+ {
+ return _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*reinterpret_cast<const int *>(mem)));
+ }
+};
+
+// unsigned int {{{2
+template<typename Flags> struct LoadHelper<unsigned int, unsigned short, Flags> {
+ static inline __m128i load(const unsigned short *mem, Flags)
+ {
+ return _mm_cvtepu16_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem)));
+ }
+};
+template<typename Flags> struct LoadHelper<unsigned int, unsigned char, Flags> {
+ static inline __m128i load(const unsigned char *mem, Flags)
+ {
+ return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(*reinterpret_cast<const int *>(mem)));
+ }
+};
+
+// short {{{2
+template<typename Flags> struct LoadHelper<short, unsigned short, Flags> {
+ static inline __m128i load(const unsigned short *mem, Flags f)
+ {
+ return VectorHelper<__m128i>::load(mem, f);
+ }
+};
+template<typename Flags> struct LoadHelper<short, unsigned char, Flags> {
+ static inline __m128i load(const unsigned char *mem, Flags)
+ {
+ return _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem)));
+ }
+};
+template<typename Flags> struct LoadHelper<short, signed char, Flags> {
+ static inline __m128i load(const signed char *mem, Flags)
+ {
+ return _mm_cvtepi8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem)));
+ }
+};
+
+// unsigned short {{{2
+template<typename Flags> struct LoadHelper<unsigned short, unsigned char, Flags> {
+ static inline __m128i load(const unsigned char *mem, Flags)
+ {
+ return _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast<const __m128i *>(mem)));
+ }
+};
+
+// general load, implemented via LoadHelper {{{2
+template<typename DstT> template<typename SrcT, typename Flags> inline void INTRINSIC Vector<DstT>::load(const SrcT *x, Flags f)
+{
+ d.v() = LoadHelper<DstT, SrcT, Flags>::load(x, f);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// expand/combine {{{1
+template<typename T> inline Vector<T>::Vector(const Vector<typename CtorTypeHelper<T>::Type> *a)
+ : d(VectorHelper<T>::concat(a[0].data(), a[1].data()))
+{
+}
+
+template<typename T> inline void Vector<T>::expand(Vector<typename ExpandTypeHelper<T>::Type> *x) const
+{
+ if (Size == 8u) {
+ x[0].data() = VectorHelper<T>::expand0(data());
+ x[1].data() = VectorHelper<T>::expand1(data());
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// zeroing {{{1
+template<typename T> inline void Vector<T>::setZero()
+{
+ data() = VectorHelper<VectorType>::zero();
+}
+
+template<typename T> inline void Vector<T>::setZero(const Mask &k)
+{
+ data() = VectorHelper<VectorType>::andnot_(mm128_reinterpret_cast<VectorType>(k.data()), data());
+}
+
+template<> inline void INTRINSIC Vector<double>::setQnan()
+{
+ data() = _mm_setallone_pd();
+}
+template<> inline void INTRINSIC Vector<double>::setQnan(Mask::Argument k)
+{
+ data() = _mm_or_pd(data(), k.dataD());
+}
+template<> inline void INTRINSIC Vector<float>::setQnan()
+{
+ data() = _mm_setallone_ps();
+}
+template<> inline void INTRINSIC Vector<float>::setQnan(Mask::Argument k)
+{
+ data() = _mm_or_ps(data(), k.data());
+}
+template<> inline void INTRINSIC Vector<float8>::setQnan()
+{
+ d.v()[0] = _mm_setallone_ps();
+ d.v()[1] = _mm_setallone_ps();
+}
+template<> inline void INTRINSIC Vector<float8>::setQnan(Mask::Argument k)
+{
+ d.v()[0] = _mm_or_ps(d.v()[0], k.data()[0]);
+ d.v()[1] = _mm_or_ps(d.v()[1], k.data()[1]);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// stores {{{1
+template<typename T> inline void Vector<T>::store(EntryType *mem) const
+{
+ VectorHelper<VectorType>::store(mem, data(), Aligned);
+}
+
+template<typename T> inline void Vector<T>::store(EntryType *mem, const Mask &mask) const
+{
+ VectorHelper<VectorType>::store(mem, data(), mm128_reinterpret_cast<VectorType>(mask.data()), Aligned);
+}
+
+template<typename T> template<typename A> inline void Vector<T>::store(EntryType *mem, A align) const
+{
+ VectorHelper<VectorType>::store(mem, data(), align);
+}
+
+template<typename T> template<typename A> inline void Vector<T>::store(EntryType *mem, const Mask &mask, A align) const
+{
+ HV::store(mem, data(), mm128_reinterpret_cast<VectorType>(mask.data()), align);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// division {{{1
+template<typename T> inline INTRINSIC CONST Vector<T> &WriteMaskedVector<T>::operator/=(const Vector<T> &x)
+{
+ return operator=(*vec / x);
+}
+template<> inline INTRINSIC CONST int_v &WriteMaskedVector<int>::operator/=(const int_v &x)
+{
+ Vc_foreach_bit (int i, mask) {
+ vec->d.m(i) /= x.d.m(i);
+ }
+ return *vec;
+}
+template<> inline INTRINSIC CONST uint_v &WriteMaskedVector<unsigned int>::operator/=(const uint_v &x)
+{
+ Vc_foreach_bit (int i, mask) {
+ vec->d.m(i) /= x.d.m(i);
+ }
+ return *vec;
+}
+template<> inline INTRINSIC CONST short_v &WriteMaskedVector<short>::operator/=(const short_v &x)
+{
+ Vc_foreach_bit (int i, mask) {
+ vec->d.m(i) /= x.d.m(i);
+ }
+ return *vec;
+}
+template<> inline INTRINSIC CONST ushort_v &WriteMaskedVector<unsigned short>::operator/=(const ushort_v &x)
+{
+ Vc_foreach_bit (int i, mask) {
+ vec->d.m(i) /= x.d.m(i);
+ }
+ return *vec;
+}
+
+template<typename T> inline Vector<T> &Vector<T>::operator/=(EntryType x)
+{
+ if (VectorTraits<T>::HasVectorDivision) {
+ return operator/=(Vector<T>(x));
+ }
+ for_all_vector_entries(i,
+ d.m(i) /= x;
+ );
+ return *this;
+}
+
+template<typename T> template<typename TT> inline PURE INTRINSIC VC_EXACT_TYPE(TT, typename DetermineEntryType<T>::Type, Vector<T>) Vector<T>::operator/(TT x) const
+{
+ if (VectorTraits<T>::HasVectorDivision) {
+ return operator/(Vector<T>(x));
+ }
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = d.m(i) / x;
+ );
+ return r;
+}
+
+template<typename T> inline Vector<T> &Vector<T>::operator/=(const Vector<T> &x)
+{
+ for_all_vector_entries(i,
+ d.m(i) /= x.d.m(i);
+ );
+ return *this;
+}
+
+template<typename T> inline Vector<T> Vector<T>::operator/(const Vector<T> &x) const
+{
+ Vector<T> r;
+ for_all_vector_entries(i,
+ r.d.m(i) = d.m(i) / x.d.m(i);
+ );
+ return r;
+}
+
+template<> inline Vector<short> &Vector<short>::operator/=(const Vector<short> &x)
+{
+ __m128 lo = _mm_cvtepi32_ps(VectorHelper<short>::expand0(d.v()));
+ __m128 hi = _mm_cvtepi32_ps(VectorHelper<short>::expand1(d.v()));
+ lo = _mm_div_ps(lo, _mm_cvtepi32_ps(VectorHelper<short>::expand0(x.d.v())));
+ hi = _mm_div_ps(hi, _mm_cvtepi32_ps(VectorHelper<short>::expand1(x.d.v())));
+ d.v() = _mm_packs_epi32(_mm_cvtps_epi32(lo), _mm_cvtps_epi32(hi));
+ return *this;
+}
+
+template<> inline Vector<short> ALWAYS_INLINE Vector<short>::operator/(const Vector<short> &x) const
+{
+ __m128 lo = _mm_cvtepi32_ps(VectorHelper<short>::expand0(d.v()));
+ __m128 hi = _mm_cvtepi32_ps(VectorHelper<short>::expand1(d.v()));
+ lo = _mm_div_ps(lo, _mm_cvtepi32_ps(VectorHelper<short>::expand0(x.d.v())));
+ hi = _mm_div_ps(hi, _mm_cvtepi32_ps(VectorHelper<short>::expand1(x.d.v())));
+ return _mm_packs_epi32(_mm_cvtps_epi32(lo), _mm_cvtps_epi32(hi));
+}
+
+template<> inline Vector<unsigned short> &Vector<unsigned short>::operator/=(const Vector<unsigned short> &x)
+{
+ __m128 lo = _mm_cvtepi32_ps(VectorHelper<short>::expand0(d.v()));
+ __m128 hi = _mm_cvtepi32_ps(VectorHelper<short>::expand1(d.v()));
+ lo = _mm_div_ps(lo, _mm_cvtepi32_ps(VectorHelper<short>::expand0(x.d.v())));
+ hi = _mm_div_ps(hi, _mm_cvtepi32_ps(VectorHelper<short>::expand1(x.d.v())));
+ d.v() = _mm_packs_epi32(_mm_cvtps_epi32(lo), _mm_cvtps_epi32(hi));
+ return *this;
+}
+
+template<> inline Vector<unsigned short> ALWAYS_INLINE Vector<unsigned short>::operator/(const Vector<unsigned short> &x) const
+{
+ __m128 lo = _mm_cvtepi32_ps(VectorHelper<short>::expand0(d.v()));
+ __m128 hi = _mm_cvtepi32_ps(VectorHelper<short>::expand1(d.v()));
+ lo = _mm_div_ps(lo, _mm_cvtepi32_ps(VectorHelper<short>::expand0(x.d.v())));
+ hi = _mm_div_ps(hi, _mm_cvtepi32_ps(VectorHelper<short>::expand1(x.d.v())));
+ return _mm_packs_epi32(_mm_cvtps_epi32(lo), _mm_cvtps_epi32(hi));
+}
+
+template<> inline Vector<float> &Vector<float>::operator/=(const Vector<float> &x)
+{
+ d.v() = _mm_div_ps(d.v(), x.d.v());
+ return *this;
+}
+
+template<> inline Vector<float> Vector<float>::operator/(const Vector<float> &x) const
+{
+ return _mm_div_ps(d.v(), x.d.v());
+}
+
+template<> inline Vector<float8> &Vector<float8>::operator/=(const Vector<float8> &x)
+{
+ d.v()[0] = _mm_div_ps(d.v()[0], x.d.v()[0]);
+ d.v()[1] = _mm_div_ps(d.v()[1], x.d.v()[1]);
+ return *this;
+}
+
+template<> inline Vector<float8> Vector<float8>::operator/(const Vector<float8> &x) const
+{
+ Vector<float8> r;
+ r.d.v()[0] = _mm_div_ps(d.v()[0], x.d.v()[0]);
+ r.d.v()[1] = _mm_div_ps(d.v()[1], x.d.v()[1]);
+ return r;
+}
+
+template<> inline Vector<double> &Vector<double>::operator/=(const Vector<double> &x)
+{
+ d.v() = _mm_div_pd(d.v(), x.d.v());
+ return *this;
+}
+
+template<> inline Vector<double> Vector<double>::operator/(const Vector<double> &x) const
+{
+ return _mm_div_pd(d.v(), x.d.v());
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// operator- {{{1
+template<> inline Vector<double> PURE ALWAYS_INLINE FLATTEN Vector<double>::operator-() const
+{
+ return _mm_xor_pd(d.v(), _mm_setsignmask_pd());
+}
+template<> inline Vector<float> PURE ALWAYS_INLINE FLATTEN Vector<float>::operator-() const
+{
+ return _mm_xor_ps(d.v(), _mm_setsignmask_ps());
+}
+template<> inline Vector<float8> PURE ALWAYS_INLINE FLATTEN Vector<float8>::operator-() const
+{
+ return M256::create(
+ _mm_xor_ps(d.v()[0], _mm_setsignmask_ps()),
+ _mm_xor_ps(d.v()[1], _mm_setsignmask_ps()));
+}
+template<> inline Vector<int> PURE ALWAYS_INLINE FLATTEN Vector<int>::operator-() const
+{
+#ifdef VC_IMPL_SSSE3
+ return _mm_sign_epi32(d.v(), _mm_setallone_si128());
+#else
+ return _mm_add_epi32(_mm_xor_si128(d.v(), _mm_setallone_si128()), _mm_setone_epi32());
+#endif
+}
+template<> inline Vector<int> PURE ALWAYS_INLINE FLATTEN Vector<unsigned int>::operator-() const
+{
+#ifdef VC_IMPL_SSSE3
+ return _mm_sign_epi32(d.v(), _mm_setallone_si128());
+#else
+ return _mm_add_epi32(_mm_xor_si128(d.v(), _mm_setallone_si128()), _mm_setone_epi32());
+#endif
+}
+template<> inline Vector<short> PURE ALWAYS_INLINE FLATTEN Vector<short>::operator-() const
+{
+#ifdef VC_IMPL_SSSE3
+ return _mm_sign_epi16(d.v(), _mm_setallone_si128());
+#else
+ return _mm_mullo_epi16(d.v(), _mm_setallone_si128());
+#endif
+}
+template<> inline Vector<short> PURE ALWAYS_INLINE FLATTEN Vector<unsigned short>::operator-() const
+{
+#ifdef VC_IMPL_SSSE3
+ return _mm_sign_epi16(d.v(), _mm_setallone_si128());
+#else
+ return _mm_mullo_epi16(d.v(), _mm_setallone_si128());
+#endif
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// integer ops {{{1
+#define OP_IMPL(T, symbol, fun) \
+template<> inline Vector<T> &Vector<T>::operator symbol##=(const Vector<T> &x) \
+{ \
+ d.v() = VectorHelper<T>::fun(d.v(), x.d.v()); \
+ return *this; \
+} \
+template<> inline Vector<T> Vector<T>::operator symbol(const Vector<T> &x) const \
+{ \
+ return VectorHelper<T>::fun(d.v(), x.d.v()); \
+}
+OP_IMPL(int, &, and_)
+OP_IMPL(int, |, or_)
+OP_IMPL(int, ^, xor_)
+OP_IMPL(unsigned int, &, and_)
+OP_IMPL(unsigned int, |, or_)
+OP_IMPL(unsigned int, ^, xor_)
+OP_IMPL(short, &, and_)
+OP_IMPL(short, |, or_)
+OP_IMPL(short, ^, xor_)
+OP_IMPL(unsigned short, &, and_)
+OP_IMPL(unsigned short, |, or_)
+OP_IMPL(unsigned short, ^, xor_)
+OP_IMPL(float, &, and_)
+OP_IMPL(float, |, or_)
+OP_IMPL(float, ^, xor_)
+OP_IMPL(float8, &, and_)
+OP_IMPL(float8, |, or_)
+OP_IMPL(float8, ^, xor_)
+OP_IMPL(double, &, and_)
+OP_IMPL(double, |, or_)
+OP_IMPL(double, ^, xor_)
+#undef OP_IMPL
+
+#ifdef VC_IMPL_XOP
+static inline INTRINSIC CONST __m128i shiftLeft (const int_v &value, const int_v &count) { return _mm_sha_epi32(value.data(), count.data()); }
+static inline INTRINSIC CONST __m128i shiftLeft (const uint_v &value, const uint_v &count) { return _mm_shl_epi32(value.data(), count.data()); }
+static inline INTRINSIC CONST __m128i shiftLeft (const short_v &value, const short_v &count) { return _mm_sha_epi16(value.data(), count.data()); }
+static inline INTRINSIC CONST __m128i shiftLeft (const ushort_v &value, const ushort_v &count) { return _mm_shl_epi16(value.data(), count.data()); }
+static inline INTRINSIC CONST __m128i shiftRight(const int_v &value, const int_v &count) { return shiftLeft(value, -count ); }
+static inline INTRINSIC CONST __m128i shiftRight(const uint_v &value, const uint_v &count) { return shiftLeft(value, uint_v(-count)); }
+static inline INTRINSIC CONST __m128i shiftRight(const short_v &value, const short_v &count) { return shiftLeft(value, -count ); }
+static inline INTRINSIC CONST __m128i shiftRight(const ushort_v &value, const ushort_v &count) { return shiftLeft(value, ushort_v(-count)); }
+
+#define _VC_OP(T, symbol, impl) \
+template<> inline INTRINSIC T &T::operator symbol##=(T::AsArg shift) \
+{ \
+ d.v() = impl(*this, shift); \
+ return *this; \
+} \
+template<> inline INTRINSIC T T::operator symbol (T::AsArg shift) const \
+{ \
+ return impl(*this, shift); \
+}
+VC_APPLY_2(VC_LIST_INT_VECTOR_TYPES, _VC_OP, <<, shiftLeft)
+VC_APPLY_2(VC_LIST_INT_VECTOR_TYPES, _VC_OP, >>, shiftRight)
+#undef _VC_OP
+#else
+#if defined(VC_GCC) && VC_GCC == 0x40600 && VC_IMPL_XOP
+#define VC_WORKAROUND_IN
+#define VC_WORKAROUND __attribute__((optimize("no-tree-vectorize"),weak))
+#else
+#define VC_WORKAROUND_IN inline
+#define VC_WORKAROUND INTRINSIC
+#endif
+
+#define OP_IMPL(T, symbol) \
+template<> VC_WORKAROUND_IN Vector<T> VC_WORKAROUND &Vector<T>::operator symbol##=(Vector<T>::AsArg x) \
+{ \
+ for_all_vector_entries(i, \
+ d.m(i) symbol##= x.d.m(i); \
+ ); \
+ return *this; \
+} \
+template<> inline Vector<T> Vector<T>::operator symbol(Vector<T>::AsArg x) const \
+{ \
+ Vector<T> r; \
+ for_all_vector_entries(i, \
+ r.d.m(i) = d.m(i) symbol x.d.m(i); \
+ ); \
+ return r; \
+}
+OP_IMPL(int, <<)
+OP_IMPL(int, >>)
+OP_IMPL(unsigned int, <<)
+OP_IMPL(unsigned int, >>)
+OP_IMPL(short, <<)
+OP_IMPL(short, >>)
+OP_IMPL(unsigned short, <<)
+OP_IMPL(unsigned short, >>)
+#undef OP_IMPL
+#undef VC_WORKAROUND
+#undef VC_WORKAROUND_IN
+#endif
+
+template<typename T> inline Vector<T> &Vector<T>::operator>>=(int shift) {
+ d.v() = VectorHelper<T>::shiftRight(d.v(), shift);
+ return *this;
+}
+template<typename T> inline Vector<T> Vector<T>::operator>>(int shift) const {
+ return VectorHelper<T>::shiftRight(d.v(), shift);
+}
+template<typename T> inline Vector<T> &Vector<T>::operator<<=(int shift) {
+ d.v() = VectorHelper<T>::shiftLeft(d.v(), shift);
+ return *this;
+}
+template<typename T> inline Vector<T> Vector<T>::operator<<(int shift) const {
+ return VectorHelper<T>::shiftLeft(d.v(), shift);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// swizzles {{{1
+template<typename T> inline const Vector<T> INTRINSIC CONST &Vector<T>::abcd() const { return *this; }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::cdab() const { return Mem::permute<X2, X3, X0, X1>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::badc() const { return Mem::permute<X1, X0, X3, X2>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::aaaa() const { return Mem::permute<X0, X0, X0, X0>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::bbbb() const { return Mem::permute<X1, X1, X1, X1>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::cccc() const { return Mem::permute<X2, X2, X2, X2>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dddd() const { return Mem::permute<X3, X3, X3, X3>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::bcad() const { return Mem::permute<X1, X2, X0, X3>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::bcda() const { return Mem::permute<X1, X2, X3, X0>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dabc() const { return Mem::permute<X3, X0, X1, X2>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::acbd() const { return Mem::permute<X0, X2, X1, X3>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dbca() const { return Mem::permute<X3, X1, X2, X0>(data()); }
+template<typename T> inline const Vector<T> INTRINSIC CONST Vector<T>::dcba() const { return Mem::permute<X3, X2, X1, X0>(data()); }
+
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::cdab() const { return M256::create(Mem::permute<X2, X3, X0, X1>(d.v()[0]), Mem::permute<X2, X3, X0, X1>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::badc() const { return M256::create(Mem::permute<X1, X0, X3, X2>(d.v()[0]), Mem::permute<X1, X0, X3, X2>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::aaaa() const { return M256::create(Mem::permute<X0, X0, X0, X0>(d.v()[0]), Mem::permute<X0, X0, X0, X0>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::bbbb() const { return M256::create(Mem::permute<X1, X1, X1, X1>(d.v()[0]), Mem::permute<X1, X1, X1, X1>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::cccc() const { return M256::create(Mem::permute<X2, X2, X2, X2>(d.v()[0]), Mem::permute<X2, X2, X2, X2>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::dddd() const { return M256::create(Mem::permute<X3, X3, X3, X3>(d.v()[0]), Mem::permute<X3, X3, X3, X3>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::bcad() const { return M256::create(Mem::permute<X1, X2, X0, X3>(d.v()[0]), Mem::permute<X1, X2, X0, X3>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::bcda() const { return M256::create(Mem::permute<X1, X2, X3, X0>(d.v()[0]), Mem::permute<X1, X2, X3, X0>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::dabc() const { return M256::create(Mem::permute<X3, X0, X1, X2>(d.v()[0]), Mem::permute<X3, X0, X1, X2>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::acbd() const { return M256::create(Mem::permute<X0, X2, X1, X3>(d.v()[0]), Mem::permute<X0, X2, X1, X3>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::dbca() const { return M256::create(Mem::permute<X3, X1, X2, X0>(d.v()[0]), Mem::permute<X3, X1, X2, X0>(d.v()[1])); }
+template<> inline const sfloat_v INTRINSIC CONST Vector<sfloat>::dcba() const { return M256::create(Mem::permute<X3, X2, X1, X0>(d.v()[0]), Mem::permute<X3, X2, X1, X0>(d.v()[1])); }
+
+#define VC_SWIZZLES_16BIT_IMPL(T) \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::cdab() const { return Mem::permute<X2, X3, X0, X1, X6, X7, X4, X5>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::badc() const { return Mem::permute<X1, X0, X3, X2, X5, X4, X7, X6>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::aaaa() const { return Mem::permute<X0, X0, X0, X0, X4, X4, X4, X4>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::bbbb() const { return Mem::permute<X1, X1, X1, X1, X5, X5, X5, X5>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::cccc() const { return Mem::permute<X2, X2, X2, X2, X6, X6, X6, X6>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dddd() const { return Mem::permute<X3, X3, X3, X3, X7, X7, X7, X7>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::bcad() const { return Mem::permute<X1, X2, X0, X3, X5, X6, X4, X7>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::bcda() const { return Mem::permute<X1, X2, X3, X0, X5, X6, X7, X4>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dabc() const { return Mem::permute<X3, X0, X1, X2, X7, X4, X5, X6>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::acbd() const { return Mem::permute<X0, X2, X1, X3, X4, X6, X5, X7>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dbca() const { return Mem::permute<X3, X1, X2, X0, X7, X5, X6, X4>(data()); } \
+template<> inline const Vector<T> INTRINSIC CONST Vector<T>::dcba() const { return Mem::permute<X3, X2, X1, X0, X7, X6, X5, X4>(data()); }
+VC_SWIZZLES_16BIT_IMPL(short)
+VC_SWIZZLES_16BIT_IMPL(unsigned short)
+#undef VC_SWIZZLES_16BIT_IMPL
+
+// operators {{{1
+#include "../common/operators.h"
+// }}}1
+// gathers {{{1
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, const IndexT *indexes)
+{
+ gather(mem, indexes);
+}
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, const Vector<IndexT> indexes)
+{
+ gather(mem, indexes);
+}
+
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, const IndexT *indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(mem, indexes, mask);
+}
+
+template<typename T> template<typename IndexT> inline ALWAYS_INLINE Vector<T>::Vector(const EntryType *mem, const Vector<IndexT> indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(mem, indexes, mask);
+}
+
+template<typename T> template<typename S1, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ gather(array, member1, indexes);
+}
+template<typename T> template<typename S1, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType S1::* member1, const IT indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(array, member1, indexes, mask);
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ gather(array, member1, member2, indexes);
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(array, member1, member2, indexes, mask);
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ gather(array, ptrMember1, outerIndexes, innerIndexes);
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline ALWAYS_INLINE Vector<T>::Vector(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes, MaskArg mask)
+ : d(HT::zero())
+{
+ gather(array, ptrMember1, outerIndexes, innerIndexes, mask);
+}
+
+template<typename T, size_t Size> struct IndexSizeChecker { static void check() {} };
+template<typename T, size_t Size> struct IndexSizeChecker<Vector<T>, Size>
+{
+ static void check() {
+ VC_STATIC_ASSERT(Vector<T>::Size >= Size, IndexVector_must_have_greater_or_equal_number_of_entries);
+ }
+};
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const EntryType *mem, const Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_pd(mem[indexes[0]], mem[indexes[1]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const EntryType *mem, const Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_ps(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<float8>::gather(const EntryType *mem, const Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v()[0] = _mm_setr_ps(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]]);
+ d.v()[1] = _mm_setr_ps(mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const EntryType *mem, const Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_epi32(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const EntryType *mem, const Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_epi32(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const EntryType *mem, const Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_epi16(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+template<> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const EntryType *mem, const Index indexes)
+{
+ IndexSizeChecker<Index, Size>::check();
+ d.v() = _mm_setr_epi16(mem[indexes[0]], mem[indexes[1]], mem[indexes[2]], mem[indexes[3]],
+ mem[indexes[4]], mem[indexes[5]], mem[indexes[6]], mem[indexes[7]]);
+}
+
+#ifdef VC_USE_SET_GATHERS
+template<typename T> template<typename IT> inline void ALWAYS_INLINE Vector<T>::gather(const EntryType *mem, Vector<IT> indexes, MaskArg mask)
+{
+ IndexSizeChecker<Vector<IT>, Size>::check();
+ indexes.setZero(!static_cast<typename Vector<IT>::Mask>(mask));
+ (*this)(mask) = Vector<T>(mem, indexes);
+}
+#endif
+
+#ifdef VC_USE_BSF_GATHERS
+#define VC_MASKED_GATHER \
+ int bits = mask.toInt(); \
+ while (bits) { \
+ const int i = _bit_scan_forward(bits); \
+ bits &= ~(1 << i); /* btr? */ \
+ d.m(i) = ith_value(i); \
+ }
+#elif defined(VC_USE_POPCNT_BSF_GATHERS)
+#define VC_MASKED_GATHER \
+ unsigned int bits = mask.toInt(); \
+ unsigned int low, high = 0; \
+ switch (mask.count()) { \
+ case 8: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ high = (1 << high); \
+ case 7: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ d.m(low) = ith_value(low); \
+ case 6: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ high = (1 << high); \
+ case 5: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ d.m(low) = ith_value(low); \
+ case 4: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ high = (1 << high); \
+ case 3: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ d.m(low) = ith_value(low); \
+ case 2: \
+ high = _bit_scan_reverse(bits); \
+ d.m(high) = ith_value(high); \
+ case 1: \
+ low = _bit_scan_forward(bits); \
+ d.m(low) = ith_value(low); \
+ case 0: \
+ break; \
+ }
+#else
+#define VC_MASKED_GATHER \
+ if (mask.isEmpty()) { \
+ return; \
+ } \
+ for_all_vector_entries(i, \
+ if (mask[i]) d.m(i) = ith_value(i); \
+ );
+#endif
+
+template<typename T> template<typename Index>
+inline void INTRINSIC Vector<T>::gather(const EntryType *mem, Index indexes, MaskArg mask)
+{
+ IndexSizeChecker<Index, Size>::check();
+#define ith_value(_i_) (mem[indexes[_i_]])
+ VC_MASKED_GATHER
+#undef ith_value
+}
+
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_pd(array[indexes[0]].*(member1), array[indexes[1]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_ps(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<float8>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v()[0] = _mm_setr_ps(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1));
+ d.v()[1] = _mm_setr_ps(array[indexes[4]].*(member1), array[indexes[5]].*(member1), array[indexes[6]].*(member1),
+ array[indexes[7]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi32(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi32(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1), array[indexes[1]].*(member1), array[indexes[2]].*(member1),
+ array[indexes[3]].*(member1), array[indexes[4]].*(member1), array[indexes[5]].*(member1),
+ array[indexes[6]].*(member1), array[indexes[7]].*(member1));
+}
+template<typename T> template<typename S1, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<T>::gather(const S1 *array, const EntryType S1::* member1, const IT indexes, MaskArg mask)
+{
+ IndexSizeChecker<IT, Size>::check();
+#define ith_value(_i_) (array[indexes[_i_]].*(member1))
+ VC_MASKED_GATHER
+#undef ith_value
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_pd(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_ps(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<float8>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v()[0] = _mm_setr_ps(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2),
+ array[indexes[2]].*(member1).*(member2), array[indexes[3]].*(member1).*(member2));
+ d.v()[1] = _mm_setr_ps(array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi32(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2),
+ array[indexes[2]].*(member1).*(member2), array[indexes[3]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi32(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2),
+ array[indexes[2]].*(member1).*(member2), array[indexes[3]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes)
+{
+ IndexSizeChecker<IT, Size>::check();
+ d.v() = _mm_setr_epi16(array[indexes[0]].*(member1).*(member2), array[indexes[1]].*(member1).*(member2), array[indexes[2]].*(member1).*(member2),
+ array[indexes[3]].*(member1).*(member2), array[indexes[4]].*(member1).*(member2), array[indexes[5]].*(member1).*(member2),
+ array[indexes[6]].*(member1).*(member2), array[indexes[7]].*(member1).*(member2));
+}
+template<typename T> template<typename S1, typename S2, typename IT>
+inline void ALWAYS_INLINE FLATTEN Vector<T>::gather(const S1 *array, const S2 S1::* member1, const EntryType S2::* member2, const IT indexes, MaskArg mask)
+{
+ IndexSizeChecker<IT, Size>::check();
+#define ith_value(_i_) (array[indexes[_i_]].*(member1).*(member2))
+ VC_MASKED_GATHER
+#undef ith_value
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<double>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_pd((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]],
+ (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<float>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_ps((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<float8>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v()[0] = _mm_setr_ps((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]]);
+ d.v()[1] = _mm_setr_ps((array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<int>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_epi32((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned int>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_epi32((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<short>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_epi16((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<unsigned short>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+ d.v() = _mm_setr_epi16((array[outerIndexes[0]].*(ptrMember1))[innerIndexes[0]], (array[outerIndexes[1]].*(ptrMember1))[innerIndexes[1]],
+ (array[outerIndexes[2]].*(ptrMember1))[innerIndexes[2]], (array[outerIndexes[3]].*(ptrMember1))[innerIndexes[3]],
+ (array[outerIndexes[4]].*(ptrMember1))[innerIndexes[4]], (array[outerIndexes[5]].*(ptrMember1))[innerIndexes[5]],
+ (array[outerIndexes[6]].*(ptrMember1))[innerIndexes[6]], (array[outerIndexes[7]].*(ptrMember1))[innerIndexes[7]]);
+}
+template<typename T> template<typename S1, typename IT1, typename IT2>
+inline void ALWAYS_INLINE FLATTEN Vector<T>::gather(const S1 *array, const EntryType *const S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes, MaskArg mask)
+{
+ IndexSizeChecker<IT1, Size>::check();
+ IndexSizeChecker<IT2, Size>::check();
+#define ith_value(_i_) (array[outerIndexes[_i_]].*(ptrMember1))[innerIndexes[_i_]]
+ VC_MASKED_GATHER
+#undef ith_value
+}
+// scatters {{{1
+#undef VC_MASKED_GATHER
+#ifdef VC_USE_BSF_SCATTERS
+#define VC_MASKED_SCATTER \
+ int bits = mask.toInt(); \
+ while (bits) { \
+ const int i = _bit_scan_forward(bits); \
+ bits ^= (1 << i); /* btr? */ \
+ ith_value(i) = d.m(i); \
+ }
+#elif defined(VC_USE_POPCNT_BSF_SCATTERS)
+#define VC_MASKED_SCATTER \
+ unsigned int bits = mask.toInt(); \
+ unsigned int low, high = 0; \
+ switch (mask.count()) { \
+ case 8: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ high = (1 << high); \
+ case 7: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ ith_value(low) = d.m(low); \
+ case 6: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ high = (1 << high); \
+ case 5: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ ith_value(low) = d.m(low); \
+ case 4: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ high = (1 << high); \
+ case 3: \
+ low = _bit_scan_forward(bits); \
+ bits ^= high | (1 << low); \
+ ith_value(low) = d.m(low); \
+ case 2: \
+ high = _bit_scan_reverse(bits); \
+ ith_value(high) = d.m(high); \
+ case 1: \
+ low = _bit_scan_forward(bits); \
+ ith_value(low) = d.m(low); \
+ case 0: \
+ break; \
+ }
+#else
+#define VC_MASKED_SCATTER \
+ if (mask.isEmpty()) { \
+ return; \
+ } \
+ for_all_vector_entries(i, \
+ if (mask[i]) ith_value(i) = d.m(i); \
+ );
+#endif
+
+template<typename T> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(EntryType *mem, const Index indexes) const
+{
+ for_all_vector_entries(i,
+ mem[indexes[i]] = d.m(i);
+ );
+}
+template<typename T> template<typename Index> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(EntryType *mem, const Index indexes, MaskArg mask) const
+{
+#define ith_value(_i_) mem[indexes[_i_]]
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+template<typename T> template<typename S1, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType S1::* member1, const IT indexes) const
+{
+ for_all_vector_entries(i,
+ array[indexes[i]].*(member1) = d.m(i);
+ );
+}
+template<typename T> template<typename S1, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType S1::* member1, const IT indexes, MaskArg mask) const
+{
+#define ith_value(_i_) array[indexes[_i_]].*(member1)
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, const IT indexes) const
+{
+ for_all_vector_entries(i,
+ array[indexes[i]].*(member1).*(member2) = d.m(i);
+ );
+}
+template<typename T> template<typename S1, typename S2, typename IT> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, S2 S1::* member1, EntryType S2::* member2, const IT indexes, MaskArg mask) const
+{
+#define ith_value(_i_) array[indexes[_i_]].*(member1).*(member2)
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType *S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes) const
+{
+ for_all_vector_entries(i,
+ (array[innerIndexes[i]].*(ptrMember1))[outerIndexes[i]] = d.m(i);
+ );
+}
+template<typename T> template<typename S1, typename IT1, typename IT2> inline void ALWAYS_INLINE FLATTEN Vector<T>::scatter(S1 *array, EntryType *S1::* ptrMember1, const IT1 outerIndexes, const IT2 innerIndexes, MaskArg mask) const
+{
+#define ith_value(_i_) (array[outerIndexes[_i_]].*(ptrMember1))[innerIndexes[_i_]]
+ VC_MASKED_SCATTER
+#undef ith_value
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// operator[] {{{1
+template<typename T> inline typename Vector<T>::EntryType PURE INTRINSIC Vector<T>::operator[](size_t index) const
+{
+ return d.m(index);
+}
+#ifdef VC_GCC
+template<> inline double PURE INTRINSIC Vector<double>::operator[](size_t index) const
+{
+ if (__builtin_constant_p(index)) {
+ return extract_double_imm(d.v(), index);
+ }
+ return d.m(index);
+}
+template<> inline float PURE INTRINSIC Vector<float>::operator[](size_t index) const
+{
+ return extract_float(d.v(), index);
+}
+template<> inline float PURE INTRINSIC Vector<float8>::operator[](size_t index) const
+{
+ if (__builtin_constant_p(index)) {
+ if (index < 4) {
+ return extract_float_imm(d.v()[0], index);
+ }
+ return extract_float_imm(d.v()[1], index - 4);
+ }
+ return d.m(index);
+}
+template<> inline int PURE INTRINSIC Vector<int>::operator[](size_t index) const
+{
+ if (__builtin_constant_p(index)) {
+#if VC_GCC >= 0x40601 || !defined(VC_USE_VEX_CODING) // GCC < 4.6.1 incorrectly uses vmovq instead of movq for the following
+#ifdef __x86_64__
+ if (index == 0) return _mm_cvtsi128_si64(d.v()) & 0xFFFFFFFFull;
+ if (index == 1) return _mm_cvtsi128_si64(d.v()) >> 32;
+#else
+ if (index == 0) return _mm_cvtsi128_si32(d.v());
+#endif
+#endif
+#ifdef VC_IMPL_SSE4_1
+ return _mm_extract_epi32(d.v(), index);
+#else
+ return _mm_cvtsi128_si32(_mm_srli_si128(d.v(), index * 4));
+#endif
+ }
+ return d.m(index);
+}
+template<> inline unsigned int PURE INTRINSIC Vector<unsigned int>::operator[](size_t index) const
+{
+ if (__builtin_constant_p(index)) {
+#if VC_GCC >= 0x40601 || !defined(VC_USE_VEX_CODING) // GCC < 4.6.1 incorrectly uses vmovq instead of movq for the following
+#ifdef __x86_64__
+ if (index == 0) return _mm_cvtsi128_si64(d.v()) & 0xFFFFFFFFull;
+ if (index == 1) return _mm_cvtsi128_si64(d.v()) >> 32;
+#else
+ if (index == 0) return _mm_cvtsi128_si32(d.v());
+#endif
+#endif
+#ifdef VC_IMPL_SSE4_1
+ return _mm_extract_epi32(d.v(), index);
+#else
+ return _mm_cvtsi128_si32(_mm_srli_si128(d.v(), index * 4));
+#endif
+ }
+ return d.m(index);
+}
+template<> inline short PURE INTRINSIC Vector<short>::operator[](size_t index) const
+{
+ if (__builtin_constant_p(index)) {
+ return _mm_extract_epi16(d.v(), index);
+ }
+ return d.m(index);
+}
+template<> inline unsigned short PURE INTRINSIC Vector<unsigned short>::operator[](size_t index) const
+{
+ if (__builtin_constant_p(index)) {
+ return _mm_extract_epi16(d.v(), index);
+ }
+ return d.m(index);
+}
+#endif // GCC
+///////////////////////////////////////////////////////////////////////////////////////////
+// horizontal ops {{{1
+#ifndef VC_IMPL_SSE4_1
+// without SSE4.1 integer multiplication is slow and we rather multiply the scalars
+template<> inline int INTRINSIC Vector<int>::product() const
+{
+ return (d.m(0) * d.m(1)) * (d.m(2) * d.m(3));
+}
+template<> inline unsigned int INTRINSIC Vector<unsigned int>::product() const
+{
+ return (d.m(0) * d.m(1)) * (d.m(2) * d.m(3));
+}
+#endif
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::min(MaskArg m) const
+{
+ Vector<T> tmp = std::numeric_limits<Vector<T> >::max();
+ tmp(m) = *this;
+ return tmp.min();
+}
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::max(MaskArg m) const
+{
+ Vector<T> tmp = std::numeric_limits<Vector<T> >::min();
+ tmp(m) = *this;
+ return tmp.max();
+}
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::product(MaskArg m) const
+{
+ Vector<T> tmp(VectorSpecialInitializerOne::One);
+ tmp(m) = *this;
+ return tmp.product();
+}
+template<typename T> inline typename Vector<T>::EntryType Vector<T>::sum(MaskArg m) const
+{
+ Vector<T> tmp(VectorSpecialInitializerZero::Zero);
+ tmp(m) = *this;
+ return tmp.sum();
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// copySign {{{1
+template<> inline Vector<float> INTRINSIC Vector<float>::copySign(Vector<float>::AsArg reference) const
+{
+ return _mm_or_ps(
+ _mm_and_ps(reference.d.v(), _mm_setsignmask_ps()),
+ _mm_and_ps(d.v(), _mm_setabsmask_ps())
+ );
+}
+template<> inline Vector<float8> INTRINSIC Vector<float8>::copySign(Vector<float8>::AsArg reference) const
+{
+ return M256::create( _mm_or_ps(
+ _mm_and_ps(reference.d.v()[0], _mm_setsignmask_ps()),
+ _mm_and_ps(d.v()[0], _mm_setabsmask_ps())
+ ), _mm_or_ps(
+ _mm_and_ps(reference.d.v()[1], _mm_setsignmask_ps()),
+ _mm_and_ps(d.v()[1], _mm_setabsmask_ps())
+ )
+ );
+}
+template<> inline Vector<double> INTRINSIC Vector<double>::copySign(Vector<double>::AsArg reference) const
+{
+ return _mm_or_pd(
+ _mm_and_pd(reference.d.v(), _mm_setsignmask_pd()),
+ _mm_and_pd(d.v(), _mm_setabsmask_pd())
+ );
+}//}}}1
+// exponent {{{1
+template<> inline Vector<float> INTRINSIC Vector<float>::exponent() const
+{
+ VC_ASSERT((*this > 0.f).isFull());
+ __m128i tmp = _mm_srli_epi32(_mm_castps_si128(d.v()), 23);
+ tmp = _mm_sub_epi32(tmp, _mm_set1_epi32(0x7f));
+ return _mm_cvtepi32_ps(tmp);
+}
+template<> inline Vector<float8> INTRINSIC Vector<float8>::exponent() const
+{
+ VC_ASSERT((*this > 0.f).isFull());
+ __m128i tmp0 = _mm_srli_epi32(_mm_castps_si128(d.v()[0]), 23);
+ __m128i tmp1 = _mm_srli_epi32(_mm_castps_si128(d.v()[1]), 23);
+ tmp0 = _mm_sub_epi32(tmp0, _mm_set1_epi32(0x7f));
+ tmp1 = _mm_sub_epi32(tmp1, _mm_set1_epi32(0x7f));
+ return M256::create( _mm_cvtepi32_ps(tmp0), _mm_cvtepi32_ps(tmp1));
+}
+template<> inline Vector<double> INTRINSIC Vector<double>::exponent() const
+{
+ VC_ASSERT((*this > 0.).isFull());
+ __m128i tmp = _mm_srli_epi64(_mm_castpd_si128(d.v()), 52);
+ tmp = _mm_sub_epi32(tmp, _mm_set1_epi32(0x3ff));
+ return _mm_cvtepi32_pd(_mm_shuffle_epi32(tmp, 0x08));
+}
+// }}}1
+// Random {{{1
+static inline ALWAYS_INLINE void _doRandomStep(Vector<unsigned int> &state0,
+ Vector<unsigned int> &state1)
+{
+ state0.load(&Vc::RandomState[0]);
+ state1.load(&Vc::RandomState[uint_v::Size]);
+ (state1 * 0xdeece66du + 11).store(&Vc::RandomState[uint_v::Size]);
+ uint_v(_mm_xor_si128((state0 * 0xdeece66du + 11).data(), _mm_srli_epi32(state1.data(), 16))).store(&Vc::RandomState[0]);
+}
+
+template<typename T> inline ALWAYS_INLINE Vector<T> Vector<T>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ return state0.reinterpretCast<Vector<T> >();
+}
+
+template<> inline ALWAYS_INLINE Vector<float> Vector<float>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ return _mm_sub_ps(_mm_or_ps(_mm_castsi128_ps(_mm_srli_epi32(state0.data(), 2)), HT::one()), HT::one());
+}
+
+template<> inline ALWAYS_INLINE Vector<float8> Vector<float8>::Random()
+{
+ Vector<unsigned int> state0, state1;
+ _doRandomStep(state0, state1);
+ state1 ^= state0 >> 16;
+ return M256::create(
+ _mm_sub_ps(_mm_or_ps(_mm_castsi128_ps(_mm_srli_epi32(state0.data(), 2)), VectorHelper<float>::one()), VectorHelper<float>::one()),
+ _mm_sub_ps(_mm_or_ps(_mm_castsi128_ps(_mm_srli_epi32(state1.data(), 2)), VectorHelper<float>::one()), VectorHelper<float>::one())
+ );
+}
+
+template<> inline ALWAYS_INLINE Vector<double> Vector<double>::Random()
+{
+ typedef unsigned long long uint64 MAY_ALIAS;
+ uint64 state0 = *reinterpret_cast<const uint64 *>(&Vc::RandomState[8]);
+ uint64 state1 = *reinterpret_cast<const uint64 *>(&Vc::RandomState[10]);
+ const __m128i state = _mm_load_si128(reinterpret_cast<const __m128i *>(&Vc::RandomState[8]));
+ *reinterpret_cast<uint64 *>(&Vc::RandomState[ 8]) = (state0 * 0x5deece66dull + 11);
+ *reinterpret_cast<uint64 *>(&Vc::RandomState[10]) = (state1 * 0x5deece66dull + 11);
+ return (Vector<double>(_mm_castsi128_pd(_mm_srli_epi64(state, 12))) | One()) - One();
+}
+// }}}1
+} // namespace SSE
+} // namespace Vc
+
+#include "undomacros.h"
+
+// vim: foldmethod=marker
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef SSE_VECTORHELPER_H
+#define SSE_VECTORHELPER_H
+
+#include "types.h"
+#include <limits>
+
+namespace Vc
+{
+namespace SSE
+{
+ template<typename VectorType, unsigned int Size> struct SortHelper
+ {
+ static VectorType sort(VectorType) PURE;
+ };
+ template<unsigned int Size> struct SortHelper<M256, Size>
+ {
+ static M256 sort(const M256 &) PURE;
+ };
+
+#undef OP_DECL
+#undef PARENT_DATA
+#undef PARENT_DATA_CONST
+
+#define OP0(name, code) static inline VectorType name() PURE { return code; }
+#define OP1(name, code) static inline VectorType name(const VectorType &a) PURE { return code; }
+#define OP2(name, code) static inline VectorType name(const VectorType &a, const VectorType &b) PURE { return code; }
+#define OP3(name, code) static inline VectorType name(const VectorType &a, const VectorType &b, const VectorType &c) PURE { return code; }
+
+ template<> struct VectorHelper<_M128>
+ {
+ typedef _M128 VectorType;
+ template<typename A> static VectorType load(const float *x, A) PURE;
+ static void store(float *mem, const VectorType x, AlignedFlag);
+ static void store(float *mem, const VectorType x, UnalignedFlag);
+ static void store(float *mem, const VectorType x, StreamingAndAlignedFlag);
+ static void store(float *mem, const VectorType x, StreamingAndUnalignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, AlignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, UnalignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag);
+ static void store(float *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag);
+
+ OP0(allone, _mm_setallone_ps())
+ OP0(zero, _mm_setzero_ps())
+ OP2(or_, _mm_or_ps(a, b))
+ OP2(xor_, _mm_xor_ps(a, b))
+ OP2(and_, _mm_and_ps(a, b))
+ OP2(andnot_, _mm_andnot_ps(a, b))
+ OP3(blend, _mm_blendv_ps(a, b, c))
+ };
+
+
+ template<> struct VectorHelper<M256>
+ {
+ typedef M256 VectorType;
+ template<typename A> static VectorType load(const float *x, A) PURE;
+ static void store(float *mem, const VectorType &x, AlignedFlag);
+ static void store(float *mem, const VectorType &x, UnalignedFlag);
+ static void store(float *mem, const VectorType &x, StreamingAndAlignedFlag);
+ static void store(float *mem, const VectorType &x, StreamingAndUnalignedFlag);
+ static void store(float *mem, const VectorType &x, const VectorType &m, AlignedFlag);
+ static void store(float *mem, const VectorType &x, const VectorType &m, UnalignedFlag);
+ static void store(float *mem, const VectorType &x, const VectorType &m, StreamingAndAlignedFlag);
+ static void store(float *mem, const VectorType &x, const VectorType &m, StreamingAndUnalignedFlag);
+
+ OP0(allone, VectorType::create(_mm_setallone_ps(), _mm_setallone_ps()))
+ OP0(zero, VectorType::create(_mm_setzero_ps(), _mm_setzero_ps()))
+ OP2(or_, VectorType::create(_mm_or_ps(a[0], b[0]), _mm_or_ps(a[1], b[1])))
+ OP2(xor_, VectorType::create(_mm_xor_ps(a[0], b[0]), _mm_xor_ps(a[1], b[1])))
+ OP2(and_, VectorType::create(_mm_and_ps(a[0], b[0]), _mm_and_ps(a[1], b[1])))
+ OP2(andnot_, VectorType::create(_mm_andnot_ps(a[0], b[0]), _mm_andnot_ps(a[1], b[1])))
+ OP3(blend, VectorType::create(_mm_blendv_ps(a[0], b[0], c[0]), _mm_blendv_ps(a[1], b[1], c[1])))
+ };
+
+ template<> struct VectorHelper<_M128D>
+ {
+ typedef _M128D VectorType;
+ template<typename A> static VectorType load(const double *x, A) PURE;
+ static void store(double *mem, const VectorType x, AlignedFlag);
+ static void store(double *mem, const VectorType x, UnalignedFlag);
+ static void store(double *mem, const VectorType x, StreamingAndAlignedFlag);
+ static void store(double *mem, const VectorType x, StreamingAndUnalignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, AlignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, UnalignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag);
+ static void store(double *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag);
+
+ OP0(allone, _mm_setallone_pd())
+ OP0(zero, _mm_setzero_pd())
+ OP2(or_, _mm_or_pd(a, b))
+ OP2(xor_, _mm_xor_pd(a, b))
+ OP2(and_, _mm_and_pd(a, b))
+ OP2(andnot_, _mm_andnot_pd(a, b))
+ OP3(blend, _mm_blendv_pd(a, b, c))
+ };
+
+ template<> struct VectorHelper<_M128I>
+ {
+ typedef _M128I VectorType;
+ template<typename T> static VectorType load(const T *x, AlignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, UnalignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, StreamingAndAlignedFlag) PURE;
+ template<typename T> static VectorType load(const T *x, StreamingAndUnalignedFlag) PURE;
+ template<typename T> static void store(T *mem, const VectorType x, AlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, UnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, StreamingAndAlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, StreamingAndUnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, AlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, UnalignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag);
+ template<typename T> static void store(T *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag);
+
+ OP0(allone, _mm_setallone_si128())
+ OP0(zero, _mm_setzero_si128())
+ OP2(or_, _mm_or_si128(a, b))
+ OP2(xor_, _mm_xor_si128(a, b))
+ OP2(and_, _mm_and_si128(a, b))
+ OP2(andnot_, _mm_andnot_si128(a, b))
+ OP3(blend, _mm_blendv_epi8(a, b, c))
+ };
+
+#undef OP1
+#undef OP2
+#undef OP3
+
+#define OP1(op) \
+ static inline VectorType op(const VectorType &a) PURE { return CAT(_mm_##op##_, SUFFIX)(a); }
+#define OP(op) \
+ static inline VectorType op(const VectorType &a, const VectorType &b) PURE { return CAT(_mm_##op##_ , SUFFIX)(a, b); }
+#define OP_(op) \
+ static inline VectorType op(const VectorType &a, const VectorType &b) PURE { return CAT(_mm_##op , SUFFIX)(a, b); }
+#define OPx(op, op2) \
+ static inline VectorType op(const VectorType &a, const VectorType &b) PURE { return CAT(_mm_##op2##_, SUFFIX)(a, b); }
+#define OPcmp(op) \
+ static inline VectorType cmp##op(const VectorType &a, const VectorType &b) PURE { return CAT(_mm_cmp##op##_, SUFFIX)(a, b); }
+#define OP_CAST_(op) \
+ static inline VectorType op(const VectorType &a, const VectorType &b) PURE { return CAT(_mm_castps_, SUFFIX)( \
+ _mm_##op##ps(CAT(CAT(_mm_cast, SUFFIX), _ps)(a), \
+ CAT(CAT(_mm_cast, SUFFIX), _ps)(b))); \
+ }
+#define MINMAX \
+ static inline VectorType min(VectorType a, VectorType b) PURE { return CAT(_mm_min_, SUFFIX)(a, b); } \
+ static inline VectorType max(VectorType a, VectorType b) PURE { return CAT(_mm_max_, SUFFIX)(a, b); }
+
+ template<> struct VectorHelper<double> {
+ typedef _M128D VectorType;
+ typedef double EntryType;
+#define SUFFIX pd
+
+ OP_(or_) OP_(and_) OP_(xor_)
+ static inline VectorType notMaskedToZero(VectorType a, _M128 mask) PURE { return CAT(_mm_and_, SUFFIX)(_mm_castps_pd(mask), a); }
+ static inline VectorType set(const double a) PURE { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType set(const double a, const double b) PURE { return CAT(_mm_set_, SUFFIX)(a, b); }
+ static inline VectorType zero() PURE { return CAT(_mm_setzero_, SUFFIX)(); }
+ static inline VectorType one() PURE { return CAT(_mm_setone_, SUFFIX)(); }// set(1.); }
+
+ static inline void multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) { v1 = add(mul(v1, v2), v3); }
+ static inline VectorType mul(VectorType a, VectorType b, _M128 _mask) PURE {
+ _M128D mask = _mm_castps_pd(_mask);
+ return _mm_or_pd(
+ _mm_and_pd(mask, _mm_mul_pd(a, b)),
+ _mm_andnot_pd(mask, a)
+ );
+ }
+
+ OP(add) OP(sub) OP(mul)
+ OPcmp(eq) OPcmp(neq)
+ OPcmp(lt) OPcmp(nlt)
+ OPcmp(le) OPcmp(nle)
+
+ OP1(sqrt)
+ static inline VectorType rsqrt(VectorType x) PURE {
+ return _mm_div_pd(one(), sqrt(x));
+ }
+ static inline VectorType reciprocal(VectorType x) PURE {
+ return _mm_div_pd(one(), x);
+ }
+ static inline VectorType isNaN(VectorType x) PURE {
+ return _mm_cmpunord_pd(x, x);
+ }
+ static inline VectorType isFinite(VectorType x) PURE {
+ return _mm_cmpord_pd(x, _mm_mul_pd(zero(), x));
+ }
+ static inline VectorType abs(const VectorType a) PURE {
+ return CAT(_mm_and_, SUFFIX)(a, _mm_setabsmask_pd());
+ }
+
+ MINMAX
+ static inline EntryType min(VectorType a) PURE {
+ a = _mm_min_sd(a, _mm_unpackhi_pd(a, a));
+ return _mm_cvtsd_f64(a);
+ }
+ static inline EntryType max(VectorType a) PURE {
+ a = _mm_max_sd(a, _mm_unpackhi_pd(a, a));
+ return _mm_cvtsd_f64(a);
+ }
+ static inline EntryType mul(VectorType a) PURE {
+ a = _mm_mul_sd(a, _mm_shuffle_pd(a, a, _MM_SHUFFLE2(0, 1)));
+ return _mm_cvtsd_f64(a);
+ }
+ static inline EntryType add(VectorType a) PURE {
+ a = _mm_add_sd(a, _mm_shuffle_pd(a, a, _MM_SHUFFLE2(0, 1)));
+ return _mm_cvtsd_f64(a);
+ }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) PURE {
+#if VC_IMPL_SSE4_1
+ return _mm_round_pd(a, _MM_FROUND_NINT);
+#else
+ //XXX: slow: _MM_SET_ROUNDING_MODE(_MM_ROUND_NEAREST);
+ return _mm_cvtepi32_pd(_mm_cvtpd_epi32(a));
+#endif
+ }
+ };
+
+ template<> struct VectorHelper<float> {
+ typedef float EntryType;
+ typedef _M128 VectorType;
+#define SUFFIX ps
+
+ OP_(or_) OP_(and_) OP_(xor_)
+ static inline VectorType notMaskedToZero(VectorType a, _M128 mask) PURE { return CAT(_mm_and_, SUFFIX)(mask, a); }
+ static inline VectorType set(const float a) PURE { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType set(const float a, const float b, const float c, const float d) PURE { return CAT(_mm_set_, SUFFIX)(a, b, c, d); }
+ static inline VectorType zero() PURE { return CAT(_mm_setzero_, SUFFIX)(); }
+ static inline VectorType one() PURE { return CAT(_mm_setone_, SUFFIX)(); }// set(1.f); }
+ static inline _M128 concat(_M128D a, _M128D b) PURE { return _mm_movelh_ps(_mm_cvtpd_ps(a), _mm_cvtpd_ps(b)); }
+
+ static inline void multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) { v1 = add(mul(v1, v2), v3); }
+ static inline VectorType mul(VectorType a, VectorType b, _M128 mask) PURE {
+ return _mm_or_ps(
+ _mm_and_ps(mask, _mm_mul_ps(a, b)),
+ _mm_andnot_ps(mask, a)
+ );
+ }
+
+ OP(add) OP(sub) OP(mul)
+ OPcmp(eq) OPcmp(neq)
+ OPcmp(lt) OPcmp(nlt)
+ OPcmp(le) OPcmp(nle)
+
+ OP1(sqrt) OP1(rsqrt)
+ static inline VectorType isNaN(VectorType x) PURE {
+ return _mm_cmpunord_ps(x, x);
+ }
+ static inline VectorType isFinite(VectorType x) PURE {
+ return _mm_cmpord_ps(x, _mm_mul_ps(zero(), x));
+ }
+ static inline VectorType reciprocal(VectorType x) PURE {
+ return _mm_rcp_ps(x);
+ }
+ static inline VectorType abs(const VectorType a) PURE {
+ return CAT(_mm_and_, SUFFIX)(a, _mm_setabsmask_ps());
+ }
+
+ MINMAX
+ static inline EntryType min(VectorType a) PURE {
+ a = _mm_min_ps(a, _mm_movehl_ps(a, a)); // a = min(a0, a2), min(a1, a3), min(a2, a2), min(a3, a3)
+ a = _mm_min_ss(a, _mm_shuffle_ps(a, a, _MM_SHUFFLE(1, 1, 1, 1))); // a = min(a0, a1), a1, a2, a3
+ return _mm_cvtss_f32(a);
+ }
+ static inline EntryType max(VectorType a) PURE {
+ a = _mm_max_ps(a, _mm_movehl_ps(a, a)); // a = max(a0, a2), max(a1, a3), max(a2, a2), max(a3, a3)
+ a = _mm_max_ss(a, _mm_shuffle_ps(a, a, _MM_SHUFFLE(1, 1, 1, 1))); // a = max(a0, a1), a1, a2, a3
+ return _mm_cvtss_f32(a);
+ }
+ static inline EntryType mul(VectorType a) PURE {
+ a = _mm_mul_ps(a, _mm_shuffle_ps(a, a, _MM_SHUFFLE(0, 1, 2, 3)));
+ a = _mm_mul_ss(a, _mm_shuffle_ps(a, a, _MM_SHUFFLE(3, 2, 0, 1)));
+ return _mm_cvtss_f32(a);
+ }
+ static inline EntryType add(VectorType a) PURE {
+ a = _mm_add_ps(a, _mm_shuffle_ps(a, a, _MM_SHUFFLE(0, 1, 2, 3)));
+ a = _mm_add_ss(a, _mm_shuffle_ps(a, a, _MM_SHUFFLE(3, 2, 0, 1)));
+ return _mm_cvtss_f32(a);
+ }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) PURE {
+#if VC_IMPL_SSE4_1
+ return _mm_round_ps(a, _MM_FROUND_NINT);
+#else
+ //XXX slow: _MM_SET_ROUNDING_MODE(_MM_ROUND_NEAREST);
+ return _mm_cvtepi32_ps(_mm_cvtps_epi32(a));
+#endif
+ }
+ };
+
+ template<> struct VectorHelper<float8> {
+ typedef float EntryType;
+ typedef M256 VectorType;
+
+ static inline VectorType set(const float a) PURE {
+ const _M128 x = _mm_set1_ps(a);
+ return VectorType::create(x, x);
+ }
+ static inline VectorType set(const float a, const float b, const float c, const float d) PURE {
+ const _M128 x = _mm_set_ps(a, b, c, d);
+ return VectorType::create(x, x);
+ }
+ static inline VectorType set(const float a, const float b, const float c, const float d,
+ const float e, const float f, const float g, const float h) PURE {
+ return VectorType::create(_mm_set_ps(a, b, c, d), _mm_set_ps(e, f, g, h));
+ }
+ static inline VectorType zero() PURE { return VectorType::create(_mm_setzero_ps(), _mm_setzero_ps()); }
+ static inline VectorType one() PURE { return set(1.f); }
+
+#define REUSE_FLOAT_IMPL1(fun) \
+ static inline VectorType fun(const VectorType &x) PURE { \
+ return VectorType::create(VectorHelper<float>::fun(x[0]), VectorHelper<float>::fun(x[1])); \
+ }
+#define REUSE_FLOAT_IMPL2(fun) \
+ static inline VectorType fun(const VectorType &x, const VectorType &y) PURE { \
+ return VectorType::create(VectorHelper<float>::fun(x[0], y[0]), VectorHelper<float>::fun(x[1], y[1])); \
+ }
+#define REUSE_FLOAT_IMPL3(fun) \
+ static inline VectorType fun(const VectorType &x, const VectorType &y, const VectorType &z) PURE { \
+ return VectorType::create(VectorHelper<float>::fun(x[0], y[0], z[0]), VectorHelper<float>::fun(x[1], y[1], z[1])); \
+ }
+ REUSE_FLOAT_IMPL1(reciprocal)
+ REUSE_FLOAT_IMPL1(sqrt)
+ REUSE_FLOAT_IMPL1(rsqrt)
+ REUSE_FLOAT_IMPL1(isNaN)
+ REUSE_FLOAT_IMPL1(isFinite)
+ REUSE_FLOAT_IMPL1(abs)
+ REUSE_FLOAT_IMPL1(round)
+
+ REUSE_FLOAT_IMPL2(and_)
+ REUSE_FLOAT_IMPL2(or_)
+ REUSE_FLOAT_IMPL2(xor_)
+ REUSE_FLOAT_IMPL2(notMaskedToZero)
+ REUSE_FLOAT_IMPL2(add)
+ REUSE_FLOAT_IMPL2(sub)
+ REUSE_FLOAT_IMPL2(mul)
+ REUSE_FLOAT_IMPL2(cmple)
+ REUSE_FLOAT_IMPL2(cmpnle)
+ REUSE_FLOAT_IMPL2(cmplt)
+ REUSE_FLOAT_IMPL2(cmpnlt)
+ REUSE_FLOAT_IMPL2(cmpeq)
+ REUSE_FLOAT_IMPL2(cmpneq)
+ REUSE_FLOAT_IMPL2(min)
+ REUSE_FLOAT_IMPL2(max)
+
+ static inline EntryType min(const VectorType &a) PURE {
+ return VectorHelper<float>::min(VectorHelper<float>::min(a[0], a[1]));
+ }
+ static inline EntryType max(const VectorType &a) PURE {
+ return VectorHelper<float>::max(VectorHelper<float>::max(a[0], a[1]));
+ }
+ static inline EntryType mul(const VectorType &a) PURE {
+ return VectorHelper<float>::mul(VectorHelper<float>::mul(a[0], a[1]));
+ }
+ static inline EntryType add(const VectorType &a) PURE {
+ return VectorHelper<float>::add(VectorHelper<float>::add(a[0], a[1]));
+ }
+
+ static inline void multiplyAndAdd(VectorType &a, const VectorType &b, const VectorType &c) {
+ VectorHelper<float>::multiplyAndAdd(a[0], b[0], c[0]);
+ VectorHelper<float>::multiplyAndAdd(a[1], b[1], c[1]);
+ }
+ REUSE_FLOAT_IMPL3(mul)
+#undef REUSE_FLOAT_IMPL3
+#undef REUSE_FLOAT_IMPL2
+#undef REUSE_FLOAT_IMPL1
+ };
+
+ template<> struct VectorHelper<int> {
+ typedef int EntryType;
+ typedef _M128I VectorType;
+#define SUFFIX si128
+
+ OP_(or_) OP_(and_) OP_(xor_)
+ static inline VectorType zero() PURE { return CAT(_mm_setzero_, SUFFIX)(); }
+ static inline VectorType notMaskedToZero(VectorType a, _M128 mask) PURE { return CAT(_mm_and_, SUFFIX)(_mm_castps_si128(mask), a); }
+#undef SUFFIX
+#define SUFFIX epi32
+ static inline VectorType one() PURE { return CAT(_mm_setone_, SUFFIX)(); }
+
+ static inline VectorType set(const int a) PURE { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType set(const int a, const int b, const int c, const int d) PURE { return CAT(_mm_set_, SUFFIX)(a, b, c, d); }
+
+ static inline void multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) { v1 = add(mul(v1, v2), v3); }
+
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm_srai_, SUFFIX)(a, shift);
+ }
+ OP1(abs)
+
+ MINMAX
+ static inline EntryType min(VectorType a) PURE {
+ a = min(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ // using lo_epi16 for speed here
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+ static inline EntryType max(VectorType a) PURE {
+ a = max(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ // using lo_epi16 for speed here
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+ static inline EntryType add(VectorType a) PURE {
+ a = add(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+#if VC_IMPL_SSE4_1
+ static inline VectorType mul(VectorType a, VectorType b) PURE { return _mm_mullo_epi32(a, b); }
+ static inline EntryType mul(VectorType a) PURE {
+ a = mul(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+#else
+ static inline VectorType mul(const VectorType &a, const VectorType &b) PURE {
+ const VectorType &aShift = _mm_srli_si128(a, 4);
+ const VectorType &ab02 = _mm_mul_epu32(a, b); // [a0 * b0, a2 * b2]
+ const VectorType &bShift = _mm_srli_si128(b, 4);
+ const VectorType &ab13 = _mm_mul_epu32(aShift, bShift); // [a1 * b1, a3 * b3]
+ return _mm_unpacklo_epi32(_mm_shuffle_epi32(ab02, 8), _mm_shuffle_epi32(ab13, 8));
+ }
+#endif
+ static inline VectorType mul(const VectorType a, const VectorType b, _M128 _mask) PURE {
+ return _mm_blendv_epi8(a, mul(a, b), _mm_castps_si128(_mask));
+ }
+
+ OP(add) OP(sub)
+ OPcmp(eq)
+ OPcmp(lt)
+ OPcmp(gt)
+ static inline VectorType cmpneq(const VectorType &a, const VectorType &b) PURE { _M128I x = cmpeq(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmpnlt(const VectorType &a, const VectorType &b) PURE { _M128I x = cmplt(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmple (const VectorType &a, const VectorType &b) PURE { _M128I x = cmpgt(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmpnle(const VectorType &a, const VectorType &b) PURE { return cmpgt(a, b); }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) PURE { return a; }
+ };
+
+ template<> struct VectorHelper<unsigned int> {
+ typedef unsigned int EntryType;
+ typedef _M128I VectorType;
+#define SUFFIX si128
+ OP_CAST_(or_) OP_CAST_(and_) OP_CAST_(xor_)
+ static inline VectorType zero() PURE { return CAT(_mm_setzero_, SUFFIX)(); }
+ static inline VectorType notMaskedToZero(VectorType a, _M128 mask) PURE { return CAT(_mm_and_, SUFFIX)(_mm_castps_si128(mask), a); }
+
+#undef SUFFIX
+#define SUFFIX epu32
+ static inline VectorType one() PURE { return CAT(_mm_setone_, SUFFIX)(); }
+
+ MINMAX
+ static inline EntryType min(VectorType a) PURE {
+ a = min(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ // using lo_epi16 for speed here
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+ static inline EntryType max(VectorType a) PURE {
+ a = max(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ // using lo_epi16 for speed here
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+ static inline EntryType mul(VectorType a) PURE {
+ a = mul(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ // using lo_epi16 for speed here
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+ static inline EntryType add(VectorType a) PURE {
+ a = add(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ // using lo_epi16 for speed here
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ return _mm_cvtsi128_si32(a);
+ }
+
+ static inline VectorType mul(const VectorType a, const VectorType b, _M128 _mask) PURE {
+ return _mm_blendv_epi8(a, mul(a, b), _mm_castps_si128(_mask));
+ }
+ static inline VectorType mul(const VectorType &a, const VectorType &b) PURE {
+ return VectorHelper<int>::mul(a, b);
+ }
+//X template<unsigned int b> static inline VectorType mul(const VectorType a) PURE {
+//X switch (b) {
+//X case 0: return zero();
+//X case 1: return a;
+//X case 2: return _mm_slli_epi32(a, 1);
+//X case 4: return _mm_slli_epi32(a, 2);
+//X case 8: return _mm_slli_epi32(a, 3);
+//X case 16: return _mm_slli_epi32(a, 4);
+//X case 32: return _mm_slli_epi32(a, 5);
+//X case 64: return _mm_slli_epi32(a, 6);
+//X case 128: return _mm_slli_epi32(a, 7);
+//X case 256: return _mm_slli_epi32(a, 8);
+//X case 512: return _mm_slli_epi32(a, 9);
+//X case 1024: return _mm_slli_epi32(a, 10);
+//X case 2048: return _mm_slli_epi32(a, 11);
+//X }
+//X return mul(a, set(b));
+//X }
+
+#undef SUFFIX
+#define SUFFIX epi32
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm_srli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType set(const unsigned int a) PURE { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType set(const unsigned int a, const unsigned int b, const unsigned int c, const unsigned int d) PURE { return CAT(_mm_set_, SUFFIX)(a, b, c, d); }
+
+ OP(add) OP(sub)
+ OPcmp(eq)
+ static inline VectorType cmpneq(const VectorType &a, const VectorType &b) PURE { return _mm_andnot_si128(cmpeq(a, b), _mm_setallone_si128()); }
+
+#ifndef USE_INCORRECT_UNSIGNED_COMPARE
+ static inline VectorType cmplt(const VectorType &a, const VectorType &b) PURE {
+ return _mm_cmplt_epu32(a, b);
+ }
+ static inline VectorType cmpgt(const VectorType &a, const VectorType &b) PURE {
+ return _mm_cmpgt_epu32(a, b);
+ }
+#else
+ OPcmp(lt)
+ OPcmp(gt)
+#endif
+ static inline VectorType cmpnlt(const VectorType &a, const VectorType &b) PURE { return _mm_andnot_si128(cmplt(a, b), _mm_setallone_si128()); }
+ static inline VectorType cmple (const VectorType &a, const VectorType &b) PURE { return _mm_andnot_si128(cmpgt(a, b), _mm_setallone_si128()); }
+ static inline VectorType cmpnle(const VectorType &a, const VectorType &b) PURE { return cmpgt(a, b); }
+
+#undef SUFFIX
+ static inline VectorType round(VectorType a) PURE { return a; }
+ };
+
+ template<> struct VectorHelper<signed short> {
+ typedef _M128I VectorType;
+ typedef signed short EntryType;
+#define SUFFIX si128
+
+ OP_(or_) OP_(and_) OP_(xor_)
+ static inline VectorType zero() PURE { return CAT(_mm_setzero_, SUFFIX)(); }
+ static inline VectorType notMaskedToZero(VectorType a, _M128 mask) PURE { return CAT(_mm_and_, SUFFIX)(_mm_castps_si128(mask), a); }
+ static inline _M128I concat(_M128I a, _M128I b) PURE { return _mm_packs_epi32(a, b); }
+ static inline _M128I expand0(_M128I x) PURE { return _mm_srai_epi32(_mm_unpacklo_epi16(x, x), 16); }
+ static inline _M128I expand1(_M128I x) PURE { return _mm_srai_epi32(_mm_unpackhi_epi16(x, x), 16); }
+
+#undef SUFFIX
+#define SUFFIX epi16
+ static inline VectorType one() PURE { return CAT(_mm_setone_, SUFFIX)(); }
+
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm_srai_, SUFFIX)(a, shift);
+ }
+ static inline VectorType set(const EntryType a) PURE { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType set(const EntryType a, const EntryType b, const EntryType c, const EntryType d,
+ const EntryType e, const EntryType f, const EntryType g, const EntryType h) PURE {
+ return CAT(_mm_set_, SUFFIX)(a, b, c, d, e, f, g, h);
+ }
+
+ static inline void multiplyAndAdd(VectorType &v1, VectorType v2, VectorType v3) {
+ v1 = add(mul(v1, v2), v3); }
+
+ OP1(abs)
+
+ static inline VectorType mul(VectorType a, VectorType b, _M128 _mask) PURE {
+ _M128I mask = _mm_castps_si128(_mask);
+ return _mm_or_si128(
+ _mm_and_si128(mask, mul(a, b)),
+ _mm_andnot_si128(mask, a)
+ );
+ }
+ OPx(mul, mullo)
+ OP(min) OP(max)
+ static inline EntryType min(VectorType a) PURE {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = min(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType max(VectorType a) PURE {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = max(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType mul(VectorType a) PURE {
+ a = mul(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType add(VectorType a) PURE {
+ a = add(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+
+ OP(add) OP(sub)
+ OPcmp(eq)
+ OPcmp(lt)
+ OPcmp(gt)
+ static inline VectorType cmpneq(const VectorType &a, const VectorType &b) PURE { _M128I x = cmpeq(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmpnlt(const VectorType &a, const VectorType &b) PURE { _M128I x = cmplt(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmple (const VectorType &a, const VectorType &b) PURE { _M128I x = cmpgt(a, b); return _mm_andnot_si128(x, _mm_setallone_si128()); }
+ static inline VectorType cmpnle(const VectorType &a, const VectorType &b) PURE { return cmpgt(a, b); }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) PURE { return a; }
+ };
+
+ template<> struct VectorHelper<unsigned short> {
+ typedef _M128I VectorType;
+ typedef unsigned short EntryType;
+#define SUFFIX si128
+ OP_CAST_(or_) OP_CAST_(and_) OP_CAST_(xor_)
+ static inline VectorType zero() PURE { return CAT(_mm_setzero_, SUFFIX)(); }
+ static inline VectorType notMaskedToZero(VectorType a, _M128 mask) PURE { return CAT(_mm_and_, SUFFIX)(_mm_castps_si128(mask), a); }
+#if VC_IMPL_SSE4_1
+ static inline _M128I concat(_M128I a, _M128I b) PURE { return _mm_packus_epi32(a, b); }
+#else
+ // XXX too bad, but this is broken without SSE 4.1
+ static inline _M128I concat(_M128I a, _M128I b) PURE { return _mm_packs_epi32(a, b); }
+#endif
+ static inline _M128I expand0(_M128I x) PURE { return _mm_srli_epi32(_mm_unpacklo_epi16(x, x), 16); }
+ static inline _M128I expand1(_M128I x) PURE { return _mm_srli_epi32(_mm_unpackhi_epi16(x, x), 16); }
+
+#undef SUFFIX
+#define SUFFIX epu16
+ static inline VectorType one() PURE { return CAT(_mm_setone_, SUFFIX)(); }
+
+ static inline VectorType mul(VectorType a, VectorType b, _M128 _mask) PURE {
+ _M128I mask = _mm_castps_si128(_mask);
+ return _mm_or_si128(
+ _mm_and_si128(mask, mul(a, b)),
+ _mm_andnot_si128(mask, a)
+ );
+ }
+//X template<unsigned int b> static inline VectorType mul(const VectorType a) PURE {
+//X switch (b) {
+//X case 0: return zero();
+//X case 1: return a;
+//X case 2: return _mm_slli_epi16(a, 1);
+//X case 4: return _mm_slli_epi16(a, 2);
+//X case 8: return _mm_slli_epi16(a, 3);
+//X case 16: return _mm_slli_epi16(a, 4);
+//X case 32: return _mm_slli_epi16(a, 5);
+//X case 64: return _mm_slli_epi16(a, 6);
+//X case 128: return _mm_slli_epi16(a, 7);
+//X case 256: return _mm_slli_epi16(a, 8);
+//X case 512: return _mm_slli_epi16(a, 9);
+//X case 1024: return _mm_slli_epi16(a, 10);
+//X case 2048: return _mm_slli_epi16(a, 11);
+//X }
+//X return mul(a, set(b));
+//X }
+#if !defined(USE_INCORRECT_UNSIGNED_COMPARE) || VC_IMPL_SSE4_1
+ OP(min) OP(max)
+#endif
+#undef SUFFIX
+#define SUFFIX epi16
+ static inline VectorType shiftLeft(VectorType a, int shift) {
+ return CAT(_mm_slli_, SUFFIX)(a, shift);
+ }
+ static inline VectorType shiftRight(VectorType a, int shift) {
+ return CAT(_mm_srli_, SUFFIX)(a, shift);
+ }
+ OPx(mul, mullo) // should work correctly for all values
+#if defined(USE_INCORRECT_UNSIGNED_COMPARE) && !defined(VC_IMPL_SSE4_1)
+ OP(min) OP(max) // XXX breaks for values with MSB set
+#endif
+ static inline EntryType min(VectorType a) PURE {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = min(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = min(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType max(VectorType a) PURE {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = max(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = max(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType mul(VectorType a) PURE {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = mul(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = mul(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline EntryType add(VectorType a) PURE {
+ // reminder: _MM_SHUFFLE(3, 2, 1, 0) means "no change"
+ a = add(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 0, 3, 2)));
+ a = add(a, _mm_shufflelo_epi16(a, _MM_SHUFFLE(1, 1, 1, 1)));
+ return _mm_cvtsi128_si32(a); // & 0xffff is implicit
+ }
+ static inline VectorType set(const EntryType a) PURE { return CAT(_mm_set1_, SUFFIX)(a); }
+ static inline VectorType set(const EntryType a, const EntryType b, const EntryType c,
+ const EntryType d, const EntryType e, const EntryType f,
+ const EntryType g, const EntryType h) PURE {
+ return CAT(_mm_set_, SUFFIX)(a, b, c, d, e, f, g, h);
+ }
+
+ OP(add) OP(sub)
+ OPcmp(eq)
+ static inline VectorType cmpneq(const VectorType &a, const VectorType &b) PURE { return _mm_andnot_si128(cmpeq(a, b), _mm_setallone_si128()); }
+
+#ifndef USE_INCORRECT_UNSIGNED_COMPARE
+ static inline VectorType cmplt(const VectorType &a, const VectorType &b) PURE {
+ return _mm_cmplt_epu16(a, b);
+ }
+ static inline VectorType cmpgt(const VectorType &a, const VectorType &b) PURE {
+ return _mm_cmpgt_epu16(a, b);
+ }
+#else
+ OPcmp(lt)
+ OPcmp(gt)
+#endif
+ static inline VectorType cmpnlt(const VectorType &a, const VectorType &b) PURE { return _mm_andnot_si128(cmplt(a, b), _mm_setallone_si128()); }
+ static inline VectorType cmple (const VectorType &a, const VectorType &b) PURE { return _mm_andnot_si128(cmpgt(a, b), _mm_setallone_si128()); }
+ static inline VectorType cmpnle(const VectorType &a, const VectorType &b) PURE { return cmpgt(a, b); }
+#undef SUFFIX
+ static inline VectorType round(VectorType a) PURE { return a; }
+ };
+#undef OP1
+#undef OP
+#undef OP_
+#undef OPx
+#undef OPcmp
+
+} // namespace SSE
+} // namespace Vc
+
+#include "vectorhelper.tcc"
+
+#endif // SSE_VECTORHELPER_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "casts.h"
+#include <cstdlib>
+
+namespace Vc
+{
+namespace SSE
+{
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// float_v
+template<> inline _M128 VectorHelper<_M128>::load(const float *x, AlignedFlag)
+{
+ return _mm_load_ps(x);
+}
+
+template<> inline _M128 VectorHelper<_M128>::load(const float *x, UnalignedFlag)
+{
+ return _mm_loadu_ps(x);
+}
+
+template<> inline _M128 VectorHelper<_M128>::load(const float *x, StreamingAndAlignedFlag)
+{
+ return _mm_stream_load(x);
+}
+
+template<> inline _M128 VectorHelper<_M128>::load(const float *x, StreamingAndUnalignedFlag)
+{
+ return load(x, Unaligned);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// stores
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, AlignedFlag)
+{
+ _mm_store_ps(mem, x);
+}
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, UnalignedFlag)
+{
+ _mm_storeu_ps(mem, x);
+}
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, StreamingAndAlignedFlag)
+{
+ _mm_stream_ps(mem, x);
+}
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castps_si128(x), _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+}
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, const VectorType m, AlignedFlag)
+{
+ _mm_store_ps(mem, _mm_blendv_ps(_mm_load_ps(mem), x, m));
+}
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, const VectorType m, UnalignedFlag)
+{
+ _mm_storeu_ps(mem, _mm_blendv_ps(_mm_loadu_ps(mem), x, m));
+}
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castps_si128(x), _mm_castps_si128(m), reinterpret_cast<char *>(mem));
+}
+inline void VectorHelper<_M128>::store(float *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castps_si128(x), _mm_castps_si128(m), reinterpret_cast<char *>(mem));
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// sfloat_v
+template<> inline M256 VectorHelper<M256>::load(const float *x, AlignedFlag)
+{
+ return VectorType::create(_mm_load_ps(x), _mm_load_ps(x + 4));
+}
+
+template<> inline M256 VectorHelper<M256>::load(const float *x, UnalignedFlag)
+{
+ return VectorType::create(_mm_loadu_ps(x), _mm_loadu_ps(x + 4));
+}
+
+template<> inline M256 VectorHelper<M256>::load(const float *x, StreamingAndAlignedFlag)
+{
+ return VectorType::create(_mm_stream_load(&x[0]), _mm_stream_load(&x[4]));
+}
+
+template<> inline M256 VectorHelper<M256>::load(const float *x, StreamingAndUnalignedFlag)
+{
+ return load(x, Unaligned);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// stores
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, AlignedFlag)
+{
+ _mm_store_ps(mem, x[0]);
+ _mm_store_ps(mem + 4, x[1]);
+}
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, UnalignedFlag)
+{
+ _mm_storeu_ps(mem, x[0]);
+ _mm_storeu_ps(mem + 4, x[1]);
+}
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, StreamingAndAlignedFlag)
+{
+ _mm_stream_ps(mem, x[0]);
+ _mm_stream_ps(mem + 4, x[1]);
+}
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castps_si128(x[0]), _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(_mm_castps_si128(x[1]), _mm_setallone_si128(), reinterpret_cast<char *>(mem + 4));
+}
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, const VectorType &m, AlignedFlag)
+{
+ _mm_store_ps(mem, _mm_blendv_ps(_mm_load_ps(mem), x[0], m[0]));
+ _mm_store_ps(mem + 4, _mm_blendv_ps(_mm_load_ps(mem + 4), x[1], m[1]));
+}
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, const VectorType &m, UnalignedFlag)
+{
+ _mm_storeu_ps(mem, _mm_blendv_ps(_mm_loadu_ps(mem), x[0], m[0]));
+ _mm_storeu_ps(mem + 4, _mm_blendv_ps(_mm_loadu_ps(mem + 4), x[1], m[1]));
+}
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, const VectorType &m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castps_si128(x[0]), _mm_castps_si128(m[0]), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(_mm_castps_si128(x[1]), _mm_castps_si128(m[1]), reinterpret_cast<char *>(mem + 4));
+}
+inline void VectorHelper<M256>::store(float *mem, const VectorType &x, const VectorType &m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castps_si128(x[0]), _mm_castps_si128(m[0]), reinterpret_cast<char *>(mem));
+ _mm_maskmoveu_si128(_mm_castps_si128(x[1]), _mm_castps_si128(m[1]), reinterpret_cast<char *>(mem + 4));
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// double_v
+template<> inline _M128D VectorHelper<_M128D>::load(const double *x, AlignedFlag)
+{
+ return _mm_load_pd(x);
+}
+
+template<> inline _M128D VectorHelper<_M128D>::load(const double *x, UnalignedFlag)
+{
+ return _mm_loadu_pd(x);
+}
+
+template<> inline _M128D VectorHelper<_M128D>::load(const double *x, StreamingAndAlignedFlag)
+{
+ return _mm_stream_load(x);
+}
+
+template<> inline _M128D VectorHelper<_M128D>::load(const double *x, StreamingAndUnalignedFlag)
+{
+ return load(x, Unaligned);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// stores
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, AlignedFlag)
+{
+ _mm_store_pd(mem, x);
+}
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, UnalignedFlag)
+{
+ _mm_storeu_pd(mem, x);
+}
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, StreamingAndAlignedFlag)
+{
+ _mm_stream_pd(mem, x);
+}
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castpd_si128(x), _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+}
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, const VectorType m, AlignedFlag)
+{
+ _mm_store_pd(mem, _mm_blendv_pd(_mm_load_pd(mem), x, m));
+}
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, const VectorType m, UnalignedFlag)
+{
+ _mm_storeu_pd(mem, _mm_blendv_pd(_mm_loadu_pd(mem), x, m));
+}
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castpd_si128(x), _mm_castpd_si128(m), reinterpret_cast<char *>(mem));
+}
+inline void VectorHelper<_M128D>::store(double *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(_mm_castpd_si128(x), _mm_castpd_si128(m), reinterpret_cast<char *>(mem));
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// int_v, uint_v, short_v, ushort_v
+template<typename T> inline _M128I VectorHelper<_M128I>::load(const T *x, AlignedFlag)
+{
+ return _mm_load_si128(reinterpret_cast<const VectorType *>(x));
+}
+
+template<typename T> inline _M128I VectorHelper<_M128I>::load(const T *x, UnalignedFlag)
+{
+ return _mm_loadu_si128(reinterpret_cast<const VectorType *>(x));
+}
+
+template<typename T> inline _M128I VectorHelper<_M128I>::load(const T *x, StreamingAndAlignedFlag)
+{
+ return _mm_stream_load(x);
+}
+
+template<typename T> inline _M128I VectorHelper<_M128I>::load(const T *x, StreamingAndUnalignedFlag)
+{
+ return load(x, Unaligned);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// stores
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, AlignedFlag)
+{
+ _mm_store_si128(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, UnalignedFlag)
+{
+ _mm_storeu_si128(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, StreamingAndAlignedFlag)
+{
+ _mm_stream_si128(reinterpret_cast<VectorType *>(mem), x);
+}
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(x, _mm_setallone_si128(), reinterpret_cast<char *>(mem));
+}
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, const VectorType m, AlignedFlag align)
+{
+ store(mem, _mm_blendv_epi8(load(mem, align), x, m), align);
+}
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, const VectorType m, UnalignedFlag align)
+{
+ store(mem, _mm_blendv_epi8(load(mem, align), x, m), align);
+}
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, const VectorType m, StreamingAndAlignedFlag)
+{
+ _mm_maskmoveu_si128(x, m, reinterpret_cast<char *>(mem));
+}
+template<typename T> inline void VectorHelper<_M128I>::store(T *mem, const VectorType x, const VectorType m, StreamingAndUnalignedFlag)
+{
+ _mm_maskmoveu_si128(x, m, reinterpret_cast<char *>(mem));
+}
+
+ template<> inline _M128I SortHelper<_M128I, 8>::sort(_M128I x)
+ {
+ _M128I lo, hi, y;
+ // sort pairs
+ y = _mm_shufflelo_epi16(_mm_shufflehi_epi16(x, _MM_SHUFFLE(2, 3, 0, 1)), _MM_SHUFFLE(2, 3, 0, 1));
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+ x = _mm_blend_epi16(lo, hi, 0xaa);
+
+ // merge left and right quads
+ y = _mm_shufflelo_epi16(_mm_shufflehi_epi16(x, _MM_SHUFFLE(0, 1, 2, 3)), _MM_SHUFFLE(0, 1, 2, 3));
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+ x = _mm_blend_epi16(lo, hi, 0xcc);
+ y = _mm_srli_si128(x, 2);
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+ x = _mm_blend_epi16(lo, _mm_slli_si128(hi, 2), 0xaa);
+
+ // merge quads into octs
+ y = _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2));
+ y = _mm_shufflelo_epi16(y, _MM_SHUFFLE(0, 1, 2, 3));
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+
+ x = _mm_unpacklo_epi16(lo, hi);
+ y = _mm_srli_si128(x, 8);
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+
+ x = _mm_unpacklo_epi16(lo, hi);
+ y = _mm_srli_si128(x, 8);
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+
+ return _mm_unpacklo_epi16(lo, hi);
+ }
+ template<> inline _M128I SortHelper<_M128I, 4>::sort(_M128I x)
+ {
+ /*
+ // in 16,67% of the cases the merge can be replaced by an append
+
+ // x = [a b c d]
+ // y = [c d a b]
+ _M128I y = _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2));
+ _M128I l = _mm_min_epi32(x, y); // min[ac bd ac bd]
+ _M128I h = _mm_max_epi32(x, y); // max[ac bd ac bd]
+ if (IS_UNLIKELY(_mm_cvtsi128_si32(h) <= l[1])) { // l[0] < h[0] < l[1] < h[1]
+ return _mm_unpacklo_epi32(l, h);
+ }
+ // h[0] > l[1]
+ */
+
+ // sort pairs
+ _M128I y = _mm_shuffle_epi32(x, _MM_SHUFFLE(2, 3, 0, 1));
+ _M128I l = _mm_min_epi32(x, y);
+ _M128I h = _mm_max_epi32(x, y);
+ x = _mm_unpacklo_epi32(l, h);
+ y = _mm_unpackhi_epi32(h, l);
+
+ // sort quads
+ l = _mm_min_epi32(x, y);
+ h = _mm_max_epi32(x, y);
+ x = _mm_unpacklo_epi32(l, h);
+ y = _mm_unpackhi_epi64(x, x);
+
+ l = _mm_min_epi32(x, y);
+ h = _mm_max_epi32(x, y);
+ return _mm_unpacklo_epi32(l, h);
+ }
+ template<> inline _M128 SortHelper<_M128, 4>::sort(_M128 x)
+ {
+ _M128 y = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));
+ _M128 l = _mm_min_ps(x, y);
+ _M128 h = _mm_max_ps(x, y);
+ x = _mm_unpacklo_ps(l, h);
+ y = _mm_unpackhi_ps(h, l);
+
+ l = _mm_min_ps(x, y);
+ h = _mm_max_ps(x, y);
+ x = _mm_unpacklo_ps(l, h);
+ y = _mm_movehl_ps(x, x);
+
+ l = _mm_min_ps(x, y);
+ h = _mm_max_ps(x, y);
+ return _mm_unpacklo_ps(l, h);
+//X _M128 k = _mm_cmpgt_ps(x, y);
+//X k = _mm_shuffle_ps(k, k, _MM_SHUFFLE(2, 2, 0, 0));
+//X x = _mm_blendv_ps(x, y, k);
+//X y = _mm_shuffle_ps(x, x, _MM_SHUFFLE(1, 0, 3, 2));
+//X k = _mm_cmpgt_ps(x, y);
+//X k = _mm_shuffle_ps(k, k, _MM_SHUFFLE(1, 0, 1, 0));
+//X x = _mm_blendv_ps(x, y, k);
+//X y = _mm_shuffle_ps(x, x, _MM_SHUFFLE(3, 1, 2, 0));
+//X k = _mm_cmpgt_ps(x, y);
+//X k = _mm_shuffle_ps(k, k, _MM_SHUFFLE(0, 1, 1, 0));
+//X return _mm_blendv_ps(x, y, k);
+ }
+ template<> inline M256 SortHelper<M256, 8>::sort(const M256 &_x)
+ {
+ M256 x = _x;
+ typedef SortHelper<_M128, 4> H;
+
+ _M128 a, b, l, h;
+ a = H::sort(x[0]);
+ b = H::sort(x[1]);
+
+ // merge
+ b = _mm_shuffle_ps(b, b, _MM_SHUFFLE(0, 1, 2, 3));
+ l = _mm_min_ps(a, b);
+ h = _mm_max_ps(a, b);
+
+ a = _mm_unpacklo_ps(l, h);
+ b = _mm_unpackhi_ps(l, h);
+ l = _mm_min_ps(a, b);
+ h = _mm_max_ps(a, b);
+
+ a = _mm_unpacklo_ps(l, h);
+ b = _mm_unpackhi_ps(l, h);
+ l = _mm_min_ps(a, b);
+ h = _mm_max_ps(a, b);
+
+ x[0] = _mm_unpacklo_ps(l, h);
+ x[1] = _mm_unpackhi_ps(l, h);
+ return x;
+ }
+ template<> inline _M128D SortHelper<_M128D, 2>::sort(_M128D x)
+ {
+ const _M128D y = _mm_shuffle_pd(x, x, _MM_SHUFFLE2(0, 1));
+ return _mm_unpacklo_pd(_mm_min_sd(x, y), _mm_max_sd(x, y));
+ }
+
+ // can be used to multiply with a constant. For some special constants it doesn't need an extra
+ // vector but can use a shift instead, basically encoding the factor in the instruction.
+ template<typename IndexType, unsigned int constant> inline IndexType mulConst(const IndexType &x) {
+ typedef VectorHelper<typename IndexType::EntryType> H;
+ switch (constant) {
+ case 0: return H::zero();
+ case 1: return x;
+ case 2: return H::slli(x.data(), 1);
+ case 4: return H::slli(x.data(), 2);
+ case 8: return H::slli(x.data(), 3);
+ case 16: return H::slli(x.data(), 4);
+ case 32: return H::slli(x.data(), 5);
+ case 64: return H::slli(x.data(), 6);
+ case 128: return H::slli(x.data(), 7);
+ case 256: return H::slli(x.data(), 8);
+ case 512: return H::slli(x.data(), 9);
+ case 1024: return H::slli(x.data(), 10);
+ case 2048: return H::slli(x.data(), 11);
+ }
+#ifndef VC_IMPL_SSE4_1
+ // without SSE 4.1 int multiplication is not so nice
+ if (sizeof(typename IndexType::EntryType) == 4) {
+ switch (constant) {
+ case 3: return H::add( x.data() , H::slli(x.data(), 1));
+ case 5: return H::add( x.data() , H::slli(x.data(), 2));
+ case 9: return H::add( x.data() , H::slli(x.data(), 3));
+ case 17: return H::add( x.data() , H::slli(x.data(), 4));
+ case 33: return H::add( x.data() , H::slli(x.data(), 5));
+ case 65: return H::add( x.data() , H::slli(x.data(), 6));
+ case 129: return H::add( x.data() , H::slli(x.data(), 7));
+ case 257: return H::add( x.data() , H::slli(x.data(), 8));
+ case 513: return H::add( x.data() , H::slli(x.data(), 9));
+ case 1025: return H::add( x.data() , H::slli(x.data(), 10));
+ case 2049: return H::add( x.data() , H::slli(x.data(), 11));
+ case 6: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 2));
+ case 10: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 3));
+ case 18: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 4));
+ case 34: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 5));
+ case 66: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 6));
+ case 130: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 7));
+ case 258: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 8));
+ case 514: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 9));
+ case 1026: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 10));
+ case 2050: return H::add(H::slli(x.data(), 1), H::slli(x.data(), 11));
+ case 12: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 3));
+ case 20: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 4));
+ case 36: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 5));
+ case 68: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 6));
+ case 132: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 7));
+ case 260: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 8));
+ case 516: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 9));
+ case 1028: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 10));
+ case 2052: return H::add(H::slli(x.data(), 2), H::slli(x.data(), 11));
+ case 24: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 4));
+ case 40: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 5));
+ case 72: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 6));
+ case 136: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 7));
+ case 264: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 8));
+ case 520: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 9));
+ case 1032: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 10));
+ case 2056: return H::add(H::slli(x.data(), 3), H::slli(x.data(), 11));
+ case 48: return H::add(H::slli(x.data(), 4), H::slli(x.data(), 5));
+ case 80: return H::add(H::slli(x.data(), 4), H::slli(x.data(), 6));
+ case 144: return H::add(H::slli(x.data(), 4), H::slli(x.data(), 7));
+ case 272: return H::add(H::slli(x.data(), 4), H::slli(x.data(), 8));
+ case 528: return H::add(H::slli(x.data(), 4), H::slli(x.data(), 9));
+ case 1040: return H::add(H::slli(x.data(), 4), H::slli(x.data(), 10));
+ case 2064: return H::add(H::slli(x.data(), 4), H::slli(x.data(), 11));
+ case 96: return H::add(H::slli(x.data(), 5), H::slli(x.data(), 6));
+ case 160: return H::add(H::slli(x.data(), 5), H::slli(x.data(), 7));
+ case 288: return H::add(H::slli(x.data(), 5), H::slli(x.data(), 8));
+ case 544: return H::add(H::slli(x.data(), 5), H::slli(x.data(), 9));
+ case 1056: return H::add(H::slli(x.data(), 5), H::slli(x.data(), 10));
+ case 2080: return H::add(H::slli(x.data(), 5), H::slli(x.data(), 11));
+ case 192: return H::add(H::slli(x.data(), 6), H::slli(x.data(), 7));
+ case 320: return H::add(H::slli(x.data(), 6), H::slli(x.data(), 8));
+ case 576: return H::add(H::slli(x.data(), 6), H::slli(x.data(), 9));
+ case 1088: return H::add(H::slli(x.data(), 6), H::slli(x.data(), 10));
+ case 2112: return H::add(H::slli(x.data(), 6), H::slli(x.data(), 11));
+ case 384: return H::add(H::slli(x.data(), 7), H::slli(x.data(), 8));
+ case 640: return H::add(H::slli(x.data(), 7), H::slli(x.data(), 9));
+ case 1152: return H::add(H::slli(x.data(), 7), H::slli(x.data(), 10));
+ case 2176: return H::add(H::slli(x.data(), 7), H::slli(x.data(), 11));
+ case 768: return H::add(H::slli(x.data(), 8), H::slli(x.data(), 9));
+ case 1280: return H::add(H::slli(x.data(), 8), H::slli(x.data(), 10));
+ case 2304: return H::add(H::slli(x.data(), 8), H::slli(x.data(), 11));
+ case 1536: return H::add(H::slli(x.data(), 9), H::slli(x.data(), 10));
+ case 2560: return H::add(H::slli(x.data(), 9), H::slli(x.data(), 11));
+ case 3072: return H::add(H::slli(x.data(),10), H::slli(x.data(), 11));
+ }
+ }
+#endif
+ return H::mul(x.data(), H::set(constant));
+ }
+} // namespace SSE
+} // namespace Vc
--- /dev/null
+#ifdef __GNUC__
+#warning "Use of the Vc/uint_v header is deprecated. The header file will be removed in a future version of Vc."
+#endif
--- /dev/null
+#ifdef __GNUC__
+#warning "Use of the Vc/ushort_v header is deprecated. The header file will be removed in a future version of Vc."
+#endif
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VECTOR_H
+#define VECTOR_H
+
+#include "global.h"
+#include "internal/namespace.h"
+
+#if VC_IMPL_Scalar
+# include "scalar/vector.h"
+# include "scalar/helperimpl.h"
+#elif VC_IMPL_AVX
+# include "avx/vector.h"
+# include "avx/helperimpl.h"
+#elif VC_IMPL_SSE
+# include "sse/vector.h"
+# include "sse/helperimpl.h"
+#endif
+
+#ifdef isfinite
+#undef isfinite
+#endif
+#ifdef isnan
+#undef isnan
+#endif
+
+namespace Vc
+{
+ using VECTOR_NAMESPACE::VectorAlignment;
+ using VECTOR_NAMESPACE::VectorAlignedBaseT;
+ typedef VectorAlignedBaseT<> VectorAlignedBase;
+ using namespace VectorSpecialInitializerZero;
+ using namespace VectorSpecialInitializerOne;
+ using namespace VectorSpecialInitializerIndexesFromZero;
+ using VECTOR_NAMESPACE::min;
+ using VECTOR_NAMESPACE::max;
+ using VECTOR_NAMESPACE::sqrt;
+ using VECTOR_NAMESPACE::rsqrt;
+ using VECTOR_NAMESPACE::abs;
+ using VECTOR_NAMESPACE::sin;
+ using VECTOR_NAMESPACE::asin;
+ using VECTOR_NAMESPACE::cos;
+ using VECTOR_NAMESPACE::sincos;
+ using VECTOR_NAMESPACE::floor;
+ using VECTOR_NAMESPACE::ceil;
+ using VECTOR_NAMESPACE::exp;
+ using VECTOR_NAMESPACE::log;
+ using VECTOR_NAMESPACE::log2;
+ using VECTOR_NAMESPACE::log10;
+ using VECTOR_NAMESPACE::reciprocal;
+ using VECTOR_NAMESPACE::atan;
+ using VECTOR_NAMESPACE::atan2;
+ using VECTOR_NAMESPACE::frexp;
+ using VECTOR_NAMESPACE::ldexp;
+ using VECTOR_NAMESPACE::round;
+ using VECTOR_NAMESPACE::isfinite;
+ using VECTOR_NAMESPACE::isnan;
+ using VECTOR_NAMESPACE::forceToRegisters;
+ using VECTOR_NAMESPACE::Vector;
+
+ typedef VECTOR_NAMESPACE::double_v double_v;
+ typedef double_v::Mask double_m;
+ typedef VECTOR_NAMESPACE::sfloat_v sfloat_v;
+ typedef sfloat_v::Mask sfloat_m;
+ typedef VECTOR_NAMESPACE::float_v float_v;
+ typedef float_v::Mask float_m;
+ typedef VECTOR_NAMESPACE::int_v int_v;
+ typedef int_v::Mask int_m;
+ typedef VECTOR_NAMESPACE::uint_v uint_v;
+ typedef uint_v::Mask uint_m;
+ typedef VECTOR_NAMESPACE::short_v short_v;
+ typedef short_v::Mask short_m;
+ typedef VECTOR_NAMESPACE::ushort_v ushort_v;
+ typedef ushort_v::Mask ushort_m;
+
+ namespace {
+ VC_STATIC_ASSERT_NC(double_v::Size == VC_DOUBLE_V_SIZE, VC_DOUBLE_V_SIZE_MACRO_WRONG);
+ VC_STATIC_ASSERT_NC(float_v::Size == VC_FLOAT_V_SIZE , VC_FLOAT_V_SIZE_MACRO_WRONG );
+ VC_STATIC_ASSERT_NC(sfloat_v::Size == VC_SFLOAT_V_SIZE, VC_SFLOAT_V_SIZE_MACRO_WRONG);
+ VC_STATIC_ASSERT_NC(int_v::Size == VC_INT_V_SIZE , VC_INT_V_SIZE_MACRO_WRONG );
+ VC_STATIC_ASSERT_NC(uint_v::Size == VC_UINT_V_SIZE , VC_UINT_V_SIZE_MACRO_WRONG );
+ VC_STATIC_ASSERT_NC(short_v::Size == VC_SHORT_V_SIZE , VC_SHORT_V_SIZE_MACRO_WRONG );
+ VC_STATIC_ASSERT_NC(ushort_v::Size == VC_USHORT_V_SIZE, VC_USHORT_V_SIZE_MACRO_WRONG);
+ }
+} // namespace Vc
+
+#ifndef VC_NO_STD_FUNCTIONS
+namespace std
+{
+ using Vc::min;
+ using Vc::max;
+
+ using Vc::abs;
+ using Vc::asin;
+ using Vc::atan;
+ using Vc::atan2;
+ using Vc::ceil;
+ using Vc::cos;
+ using Vc::exp;
+ using Vc::floor;
+ using Vc::frexp;
+ using Vc::ldexp;
+ using Vc::log;
+ using Vc::log10;
+ using Vc::log2;
+ using Vc::round;
+ using Vc::sin;
+ using Vc::sqrt;
+
+ using Vc::isfinite;
+ using Vc::isnan;
+} // namespace std
+#endif
+
+#ifndef VC_CLEAN_NAMESPACE
+#define foreach_bit(_it_, _mask_) Vc_foreach_bit(_it_, _mask_)
+#endif
+
+#undef VECTOR_NAMESPACE
+
+#endif // VECTOR_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef VC_VERSION_H
+#define VC_VERSION_H
+
+#define VC_VERSION_STRING "0.6.70-dev"
+#define VC_VERSION_NUMBER 0x00068d
+#define VC_VERSION_CHECK(major, minor, patch) ((major << 16) | (minor << 8) | (patch << 1))
+#define VC_LIBRARY_ABI_VERSION 1
+
+namespace Vc
+{
+ static inline const char *versionString() {
+ return VC_VERSION_STRING;
+ }
+
+ static inline unsigned int versionNumber() {
+ return VC_VERSION_NUMBER;
+ }
+
+#if !defined(VC_NO_VERSION_CHECK) && !defined(VC_COMPILE_LIB)
+ void checkLibraryAbi(unsigned int compileTimeAbi, unsigned int versionNumber, const char *versionString);
+ namespace {
+ static struct runLibraryAbiCheck
+ {
+ runLibraryAbiCheck() {
+ checkLibraryAbi(VC_LIBRARY_ABI_VERSION, VC_VERSION_NUMBER, VC_VERSION_STRING);
+ }
+ } _runLibraryAbiCheck;
+ }
+#endif
+} // namespace Vc
+
+#endif // VC_VERSION_H
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "intrinsics.h"
+#include "casts.h"
+#include "sorthelper.h"
+#include "macros.h"
+
+namespace Vc
+{
+namespace AVX
+{
+
+template<> __m128i SortHelper<short>::sort(__m128i x)
+{
+ __m128i lo, hi, y;
+ // sort pairs
+ y = _mm_shufflelo_epi16(_mm_shufflehi_epi16(x, _MM_SHUFFLE(2, 3, 0, 1)), _MM_SHUFFLE(2, 3, 0, 1));
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+ x = _mm_blend_epi16(lo, hi, 0xaa);
+
+ // merge left and right quads
+ y = _mm_shufflelo_epi16(_mm_shufflehi_epi16(x, _MM_SHUFFLE(0, 1, 2, 3)), _MM_SHUFFLE(0, 1, 2, 3));
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+ x = _mm_blend_epi16(lo, hi, 0xcc);
+ y = _mm_srli_si128(x, 2);
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+ x = _mm_blend_epi16(lo, _mm_slli_si128(hi, 2), 0xaa);
+
+ // merge quads into octs
+ y = _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2));
+ y = _mm_shufflelo_epi16(y, _MM_SHUFFLE(0, 1, 2, 3));
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+
+ x = _mm_unpacklo_epi16(lo, hi);
+ y = _mm_srli_si128(x, 8);
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+
+ x = _mm_unpacklo_epi16(lo, hi);
+ y = _mm_srli_si128(x, 8);
+ lo = _mm_min_epi16(x, y);
+ hi = _mm_max_epi16(x, y);
+
+ return _mm_unpacklo_epi16(lo, hi);
+}
+template<> __m128i SortHelper<unsigned short>::sort(__m128i x)
+{
+ __m128i lo, hi, y;
+ // sort pairs
+ y = _mm_shufflelo_epi16(_mm_shufflehi_epi16(x, _MM_SHUFFLE(2, 3, 0, 1)), _MM_SHUFFLE(2, 3, 0, 1));
+ lo = _mm_min_epu16(x, y);
+ hi = _mm_max_epu16(x, y);
+ x = _mm_blend_epi16(lo, hi, 0xaa);
+
+ // merge left and right quads
+ y = _mm_shufflelo_epi16(_mm_shufflehi_epi16(x, _MM_SHUFFLE(0, 1, 2, 3)), _MM_SHUFFLE(0, 1, 2, 3));
+ lo = _mm_min_epu16(x, y);
+ hi = _mm_max_epu16(x, y);
+ x = _mm_blend_epi16(lo, hi, 0xcc);
+ y = _mm_srli_si128(x, 2);
+ lo = _mm_min_epu16(x, y);
+ hi = _mm_max_epu16(x, y);
+ x = _mm_blend_epi16(lo, _mm_slli_si128(hi, 2), 0xaa);
+
+ // merge quads into octs
+ y = _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2));
+ y = _mm_shufflelo_epi16(y, _MM_SHUFFLE(0, 1, 2, 3));
+ lo = _mm_min_epu16(x, y);
+ hi = _mm_max_epu16(x, y);
+
+ x = _mm_unpacklo_epi16(lo, hi);
+ y = _mm_srli_si128(x, 8);
+ lo = _mm_min_epu16(x, y);
+ hi = _mm_max_epu16(x, y);
+
+ x = _mm_unpacklo_epi16(lo, hi);
+ y = _mm_srli_si128(x, 8);
+ lo = _mm_min_epu16(x, y);
+ hi = _mm_max_epu16(x, y);
+
+ return _mm_unpacklo_epi16(lo, hi);
+}
+
+template<> __m256i SortHelper<int>::sort(__m256i hgfedcba)
+{
+ const __m128i hgfe = hi128(hgfedcba);
+ const __m128i dcba = lo128(hgfedcba);
+ __m128i l = _mm_min_epi32(hgfe, dcba); // ↓hd ↓gc ↓fb ↓ea
+ __m128i h = _mm_max_epi32(hgfe, dcba); // ↑hd ↑gc ↑fb ↑ea
+
+ __m128i x = _mm_unpacklo_epi32(l, h); // ↑fb ↓fb ↑ea ↓ea
+ __m128i y = _mm_unpackhi_epi32(l, h); // ↑hd ↓hd ↑gc ↓gc
+
+ l = _mm_min_epi32(x, y); // ↓(↑fb,↑hd) ↓hfdb ↓(↑ea,↑gc) ↓geca
+ h = _mm_max_epi32(x, y); // ↑hfdb ↑(↓fb,↓hd) ↑geca ↑(↓ea,↓gc)
+
+ x = _mm_min_epi32(l, Reg::permute<X2, X2, X0, X0>(h)); // 2(hfdb) 1(hfdb) 2(geca) 1(geca)
+ y = _mm_max_epi32(h, Reg::permute<X3, X3, X1, X1>(l)); // 4(hfdb) 3(hfdb) 4(geca) 3(geca)
+
+ __m128i b = Reg::shuffle<Y0, Y1, X0, X1>(y, x); // b3 <= b2 <= b1 <= b0
+ __m128i a = _mm_unpackhi_epi64(x, y); // a3 >= a2 >= a1 >= a0
+
+ if (VC_IS_UNLIKELY(_mm_extract_epi32(x, 2) >= _mm_extract_epi32(y, 1))) {
+ return concat(Reg::permute<X0, X1, X2, X3>(b), a);
+ } else if (VC_IS_UNLIKELY(_mm_extract_epi32(x, 0) >= _mm_extract_epi32(y, 3))) {
+ return concat(a, Reg::permute<X0, X1, X2, X3>(b));
+ }
+
+ // merge
+ l = _mm_min_epi32(a, b); // ↓a3b3 ↓a2b2 ↓a1b1 ↓a0b0
+ h = _mm_max_epi32(a, b); // ↑a3b3 ↑a2b2 ↑a1b1 ↑a0b0
+
+ a = _mm_unpacklo_epi32(l, h); // ↑a1b1 ↓a1b1 ↑a0b0 ↓a0b0
+ b = _mm_unpackhi_epi32(l, h); // ↑a3b3 ↓a3b3 ↑a2b2 ↓a2b2
+ l = _mm_min_epi32(a, b); // ↓(↑a1b1,↑a3b3) ↓a1b3 ↓(↑a0b0,↑a2b2) ↓a0b2
+ h = _mm_max_epi32(a, b); // ↑a3b1 ↑(↓a1b1,↓a3b3) ↑a2b0 ↑(↓a0b0,↓a2b2)
+
+ a = _mm_unpacklo_epi32(l, h); // ↑a2b0 ↓(↑a0b0,↑a2b2) ↑(↓a0b0,↓a2b2) ↓a0b2
+ b = _mm_unpackhi_epi32(l, h); // ↑a3b1 ↓(↑a1b1,↑a3b3) ↑(↓a1b1,↓a3b3) ↓a1b3
+ l = _mm_min_epi32(a, b); // ↓(↑a2b0,↑a3b1) ↓(↑a0b0,↑a2b2,↑a1b1,↑a3b3) ↓(↑(↓a0b0,↓a2b2) ↑(↓a1b1,↓a3b3)) ↓a0b3
+ h = _mm_max_epi32(a, b); // ↑a3b0 ↑(↓(↑a0b0,↑a2b2) ↓(↑a1b1,↑a3b3)) ↑(↓a0b0,↓a2b2,↓a1b1,↓a3b3) ↑(↓a0b2,↓a1b3)
+
+ return concat(_mm_unpacklo_epi32(l, h), _mm_unpackhi_epi32(l, h));
+}
+
+template<> __m256i SortHelper<unsigned int>::sort(__m256i hgfedcba)
+{
+ const __m128i hgfe = hi128(hgfedcba);
+ const __m128i dcba = lo128(hgfedcba);
+ __m128i l = _mm_min_epu32(hgfe, dcba); // ↓hd ↓gc ↓fb ↓ea
+ __m128i h = _mm_max_epu32(hgfe, dcba); // ↑hd ↑gc ↑fb ↑ea
+
+ __m128i x = _mm_unpacklo_epi32(l, h); // ↑fb ↓fb ↑ea ↓ea
+ __m128i y = _mm_unpackhi_epi32(l, h); // ↑hd ↓hd ↑gc ↓gc
+
+ l = _mm_min_epu32(x, y); // ↓(↑fb,↑hd) ↓hfdb ↓(↑ea,↑gc) ↓geca
+ h = _mm_max_epu32(x, y); // ↑hfdb ↑(↓fb,↓hd) ↑geca ↑(↓ea,↓gc)
+
+ x = _mm_min_epu32(l, Reg::permute<X2, X2, X0, X0>(h)); // 2(hfdb) 1(hfdb) 2(geca) 1(geca)
+ y = _mm_max_epu32(h, Reg::permute<X3, X3, X1, X1>(l)); // 4(hfdb) 3(hfdb) 4(geca) 3(geca)
+
+ __m128i b = Reg::shuffle<Y0, Y1, X0, X1>(y, x); // b3 <= b2 <= b1 <= b0
+ __m128i a = _mm_unpackhi_epi64(x, y); // a3 >= a2 >= a1 >= a0
+
+ if (VC_IS_UNLIKELY(_mm_extract_epu32(x, 2) >= _mm_extract_epu32(y, 1))) {
+ return concat(Reg::permute<X0, X1, X2, X3>(b), a);
+ } else if (VC_IS_UNLIKELY(_mm_extract_epu32(x, 0) >= _mm_extract_epu32(y, 3))) {
+ return concat(a, Reg::permute<X0, X1, X2, X3>(b));
+ }
+
+ // merge
+ l = _mm_min_epu32(a, b); // ↓a3b3 ↓a2b2 ↓a1b1 ↓a0b0
+ h = _mm_max_epu32(a, b); // ↑a3b3 ↑a2b2 ↑a1b1 ↑a0b0
+
+ a = _mm_unpacklo_epi32(l, h); // ↑a1b1 ↓a1b1 ↑a0b0 ↓a0b0
+ b = _mm_unpackhi_epi32(l, h); // ↑a3b3 ↓a3b3 ↑a2b2 ↓a2b2
+ l = _mm_min_epu32(a, b); // ↓(↑a1b1,↑a3b3) ↓a1b3 ↓(↑a0b0,↑a2b2) ↓a0b2
+ h = _mm_max_epu32(a, b); // ↑a3b1 ↑(↓a1b1,↓a3b3) ↑a2b0 ↑(↓a0b0,↓a2b2)
+
+ a = _mm_unpacklo_epi32(l, h); // ↑a2b0 ↓(↑a0b0,↑a2b2) ↑(↓a0b0,↓a2b2) ↓a0b2
+ b = _mm_unpackhi_epi32(l, h); // ↑a3b1 ↓(↑a1b1,↑a3b3) ↑(↓a1b1,↓a3b3) ↓a1b3
+ l = _mm_min_epu32(a, b); // ↓(↑a2b0,↑a3b1) ↓(↑a0b0,↑a2b2,↑a1b1,↑a3b3) ↓(↑(↓a0b0,↓a2b2) ↑(↓a1b1,↓a3b3)) ↓a0b3
+ h = _mm_max_epu32(a, b); // ↑a3b0 ↑(↓(↑a0b0,↑a2b2) ↓(↑a1b1,↑a3b3)) ↑(↓a0b0,↓a2b2,↓a1b1,↓a3b3) ↑(↓a0b2,↓a1b3)
+
+ return concat(_mm_unpacklo_epi32(l, h), _mm_unpackhi_epi32(l, h));
+}
+
+template<> __m256 SortHelper<float>::sort(__m256 hgfedcba)
+{
+ const __m128 hgfe = hi128(hgfedcba);
+ const __m128 dcba = lo128(hgfedcba);
+ __m128 l = _mm_min_ps(hgfe, dcba); // ↓hd ↓gc ↓fb ↓ea
+ __m128 h = _mm_max_ps(hgfe, dcba); // ↑hd ↑gc ↑fb ↑ea
+
+ __m128 x = _mm_unpacklo_ps(l, h); // ↑fb ↓fb ↑ea ↓ea
+ __m128 y = _mm_unpackhi_ps(l, h); // ↑hd ↓hd ↑gc ↓gc
+
+ l = _mm_min_ps(x, y); // ↓(↑fb,↑hd) ↓hfdb ↓(↑ea,↑gc) ↓geca
+ h = _mm_max_ps(x, y); // ↑hfdb ↑(↓fb,↓hd) ↑geca ↑(↓ea,↓gc)
+
+ x = _mm_min_ps(l, Reg::permute<X2, X2, X0, X0>(h)); // 2(hfdb) 1(hfdb) 2(geca) 1(geca)
+ y = _mm_max_ps(h, Reg::permute<X3, X3, X1, X1>(l)); // 4(hfdb) 3(hfdb) 4(geca) 3(geca)
+
+ __m128 a = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(x), _mm_castps_pd(y))); // a3 >= a2 >= a1 >= a0
+ __m128 b = Reg::shuffle<Y0, Y1, X0, X1>(y, x); // b3 <= b2 <= b1 <= b0
+
+ // merge
+ l = _mm_min_ps(a, b); // ↓a3b3 ↓a2b2 ↓a1b1 ↓a0b0
+ h = _mm_max_ps(a, b); // ↑a3b3 ↑a2b2 ↑a1b1 ↑a0b0
+
+ a = _mm_unpacklo_ps(l, h); // ↑a1b1 ↓a1b1 ↑a0b0 ↓a0b0
+ b = _mm_unpackhi_ps(l, h); // ↑a3b3 ↓a3b3 ↑a2b2 ↓a2b2
+ l = _mm_min_ps(a, b); // ↓(↑a1b1,↑a3b3) ↓a1b3 ↓(↑a0b0,↑a2b2) ↓a0b2
+ h = _mm_max_ps(a, b); // ↑a3b1 ↑(↓a1b1,↓a3b3) ↑a2b0 ↑(↓a0b0,↓a2b2)
+
+ a = _mm_unpacklo_ps(l, h); // ↑a2b0 ↓(↑a0b0,↑a2b2) ↑(↓a0b0,↓a2b2) ↓a0b2
+ b = _mm_unpackhi_ps(l, h); // ↑a3b1 ↓(↑a1b1,↑a3b3) ↑(↓a1b1,↓a3b3) ↓a1b3
+ l = _mm_min_ps(a, b); // ↓(↑a2b0,↑a3b1) ↓(↑a0b0,↑a2b2,↑a1b1,↑a3b3) ↓(↑(↓a0b0,↓a2b2) ↑(↓a1b1,↓a3b3)) ↓a0b3
+ h = _mm_max_ps(a, b); // ↑a3b0 ↑(↓(↑a0b0,↑a2b2) ↓(↑a1b1,↑a3b3)) ↑(↓a0b0,↓a2b2,↓a1b1,↓a3b3) ↑(↓a0b2,↓a1b3)
+
+ return concat(_mm_unpacklo_ps(l, h), _mm_unpackhi_ps(l, h));
+}
+
+template<> __m256 SortHelper<sfloat>::sort(__m256 hgfedcba)
+{
+ return SortHelper<float>::sort(hgfedcba);
+}
+
+template<> void SortHelper<double>::sort(__m256d &VC_RESTRICT x, __m256d &VC_RESTRICT y)
+{
+ __m256d l = _mm256_min_pd(x, y); // ↓x3y3 ↓x2y2 ↓x1y1 ↓x0y0
+ __m256d h = _mm256_max_pd(x, y); // ↑x3y3 ↑x2y2 ↑x1y1 ↑x0y0
+ x = _mm256_unpacklo_pd(l, h); // ↑x2y2 ↓x2y2 ↑x0y0 ↓x0y0
+ y = _mm256_unpackhi_pd(l, h); // ↑x3y3 ↓x3y3 ↑x1y1 ↓x1y1
+ l = _mm256_min_pd(x, y); // ↓(↑x2y2,↑x3y3) ↓x3x2y3y2 ↓(↑x0y0,↑x1y1) ↓x1x0y1y0
+ h = _mm256_max_pd(x, y); // ↑x3x2y3y2 ↑(↓x2y2,↓x3y3) ↑x1x0y1y0 ↑(↓x0y0,↓x1y1)
+ x = _mm256_unpacklo_pd(l, h); // ↑(↓x2y2,↓x3y3) ↓x3x2y3y2 ↑(↓x0y0,↓x1y1) ↓x1x0y1y0
+ y = _mm256_unpackhi_pd(h, l); // ↓(↑x2y2,↑x3y3) ↑x3x2y3y2 ↓(↑x0y0,↑x1y1) ↑x1x0y1y0
+ l = _mm256_min_pd(x, y); // ↓(↑(↓x2y2,↓x3y3) ↓(↑x2y2,↑x3y3)) ↓x3x2y3y2 ↓(↑(↓x0y0,↓x1y1) ↓(↑x0y0,↑x1y1)) ↓x1x0y1y0
+ h = _mm256_max_pd(x, y); // ↑(↑(↓x2y2,↓x3y3) ↓(↑x2y2,↑x3y3)) ↑x3x2y3y2 ↑(↑(↓x0y0,↓x1y1) ↓(↑x0y0,↑x1y1)) ↑x1x0y1y0
+ __m256d a = Reg::permute<X2, X3, X1, X0>(Reg::permute128<X0, X1>(h, h)); // h0 h1 h3 h2
+ __m256d b = Reg::permute<X2, X3, X1, X0>(l); // l2 l3 l1 l0
+
+ // a3 >= a2 >= b1 >= b0
+ // b3 <= b2 <= a1 <= a0
+
+ // merge
+ l = _mm256_min_pd(a, b); // ↓a3b3 ↓a2b2 ↓a1b1 ↓a0b0
+ h = _mm256_min_pd(a, b); // ↑a3b3 ↑a2b2 ↑a1b1 ↑a0b0
+
+ x = _mm256_unpacklo_pd(l, h); // ↑a2b2 ↓a2b2 ↑a0b0 ↓a0b0
+ y = _mm256_unpackhi_pd(l, h); // ↑a3b3 ↓a3b3 ↑a1b1 ↓a1b1
+ l = _mm256_min_pd(x, y); // ↓(↑a2b2,↑a3b3) ↓a2b3 ↓(↑a0b0,↑a1b1) ↓a1b0
+ h = _mm256_min_pd(x, y); // ↑a3b2 ↑(↓a2b2,↓a3b3) ↑a0b1 ↑(↓a0b0,↓a1b1)
+
+ x = Reg::permute128<Y0, X0>(l, h); // ↑a0b1 ↑(↓a0b0,↓a1b1) ↓(↑a0b0,↑a1b1) ↓a1b0
+ y = Reg::permute128<Y1, X1>(l, h); // ↑a3b2 ↑(↓a2b2,↓a3b3) ↓(↑a2b2,↑a3b3) ↓a2b3
+ l = _mm256_min_pd(x, y); // ↓(↑a0b1,↑a3b2) ↓(↑(↓a0b0,↓a1b1) ↑(↓a2b2,↓a3b3)) ↓(↑a0b0,↑a1b1,↑a2b2,↑a3b3) ↓b0b3
+ h = _mm256_min_pd(x, y); // ↑a0a3 ↑(↓a0b0,↓a1b1,↓a2b2,↓a3b3) ↑(↓(↑a0b0,↑a1b1) ↓(↑a2b2,↑a3b3)) ↑(↓a1b0,↓a2b3)
+
+ x = _mm256_unpacklo_pd(l, h); // h2 l2 h0 l0
+ y = _mm256_unpackhi_pd(l, h); // h3 l3 h1 l1
+}
+template<> __m256d SortHelper<double>::sort(__m256d dcba)
+{
+ /*
+ * to find the second largest number find
+ * max(min(max(ab),max(cd)), min(max(ad),max(bc)))
+ * or
+ * max(max(min(ab),min(cd)), min(max(ab),max(cd)))
+ *
+ const __m256d adcb = avx_cast<__m256d>(concat(_mm_alignr_epi8(avx_cast<__m128i>(dc), avx_cast<__m128i>(ba), 8), _mm_alignr_epi8(avx_cast<__m128i>(ba), avx_cast<__m128i>(dc), 8)));
+ const __m256d l = _mm256_min_pd(dcba, adcb); // min(ad cd bc ab)
+ const __m256d h = _mm256_max_pd(dcba, adcb); // max(ad cd bc ab)
+ // max(h3, h1)
+ // max(min(h0,h2), min(h3,h1))
+ // min(max(l0,l2), max(l3,l1))
+ // min(l3, l1)
+
+ const __m256d ll = _mm256_min_pd(h, Reg::permute128<X0, X1>(h, h)); // min(h3h1 h2h0 h1h3 h0h2)
+ //const __m256d hh = _mm256_max_pd(h3 ll1_3 l1 l0, h1 ll0_2 l3 l2);
+ const __m256d hh = _mm256_max_pd(
+ Reg::permute128<X1, Y0>(_mm256_unpackhi_pd(ll, h), l),
+ Reg::permute128<X0, Y1>(_mm256_blend_pd(h ll, 0x1), l));
+ _mm256_min_pd(hh0, hh1
+ */
+
+ //////////////////////////////////////////////////////////////////////////////////
+ // max(max(ac), max(bd))
+ // max(max(min(ac),min(bd)), min(max(ac),max(bd)))
+ // min(max(min(ac),min(bd)), min(max(ac),max(bd)))
+ // min(min(ac), min(bd))
+ __m128d l = _mm_min_pd(lo128(dcba), hi128(dcba)); // min(bd) min(ac)
+ __m128d h = _mm_max_pd(lo128(dcba), hi128(dcba)); // max(bd) max(ac)
+ __m128d h0_l0 = _mm_unpacklo_pd(l, h);
+ __m128d h1_l1 = _mm_unpackhi_pd(l, h);
+ l = _mm_min_pd(h0_l0, h1_l1);
+ h = _mm_max_pd(h0_l0, h1_l1);
+ return concat(
+ _mm_min_pd(l, Reg::permute<X0, X0>(h)),
+ _mm_max_pd(h, Reg::permute<X1, X1>(l))
+ );
+ // extract: 1 cycle
+ // min/max: 4 cycles
+ // unpacklo/hi: 2 cycles
+ // min/max: 4 cycles
+ // permute: 1 cycle
+ // min/max: 4 cycles
+ // insert: 1 cycle
+ // ----------------------
+ // total: 17 cycles
+
+ /*
+ __m256d cdab = Reg::permute<X2, X3, X0, X1>(dcba);
+ __m256d l = _mm256_min_pd(dcba, cdab);
+ __m256d h = _mm256_max_pd(dcba, cdab);
+ __m256d maxmin_ba = Reg::permute128<X0, Y0>(l, h);
+ __m256d maxmin_dc = Reg::permute128<X1, Y1>(l, h);
+
+ l = _mm256_min_pd(maxmin_ba, maxmin_dc);
+ h = _mm256_max_pd(maxmin_ba, maxmin_dc);
+
+ return _mm256_blend_pd(h, l, 0x55);
+ */
+
+ /*
+ // a b c d
+ // b a d c
+ // sort pairs
+ __m256d y, l, h;
+ __m128d l2, h2;
+ y = shuffle<X1, Y0, X3, Y2>(x, x);
+ l = _mm256_min_pd(x, y); // min[ab ab cd cd]
+ h = _mm256_max_pd(x, y); // max[ab ab cd cd]
+
+ // 1 of 2 is at [0]
+ // 1 of 4 is at [1]
+ // 1 of 4 is at [2]
+ // 1 of 2 is at [3]
+
+ // don't be fooled by unpack here. It works differently for AVX pd than for SSE ps
+ x = _mm256_unpacklo_pd(l, h); // l_ab h_ab l_cd h_cd
+ l2 = _mm_min_pd(lo128(x), hi128(x)); // l_abcd l(h_ab hcd)
+ h2 = _mm_max_pd(lo128(x), hi128(x)); // h(l_ab l_cd) h_abcd
+
+ // either it is:
+ return concat(l2, h2);
+ // or:
+ // concat(_mm_unpacklo_pd(l2, h2), _mm_unpackhi_pd(l2, h2));
+
+ // I'd like to have four useful compares
+ const __m128d dc = hi128(dcba);
+ const __m128d ba = lo128(dcba);
+ const __m256d adcb = avx_cast<__m256d>(concat(_mm_alignr_epi8(avx_cast<__m128i>(dc), avx_cast<__m128i>(ba), 8), _mm_alignr_epi8(avx_cast<__m128i>(ba), avx_cast<__m128i>(dc), 8)));
+
+ const int extraCmp = _mm_movemask_pd(_mm_cmpgt_pd(dc, ba));
+ // 0x0: d <= b && c <= a
+ // 0x1: d <= b && c > a
+ // 0x2: d > b && c <= a
+ // 0x3: d > b && c > a
+
+ switch (_mm256_movemask_pd(_mm256_cmpgt_pd(dcba, adcb))) {
+ // impossible: 0x0, 0xf
+ case 0x1: // a <= b && b <= c && c <= d && d > a
+ // abcd
+ return Reg::permute<X2, X3, X0, X1>(Reg::permute<X0, X1>(dcba, dcba));
+ case 0x2: // a <= b && b <= c && c > d && d <= a
+ // dabc
+ return Reg::permute<X2, X3, X0, X1>(adcb);
+ case 0x3: // a <= b && b <= c && c > d && d > a
+ // a[bd]c
+ if (extraCmp & 2) {
+ // abdc
+ return Reg::permute<X2, X3, X1, X0>(Reg::permute<X0, X1>(dcba, dcba));
+ } else {
+ // adbc
+ return Reg::permute<X3, X2, X0, X1>(adcb);
+ }
+ case 0x4: // a <= b && b > c && c <= d && d <= a
+ // cdab;
+ return Reg::permute<X2, X3, X0, X1>(dcba);
+ case 0x5: // a <= b && b > c && c <= d && d > a
+ // [ac] < [bd]
+ switch (extraCmp) {
+ case 0x0: // d <= b && c <= a
+ // cadb
+ return shuffle<>(dcba, bcda);
+ case 0x1: // d <= b && c > a
+ case 0x2: // d > b && c <= a
+ case 0x3: // d > b && c > a
+ }
+ case 0x6: // a <= b && b > c && c > d && d <= a
+ // d[ac]b
+ case 0x7: // a <= b && b > c && c > d && d > a
+ // adcb;
+ return permute<X1, X0, X3, X2>(permute128<X1, X0>(bcda, bcda));
+ case 0x8: // a > b && b <= c && c <= d && d <= a
+ return bcda;
+ case 0x9: // a > b && b <= c && c <= d && d > a
+ // b[ac]d;
+ case 0xa: // a > b && b <= c && c > d && d <= a
+ // [ac] > [bd]
+ case 0xb: // a > b && b <= c && c > d && d > a
+ // badc;
+ return permute128<X1, X0>(dcba);
+ case 0xc: // a > b && b > c && c <= d && d <= a
+ // c[bd]a;
+ case 0xd: // a > b && b > c && c <= d && d > a
+ // cbad;
+ return permute<X1, X0, X3, X2>(bcda);
+ case 0xe: // a > b && b > c && c > d && d <= a
+ return dcba;
+ }
+ */
+}
+
+} // namespace AVX
+} // namespace Vc
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2011-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include <Vc/cpuid.h>
+
+namespace Vc
+{
+CpuId::uint CpuId::s_ecx0 = 0;
+CpuId::uint CpuId::s_logicalProcessors = 0;
+CpuId::uint CpuId::s_processorFeaturesC = 0;
+CpuId::uint CpuId::s_processorFeaturesD = 0;
+CpuId::uint CpuId::s_processorFeatures8C = 0;
+CpuId::uint CpuId::s_processorFeatures8D = 0;
+CpuId::uint CpuId::s_L1Instruction = 0;
+CpuId::uint CpuId::s_L1Data = 0;
+CpuId::uint CpuId::s_L2Data = 0;
+CpuId::uint CpuId::s_L3Data = 0;
+CpuId::ushort CpuId::s_L1InstructionLineSize = 0;
+CpuId::ushort CpuId::s_L1DataLineSize = 0;
+CpuId::ushort CpuId::s_L2DataLineSize = 0;
+CpuId::ushort CpuId::s_L3DataLineSize = 0;
+CpuId::uint CpuId::s_L1Associativity = 0;
+CpuId::uint CpuId::s_L2Associativity = 0;
+CpuId::uint CpuId::s_L3Associativity = 0;
+CpuId::ushort CpuId::s_prefetch = 32; // The Intel ORM says that if CPUID(2) doesn't set the prefetch size it is 32
+CpuId::uchar CpuId::s_brandIndex = 0;
+CpuId::uchar CpuId::s_cacheLineSize = 0;
+CpuId::uchar CpuId::s_processorModel = 0;
+CpuId::uchar CpuId::s_processorFamily = 0;
+CpuId::ProcessorType CpuId::s_processorType = CpuId::IntelReserved;
+bool CpuId::s_noL2orL3 = false;
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#define CPUID(leaf) \
+ do { \
+ int out[4]; \
+ __cpuid(out, leaf); \
+ eax = out[0]; \
+ ebx = out[1]; \
+ ecx = out[2]; \
+ edx = out[3]; \
+ } while (false)
+#define CPUID_C(leaf, _ecx_) \
+ do { \
+ int out[4]; \
+ __cpuidex(out, leaf, _ecx_); \
+ eax = out[0]; \
+ ebx = out[1]; \
+ ecx = out[2]; \
+ edx = out[3]; \
+ } while (false)
+#else
+#define CPUID(leaf) \
+ __asm__("mov $" #leaf ",%%eax\n\tcpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx))
+#define CPUID_C(leaf, _ecx_) \
+ __asm__("mov $" #leaf ",%%eax\n\tcpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "c"(_ecx_))
+#endif
+static unsigned int CpuIdAmdAssociativityTable(int bits)
+{
+ switch (bits) {
+ case 0x0: return 0;
+ case 0x1: return 1;
+ case 0x2: return 2;
+ case 0x4: return 4;
+ case 0x6: return 8;
+ case 0x8: return 16;
+ case 0xA: return 32;
+ case 0xB: return 48;
+ case 0xC: return 64;
+ case 0xD: return 96;
+ case 0xE: return 128;
+ case 0xF: return 0xff;
+ }
+ return 0xffffffffu;
+}
+
+void CpuId::init()
+{
+ {
+ static bool done = false;
+ if (done) return;
+ done = true;
+ }
+ uint eax, ebx, ecx, edx;
+
+ CPUID(0);
+ s_ecx0 = ecx;
+
+ CPUID(1);
+ s_processorFeaturesC = ecx;
+ s_processorFeaturesD = edx;
+ s_processorModel = (eax & 0x000000f0) >> 4;
+ s_processorFamily = (eax & 0x00000f00) >> 8;
+ if (isAmd()) {
+ if (s_processorFamily >= 0xf) {
+ const uchar processorFamilyExt = (eax & 0x0ff00000) >> 20;
+ s_processorFamily += processorFamilyExt;
+ const uchar processorModelExt = (eax & 0x000f0000) >> 12;
+ s_processorModel += processorModelExt;
+ }
+ } else if (s_processorFamily == 0xf) {
+ const uchar processorFamilyExt = (eax & 0x0ff00000) >> 20;
+ s_processorFamily += processorFamilyExt;
+ const uchar processorModelExt = (eax & 0x000f0000) >> 12;
+ s_processorModel += processorModelExt;
+ } else if (s_processorFamily == 0x6) {
+ const uchar processorModelExt = (eax & 0x000f0000) >> 12;
+ s_processorModel += processorModelExt;
+ }
+ s_processorType = static_cast<ProcessorType>((eax & 0x00003000) >> 12);
+
+ s_brandIndex = ebx & 0xff;
+ ebx >>= 8;
+ s_cacheLineSize = ebx & 0xff;
+ ebx >>= 8;
+ s_logicalProcessors = ebx & 0xff;
+
+ CPUID(0x80000001);
+ s_processorFeatures8C = ecx;
+ s_processorFeatures8D = edx;
+
+ if (isAmd()) {
+ s_prefetch = cacheLineSize();
+
+ CPUID(0x80000005);
+ s_L1DataLineSize = ecx & 0xff;
+ s_L1Data = (ecx >> 24) * 1024;
+ s_L1Associativity = (ecx >> 16) & 0xff;
+ s_L1InstructionLineSize = edx & 0xff;
+ s_L1Instruction = (edx >> 24) * 1024;
+
+ CPUID(0x80000006);
+ s_L2DataLineSize = ecx & 0xff;
+ s_L2Data = (ecx >> 16) * 1024;
+ s_L2Associativity = CpuIdAmdAssociativityTable((ecx >> 12) & 0xf);
+ s_L3DataLineSize = edx & 0xff;
+ s_L3Data = (edx >> 18) * 512 * 1024;
+ s_L3Associativity = CpuIdAmdAssociativityTable((ecx >> 12) & 0xf);
+ return;
+ }
+
+ // Intel only
+ int repeat = 0;
+ bool checkLeaf4 = false;
+ do {
+ CPUID(2);
+ if (repeat == 0) {
+ repeat = eax & 0xff;
+ }
+ if (0 == (0x80000000u & eax)) {
+ for (int i = 0; i < 3; ++i) {
+ eax >>= 8;
+ interpret(eax & 0xff, &checkLeaf4);
+ }
+ }
+ if (0 == (0x80000000u & ebx)) {
+ for (int i = 0; i < 4; ++i) {
+ interpret(ebx & 0xff, &checkLeaf4);
+ ebx >>= 8;
+ }
+ }
+ if (0 == (0x80000000u & ecx)) {
+ for (int i = 0; i < 4; ++i) {
+ interpret(ecx & 0xff, &checkLeaf4);
+ ecx >>= 8;
+ }
+ }
+ if (0 == (0x80000000u & edx)) {
+ for (int i = 0; i < 4; ++i) {
+ interpret(edx & 0xff, &checkLeaf4);
+ edx >>= 8;
+ }
+ }
+ } while (--repeat > 0);
+ if (checkLeaf4) {
+ s_prefetch = cacheLineSize();
+ if (s_prefetch == 0) {
+ s_prefetch = 64;
+ }
+ eax = 1;
+ for (int i = 0; eax & 0x1f; ++i) {
+ CPUID_C(4, i);
+ const int cacheLevel = (eax >> 5) & 7;
+ //const int sharedBy = 1 + ((eax >> 14) & 0xfff);
+ const int linesize = 1 + (ebx & 0xfff); ebx >>= 12;
+ const int partitions = 1 + (ebx & 0x3ff); ebx >>= 10;
+ const int ways = 1 + (ebx & 0x3ff);
+ const int sets = 1 + ecx;
+ const int size = ways * partitions * linesize * sets;
+ switch (eax & 0x1f) {
+ case 1: // data cache
+ switch (cacheLevel) {
+ case 1:
+ s_L1Data = size;
+ s_L1DataLineSize = linesize;
+ s_L1Associativity = ways;
+ break;
+ case 2:
+ s_L2Data = size;
+ s_L2DataLineSize = linesize;
+ s_L2Associativity = ways;
+ break;
+ case 3:
+ s_L3Data = size;
+ s_L3DataLineSize = linesize;
+ s_L3Associativity = ways;
+ break;
+ }
+ break;
+ case 2: // instruction cache
+ switch (cacheLevel) {
+ case 1:
+ s_L1Instruction = size;
+ s_L1InstructionLineSize = linesize;
+ break;
+ }
+ break;
+ case 3: // unified cache
+ switch (cacheLevel) {
+ case 1:
+ s_L1Data = size;// / sharedBy;
+ s_L1DataLineSize = linesize;
+ s_L1Associativity = ways;
+ break;
+ case 2:
+ s_L2Data = size;// / sharedBy;
+ s_L2DataLineSize = linesize;
+ s_L2Associativity = ways;
+ break;
+ case 3:
+ s_L3Data = size;// / sharedBy;
+ s_L3DataLineSize = linesize;
+ s_L3Associativity = ways;
+ break;
+ }
+ break;
+ case 0: // no more caches
+ break;
+ default: // reserved
+ break;
+ }
+ }
+ }
+}
+
+void CpuId::interpret(uchar byte, bool *checkLeaf4)
+{
+ switch (byte) {
+ case 0x06:
+ s_L1Instruction = 8 * 1024;
+ s_L1InstructionLineSize = 32;
+ s_L1Associativity = 4;
+ break;
+ case 0x08:
+ s_L1Instruction = 16 * 1024;
+ s_L1InstructionLineSize = 32;
+ s_L1Associativity = 4;
+ break;
+ case 0x09:
+ s_L1Instruction = 32 * 1024;
+ s_L1InstructionLineSize = 64;
+ s_L1Associativity = 4;
+ break;
+ case 0x0A:
+ s_L1Data = 8 * 1024;
+ s_L1DataLineSize = 32;
+ s_L1Associativity = 2;
+ break;
+ case 0x0C:
+ s_L1Data = 16 * 1024;
+ s_L1DataLineSize = 32;
+ s_L1Associativity = 4;
+ break;
+ case 0x0D:
+ s_L1Data = 16 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 4;
+ break;
+ case 0x0E:
+ s_L1Data = 24 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 6;
+ break;
+ case 0x21:
+ s_L2Data = 256 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0x22:
+ s_L3Data = 512 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 4;
+ break;
+ case 0x23:
+ s_L3Data = 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 8;
+ break;
+ case 0x25:
+ s_L3Data = 2 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 8;
+ break;
+ case 0x29:
+ s_L3Data = 4 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 8;
+ break;
+ case 0x2C:
+ s_L1Data = 32 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 8;
+ break;
+ case 0x30:
+ s_L1Data = 32 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 8;
+ break;
+ case 0x40:
+ s_noL2orL3 = true;
+ break;
+ case 0x41:
+ s_L2Data = 128 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 4;
+ break;
+ case 0x42:
+ s_L2Data = 256 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 4;
+ break;
+ case 0x43:
+ s_L2Data = 512 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 4;
+ break;
+ case 0x44:
+ s_L2Data = 1024 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 4;
+ break;
+ case 0x45:
+ s_L2Data = 2 * 1024 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 4;
+ break;
+ case 0x46:
+ s_L3Data = 4 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 4;
+ break;
+ case 0x47:
+ s_L3Data = 8 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 8;
+ break;
+ case 0x48:
+ s_L2Data = 3 * 1024 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 12;
+ break;
+ case 0x49:
+ if (s_processorFamily == 0xf && s_processorModel == 0x6) {
+ s_L3Data = 4 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 16;
+ } else {
+ s_L2Data = 4 * 1024 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 16;
+ }
+ break;
+ case 0x4A:
+ s_L3Data = 6 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 12;
+ break;
+ case 0x4B:
+ s_L3Data = 8 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 16;
+ break;
+ case 0x4C:
+ s_L3Data = 12 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 12;
+ break;
+ case 0x4D:
+ s_L3Data = 16 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 16;
+ break;
+ case 0x4E:
+ s_L2Data = 6 * 1024 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 24;
+ break;
+ case 0x60:
+ s_L1Data = 16 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 8;
+ break;
+ case 0x66:
+ s_L1Data = 8 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 4;
+ break;
+ case 0x67:
+ s_L1Data = 16 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 4;
+ break;
+ case 0x68:
+ s_L1Data = 32 * 1024;
+ s_L1DataLineSize = 64;
+ s_L1Associativity = 4;
+ break;
+ case 0x78:
+ s_L2Data = 1024 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 4;
+ break;
+ case 0x79:
+ s_L2Data = 128 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0x7A:
+ s_L2Data = 256 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0x7B:
+ s_L2Data = 512 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0x7C:
+ s_L2Data = 1024 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0x7D:
+ s_L2Data = 2 * 1024 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0x7F:
+ s_L2Data = 512 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 2;
+ break;
+ case 0x80:
+ s_L2Data = 512 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0x82:
+ s_L2Data = 256 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 8;
+ break;
+ case 0x83:
+ s_L2Data = 512 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 8;
+ break;
+ case 0x84:
+ s_L2Data = 1024 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 8;
+ break;
+ case 0x85:
+ s_L2Data = 2 * 1024 * 1024;
+ s_L2DataLineSize = 32;
+ s_L2Associativity = 8;
+ break;
+ case 0x86:
+ s_L2Data = 512 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 4;
+ break;
+ case 0x87:
+ s_L2Data = 1024 * 1024;
+ s_L2DataLineSize = 64;
+ s_L2Associativity = 8;
+ break;
+ case 0xD0:
+ s_L3Data = 512 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 4;
+ break;
+ case 0xD1:
+ s_L3Data = 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 4;
+ break;
+ case 0xD2:
+ s_L3Data = 2 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 4;
+ break;
+ case 0xD6:
+ s_L3Data = 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 8;
+ break;
+ case 0xD7:
+ s_L3Data = 2 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 8;
+ break;
+ case 0xD8:
+ s_L3Data = 4 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 8;
+ break;
+ case 0xDC:
+ s_L3Data = 3 * 512 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 12;
+ break;
+ case 0xDD:
+ s_L3Data = 3 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 12;
+ break;
+ case 0xDE:
+ s_L3Data = 6 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 12;
+ break;
+ case 0xE2:
+ s_L3Data = 2 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 16;
+ break;
+ case 0xE3:
+ s_L3Data = 4 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 16;
+ break;
+ case 0xE4:
+ s_L3Data = 8 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 16;
+ break;
+ case 0xEA:
+ s_L3Data = 12 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 24;
+ break;
+ case 0xEB:
+ s_L3Data = 18 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 24;
+ break;
+ case 0xEC:
+ s_L3Data = 24 * 1024 * 1024;
+ s_L3DataLineSize = 64;
+ s_L3Associativity = 24;
+ break;
+ case 0xF0:
+ s_prefetch = 64;
+ break;
+ case 0xF1:
+ s_prefetch = 128;
+ break;
+ case 0xFF:
+ // we have to use CPUID(4) to find out
+ *checkLeaf4 = true;
+ break;
+ default:
+ break;
+ }
+}
+} // namespace Vc
+
+// vim: sw=4 sts=4 et tw=100
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2010-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include <Vc/global.h>
+#include <Vc/cpuid.h>
+#include "common/support.h"
+
+#ifdef VC_MSVC
+#include <intrin.h>
+#endif
+
+namespace Vc
+{
+
+#ifdef VC_GCC
+ __attribute__((target("no-sse2,no-avx")))
+#endif
+bool isImplementationSupported(Implementation impl)
+{
+ CpuId::init();
+ // for AVX we need to check for OSXSAVE and AVX
+
+ switch (impl) {
+ case ScalarImpl:
+ return true;
+ case SSE2Impl:
+ return CpuId::hasSse2();
+ case SSE3Impl:
+ return CpuId::hasSse3();
+ case SSSE3Impl:
+ return CpuId::hasSsse3();
+ case SSE41Impl:
+ return CpuId::hasSse41();
+ case SSE42Impl:
+ return CpuId::hasSse42();
+ case SSE4aImpl:
+ return CpuId::hasSse4a();
+ case XopImpl:
+ return CpuId::hasXop();
+ case Fma4Impl:
+ return CpuId::hasFma4();
+ case AVXImpl:
+#if defined(VC_MSVC) && VC_MSVC >= 160040219 // MSVC 2010 SP1 introduced _xgetbv
+ unsigned long long xcrFeatureMask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
+ return (xcrFeatureMask & 0x6) != 0;
+#elif !defined(VC_NO_XGETBV)
+ if (CpuId::hasOsxsave() && CpuId::hasAvx()) {
+ unsigned int eax;
+ asm("xgetbv" : "=a"(eax) : "c"(0) : "edx");
+ return (eax & 0x06) == 0x06;
+ }
+#endif
+ return false;
+ }
+ return false;
+}
+
+} // namespace Vc
+
+// vim: sw=4 sts=4 et tw=100
--- /dev/null
+/* This file is part of the Vc library.
+
+ Copyright (C) 2009-2012 Matthias Kretz <kretz@kde.org>
+
+ Vc is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of
+ the License, or (at your option) any later version.
+
+ Vc is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Vc. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#ifndef V_ALIGN
+# ifdef __GNUC__
+# define V_ALIGN(n) __attribute__((aligned(n)))
+# else
+# define V_ALIGN(n) __declspec(align(n))
+# endif
+#endif
+
+#include "common/const.h"
+#include "avx/const_data.h"
+#include "sse/const_data.h"
+#include <Vc/version.h>
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+namespace Vc
+{
+namespace AVX
+{
+ // cacheline 1
+ V_ALIGN(64) extern const unsigned int _IndexesFromZero32[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
+ V_ALIGN(16) extern const unsigned short _IndexesFromZero16[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
+ V_ALIGN(16) extern const unsigned char _IndexesFromZero8 [16]= { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+
+ // cacheline 2
+ template<> const double c_sin<double>::data[8] = {
+ 0.15915494309189533576888376337251436, // 1 over 2pi
+ 6.2831853071795864769252867665590058, // 2pi
+ 1.5707963267948966192313216916397514, // pi over 2
+ 3.1415926535897932384626433832795029, // pi
+ 1.666666666666666574148081281236954964697360992431640625e-01, // 1 over 3!
+ 8.33333333333333321768510160154619370587170124053955078125e-03, // 1 over 5!
+ 1.984126984126984125263171154784913596813566982746124267578125e-04, // 1 over 7!
+ 2.755731922398589251095059327045788677423843182623386383056640625e-06 // 1 over 9!
+ };
+
+ // cacheline 3
+ template<> const float c_sin<float>::data[8] = {
+ 1.59154936671257019e-01f, // 1 over 2pi
+ 6.28318548202514648f, // 2pi
+ 1.57079637050628662f, // pi over 2
+ 3.14159274101257324f, // pi
+ 1.66666671633720398e-01f, // 1 over 3!
+ 8.33333376795053482e-03f, // 1 over 5!
+ 1.98412701138295233e-04f, // 1 over 7!
+ 2.75573188446287531e-06f // 1 over 9!
+ };
+
+ const unsigned int c_general::absMaskFloat[2] = { 0xffffffffu, 0x7fffffffu };
+ const unsigned int c_general::signMaskFloat[2] = { 0x0u, 0x80000000u };
+ const float c_general::oneFloat = 1.f;
+ const unsigned short c_general::minShort[2] = { 0x8000u, 0x8000u };
+ const unsigned short c_general::one16[2] = { 1, 1 };
+ const float c_general::_2power31 = 1u << 31;
+
+ // cacheline 4
+ const double c_general::oneDouble = 1.;
+ const unsigned long long c_general::frexpMask = 0xbfefffffffffffffull;
+
+ const unsigned long long c_log<double>::data[21] = {
+ 0x000003ff000003ffull // bias TODO: remove
+ , 0x7ff0000000000000ull // exponentMask (+inf)
+
+ , 0x3f1ab4c293c31bb0ull // P[0]
+ , 0x3fdfd6f53f5652f2ull // P[1]
+ , 0x4012d2baed926911ull // P[2]
+ , 0x402cff72c63eeb2eull // P[3]
+ , 0x4031efd6924bc84dull // P[4]
+ , 0x401ed5637d7edcf8ull // P[5]
+
+ , 0x40269320ae97ef8eull // Q[0]
+ , 0x40469d2c4e19c033ull // Q[1]
+ , 0x4054bf33a326bdbdull // Q[2]
+ , 0x4051c9e2eb5eae21ull // Q[3]
+ , 0x4037200a9e1f25b2ull // Q[4]
+
+ , 0xfff0000000000000ull // -inf
+ , 0x0010000000000000ull // min()
+ , 0x3fe6a09e667f3bcdull // 1/sqrt(2)
+ , 0x3fe6300000000000ull // round(ln(2) * 512) / 512
+ , 0xbf2bd0105c610ca8ull // ln(2) - round(ln(2) * 512) / 512
+ , 0x3fe0000000000000ull // 0.5
+ , 0x3fdbcb7b1526e50eull // log10(e)
+ , 0x3ff71547652b82feull // log2(e)
+ };
+
+ template<> const unsigned int c_log<float>::data[21] = {
+ 0x0000007fu // bias TODO: remove
+ , 0x7f800000u // exponentMask (+inf)
+
+ , 0x3d9021bbu // 7.0376836292e-2f // P[0]
+ , 0xbdebd1b8u // -1.1514610310e-1f // P[1]
+ , 0x3def251au // 1.1676998740e-1f // P[2]
+ , 0xbdfe5d4fu // -1.2420140846e-1f // P[3]
+ , 0x3e11e9bfu // 1.4249322787e-1f // P[4]
+ , 0xbe2aae50u // -1.6668057665e-1f // P[5]
+ , 0x3e4cceacu // 2.0000714765e-1f // P[6]
+ , 0xbe7ffffcu // -2.4999993993e-1f // P[7]
+ , 0x3eaaaaaau // 3.3333331174e-1f // P[8]
+ , 0 // padding because of c_log<double>
+ , 0 // padding because of c_log<double>
+
+ , 0xff800000u // -inf
+ , 0x00800000u // min()
+ , 0x3f3504f3u // 1/sqrt(2)
+ , 0x3f318000u // round(ln(2) * 512) / 512
+ , 0xb95e8083u // ln(2) - round(ln(2) * 512) / 512
+ , 0x3f000000u // 0.5
+ , 0x3ede5bd9u // log10(e)
+ , 0x3fb8aa3bu // log2(e)
+ };
+} // namespace AVX
+
+namespace SSE
+{
+ // cacheline 1
+ V_ALIGN(64) const int c_general::absMaskFloat[4] = { 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff };
+ V_ALIGN(16) const unsigned int c_general::signMaskFloat[4] = { 0x80000000, 0x80000000, 0x80000000, 0x80000000 };
+ V_ALIGN(16) const short c_general::minShort[8] = { -0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000, -0x8000 };
+ V_ALIGN(16) extern const unsigned short _IndexesFromZero8[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
+
+ // cacheline 2
+ V_ALIGN(16) extern const unsigned int _IndexesFromZero4[4] = { 0, 1, 2, 3 };
+ V_ALIGN(16) const unsigned short c_general::one16[8] = { 1, 1, 1, 1, 1, 1, 1, 1 };
+ V_ALIGN(16) const unsigned int c_general::one32[4] = { 1, 1, 1, 1 };
+ V_ALIGN(16) const float c_general::oneFloat[4] = { 1.f, 1.f, 1.f, 1.f };
+
+ // cacheline 3
+ V_ALIGN(16) const double c_general::oneDouble[2] = { 1., 1. };
+ V_ALIGN(16) const long long c_general::absMaskDouble[2] = { 0x7fffffffffffffffll, 0x7fffffffffffffffll };
+ V_ALIGN(16) const unsigned long long c_general::signMaskDouble[2] = { 0x8000000000000000ull, 0x8000000000000000ull };
+ V_ALIGN(16) const unsigned long long c_general::frexpMask[2] = { 0xbfefffffffffffffull, 0xbfefffffffffffffull };
+
+ template<> const float c_sin<float>::data[4 * 8] = {
+ // cacheline 4
+ // 1 over 2pi
+ 1.59154936671257019e-01f, 1.59154936671257019e-01f, 1.59154936671257019e-01f, 1.59154936671257019e-01f,
+ // 2pi
+ 6.28318548202514648f, 6.28318548202514648f, 6.28318548202514648f, 6.28318548202514648f,
+ // pi over 2
+ 1.57079637050628662f, 1.57079637050628662f, 1.57079637050628662f, 1.57079637050628662f,
+ // pi
+ 3.14159274101257324f, 3.14159274101257324f, 3.14159274101257324f, 3.14159274101257324f,
+
+ // cacheline 5
+ // 1 over 3!
+ 1.66666671633720398e-01f, 1.66666671633720398e-01f, 1.66666671633720398e-01f, 1.66666671633720398e-01f,
+ // 1 over 5!
+ 8.33333376795053482e-03f, 8.33333376795053482e-03f, 8.33333376795053482e-03f, 8.33333376795053482e-03f,
+ // 1 over 7!
+ 1.98412701138295233e-04f, 1.98412701138295233e-04f, 1.98412701138295233e-04f, 1.98412701138295233e-04f,
+ // 1 over 9!
+ 2.75573188446287531e-06f, 2.75573188446287531e-06f, 2.75573188446287531e-06f, 2.75573188446287531e-06f
+ };
+
+ template<> const double c_sin<double>::data[2 * 8] = {
+ // cacheline 6
+ // 1 over 2pi
+ 0.15915494309189533576888376337251436, 0.15915494309189533576888376337251436,
+ // 2pi
+ 6.2831853071795864769252867665590058 , 6.2831853071795864769252867665590058 ,
+ // pi over 2
+ 1.5707963267948966192313216916397514 , 1.5707963267948966192313216916397514 ,
+ // pi
+ 3.1415926535897932384626433832795029 , 3.1415926535897932384626433832795029 ,
+
+ // cacheline 7
+ // 1 over 3!
+ 1.666666666666666574148081281236954964697360992431640625e-01, 1.666666666666666574148081281236954964697360992431640625e-01,
+ // 1 over 5!
+ 8.33333333333333321768510160154619370587170124053955078125e-03, 8.33333333333333321768510160154619370587170124053955078125e-03,
+ // 1 over 7!
+ 1.984126984126984125263171154784913596813566982746124267578125e-04, 1.984126984126984125263171154784913596813566982746124267578125e-04,
+ // 1 over 9!
+ 2.755731922398589251095059327045788677423843182623386383056640625e-06, 2.755731922398589251095059327045788677423843182623386383056640625e-06
+ };
+
+ // cacheline 8
+ V_ALIGN(16) extern const unsigned char _IndexesFromZero16[16] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+
+ V_ALIGN(64) const unsigned long long c_log<double>::data[21 * 2] = {
+ /* 0*/ 0x000003ff000003ffull, 0x000003ff000003ffull // bias TODO: remove
+ /* 1*/ , 0x7ff0000000000000ull, 0x7ff0000000000000ull // exponentMask (+inf)
+
+ /* 2*/ , 0x3f1ab4c293c31bb0ull, 0x3f1ab4c293c31bb0ull // P[0]
+ /* 3*/ , 0x3fdfd6f53f5652f2ull, 0x3fdfd6f53f5652f2ull // P[1]
+ /* 4*/ , 0x4012d2baed926911ull, 0x4012d2baed926911ull // P[2]
+ /* 5*/ , 0x402cff72c63eeb2eull, 0x402cff72c63eeb2eull // P[3]
+ /* 6*/ , 0x4031efd6924bc84dull, 0x4031efd6924bc84dull // P[4]
+ /* 7*/ , 0x401ed5637d7edcf8ull, 0x401ed5637d7edcf8ull // P[5]
+
+ /* 8*/ , 0x40269320ae97ef8eull, 0x40269320ae97ef8eull // Q[0]
+ /* 9*/ , 0x40469d2c4e19c033ull, 0x40469d2c4e19c033ull // Q[1]
+ /*10*/ , 0x4054bf33a326bdbdull, 0x4054bf33a326bdbdull // Q[2]
+ /*11*/ , 0x4051c9e2eb5eae21ull, 0x4051c9e2eb5eae21ull // Q[3]
+ /*12*/ , 0x4037200a9e1f25b2ull, 0x4037200a9e1f25b2ull // Q[4]
+
+ /*13*/ , 0xfff0000000000000ull, 0xfff0000000000000ull // -inf
+ /*14*/ , 0x0010000000000000ull, 0x0010000000000000ull // min()
+ /*15*/ , 0x3fe6a09e667f3bcdull, 0x3fe6a09e667f3bcdull // 1/sqrt(2)
+ /*16*/ , 0x3fe6300000000000ull, 0x3fe6300000000000ull // round(ln(2) * 512) / 512
+ /*17*/ , 0xbf2bd0105c610ca8ull, 0xbf2bd0105c610ca8ull // ln(2) - round(ln(2) * 512) / 512
+ /*18*/ , 0x3fe0000000000000ull, 0x3fe0000000000000ull // 0.5
+ /*19*/ , 0x3fdbcb7b1526e50eull, 0x3fdbcb7b1526e50eull // log10(e)
+ /*20*/ , 0x3ff71547652b82feull, 0x3ff71547652b82feull // log2(e)
+ };
+
+ template<> V_ALIGN(64) const unsigned int c_log<float>::data[21 * 4] = {
+ 0x0000007fu, 0x0000007fu, 0x0000007fu, 0x0000007fu, // bias TODO: remove
+ 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, // exponentMask (+inf)
+
+ 0x3d9021bbu, 0x3d9021bbu, 0x3d9021bbu, 0x3d9021bbu, // 7.0376836292e-2f // P[0]
+ 0xbdebd1b8u, 0xbdebd1b8u, 0xbdebd1b8u, 0xbdebd1b8u, // -1.1514610310e-1f // P[1]
+ 0x3def251au, 0x3def251au, 0x3def251au, 0x3def251au, // 1.1676998740e-1f // P[2]
+ 0xbdfe5d4fu, 0xbdfe5d4fu, 0xbdfe5d4fu, 0xbdfe5d4fu, // -1.2420140846e-1f // P[3]
+ 0x3e11e9bfu, 0x3e11e9bfu, 0x3e11e9bfu, 0x3e11e9bfu, // 1.4249322787e-1f // P[4]
+ 0xbe2aae50u, 0xbe2aae50u, 0xbe2aae50u, 0xbe2aae50u, // -1.6668057665e-1f // P[5]
+ 0x3e4cceacu, 0x3e4cceacu, 0x3e4cceacu, 0x3e4cceacu, // 2.0000714765e-1f // P[6]
+ 0xbe7ffffcu, 0xbe7ffffcu, 0xbe7ffffcu, 0xbe7ffffcu, // -2.4999993993e-1f // P[7]
+ 0x3eaaaaaau, 0x3eaaaaaau, 0x3eaaaaaau, 0x3eaaaaaau, // 3.3333331174e-1f // P[8]
+ 0, 0, 0, 0, // padding because of c_log<double>
+ 0, 0, 0, 0, // padding because of c_log<double>
+
+ 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, // -inf
+ 0x00800000u, 0x00800000u, 0x00800000u, 0x00800000u, // min()
+ 0x3f3504f3u, 0x3f3504f3u, 0x3f3504f3u, 0x3f3504f3u, // 1/sqrt(2)
+ // ln(2) = 0x3fe62e42fefa39ef
+ // ln(2) = Vc_buildDouble( 1, 0x00062e42fefa39ef, -1)
+ // = Vc_buildFloat( 1, 0x00317217(f7d), -1) + Vc_buildFloat( 1, 0x0077d1cd, -25)
+ // = Vc_buildFloat( 1, 0x00318000(000), -1) + Vc_buildFloat(-1, 0x005e8083, -13)
+ 0x3f318000u, 0x3f318000u, 0x3f318000u, 0x3f318000u, // round(ln(2) * 512) / 512
+ 0xb95e8083u, 0xb95e8083u, 0xb95e8083u, 0xb95e8083u, // ln(2) - round(ln(2) * 512) / 512
+ 0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u, // 0.5
+ 0x3ede5bd9u, 0x3ede5bd9u, 0x3ede5bd9u, 0x3ede5bd9u, // log10(e)
+ 0x3fb8aa3bu, 0x3fb8aa3bu, 0x3fb8aa3bu, 0x3fb8aa3bu, // log2(e)
+ // log10(2) = 0x3fd34413509f79ff
+ // = Vc_buildDouble( 1, 0x00034413509f79ff, -2)
+ // = Vc_buildFloat( 1, 0x001a209a(84fbcff8), -2) + Vc_buildFloat( 1, 0x0004fbcff(8), -26)
+ //Vc_buildFloat( 1, 0x001a209a, -2), // log10(2)
+ //Vc_buildFloat( 1, 0x001a209a, -2), // log10(2)
+ //Vc_buildFloat( 1, 0x001a209a, -2), // log10(2)
+ //Vc_buildFloat( 1, 0x001a209a, -2), // log10(2)
+ };
+} // namespace SSE
+
+V_ALIGN(64) unsigned int RandomState[16] = {
+ 0x5a383a4fu, 0xc68bd45eu, 0x691d6d86u, 0xb367e14fu,
+ 0xd689dbaau, 0xfde442aau, 0x3d265423u, 0x1a77885cu,
+ 0x36ed2684u, 0xfb1f049du, 0x19e52f31u, 0x821e4dd7u,
+ 0x23996d25u, 0x5962725au, 0x6aced4ceu, 0xd4c610f3u
+};
+
+// dummy symbol to emit warnings with GCC 4.3
+namespace Warnings {
+ void _operator_bracket_warning() {}
+} // namespace Warnings
+
+const char LIBRARY_VERSION[] = VC_VERSION_STRING;
+const unsigned int LIBRARY_VERSION_NUMBER = VC_VERSION_NUMBER;
+const unsigned int LIBRARY_ABI_VERSION = VC_LIBRARY_ABI_VERSION;
+
+void checkLibraryAbi(unsigned int compileTimeAbi, unsigned int versionNumber, const char *compileTimeVersion) {
+ if (LIBRARY_ABI_VERSION != compileTimeAbi || LIBRARY_VERSION_NUMBER < versionNumber) {
+ printf("The versions of libVc.a (%s) and Vc/version.h (%s) are incompatible. Aborting.\n", LIBRARY_VERSION, compileTimeVersion);
+ abort();
+ }
+}
+
+} // namespace Vc
+
+#undef V_ALIGN
git://git.uio.no / u / mrichter / AliRoot.git / commitdiff