Skip to content

Commit 523fe5f

Browse files
committed
Fix aarch64/arm64 Linux wheel build failure due to missing libatomic linking
Problem The tokenizers library fails to build on aarch64 Linux systems during CI with: /opt/rh/gcc-toolset-13/root/usr/libexec/gcc/aarch64-redhat-linux/13/ld: cannot find -latomic: No such file or directory Root Cause - On aarch64, certain atomic operations require explicit linking with libatomic - The sentencepiece dependency detects libatomic (Found atomic: /usr/lib64/libatomic.so.1) but incorrectly adds just the string "atomic" to link flags instead of the actual library path - This causes the linker to fail when building the Python extension Solution Added proper libatomic detection and linking for aarch64/arm64 systems in two places: 1. For the tokenizers static library 2. For the pytorch_tokenizers_cpp Python extension module The fix uses CMake's find_library to locate libatomic and explicitly links it when building on aarch64/arm64 architectures.
1 parent 2dd303e commit 523fe5f

File tree

1 file changed

+26
-2
lines changed

1 file changed

+26
-2
lines changed

CMakeLists.txt

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ include(CMakePackageConfigHelpers)
2828
include(Utils.cmake)
2929

3030
# Ignore weak attribute warning
31-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
31+
if(NOT MSVC)
32+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
33+
endif()
3234

3335
set(ABSL_ENABLE_INSTALL ON)
3436
set(ABSL_PROPAGATE_CXX_STD ON)
@@ -49,7 +51,7 @@ endif()
4951

5052
add_subdirectory(
5153
${CMAKE_CURRENT_SOURCE_DIR}/third-party/sentencepiece
52-
${CMAKE_CURRENT_BINARY_DIR}/sentencepiece-build
54+
${CMAKE_CURRENT_BINARY_DIR}/sp-build
5355
EXCLUDE_FROM_ALL
5456
)
5557

@@ -65,6 +67,7 @@ set(tokenizers_source_files
6567
${CMAKE_CURRENT_SOURCE_DIR}/src/re2_regex.cpp
6668
${CMAKE_CURRENT_SOURCE_DIR}/src/regex.cpp
6769
${CMAKE_CURRENT_SOURCE_DIR}/src/sentencepiece.cpp
70+
${CMAKE_CURRENT_SOURCE_DIR}/src/tekken.cpp
6871
${CMAKE_CURRENT_SOURCE_DIR}/src/tiktoken.cpp
6972
${CMAKE_CURRENT_SOURCE_DIR}/src/token_decoder.cpp
7073
)
@@ -91,6 +94,15 @@ target_include_directories(
9194
)
9295
target_link_libraries(tokenizers PUBLIC sentencepiece-static re2::re2)
9396

97+
# Link with atomic library on aarch64/arm64 systems
98+
# Some aarch64 systems require explicit linking with libatomic for certain atomic operations
99+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
100+
find_library(ATOMIC_LIBRARY atomic)
101+
if(ATOMIC_LIBRARY)
102+
target_link_libraries(tokenizers PUBLIC ${ATOMIC_LIBRARY})
103+
endif()
104+
endif()
105+
94106
if(SUPPORT_REGEX_LOOKAHEAD)
95107
set(PCRE2_STATIC_PIC ON)
96108
set(PCRE2_BUILD_PCRE2_8 ON)
@@ -157,6 +169,18 @@ if(TOKENIZERS_BUILD_PYTHON)
157169
# Link with the tokenizers library
158170
target_link_libraries(pytorch_tokenizers_cpp PRIVATE tokenizers)
159171

172+
# Link with atomic library on aarch64/arm64 systems
173+
# Some aarch64 systems require explicit linking with libatomic for certain atomic operations
174+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
175+
find_library(ATOMIC_LIBRARY atomic)
176+
if(ATOMIC_LIBRARY)
177+
target_link_libraries(pytorch_tokenizers_cpp PRIVATE ${ATOMIC_LIBRARY})
178+
message(STATUS "Found libatomic: ${ATOMIC_LIBRARY}")
179+
else()
180+
message(WARNING "libatomic not found on aarch64 - build may fail")
181+
endif()
182+
endif()
183+
160184
# Set properties for the Python extension
161185
target_compile_definitions(pytorch_tokenizers_cpp PRIVATE VERSION_INFO=${PROJECT_VERSION})
162186

0 commit comments

Comments
 (0)