diff --git a/CMakeLists.txt b/CMakeLists.txt
index 84fdbe0..31a5470 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,15 +15,9 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
 set(CMAKE_SKIP_BUILD_RPATH FALSE)
 set(BUILD_RPATH_USE_ORIGIN TRUE)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+set(CMAKE_INSTALL_RPATH "$ORIGIN")
 set(CMAKE_BUILD_RPATH "$ORIGIN")
 
-if(NOT APPLE)
-  set(CMAKE_INSTALL_RPATH "$ORIGIN")
-else()
-  set(CMAKE_MACOSX_RPATH ON)
-  set(CMAKE_INSTALL_RPATH "@loader_path")
-endif()
-
 set(BUILD_SHARED_LIBS ON)
 if(WIN32)
   message(STATUS "Set BUILD_SHARED_LIBS to OFF for Windows")
diff --git a/kaldifeat/python/csrc/CMakeLists.txt b/kaldifeat/python/csrc/CMakeLists.txt
index c80637c..1403e6d 100644
--- a/kaldifeat/python/csrc/CMakeLists.txt
+++ b/kaldifeat/python/csrc/CMakeLists.txt
@@ -10,6 +10,17 @@ pybind11_add_module(_kaldifeat
   online-feature.cc
   utils.cc
 )
+
+if(APPLE)
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())"
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    OUTPUT_VARIABLE PYTHON_SITE_PACKAGE_DIR
+  )
+  message(STATUS "PYTHON_SITE_PACKAGE_DIR: ${PYTHON_SITE_PACKAGE_DIR}")
+  target_link_libraries(_kaldifeat PRIVATE "-Wl,-rpath,${PYTHON_SITE_PACKAGE_DIR}")
+endif()
+
 target_link_libraries(_kaldifeat PRIVATE kaldifeat_core)
 if(UNIX AND NOT APPLE)
   target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/libtorch_python.so)