diff --git a/CMakeLists.txt b/CMakeLists.txt
index d418afe..84a406c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,12 +32,15 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules)
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
 
 
-option(BUILD_TESTS "Whether to build tests or not" ON)
+option(kaldifeat_BUILD_TESTS "Whether to build tests or not" ON)
+
+if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
+  include(pybind11)
+endif()
 
-include(pybind11)
 include(torch)
 
-if(BUILD_TESTS)
+if(kaldifeat_BUILD_TESTS)
   include(googletest)
   enable_testing()
 endif()
diff --git a/kaldifeat/csrc/CMakeLists.txt b/kaldifeat/csrc/CMakeLists.txt
index 13b9f9b..8dce57c 100644
--- a/kaldifeat/csrc/CMakeLists.txt
+++ b/kaldifeat/csrc/CMakeLists.txt
@@ -36,7 +36,7 @@ function(kaldifeat_add_test source)
   )
 endfunction()
 
-if(BUILD_TESTS)
+if(kaldifeat_BUILD_TESTS)
   # please sort the source files alphabetically
   set(test_srcs
     feature-window-test.cc