[Hexagon] Do not auto-build apps when building TVM (#9970)

Krzysztof Parzyszek · web-flow · commit 73bbfbb47819 · 2022-01-24T13:08:51.000-06:00
* [Hexagon] Do not auto-build apps when building TVM

The Hexagon cmakes have recently become unwieldy due to a complex
network of dependencies between various automatically built components.
This was in large part because of trying to automatically build some
apps, which then tried to build TVM runtimes again, but with their
own configurations.

This patch removes the ability to automatically build any Hexagon-
-related apps from the main TVM build. The following cmake options
are now deprecated:
  - `USE_HEXAGON_LAUNCHER`
  - `USE_HEXAGON_PROXY_RPC`

In order to build the binaries needed for HexagonLauncher from
tvm.contrib.hexagon:
  - Build TVM+runtime for x86, with codegen for Hexagon enabled.
    This can be done via `USE_HEXAGON_DEVICE=sim` or `target`.
  - Build Android runtime and tvm_rpc with `-DUSE_RPC=ON`,
    `-DUSE_CPP_RPC=ON`, and `-DUSE_HEXAGON_RPC=ON`.
  - Build Hexagon runtime with `-DUSE_HEXAGON_RPC=ON`, and
    `-DBUILD_STATIC_RUNTIME=ON`.

* Add README.md

* Restart CI

* Add optional variable to set output directory
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -32,8 +32,6 @@ tvm_option(ROCM_PATH "The path to rocm" /opt/rocm)
 tvm_option(USE_HEXAGON_DEVICE "Build with Hexagon device support in TVM runtime" OFF)
 tvm_option(USE_HEXAGON_SDK "Path to the Hexagon SDK root (required for Hexagon support in TVM runtime or for building TVM runtime for Hexagon)" /path/to/sdk)
 tvm_option(USE_HEXAGON_RPC "Enable Hexagon RPC using minRPC implementation over Android." OFF)
-tvm_option(USE_HEXAGON_LAUNCHER "Build the Hexagon graph launcher application" OFF)
-tvm_option(USE_HEXAGON_PROXY_RPC "Build the Hexagon Proxy RPC server application" OFF)
 tvm_option(USE_RPC "Build with RPC" ON)
 tvm_option(USE_THREADS "Build with thread support" ON)
 tvm_option(USE_LLVM "Build with LLVM, can be set to specific llvm-config path" OFF)
@@ -323,7 +321,7 @@ if(BUILD_FOR_HEXAGON)
   # runtime, since it would cause multiple definition errors with the
   # static one.
   if(NOT BUILD_STATIC_RUNTIME)
-    list(APPEND RUNTIME_SRCS src/runtime/hexagon/hexagon_posix.cc)
+    list(APPEND RUNTIME_SRCS src/runtime/hexagon/android/hexagon_posix.cc)
     # Allow undefined symbols (there will be some from libc).
     set(TVM_NO_UNDEFINED_SYMBOLS "")
   endif()
@@ -559,12 +557,6 @@ else()
   target_compile_definitions(tvm_libinfo_objs PRIVATE "USE_FALLBACK_STL_MAP=0")
 endif(USE_FALLBACK_STL_MAP)
 
-if(BUILD_FOR_HEXAGON)
-  # Wrap pthread_create to allow setting custom stack size.
-  set_property(TARGET tvm_runtime APPEND PROPERTY LINK_FLAGS
-                        "-Wl,--wrap=pthread_create")
-endif()
-
 if(USE_THREADS AND NOT BUILD_FOR_HEXAGON)
   message(STATUS "Build with thread support...")
   set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
@@ -699,6 +691,18 @@ if(APPLE AND TVM_IS_DEBUG_BUILD)
 		  )
 endif()
 
+if(BUILD_FOR_HEXAGON)
+  # Wrap pthread_create to allow setting custom stack size.
+  set_property(TARGET tvm_runtime APPEND PROPERTY LINK_FLAGS
+                        "-Wl,--wrap=pthread_create")
+  # Link tvm_runtime into the RPC skel library. Make sure it's built
+  # as a part of the "runtime" target.
+  if(USE_HEXAGON_RPC)
+    target_link_libraries(hexagon_rpc_skel -Wl,--whole-archive tvm_runtime -Wl,--no-whole-archive)
+    add_dependencies(runtime hexagon_rpc_skel)
+  endif()
+endif()
+
 #Caches the build.
 #Note that ccache-3.x doesn't support nvcc well, so CUDA kernels may never hit the cache and still
 #need to be re-compiled every time. Using ccache 4.0+ can resolve this issue.
diff --git a/apps/hexagon_api/CMakeLists.txt b/apps/hexagon_api/CMakeLists.txt
@@ -0,0 +1,90 @@
+cmake_minimum_required(VERSION 3.2)
+
+project(hexagon_api)
+
+include(ExternalProject)
+
+# Required variables:
+#   ANDROID_ABI
+#   ANDROID_PLATFORM
+#   USE_ANDROID_TOOLCHAIN (Android toolchain .cmake file)
+#   USE_HEXAGON_ARCH
+#   USE_HEXAGON_SDK
+#   USE_HEXAGON_TOOLCHAIN (Path to Hexagon toolchain ending with "Tools")
+# Optional variable:
+#   USE_OUTPUT_BINARY_DIR (Path to copy the output binaries to)
+
+set(TVM_SOURCE_DIR "${CMAKE_SOURCE_DIR}/../..")
+
+if(DEFINED USE_OUTPUT_BINARY_DIR)
+  set(HEXAGON_API_BINARY_DIR "${USE_OUTPUT_BINARY_DIR}")
+else()
+  set(HEXAGON_API_BINARY_DIR "${CMAKE_BINARY_DIR}/hexagon_rpc")
+endif()
+file(MAKE_DIRECTORY ${HEXAGON_API_BINARY_DIR})
+
+# Build Android binaries:
+# - libtvm_runtime.so
+# - tvm_rpc_android
+
+ExternalProject_Add(android_tvm_runtime_rpc
+  SOURCE_DIR "${TVM_SOURCE_DIR}"
+  BUILD_COMMAND $(MAKE) runtime tvm_rpc
+  CMAKE_ARGS
+    "-DCMAKE_TOOLCHAIN_FILE=${USE_ANDROID_TOOLCHAIN}"
+    "-DANDROID_PLATFORM=${ANDROID_PLATFORM}"
+    "-DANDROID_ABI=${ANDROID_ABI}"
+    "-DUSE_HEXAGON_SDK=${USE_HEXAGON_SDK}"
+    "-DUSE_HEXAGON_ARCH=${USE_HEXAGON_ARCH}"
+    "-DCMAKE_CXX_STANDARD=14"
+    "-DUSE_LIBBACKTRACE=OFF"
+    "-DUSE_LLVM=OFF"
+    "-DUSE_RPC=ON"
+    "-DUSE_CPP_RPC=ON"
+    "-DUSE_HEXAGON_RPC=ON"
+  INSTALL_COMMAND ""
+  BUILD_ALWAYS ON
+)
+ExternalProject_Get_Property(android_tvm_runtime_rpc BINARY_DIR)
+ExternalProject_Add_Step(android_tvm_runtime_rpc copy_runtime
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${BINARY_DIR}/libtvm_runtime.so
+    ${HEXAGON_API_BINARY_DIR}
+  DEPENDEES install
+)
+ExternalProject_Add_Step(android_tvm_runtime_rpc copy_rpc_server
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${BINARY_DIR}/tvm_rpc
+    ${HEXAGON_API_BINARY_DIR}/tvm_rpc_android
+  DEPENDEES install
+)
+
+
+# Build Hexagon binaries:
+# - libhexagon_rpc_skel.so
+# - libtvm_runtime.a
+
+ExternalProject_Add(hexagon_tvm_runtime_rpc
+  SOURCE_DIR "${TVM_SOURCE_DIR}"
+  BUILD_COMMAND $(MAKE) runtime
+  CMAKE_ARGS
+    "-DCMAKE_C_COMPILER=${USE_HEXAGON_TOOLCHAIN}/bin/hexagon-clang"
+    "-DCMAKE_CXX_COMPILER=${USE_HEXAGON_TOOLCHAIN}/bin/hexagon-clang++"
+    "-DUSE_HEXAGON_SDK=${USE_HEXAGON_SDK}"
+    "-DUSE_HEXAGON_ARCH=${USE_HEXAGON_ARCH}"
+    "-DUSE_LIBBACKTRACE=OFF"
+    "-DUSE_RPC=OFF"
+    "-DUSE_HEXAGON_RPC=ON"
+    "-DBUILD_STATIC_RUNTIME=ON"
+  INSTALL_COMMAND ""
+  BUILD_ALWAYS ON
+)
+ExternalProject_Get_Property(hexagon_tvm_runtime_rpc BINARY_DIR)
+ExternalProject_Add_Step(hexagon_tvm_runtime_rpc copy_binaries
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${BINARY_DIR}/libtvm_runtime.a
+    ${BINARY_DIR}/libhexagon_rpc_skel.so
+    ${HEXAGON_API_BINARY_DIR}
+  DEPENDEES install
+)
+
diff --git a/apps/hexagon_api/README.md b/apps/hexagon_api/README.md
@@ -0,0 +1,58 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# Hexagon API app
+
+This is a meta-app that build the necessary binaries for use with
+the `HexagonLauncher` utility from `tvm.contrib.hexagon`.
+
+It will build the TVM runtime for Android, the RPC server application
+for Android, and the RPC library for Hexagon with the TVM runtime for
+Hexagon built into it.
+
+## Configuration
+
+There is a set of configuration variables that are required for cmake:
+- `ANDROID_ABI`: Set this to `arm64-v8a`.
+- `ANDROID_PLATFORM`: This can be `android-28`.
+- `USE_ANDROID_TOOLCHAIN`: The path to the Android toolchain file, i.e.
+`android.toolchain.cmake`. This file is a part of the Android NDK.
+- `USE_HEXAGON_ARCH`: The version string of the Hexagon architecture
+to use, i.e. vNN. The typical setting would be `v68` or later.
+- `USE_HEXAGON_SDK`: The path to the Hexagon SDK. Set this path in such
+a way that `${USE_HEXAGON_SDK}/setup_sdk_env.source` exists.
+- `USE_HEXAGON_TOOLCHAIN`: Path to Hexagon toolchain. It can be the
+Hexagon toolchain included in the SDK, for example
+`${USE_HEXAGON_TOOLCHAIN}/tools/HEXAGON_Tools/x.y.z/Tools`.  The `x.y.z`
+in the path is the toolchain version number, which is specific to the
+version of the SDK.
+
+Additionally, the variable `USE_OUTPUT_BINARY_DIR` can be set to indicate
+the location where the generated binaries will be placed. If not set, it
+defaults to `hexagon_rpc` subdirectory in the current build directory.
+
+
+## Build
+
+The build will generate the following binaries:
+- `tvm_runtime.so`: TVM runtime for Android (shared library).
+- `tvm_rpc_android`: RPC server for Android.
+- `libhexagon_rpc_skel.so`: RPC library for Hexagon.
+- `libtvm_runtime.a`: TVM runtime for Hexagon (static library).
+
+The RPC library for Hexagon contains the TVM runtime, so the static
+TVM runtime for Hexagon is not strictly necessary.
diff --git a/apps/hexagon_launcher/README.md b/apps/hexagon_launcher/README.md
@@ -31,43 +31,6 @@ The supported Snapdragon architectures are 855, 865, and 888.
 Android NDK can be downloaded from https://developer.android.com/ndk.
 Hexagon SDK is available at //developer.qualcomm.com/software/hexagon-dsp-sdk.
 
-### Compilation with TVM
-
-Building the Hexagon launcher application as a component of the main TVM build
-used for Hexagon codegen can be achieved by setting `USE_HEXAGON_LAUNCHER=ON`.
-This option will compile core tvm, the android launcher binary and its corresponding
-tvm_runtime, as well as the Hexagon launcher shared library and its corresponding
-tvm_runtime. As described in the [Manual compilation](#Manual compilation) section
-each component requires Hexagon and android dependencies. When building the launcher
-along with TVM these configurations must be providing when invoking cmake. A minimal
-example invocation for compiling TVM along with the Hexagon launcher is included below:
-
-```
-cmake -DCMAKE_C_COMPILER=/path/to/clang \
-      -DCMAKE_CXX_COMPILER=/path/to/clang++ \
-      -DCMAKE_CXX_FLAGS='-stdlib=libc++' \
-      -DCMAKE_CXX_STANDARD=14 \
-      -DUSE_LLVM=/path/to/llvm/bin/llvm-config \
-      -DUSE_HEXAGON_ARCH=v65|v66|v68 \
-      -DUSE_HEXAGON_LAUNCHER=ON \
-      -DUSE_HEXAGON_SDK=/path/to/hexagon/SDK \
-      -DUSE_HEXAGON_TOOLCHAIN=/path/to/hexagon/toolchain/ ..
-      -DANDROID_ABI=arm64-v8a \
-      -DANDROID_PLATFORM=android-28 \
-      -DUSE_ANDROID_TOOLCHAIN=/path/to/android-ndk/build/cmake/android.toolchain.cmake \
-      ..
-```
-
-where `v65|v66|v68` means "one of" these architecture versions.
-The Hexagon launcher application is an android binary and thus requires the use
-of an android toolchain for compilation. Similarly, the Hexagon tvm runtime
-requires the use of the Hexagon toolchain and depends on the Hexagon SDK. The
-resulting hexagon launcher binaries can be found in the `apps_hexagon_launcher`
-subdirectory of the cmake build directory. The above command
-will build support for Hexagon codegen in the TVM library that requires 
-`USE_LLVM` to be set to an llvm-config that has the Hexagon target built in.
-
-
 ### Manual compilation
 
 Since some source files are shared between the Hexagon and android builds,
diff --git a/cmake/modules/Hexagon.cmake b/cmake/modules/Hexagon.cmake
diff --git a/src/runtime/hexagon/proxy_rpc/device_api.cc b/src/runtime/hexagon/proxy_rpc/device_api.cc
