[microTVM] Add support for host-driven AoT Executor (apache#11044)

* Generate AOT Metadata when targeting C runtime and packed API.

* Also copy metadata.h and metadata_base.h to standalone_crt.

* add support for get_input_index as well as setting up get_input_info as unsupported

* add support for tvm.aot_executor.create in C runtime

* changes in-progress to unit tests

* Include get_c_metadata in emitted function list

* make CRT error codes generic for graph or AoT executor, fix AoT lib link order

* add AoT executor creation and initializaion, as well as support for get_input_index()

* add allocation of inputs, outputs, and pools; add get_input(), but shape encoded in metadata appears to be incorrect;

* add support to test_aot_executor for get_input()

* fix numpy array shape so that get_input() works properly

* implement run(), get_output(), get_num_inputs(), and get_num_outputs(); test_aot_executor() is now passing;

* fix up some issues from rebase with main

* clean up logging and test_graph_executor()

* lint clean-up

* more lint clean-up

* fix i386 build errors

* first set of changes addressing PR feedback

* more PR feedback: device pass-by-value, docstring entries, return variable name

* add mangling of get_c_metadata() name to avoid function name collisions

* only mangle get_c_metadata() when using C runtime

* add static specifier to all kTvmgenMetadata variables to avoid namespace collisions

* use TVM_IS_CPP_RUNTIME preprocessor define to deteremine whether or not to include metadata.h c++ code

* add TVM_IS_CPP_RUNTIME define for cpptest

* add TVM_IS_CPP_RUNTIME to apps/bundle_deploy

* add TVM_IS_CPP_RUNTIME web/Makefile

* update number of expected generated C files for AoT source files

* break out metadata data structures into separate metadata_types.h header to avoid c/c++ issues and remove the need for the TVM_IS_CPP_RUNTIME define

* remove TVM_IS_CPP_RUNTIME from web makefile

* fix metadata.h include-order lint issue

* correct error mask bits

* address PR feedback

* trigger build

* trigger build

* trigger build

* trigger build

* add alternate name for test_graph_executor() too see if it runs in CI

* fix lint

* revert alternate test code

Co-authored-by: Andrew Reusch <areusch@gmail.com>

Author: 2 people

cmake/modules/StandaloneCrt.cmake

@@ -28,9 +28,12 @@ if(USE_MICRO)
          "3rdparty/dlpack/include *.h -> include"
          "3rdparty/dmlc-core/include *.h -> include"
          "include/tvm/runtime c_*_api.h -> include/tvm/runtime"
+         "include/tvm/runtime metadata_types.h -> include/tvm/runtime"
          "include/tvm/runtime/crt *.h -> include/tvm/runtime/crt"
          "src/runtime/crt Makefile -> ."
          "src/runtime/crt/include *.h -> include"
+         "src/runtime/crt/aot_executor *.c -> src/runtime/crt/aot_executor"
+         "src/runtime/crt/aot_executor_module *.c -> src/runtime/crt/aot_executor_module"
          "src/runtime/crt/common *.c -> src/runtime/crt/common"
          "src/runtime/crt/graph_executor *.c -> src/runtime/crt/graph_executor"
          "src/runtime/crt/graph_executor_module *.c -> src/runtime/crt/graph_executor_module"

include/tvm/runtime/c_runtime_api.h

@@ -298,7 +298,7 @@ TVM_DLL int TVMCbArgToReturn(TVMValue* value, int* code);
  * \param type_codes The type codes of the arguments
  * \param num_args Number of arguments.
  * \param ret The return value handle.
- * \param resource_handle The handle additional resouce handle from fron-end.
+ * \param resource_handle The handle additional resouce handle from front-end.
  * \return 0 if success, -1 if failure happens, set error via TVMAPISetLastError.
  * \sa TVMCFuncSetReturn
  */
@@ -307,7 +307,7 @@ typedef int (*TVMPackedCFunc)(TVMValue* args, int* type_codes, int num_args, TVM
 
 /*!
  * \brief C callback to free the resource handle in C packed function.
- * \param resource_handle The handle additional resouce handle from fron-end.
+ * \param resource_handle The handle additional resouce handle from front-end.
  */
 typedef void (*TVMPackedCFuncFinalizer)(void* resource_handle);
 

include/tvm/runtime/crt/aot_executor.h

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file aot_executor.h
+ * \brief AoT Executor
+ */
+#ifndef TVM_RUNTIME_CRT_AOT_EXECUTOR_H_
+#define TVM_RUNTIME_CRT_AOT_EXECUTOR_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <dlpack/dlpack.h>
+#include <tvm/runtime/crt/internal/common/ndarray.h>
+#include <tvm/runtime/metadata_types.h>
+
+typedef struct TVMMetadata TVMMetadata;
+
+typedef struct TVMAotExecutor {
+  /*! \brief The top-level metadata structure supplied by the generated code */
+  const TVMMetadata* metadata;
+  /*! \brief The code module that contains the compiled model */
+  TVMModuleHandle module_handle;
+  /*! \brief The device type */
+  DLDevice device;
+  /*! \brief List of allocated arguments, input(s), output(s), and pool(s)*/
+  TVMNDArray* args;
+  int64_t num_args;
+} TVMAotExecutor;
+
+/*!
+ * \brief Allocate a new AotExecutor with TVMPlatformMemoryAllocate and initialize it.
+ *
+ * \param module_handle TVM Module that exposes the functions to call.
+ * \param device Runtime execution device, only supports device type kDLCPU, index 0.
+ * \param executor Pointer which receives a pointer to the newly-created instance.
+ * \param module_name TVM Module name prefix, typically "default".
+ * \return 0 if successful.
+ */
+int TVMAotExecutor_Create(TVMModuleHandle module_handle, const DLDevice device,
+                          TVMAotExecutor** executor, const char* module_name);
+
+/*!
+ * \brief Release the AoT executor created by TVMAotExecutor_Create().
+ *
+ * \param executor Pointer to executor instance, created by TVMAotExecutor_Create().
+ * \param device Runtime execution device, only supports device type kDLCPU, index 0.
+ * \return 0 if successful.
+ */
+int TVMAotExecutor_Release(TVMAotExecutor* executor, const DLDevice device);
+
+/*!
+ * \brief Return the number of inputs.
+ *
+ * \param executor Pointer to executor instance, created by TVMAotExecutor_Create().
+ * \return Number of inputs.
+ */
+int TVMAotExecutor_GetNumInputs(TVMAotExecutor* executor);
+
+/*!
+ * \brief Return the number of outputs.
+ *
+ * \param executor Pointer to executor instance, created by TVMAotExecutor_Create().
+ * \return Number of outputs.
+ */
+int TVMAotExecutor_GetNumOutputs(TVMAotExecutor* executor);
+
+/*!
+ * \brief Return the input index of the specified input name
+ *
+ * \param executor Pointer to executor instance, created by TVMAotExecutor_Create().
+ * \param name Input name for retrieving index.
+ * \return Input index.
+ */
+int TVMAotExecutor_GetInputIndex(TVMAotExecutor* executor, const char* name);
+
+/*!
+ * \brief Run the generated program.
+ *
+ * \param executor Pointer to executor instance, created by TVMAotExecutor_Create().
+ * \return 0 if successful.
+ */
+int TVMAotExecutor_Run(TVMAotExecutor* executor);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TVM_RUNTIME_CRT_AOT_EXECUTOR_H_

include/tvm/runtime/crt/aot_executor_module.h

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file graph_executor.h
+ * \brief Tiny AoT executor
+ */
+#ifndef TVM_RUNTIME_CRT_AOT_EXECUTOR_MODULE_H_
+#define TVM_RUNTIME_CRT_AOT_EXECUTOR_MODULE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <tvm/runtime/crt/error_codes.h>
+
+/*!
+ * \brief Register the "tvm.aot_executor.create" constructor PackedFunc.
+ */
+tvm_crt_error_t TVMAotExecutorModule_Register();
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TVM_RUNTIME_CRT_AOT_EXECUTOR_MODULE_H_

include/tvm/runtime/crt/error_codes.h

@@ -42,7 +42,7 @@ typedef enum {
   kTvmErrorCategorySession = 4,
   kTvmErrorCategoryPlatform = 5,
   kTvmErrorCategoryGenerated = 6,
-  kTvmErrorCategoryGraphExecutor = 7,
+  kTvmErrorCategoryExecutor = 7,
   kTvmErrorCategoryFunctionCall = 8,
   kTvmErrorCategoryTimeEvaluator = 9,
 } tvm_crt_error_category_t;
@@ -84,10 +84,10 @@ typedef enum {
   // Common error codes returned from generated functions.
   kTvmErrorGeneratedInvalidStorageId = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryGenerated, 0),
 
-  // Graph executor
-  kTvmErrorGraphModuleAlreadyCreated = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryGraphExecutor, 0),
-  kTvmErrorGraphModuleBadContext = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryGraphExecutor, 1),
-  kTvmErrorGraphModuleNoSuchInput = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryGraphExecutor, 2),
+  // Graph or AoT executor
+  kTvmErrorExecutorModuleAlreadyCreated = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryExecutor, 0),
+  kTvmErrorExecutorModuleBadContext = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryExecutor, 1),
+  kTvmErrorExecutorModuleNoSuchInput = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryExecutor, 2),
 
   // Function Calls - common problems encountered calling functions.
   kTvmErrorFunctionCallNumArguments = DEFINE_TVM_CRT_ERROR(kTvmErrorCategoryFunctionCall, 0),
@@ -100,7 +100,7 @@ typedef enum {
 
   // System errors are always negative integers; this mask indicates presence of a system error.
   // Cast tvm_crt_error_t to a signed integer to interpret the negative error code.
-  kTvmErrorSystemErrorMask = (1 << (sizeof(int) * 4 - 1)),
+  kTvmErrorSystemErrorMask = (1 << (sizeof(int) * 8 - 1)),
 } tvm_crt_error_t;
 
 #ifdef __cplusplus

include/tvm/runtime/crt/graph_executor_module.h

@@ -18,7 +18,7 @@
  */
 
 /*!
- * \file graph_executor.h
+ * \file graph_executor_module.h
  * \brief Tiny graph executor that can run graph containing only tvm PackedFunc.
  */
 #ifndef TVM_RUNTIME_CRT_GRAPH_EXECUTOR_MODULE_H_

include/tvm/runtime/metadata.h

@@ -24,22 +24,19 @@
 #ifndef TVM_RUNTIME_METADATA_H_
 #define TVM_RUNTIME_METADATA_H_
 
-#include <inttypes.h>
-#ifdef __cplusplus
-#include <memory>
-#include <string>
-#include <vector>
-#endif
 #include <tvm/runtime/c_runtime_api.h>
-#ifdef __cplusplus
 #include <tvm/runtime/metadata_base.h>
+#include <tvm/runtime/metadata_types.h>
+#include <tvm/runtime/object.h>
 #include <tvm/support/span.h>
-#endif
+
+#include <memory>
+#include <string>
+#include <vector>
 
 // Version number recorded in emitted artifacts for runtime checking.
 #define TVM_METADATA_VERSION 1
 
-#ifdef __cplusplus
 namespace tvm {
 namespace runtime {
 namespace metadata {
@@ -52,59 +49,6 @@ static const constexpr int64_t kMetadataVersion = TVM_METADATA_VERSION;
 }  // namespace runtime
 }  // namespace tvm
 
-extern "C" {
-#endif
-
-/*!
- * \brief Top-level metadata structure. Holds all other metadata types.
- */
-struct TVMMetadata {
-  /*! \brief Version identifier for this metadata. */
-  int64_t version;
-  /*! \brief Inputs to the AOT run_model function.
-   * The order of the elements is the same as in the arguments to run_model. That is to say,
-   * this array specifies the first `num_inputs` arguments to run_model.
-   */
-  const struct TVMTensorInfo* inputs;
-  /*! \brief Number of elements in `inputs` array. */
-  int64_t num_inputs;
-  /*! \brief Outputs of the AOT run_model function.
-   * The order of the elements is the same as in the arguments to run_model. That is to say,
-   * this array specifies the last `num_outputs` arguments to run_model.
-   */
-  const struct TVMTensorInfo* outputs;
-  /*! \brief Number of elements in `outputs` array. */
-  int64_t num_outputs;
-  /*! \brief Memory Pools needed by the AOT main function.
-   * The order of the elements is the same as in the arguments to run_model. That is to say,
-   * this array specifies the last `num_pools` arguments to run_model.
-   */
-  const struct TVMTensorInfo* pools;
-  /*! \brief Number of elements in `pools` array. */
-  int64_t num_pools;
-  /*! \brief Name of the model, as passed to tvm.relay.build. */
-  const char* mod_name;
-};
-
-/*!
- * \brief Describes one tensor argument to `run_model`.
- * NOTE: while TIR allows for other types of arguments, such as scalars, the AOT run_model
- * function does not currently accept these. Therefore it's not possible to express those
- * in this metadata. A future patch may modify this.
- */
-struct TVMTensorInfo {
-  /*! \brief Name of the tensor, as specified in the Relay program. */
-  const char* name;
-  /*! \brief Shape of the tensor. */
-  const int64_t* shape;
-  /*! \brief Rank of this tensor. */
-  int64_t num_shape;
-  /*! \brief Data type of one element of this tensor. */
-  DLDataType dtype;
-};
-#ifdef __cplusplus
-}  // extern "C"
-#include <tvm/runtime/object.h>
 namespace tvm {
 namespace runtime {
 namespace metadata {
@@ -166,6 +110,5 @@ class TensorInfo : public MetadataBase {
 }  // namespace metadata
 }  // namespace runtime
 }  // namespace tvm
-#endif  // defined(__cplusplus)
 
 #endif  // TVM_RUNTIME_METADATA_H_