Merge branch 'master' into benchmark

* master: (21 commits)
  [Fix][VM] Fix VM invoke with set_params (apache#4079)
  [QNN] Refactor fixed point multiplication in requantize (apache#4073)
  Fix match case in Python-side expr functor (apache#4037)
  Hide symbols from dependent libraries if HIDE_PRIVATE_SYMBOLS is ON. (apache#4041)
  Add gradient for log-softmax (apache#4069)
  [DOC] Fix typos in tutorials (apache#4066)
  dicrease the complexity of CalcDep from exponential to linear (apache#4053)
  [Relay][AlterOp] Minor refactor. (apache#4064)
  [Relay][AlterOp] Improving support for broadcast layout alteration. (apache#4040)
  Add parses support for zeros_like tflite operator (apache#4042)
  [Bugfix][TF] reset graph after getting tag of savedmodel (apache#4055)
  [Relay][VM] Add more passes to VMCompiler (apache#4058)
  [Relay][VM] Add autotvm context when compile (apache#4062)
  [Bugfix] Fix target host for vm compiler (apache#4057)
  [Relay][Training] Add gradient for Crossentropy (apache#3925)
  [llvm] switch to use Align for llvm trunk (apache#4051)
  [Relay][TopHub] Add switch to disable TopHub download (apache#4015)
  [Relay][Op] Add instance norm op (apache#4004)
  [QNN][Relay] Calling Dialect passes from inside Relay Build API. (apache#3971)
  [RELAY/PASS] Fix the extent for the post_stmt in the loop partition (apache#3734)
  ...

Author: Peter Yeh

CMakeLists.txt

@@ -294,6 +294,17 @@ target_link_libraries(tvm_topi tvm ${TVM_LINKER_LIBS} ${TVM_RUNTIME_LINKER_LIBS}
 target_link_libraries(tvm_runtime ${TVM_RUNTIME_LINKER_LIBS})
 target_link_libraries(nnvm_compiler tvm)
 
+if (HIDE_PRIVATE_SYMBOLS AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  set(HIDE_SYMBOLS_LINKER_FLAGS "-Wl,--exclude-libs,ALL")
+  # Note: 'target_link_options' with 'PRIVATE' keyword would be cleaner
+  # but it's not available until CMake 3.13. Switch to 'target_link_options'
+  # once minimum CMake version is bumped up to 3.13 or above.
+  target_link_libraries(tvm ${HIDE_SYMBOLS_LINKER_FLAGS})
+  target_link_libraries(tvm_topi ${HIDE_SYMBOLS_LINKER_FLAGS})
+  target_link_libraries(tvm_runtime ${HIDE_SYMBOLS_LINKER_FLAGS})
+  target_link_libraries(nnvm_compiler ${HIDE_SYMBOLS_LINKER_FLAGS})
+endif()
+
 # Related headers
 target_include_directories(
   tvm

include/tvm/data_layout.h

@@ -210,6 +210,28 @@ class Layout : public NodeRef {
     return ct;
   }
 
+  /*!
+   * \brief Returns a new layout where the dims have been expanded to match the primal dimensions.
+   * \param dst_layout The dst layout to which current layout has to be expanded.
+   * \return The expanded Layout.
+   */
+  inline Layout ExpandPrimal(const Layout& dst_layout) {
+    Layout new_src_layout;
+    // 1) Find the axis which are missing in the current layout. Make them the prefix.
+    std::string new_src_layout_str = "";
+    for (auto dst_axis : dst_layout->axes) {
+      if (LayoutAxis::Get(dst_axis).IsPrimal()) {
+        if (!this->Contains(LayoutAxis::Get(dst_axis))) {
+          new_src_layout_str += dst_axis->var->name_hint;
+        }
+      }
+    }
+    // 2) Now, add the primal axis of the current layout.
+    new_src_layout_str += this->name();
+    new_src_layout = Layout(new_src_layout_str);
+    return new_src_layout;
+  }
+
   /*!
    * \brief return the index of the input axis.
    *        If it is not found in the layout or the layout is undefined,

include/tvm/relay/attrs/nn.h

@@ -492,6 +492,29 @@ struct BatchNormAttrs : public tvm::AttrsNode<BatchNormAttrs> {
 };  // struct BatchNormAttrs
 
 
+/*! \brief Attributes used in instance_norm operator */
+struct InstanceNormAttrs : public tvm::AttrsNode<InstanceNormAttrs> {
+  int axis;
+  double epsilon;
+  bool center;
+  bool scale;
+
+  TVM_DECLARE_ATTRS(InstanceNormAttrs, "relay.attrs.InstanceNormAttrs") {
+    TVM_ATTR_FIELD(axis)
+      .describe("Specify which shape axis denotes the channel.")
+      .set_default(1);
+    TVM_ATTR_FIELD(epsilon)
+      .describe("Small float added to variance to avoid dividing by zero")
+      .set_default(1e-5);
+    TVM_ATTR_FIELD(center).set_default(true)
+      .describe("If true, add offset of beta to normalized tensor; "
+                "otherwise, beta is ignored.");
+    TVM_ATTR_FIELD(scale).set_default(true)
+      .describe("If true, multiply by gamma; otherwise, gamma is ignored.");
+  }
+};  // struct InstanceNormAttrs
+
+
 /*! \brief Attributes used in layer_norm operator */
 struct LayerNormAttrs : public tvm::AttrsNode<LayerNormAttrs> {
   int axis;

include/tvm/relay/op.h

@@ -153,6 +153,12 @@ class Op : public relay::Expr {
    */
   template <typename ValueType>
   inline static OpMap<ValueType> GetAttr(const std::string& attr_name);
+  /*!
+   * \brief Checks if an attr is present in the registry.
+   * \param attr_name The name of the attribute.
+   * \return bool True if the attr is present.
+   */
+  inline static bool HasAttr(const std::string& attr_name);
   /*!
    * \brief Get an Op for a given operator name.
    *  Will raise an error if the op has not been registered.
@@ -171,6 +177,12 @@ class Op : public relay::Expr {
    * \return reference to GenericOpMap
    */
   TVM_DLL static const GenericOpMap& GetGenericAttr(const std::string& key);
+  /*!
+   * \brief Checks if the key is present in the registry
+   * \param key The attribute key
+   * \return bool True if the key is present
+   */
+  TVM_DLL static const bool HasGenericAttr(const std::string& key);
 };
 
 /*! \brief Helper structure to register operators */
@@ -393,6 +405,10 @@ inline OpMap<ValueType> Op::GetAttr(const std::string& key) {
   return OpMap<ValueType>(Op::GetGenericAttr(key));
 }
 
+inline bool Op::HasAttr(const std::string& key) {
+  return Op::HasGenericAttr(key);
+}
+
 inline OpNode* OpRegistry::get() {
   return const_cast<OpNode*>(op_.operator->());
 }

include/tvm/relay/qnn/transform.h

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file tvm/relay/qnn/transform.h
+ *
+ * This file implements a pass manager for QNN ops using Relay Pass manager.
+ */
+#ifndef TVM_RELAY_QNN_TRANSFORM_H_
+#define TVM_RELAY_QNN_TRANSFORM_H_
+
+#include <tvm/runtime/c_runtime_api.h>
+#include <tvm/relay/transform.h>
+
+namespace tvm {
+namespace relay {
+
+using relay::transform::Pass;
+
+namespace qnn {
+namespace transform {
+
+/*!
+ * \brief Legalizes a QNN expr. Contains specifically two types of Legalizations. First,
+ * converts/Lowers an expression containing QNN ops to an expression containing only core Relay ops.
+ * Each QNN op is lowered to a sequence of exisiting Relay ops. This is a target-independent pass.
+ * One can register the lowering/transformation function for this op using FTVMQnnCanonicalize
+ * attr_name for FTVMLegalize op attribute. Second, as opposed to Relay Legalize, this one legalizes
+ * only QNN ops. One can register a transformation/legalization function for an op by using the
+ * FTVMQnnLegalize attr_name for FTVMLegalize op attribute. The isolation of QNN and Relay Legalize
+ * gives us separation of concerns, leading to a better software practice. The legalization can be
+ * configured to happen per target.
+ *
+ * \return The pass.
+ */
+TVM_DLL Pass Legalize();
+
+}  // namespace transform
+
+}  // namespace qnn
+}  // namespace relay
+}  // namespace tvm
+
+#endif  // TVM_RELAY_QNN_TRANSFORM_H_

python/tvm/autotvm/tophub.py

@@ -18,7 +18,8 @@
 TopHub: Tensor Operator Hub
 To get the best performance, we typically need auto-tuning for the specific devices.
 TVM releases pre-tuned parameters in TopHub for some common networks and hardware targets.
-TVM will download these parameters for you when you call nnvm.compiler.build_module .
+TVM will download these parameters for you when you call
+nnvm.compiler.build_module or relay.build.
 """
 # pylint: disable=invalid-name
 
@@ -30,6 +31,16 @@
 from .. import target as _target
 from ..contrib.download import download
 from .record import load_from_file
+from .util import EmptyContext
+
+# environment variable to read TopHub location
+AUTOTVM_TOPHUB_LOC_VAR = "TOPHUB_LOCATION"
+
+# default location of TopHub
+AUTOTVM_TOPHUB_DEFAULT_LOC = "https://raw.githubusercontent.com/uwsampl/tvm-distro/master/tophub"
+
+# value of AUTOTVM_TOPHUB_LOC_VAR to specify to not read from TopHub
+AUTOTVM_TOPHUB_NONE_LOC = "NONE"
 
 # root path to store TopHub files
 AUTOTVM_TOPHUB_ROOT_PATH = os.path.join(os.path.expanduser('~'), ".tvm", "tophub")
@@ -61,6 +72,9 @@ def _alias(name):
     }
     return table.get(name, name)
 
+def _get_tophub_location():
+    location = os.getenv(AUTOTVM_TOPHUB_LOC_VAR, None)
+    return AUTOTVM_TOPHUB_DEFAULT_LOC if location is None else location
 
 def context(target, extra_files=None):
     """Return the dispatch context with pre-tuned parameters.
@@ -75,6 +89,10 @@ def context(target, extra_files=None):
     extra_files: list of str, optional
         Extra log files to load
     """
+    tophub_location = _get_tophub_location()
+    if tophub_location == AUTOTVM_TOPHUB_NONE_LOC:
+        return EmptyContext()
+
     best_context = ApplyHistoryBest([])
 
     targets = target if isinstance(target, (list, tuple)) else [target]
@@ -94,7 +112,7 @@ def context(target, extra_files=None):
         for name in possible_names:
             name = _alias(name)
             if name in all_packages:
-                if not check_backend(name):
+                if not check_backend(tophub_location, name):
                     continue
 
                 filename = "%s_%s.log" % (name, PACKAGE_VERSION[name])
@@ -108,7 +126,7 @@ def context(target, extra_files=None):
     return best_context
 
 
-def check_backend(backend):
+def check_backend(tophub_location, backend):
     """Check whether have pre-tuned parameters of the certain target.
     If not, will download it.
 
@@ -135,18 +153,21 @@ def check_backend(backend):
     else:
         import urllib2
     try:
-        download_package(package_name)
+        download_package(tophub_location, package_name)
         return True
     except urllib2.URLError as e:
         logging.warning("Failed to download tophub package for %s: %s", backend, e)
         return False
 
 
-def download_package(package_name):
+def download_package(tophub_location, package_name):
     """Download pre-tuned parameters of operators for a backend
 
     Parameters
     ----------
+    tophub_location: str
+        The location to download TopHub parameters from
+
     package_name: str
         The name of package
     """
@@ -160,9 +181,9 @@ def download_package(package_name):
             if not os.path.isdir(path):
                 os.mkdir(path)
 
-    logger.info("Download pre-tuned parameters package %s", package_name)
-    download("https://raw.githubusercontent.com/uwsampl/tvm-distro/master/tophub/%s"
-             % package_name, os.path.join(rootpath, package_name), True, verbose=0)
+    download_url = "{0}/{1}".format(tophub_location, package_name)
+    logger.info("Download pre-tuned parameters package from %s", download_url)
+    download(download_url, os.path.join(rootpath, package_name), True, verbose=0)
 
 
 # global cache for load_reference_log

python/tvm/relay/backend/vm.py

@@ -23,6 +23,7 @@
 import numpy as np
 
 import tvm
+from tvm import autotvm
 from tvm._ffi.runtime_ctypes import TVMByteArray
 from . import _vm
 from . import vmobj as _obj
@@ -178,10 +179,24 @@ def compile(self, mod, target=None, target_host=None):
         """
         target = _update_target(target)
         target_host = None if target_host == "" else target_host
+        if not target_host:
+            for device_type, tgt in target.items():
+                if device_type.value == tvm.nd.cpu(0).device_type:
+                    target_host = tgt
+                    break
         if not target_host:
             target_host = "llvm" if tvm.module.enabled("llvm") else "stackvm"
         target_host = tvm.target.create(target_host)
-        self._compile(mod, target, target_host)
+
+        # If current dispatch context is fallback context (the default root context),
+        # then load pre-tuned parameters from TopHub
+        if isinstance(autotvm.DispatchContext.current, autotvm.FallbackContext):
+            tophub_context = autotvm.tophub.context(list(target.values()))
+        else:
+            tophub_context = autotvm.util.EmptyContext()
+
+        with tophub_context:
+            self._compile(mod, target, target_host)
         return VirtualMachine(self._get_vm())
 
 

python/tvm/relay/expr_functor.py

@@ -249,7 +249,10 @@ def visit_constructor(self, con):
         return con
 
     def visit_match(self, m):
-        return Match(self.visit(m.data), [Clause(c.lhs, self.visit(c.rhs)) for c in m.clauses])
+        return Match(
+            self.visit(m.data),
+            [Clause(c.lhs, self.visit(c.rhs)) for c in m.clauses],
+            complete=m.complete)
 
     def visit_ref_create(self, r):
         return RefCreate(self.visit(r.value))

python/tvm/relay/frontend/mxnet.py

@@ -324,6 +324,14 @@ def _mx_batch_norm(inputs, attrs):
     return _op.nn.batch_norm(*inputs, **new_attrs)
 
 
+def _mx_instance_norm(inputs, attrs):
+    assert len(inputs) == 3
+    new_attrs = {}
+    new_attrs["axis"] = attrs.get_int("axis", 1)
+    new_attrs["epsilon"] = attrs.get_float("eps", 1e-5)
+    return _op.nn.instance_norm(*inputs, **new_attrs)
+
+
 def _mx_layer_norm(inputs, attrs):
     assert len(inputs) == 3
     if attrs.get_bool("output_mean_var", False):
@@ -1133,6 +1141,7 @@ def _mx_one_hot(inputs, attrs):
     "Dropout"       : _mx_dropout,
     "BatchNorm"     : _mx_batch_norm,
     "BatchNorm_v1"  : _mx_batch_norm,
+    "InstanceNorm"  : _mx_instance_norm,
     "LayerNorm"     : _mx_layer_norm,
     "LRN"           : _mx_lrn,
     "L2Normalization"  : _mx_l2_normalize,

python/tvm/relay/frontend/onnx.py

@@ -176,6 +176,15 @@ def _impl_v1(cls, inputs, attr, params):
         return out[0]
 
 
+class InstanceNorm(OnnxOpConverter):
+    """ Operator converter for BatchNorm.
+    """
+
+    @classmethod
+    def _impl_v1(cls, inputs, attr, params):
+        return AttrCvt(op_name='instance_norm')(inputs, attr, params)
+
+
 class Conv(OnnxOpConverter):
     """ Operator converter for Conv.
     """
@@ -999,7 +1008,7 @@ def _get_convert_map(opset):
         'GlobalAveragePool': Renamer('global_avg_pool2d'),
         'GlobalMaxPool': Renamer('global_max_pool2d'),
         'BatchNormalization': BatchNorm.get_converter(opset),
-        # 'InstanceNormalization'
+        'InstanceNormalization': InstanceNorm.get_converter(opset),
         # 'LpNormalization'
         'Dropout': AttrCvt('dropout', {'ratio': 'rate'}, ignores=['is_test']),
         'Flatten': Flatten.get_converter(opset),