Add JSON serialization and deserialization of xtensor containers

SylvainCorlay · SylvainCorlay · commit 1c33912c17f3 · 2018-05-04T13:34:51.000+02:00
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -5,7 +5,7 @@ platform:
   - x86
 
 image:
-  - Visual Studio 2017 
+  - Visual Studio 2017
   - Visual Studio 2015
 
 environment:
@@ -15,7 +15,7 @@ environment:
 init:
   - "ECHO %MINICONDA%"
   - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" set VCVARPATH="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"
-  - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" set VCARGUMENT=%PLATFORM% 
+  - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" set VCARGUMENT=%PLATFORM%
   - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" if "%PLATFORM%" == "x64" set VCVARPATH="C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
   - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" if "%PLATFORM%" == "x86" set VCVARPATH="C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars32.bat"
   - echo "%VCVARPATH% %VCARGUMENT%"
@@ -30,9 +30,10 @@ install:
   - conda update -q conda
   - conda info -a
   - conda install gtest cmake -c conda-forge
-  - conda install xtl==0.4.1 -c QuantStack
+  - conda install xtl==0.4.8 -c QuantStack
   - conda install xsimd -c QuantStack
-    #- cmake -G "NMake Makefiles" -D CMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DBUILD_TESTS=ON -DXTENSOR_USE_XSIMD=ON -DDISABLE_VS2017=ON .
+  - conda install nlohmann_json -c QuantStack
+    # - cmake -G "NMake Makefiles" -D CMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DBUILD_TESTS=ON -DXTENSOR_USE_XSIMD=ON -DDISABLE_VS2017=ON .
   - cmake -G "NMake Makefiles" -D CMAKE_INSTALL_PREFIX=%MINICONDA%\\LIBRARY -DBUILD_TESTS=ON -DDISABLE_VS2017=ON .
   - nmake test_xtensor_lib
   - cd test
diff --git a/.travis.yml b/.travis.yml
@@ -114,9 +114,10 @@ install:
     - hash -r
     - conda config --set always_yes yes --set changeps1 no
     - conda update -q conda
-    - conda install gtest cmake -c conda-forge 
-    - conda install xtl==0.4.1 -c QuantStack
+    - conda install gtest cmake -c conda-forge
+    - conda install xtl==0.4.8 -c QuantStack
     - conda install xsimd -c QuantStack
+    - conda install nlohmann_json -c QuantStack
     # Testing
     - mkdir build
     - cd build
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -29,8 +29,11 @@ message(STATUS "Building xtensor v${${PROJECT_NAME}_VERSION}")
 # ============
 
 find_package(xtl 0.4.1 REQUIRED)
+
 message(STATUS "Found xtl: ${xtl_INCLUDE_DIRS}/xtl")
 
+find_package(nlohmann_json 3.1.1)
+
 # Build
 # =====
 
@@ -59,6 +62,7 @@ set(XTENSOR_HEADERS
     ${XTENSOR_INCLUDE_DIR}/xtensor/xio.hpp
     ${XTENSOR_INCLUDE_DIR}/xtensor/xiterable.hpp
     ${XTENSOR_INCLUDE_DIR}/xtensor/xiterator.hpp
+    ${XTENSOR_INCLUDE_DIR}/xtensor/xjson.hpp
     ${XTENSOR_INCLUDE_DIR}/xtensor/xlayout.hpp
     ${XTENSOR_INCLUDE_DIR}/xtensor/xmath.hpp
     ${XTENSOR_INCLUDE_DIR}/xtensor/xnoalias.hpp
diff --git a/docs/source/api/io_index.rst b/docs/source/api/io_index.rst
@@ -0,0 +1,12 @@
+.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht
+
+   Distributed under the terms of the BSD 3-Clause License.
+
+   The full license is in the file LICENSE, distributed with this software.
+
+IO Operations
+=============
+
+.. toctree::
+
+   xjson
diff --git a/docs/source/api/xjson.rst b/docs/source/api/xjson.rst
@@ -0,0 +1,16 @@
+.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht
+
+   Distributed under the terms of the BSD 3-Clause License.
+
+   The full license is in the file LICENSE, distributed with this software.
+
+xjson
+=====
+
+Defined in ``xtensor/xjson.hpp``
+
+.. doxygenfunction:: xt::to_json(nlohmann::json&, const E&);
+   :project: xtensor
+
+.. doxygenfunction:: xt::from_json(nlohmann::json&, const E&);
+   :project: xtensor
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
@@ -7,6 +7,12 @@
 Changelog
 =========
 
+0.16.0
+------
+
+- Support for JSON serialization and deserialization of xtensor expressions
+  `#830 <https://github.com/QuantStack/xtensor/pull/830>`_.
+
 0.15.9
 ------
 
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -74,6 +74,7 @@ for details.
    api/expression_index
    api/container_index
    api/function_index
+   api/io_index
    api/xmath
 
 .. toctree::
diff --git a/include/xtensor/xcontainer.hpp b/include/xtensor/xcontainer.hpp
@@ -1311,7 +1311,7 @@ namespace xt
     {
         if (compute_size(shape) != this->size())
         {
-            throw std::runtime_error("Cannot reshape with incorrect number of elements.");
+            throw std::runtime_error("Cannot reshape with incorrect number of elements. Do you mean to resize?");
         }
         if (layout == layout_type::dynamic || layout == layout_type::any)
         {
diff --git a/include/xtensor/xeval.hpp b/include/xtensor/xeval.hpp
@@ -20,10 +20,11 @@ namespace xt
         template <class T>
         using is_container = std::is_base_of<xcontainer<std::remove_const_t<T>>, T>;
     }
+
     /**
      * Force evaluation of xexpression.
      * @return xarray or xtensor depending on shape type
-     * 
+     *
      * \code{.cpp}
      * xarray<double> a = {1,2,3,4};
      * auto&& b = xt::eval(a); // b is a reference to a, no copy!
diff --git a/include/xtensor/xexpression.hpp b/include/xtensor/xexpression.hpp
@@ -113,6 +113,9 @@ namespace xt
     template <class E>
     using is_xexpression = detail::is_xexpression_impl<E>;
 
+    template <class E, class R = void>
+    using enable_xexpression = typename std::enable_if<is_xexpression<E>::value, R>::type;
+
     template <class E, class R = void>
     using disable_xexpression = typename std::enable_if<!is_xexpression<E>::value, R>::type;
 
diff --git a/include/xtensor/xjson.hpp b/include/xtensor/xjson.hpp
@@ -0,0 +1,183 @@
+/***************************************************************************
+* Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht    *
+*                                                                          *
+* Distributed under the terms of the BSD 3-Clause License.                 *
+*                                                                          *
+* The full license is in the file LICENSE, distributed with this software. *
+****************************************************************************/
+
+#ifndef XTENSOR_JSON_HPP
+#define XTENSOR_JSON_HPP
+
+#include <cstddef>
+#include <stdexcept>
+#include <utility>
+
+#include <nlohmann/json.hpp>
+
+#include "xstrided_view.hpp"
+
+namespace xt
+{
+    /*************************************
+     * to_json and from_json declaration *
+     *************************************/
+
+    template <class E>
+    enable_xexpression<E> to_json(nlohmann::json&, const E&);
+
+    template <class E>
+    enable_xcontainer_semantics<E> from_json(const nlohmann::json&, E&);
+
+    /// @cond DOXYGEN_INCLUDE_SFINAE
+    template <class E>
+    enable_xview_semantics<E> from_json(const nlohmann::json&, E&);
+    /// @endcond
+
+    /****************************************
+     * to_json and from_json implementation *
+     ****************************************/
+
+    namespace detail
+    {
+        template <class D>
+        void to_json_impl(nlohmann::json& j, const xexpression<D>& e, slice_vector& slices)
+        {
+            const auto view = strided_view(e.derived_cast(), slices);
+            if (view.dimension() == 0)
+            {
+                j = view();
+            }
+            else
+            {
+                j = nlohmann::json::array();
+                using size_type = typename D::size_type;
+                size_type nrows = view.shape()[0];
+                for (size_type i = 0; i != nrows; ++i)
+                {
+                    slices.push_back(i);
+                    nlohmann::json k;
+                    to_json_impl(k, e, slices);
+                    j.push_back(std::move(k));
+                    slices.pop_back();
+                }
+            }
+        }
+
+        template <class D>
+        inline void from_json_impl(const nlohmann::json& j, xexpression<D>& e, slice_vector& slices)
+        {
+            auto view = strided_view(e.derived_cast(), slices);
+
+            if (view.dimension() == 0)
+            {
+                view() = j;
+            }
+            else
+            {
+                using size_type = typename D::size_type;
+                size_type nrows = view.shape()[0];
+                for (auto i = 0; i != nrows; ++i)
+                {
+                    slices.push_back(i);
+                    const nlohmann::json& k = j[i];
+                    from_json_impl(k, e, slices);
+                    slices.pop_back();
+                }
+            }
+        }
+
+        inline unsigned int json_dimension(const nlohmann::json& j)
+        {
+            if (j.is_array() && j.size())
+            {
+                return 1 + json_dimension(j[0]);
+            }
+            else
+            {
+                return 0;
+            }
+        }
+
+        template <class S>
+        inline void json_shape(const nlohmann::json& j, S& s, std::size_t pos = 0)
+        {
+            if (j.is_array())
+            {
+                auto size = j.size();
+                s[pos] = size;
+                if (size)
+                {
+                    json_shape(j[0], s, pos + 1);
+                }
+            }
+        }
+    }
+
+    /**
+     * @brief JSON serialization of an xtensor expression.
+     *
+     * The to_json method is used by the nlohmann_json package for automatic
+     * serialization of user-defined types. The method is picked up by
+     * argument-dependent lookup.
+     *
+     * @param j a JSON object
+     * @param e a const \ref xexpression
+     */
+    template <class E>
+    inline enable_xexpression<E> to_json(nlohmann::json& j, const E& e)
+    {
+        auto sv = slice_vector();
+        detail::to_json_impl(j, e, sv);
+    }
+
+    /**
+     * @brief JSON deserialization of a xtensor expression with a container or
+     * a view semantics.
+     *
+     * The from_json method is used by the nlohmann_json library for automatic
+     * serialization of user-defined types. The method is picked up by
+     * argument-dependent lookup.
+     *
+     * Note: for converting a JSON object to a value, nlohmann_json requiress
+     * the value type to be default constructible, which is typically not the
+     * case for expressions with a view semantics. In this case, from_json can
+     * be called directly.
+     *
+     * @param j a const JSON object
+     * @param e an \ref xexpression
+     */
+    template <class E>
+    inline enable_xcontainer_semantics<E> from_json(const nlohmann::json& j, E& e)
+    {
+        auto dimension = detail::json_dimension(j);
+        auto s = xtl::make_sequence<typename E::shape_type>(dimension);
+        detail::json_shape(j, s);
+
+        // In the case of a container, we resize the container.
+        e.resize(s);
+
+        auto sv = slice_vector();
+        detail::from_json_impl(j, e, sv);
+    }
+
+    /// @cond DOXYGEN_INCLUDE_SFINAE
+    template <class E>
+    inline enable_xview_semantics<E> from_json(const nlohmann::json& j, E& e)
+    {
+        typename E::shape_type s;
+        detail::json_shape(j, s);
+
+        // In the case of a view, we check the size of the container.
+        if (!std::equal(s.cbegin(), s.cend(), e.shape().cbegin()))
+        {
+            throw std::runtime_error("Shape mismatch when deserializing JSON to view");
+        }
+
+        auto sv = slice_vector();
+        detail::from_json_impl(j, e, sv);
+    }
+    /// @endcond
+}
+
+#endif
diff --git a/include/xtensor/xnorm.hpp b/include/xtensor/xnorm.hpp
@@ -21,7 +21,7 @@
 
 namespace xt
 {
-/*************************************
+    /*************************************
      * norm functions for built-in types *
      *************************************/
 
diff --git a/include/xtensor/xsemantic.hpp b/include/xtensor/xsemantic.hpp
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
diff --git a/test/test_xjson.cpp b/test/test_xjson.cpp

Original file line number	Diff line number	Diff line change
`@@ -1311,7 +1311,7 @@ namespace xt`
`1311`	`1311`	`{`
`1312`	`1312`	`if (compute_size(shape) != this->size())`
`1313`	`1313`	`{`
`1314`		`- throw std::runtime_error("Cannot reshape with incorrect number of elements.");`
	`1314`	`+ throw std::runtime_error("Cannot reshape with incorrect number of elements. Do you mean to resize?");`
`1315`	`1315`	`}`
`1316`	`1316`	`if (layout == layout_type::dynamic \|\| layout == layout_type::any)`
`1317`	`1317`	`{`