From bc6de048e9f4f84598e291161c08e6d56a41ac0e Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Mon, 2 Nov 2026 23:50:53 +0000 Subject: [PATCH 01/24] Add per-dimension scalar quantized datasets --- cpp/include/cuvs/core/dataset/sq.hpp | 61 ++++- .../cuvs/preprocessing/quantize/scalar.hpp | 51 ++++- .../preprocessing/quantize/detail/scalar.cuh | 136 ++++++++++++ cpp/src/preprocessing/quantize/scalar.cu | 25 ++- cpp/tests/CMakeLists.txt | 1 + cpp/tests/preprocessing/scalar_dataset.cu | 209 ++++++++++++++++++ fern/pages/cpp_api/cpp-api-core-dataset-sq.md | 20 +- .../cpp-api-preprocessing-quantize-scalar.md | 114 ++++++++++ 8 files changed, 602 insertions(+), 15 deletions(-) create mode 100644 cpp/tests/preprocessing/scalar_dataset.cu diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index bc875784f2..ea0d7cccaf 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -16,15 +16,18 @@ #include #include +#include namespace CUVS_EXPORT cuvs { /** * Scalar-quantized dataset specification: every component of a vector becomes one code, so the - * manifest is only the range the codes are mapped back onto and the dimension is still a property + * manifest holds the quantizer and the dimension is still a property * of the data. */ -template +template > struct sq_spec { // One code per component, so the codes are laid out exactly like a dense dataset. template @@ -33,10 +36,9 @@ struct sq_spec { using math_type = MathT; - // Nothing to allocate: the whole manifest is the interval the codes are dequantized into, - // which is exactly what the scalar quantizer holds. - using manifest_type = cuvs::preprocessing::quantize::scalar::quantizer; - using manifest_view_type = manifest_type; + // The manifest is either a global interval or owning per-dimension offsets and a shared scale. + using manifest_type = QuantizerT; + using manifest_view_type = typename manifest_type::view_type; template struct apply : storage_spec { @@ -54,20 +56,30 @@ struct sq_spec { [[nodiscard]] static auto get_manifest_view(const manifest_type& manifest) noexcept -> manifest_view_type { - return manifest; + return manifest.view(); } - // NOTE: the manifest is its own view here, read-only or not, so one overload serves both the - // owning dataset and the view. + /** Borrow read-only quantizer parameters; the manifest must outlive use of the view. */ [[nodiscard]] static auto get_manifest_const_view(const manifest_type& manifest) noexcept -> manifest_view_type + { + return manifest.view(); + } + /** Return the read-only view; its owner must outlive stream work using it. */ + [[nodiscard]] static auto get_manifest_const_view(manifest_view_type manifest) noexcept + -> manifest_view_type + requires(!std::is_same_v) { return manifest; } }; }; -template -using sq_dataset = dataset>; +template > +using sq_dataset = dataset>; template using device_sq_dataset = sq_dataset>>; @@ -75,4 +87,31 @@ template using host_sq_dataset = sq_dataset>>; +/** Owns row-major INT8 codes and their per-dimension offsets/shared scale. */ +using device_per_dim_sq_dataset = + sq_dataset>, + cuvs::preprocessing::quantize::scalar::per_dim_quantizer>; + +namespace preprocessing::quantize::scalar { + +/** + * Train and encode a device FP32 corpus. Reuse result.const_manifest_view() with transform + * for query batches. For separate training/encoding, use train_per_dim and transform. + */ +inline device_per_dim_sq_dataset build_per_dim( + raft::resources const& res, + raft::device_matrix_view dataset, + params params = {}) +{ + auto quantizer = train_per_dim(res, dataset, params); + auto codes = raft::make_device_matrix(res, dataset.extent(0), dataset.extent(1)); + transform(res, quantizer.view(), dataset, codes.view()); + return device_per_dim_sq_dataset{std::move(codes), std::move(quantizer)}; +} + +} // namespace preprocessing::quantize::scalar + } // namespace CUVS_EXPORT cuvs diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 9cb0afb537..f52e380feb 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -46,6 +46,11 @@ struct params { */ template struct quantizer { + using view_type = quantizer; + + /** Return a value copy of this quantizer. */ + [[nodiscard]] auto view() const noexcept -> view_type { return *this; } + /** Minimum value of the quantization range. */ T min_; /** Maximum value of the quantization range. */ @@ -473,6 +478,50 @@ void inverse_transform(raft::resources const& res, raft::host_matrix_view dataset, raft::host_matrix_view out); +/** Read-only device offsets and a shared step size. The owner must outlive stream work. */ +struct per_dim_quantizer_const_view { + raft::device_vector_view offsets; + float scale; +}; + +/** Owns corpus-trained offsets on the device; query batches reuse its view. */ +struct per_dim_quantizer { + using view_type = per_dim_quantizer_const_view; + raft::device_vector offsets; + float scale; + + /** Borrow device offsets; this owner must outlive stream work using the view. */ + [[nodiscard]] auto view() const noexcept -> view_type { return {offsets.view(), scale}; } +}; + +/** + * Apply params.quantile independently to each dimension using all finite corpus rows. + * The default 0.99 retains the central 99%; 1.0 uses the full min/max range. + * offset[j] = (min[j] + max[j]) / 2; scale = max_j((max[j] - min[j]) / 2) / 127. + * Requires nonempty input. Constant data uses scale = 1; positive scales are floored to the + * smallest normal float. Synchronizes the resource stream to return the scale. + */ +per_dim_quantizer train_per_dim(raft::resources const& res, + raft::device_matrix_view dataset, + params params = {}); + +/** + * Encode finite FP32 values as clamp(rintf((x - offset[j]) * (1 / scale)), -127, 127). + * Rounds ties to even. Use the corpus quantizer for query batches. Custom offsets and scale + * are supported; scale and its reciprocal must be finite and positive. Input/output shapes + * must match, with width equal to offsets.size(). Enqueued on the resource stream. + */ +void transform(raft::resources const& res, + per_dim_quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out); + +/** Reconstruct offsets[j] + scale * code; same shape/scale requirements as transform. */ +void inverse_transform(raft::resources const& res, + per_dim_quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out); + /** @} */ // end of group scalar } // namespace scalar diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index 555aa7707b..e49308c5d5 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -13,6 +13,11 @@ #include #include #include +#include + +#include +#include +#include #include #include #include @@ -222,4 +227,135 @@ void inverse_transform(raft::resources const& res, } } +inline per_dim_quantizer train_per_dim(raft::resources const& res, + raft::device_matrix_view dataset, + params params) +{ + RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); + auto const dim = dataset.extent(1); + RAFT_EXPECTS(dataset.extent(0) > 0 && dim > 0, "Training requires nonempty rows and dimensions"); + auto offsets = raft::make_device_vector(res, dim); + auto maxima = raft::make_device_vector(res, dim); + auto max_half_range = raft::make_device_vector(res, 1); + if (params.quantile == 1.f) { + raft::linalg::reduce(res, + dataset, + offsets.view(), + std::numeric_limits::infinity(), + false, + raft::identity_op{}, + raft::min_op{}); + raft::linalg::reduce(res, + dataset, + maxima.view(), + -std::numeric_limits::infinity(), + false, + raft::identity_op{}, + raft::max_op{}); + } else { + auto const rows = dataset.extent(0); + auto column = raft::make_device_vector(res, rows); + auto stream = raft::resource::get_cuda_stream(res).get(); + size_t workspace_bytes = 0; + RAFT_CUDA_TRY(cub::DeviceMergeSort::SortKeys( + nullptr, workspace_bytes, column.data_handle(), rows, fp_lt_op{}, stream)); + auto workspace = raft::make_device_vector(res, workspace_bytes); + auto const upper = static_cast(std::ceil((0.5 + 0.5 * params.quantile) * rows)) - 1; + auto const lower = rows - upper - 1; + auto const* input = dataset.data_handle(); + for (int64_t j = 0; j < dim; ++j) { + raft::linalg::map_offset( + res, column.view(), [input, dim, j] __device__(int64_t i) { return input[i * dim + j]; }); + if (!raft::resource::get_dry_run_flag(res)) { + RAFT_CUDA_TRY(cub::DeviceMergeSort::SortKeys(workspace.data_handle(), + workspace_bytes, + column.data_handle(), + rows, + fp_lt_op{}, + stream)); + } + raft::copy(res, + raft::make_device_scalar_view(offsets.data_handle() + j), + raft::make_device_scalar_view(column.data_handle() + lower)); + raft::copy(res, + raft::make_device_scalar_view(maxima.data_handle() + j), + raft::make_device_scalar_view(column.data_handle() + upper)); + } + } + auto* offset_ptr = offsets.data_handle(); + auto const* max_ptr = maxima.data_handle(); + raft::linalg::map_offset(res, maxima.view(), [offset_ptr, max_ptr] __device__(int64_t j) { + // Halve before adding/subtracting to avoid overflow for large finite inputs. + float const lo = offset_ptr[j] * 0.5f; + float const hi = max_ptr[j] * 0.5f; + offset_ptr[j] = lo + hi; + return hi - lo; + }); + raft::linalg::reduce( + res, + raft::make_device_matrix_view(maxima.data_handle(), 1, dim), + max_half_range.view(), + 0.f, + false, + raft::identity_op{}, + raft::max_op{}); + float half_range = 0.f; + raft::copy(res, + raft::make_host_scalar_view(&half_range), + raft::make_device_scalar_view(max_half_range.data_handle())); + raft::resource::sync_stream(res); + float scale = + half_range > 0.f ? std::max(half_range / 127.f, std::numeric_limits::min()) : 1.f; + // Avoid upward division rounding overflowing the largest reconstructed magnitude. + if (static_cast(scale) * 127.0 > std::numeric_limits::max()) { + scale = std::nextafter(scale, 0.f); + } + return {std::move(offsets), scale}; +} + +template +void check_per_dim_shapes(per_dim_quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) +{ + RAFT_EXPECTS( + std::isfinite(quantizer.scale) && quantizer.scale > 0.f && std::isfinite(1.f / quantizer.scale), + "Scale and its reciprocal must be finite and positive"); + RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.offsets.extent(0), + "Input width must match the nonempty offsets vector"); + RAFT_EXPECTS(dataset.extent(0) == out.extent(0) && dataset.extent(1) == out.extent(1), + "Input and output shapes must match"); +} + +inline void transform(raft::resources const& res, + per_dim_quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) +{ + check_per_dim_shapes(quantizer, dataset, out); + auto const* input = dataset.data_handle(); + auto const* offsets = quantizer.offsets.data_handle(); + auto const dim = dataset.extent(1); + float const inv = 1.f / quantizer.scale; + raft::linalg::map_offset(res, out, [input, offsets, dim, inv] __device__(int64_t i) { + float const code = rintf((input[i] - offsets[i % dim]) * inv); + return static_cast(fminf(fmaxf(code, -127.f), 127.f)); + }); +} + +inline void inverse_transform(raft::resources const& res, + per_dim_quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) +{ + check_per_dim_shapes(quantizer, dataset, out); + auto const* input = dataset.data_handle(); + auto const* offsets = quantizer.offsets.data_handle(); + auto const dim = dataset.extent(1); + float const scale = quantizer.scale; + raft::linalg::map_offset(res, out, [input, offsets, dim, scale] __device__(int64_t i) { + return fmaf(scale, static_cast(input[i]), offsets[i % dim]); + }); +} + } // namespace cuvs::preprocessing::quantize::scalar::detail diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index 012d36b932..43b9e10e91 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -58,4 +58,27 @@ CUVS_INST_QUANTIZATION(half, int8_t); #undef CUVS_INST_QUANTIZATION +per_dim_quantizer train_per_dim(raft::resources const& res, + raft::device_matrix_view dataset, + params params) +{ + return detail::train_per_dim(res, dataset, params); +} + +void transform(raft::resources const& res, + per_dim_quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) +{ + detail::transform(res, quantizer, dataset, out); +} + +void inverse_transform(raft::resources const& res, + per_dim_quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) +{ + detail::inverse_transform(res, quantizer, dataset, out); +} + } // namespace cuvs::preprocessing::quantize::scalar diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index f132bb21a3..f707105a09 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -477,6 +477,7 @@ ConfigureTest( ConfigureTest( NAME PREPROCESSING_TEST PATH preprocessing/scalar_quantization.cu + preprocessing/scalar_dataset.cu preprocessing/binary_quantization.cu preprocessing/spectral_embedding.cu preprocessing/product_quantization.cu diff --git a/cpp/tests/preprocessing/scalar_dataset.cu b/cpp/tests/preprocessing/scalar_dataset.cu new file mode 100644 index 0000000000..bdcf86d04d --- /dev/null +++ b/cpp/tests/preprocessing/scalar_dataset.cu @@ -0,0 +1,209 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace cuvs::preprocessing::quantize::scalar { + +template +auto copy_to_host(raft::resources const& res, const T* data, int64_t size) +{ + std::vector result(size); + auto stream = raft::resource::get_cuda_stream(res); + raft::update_host(result.data(), data, size, stream); + raft::resource::sync_stream(res); + return result; +} + +TEST(ScalarDataset, CorpusAndQueriesShareQuantizer) +{ + raft::device_resources res; + auto stream = raft::resource::get_cuda_stream(res); + std::vector corpus{0.f, 10.f, 9.f, 254.f, 14.f, 9.f}; + auto input = raft::make_device_matrix(res, 2, 3); + raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); + auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + ASSERT_EQ(encoded.n_rows(), 2); + ASSERT_EQ(encoded.dim(), 3u); + auto quantizer = encoded.const_manifest_view(); + EXPECT_FLOAT_EQ(quantizer.scale, 1.f); + EXPECT_EQ(copy_to_host(res, quantizer.offsets.data_handle(), 3), + (std::vector{127.f, 12.f, 9.f})); + EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 6), + (std::vector{-127, -2, 0, 127, 2, 0})); + + // The corpus manifest, rather than query extrema, controls ties and saturation. + std::vector queries{127.5f, 11.5f, 1000.f, 128.5f, 9.5f, -1000.f}; + raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); + auto codes = raft::make_device_matrix(res, 2, 3); + transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_EQ(copy_to_host(res, codes.data_handle(), 6), + (std::vector{0, 0, 127, 2, -2, -127})); + inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); + EXPECT_EQ(copy_to_host(res, input.data_handle(), 6), + (std::vector{127.f, 12.f, 136.f, 129.f, 10.f, -118.f})); + + // Ownership moves without copying codes or offsets, and views preserve constness. + static_assert(!std::is_copy_constructible_v); + auto moved = std::move(encoded); + device_per_dim_sq_dataset::const_view_type const_view = moved.view(); + static_assert(std::is_const_v); + EXPECT_EQ(const_view.const_manifest_view().offsets.data_handle(), + quantizer.offsets.data_handle()); + EXPECT_EQ(const_view.n_rows(), 2); + EXPECT_EQ(copy_to_host(res, const_view.data_const_view().data_handle(), 6), + (std::vector{-127, -2, 0, 127, 2, 0})); +} + +TEST(ScalarDataset, FullCorpusExtremaAndChunkedEncoding) +{ + raft::device_resources res; + constexpr int64_t rows = 4097, dim = 5; + std::vector corpus(rows * dim, 4.f); + corpus[0] = -250.f; + corpus[(rows - 1) * dim] = 258.f; + corpus[1] = 2.f; + corpus[(rows - 1) * dim + 1] = 6.f; + auto input = raft::make_device_matrix(res, rows, dim); + raft::update_device( + input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); + auto quantizer = train_per_dim(res, raft::make_const_mdspan(input.view()), params{1.f}); + EXPECT_FLOAT_EQ(quantizer.scale, 2.f); + EXPECT_EQ(copy_to_host(res, quantizer.offsets.data_handle(), dim), + (std::vector(dim, 4.f))); + auto codes = raft::make_device_matrix(res, rows, dim); + for (int64_t start = 0; start < rows; start += 1024) { + auto n = std::min(int64_t{1024}, rows - start); + transform( + res, + quantizer.view(), + raft::make_device_matrix_view( + input.data_handle() + start * dim, n, dim), + raft::make_device_matrix_view(codes.data_handle() + start * dim, n, dim)); + } + auto result = copy_to_host(res, codes.data_handle(), rows * dim); + EXPECT_EQ(result.front(), -127); + EXPECT_EQ(result[(rows - 1) * dim], 127); + EXPECT_EQ(result[1], -1); + EXPECT_EQ(result[(rows - 1) * dim + 1], 1); + EXPECT_EQ(result[1024 * dim + 2], 0); +} + +TEST(ScalarDataset, PerDimensionQuantiles) +{ + raft::device_resources res; + constexpr int64_t rows = 1000, dim = 3; + std::vector corpus(rows * dim); + for (int64_t i = 0; i < rows; ++i) { + corpus[i * dim] = static_cast(i); + corpus[i * dim + 1] = 10000.f + 2.f * static_cast(rows - i - 1); + corpus[i * dim + 2] = 9.f; + } + auto input = raft::make_device_matrix(res, rows, dim); + raft::update_device( + input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); + auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + auto quantizer = encoded.const_manifest_view(); + EXPECT_EQ(copy_to_host(res, quantizer.offsets.data_handle(), dim), + (std::vector{499.5f, 10999.f, 9.f})); + EXPECT_FLOAT_EQ(quantizer.scale, 991.f / 127.f); + auto codes = copy_to_host(res, encoded.data_const_view().data_handle(), rows * dim); + EXPECT_EQ(codes[1], 127); + EXPECT_EQ(codes[(rows - 1) * dim + 1], -127); + EXPECT_EQ(codes[2], 0); + auto trimmed = train_per_dim(res, raft::make_const_mdspan(input.view()), params{0.5f}); + EXPECT_FLOAT_EQ(trimmed.scale, 499.f / 127.f); + for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { + EXPECT_THROW(train_per_dim(res, raft::make_const_mdspan(input.view()), params{q}), + raft::logic_error); + } +} + +TEST(ScalarDataset, ConstantCorpus) +{ + raft::device_resources res; + std::vector corpus{10.f, -7.f, 10.f, -7.f}; + auto input = raft::make_device_matrix(res, 2, 2); + raft::update_device( + input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); + auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + EXPECT_FLOAT_EQ(encoded.const_manifest_view().scale, 1.f); + EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 4), + (std::vector(4, 0))); + inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); + EXPECT_EQ(copy_to_host(res, input.data_handle(), 4), corpus); +} + +TEST(ScalarDataset, ExtremeFiniteCorpus) +{ + raft::device_resources res; + float const max = std::numeric_limits::max(); + std::vector corpus{-max, max}; + auto input = raft::make_device_matrix(res, 2, 1); + raft::update_device( + input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); + auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), + (std::vector{-127, 127})); + inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); + auto decoded = copy_to_host(res, input.data_handle(), 2); + for (int i = 0; i < 2; ++i) { + EXPECT_TRUE(std::isfinite(decoded[i])); + EXPECT_FLOAT_EQ(decoded[i], corpus[i]); + } +} + +TEST(ScalarDataset, CustomParametersAndValidation) +{ + raft::device_resources res; + auto offsets = raft::make_device_vector(res, 2); + std::vector custom_offsets{100.f, -100.f}; + raft::update_device(offsets.data_handle(), + custom_offsets.data(), + custom_offsets.size(), + raft::resource::get_cuda_stream(res)); + per_dim_quantizer_const_view quantizer{raft::make_const_mdspan(offsets.view()), 2.f}; + auto input = raft::make_device_matrix(res, 1, 2); + std::vector values{104.f, -106.f}; + raft::update_device( + input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); + auto codes = raft::make_device_matrix(res, 1, 2); + transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_EQ(copy_to_host(res, codes.data_handle(), 2), (std::vector{2, -3})); + auto wrong_shape = raft::make_device_matrix(res, 2, 1); + EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), + raft::logic_error); + for (float scale : {0.f, -1.f, std::numeric_limits::infinity()}) { + quantizer.scale = scale; + EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()), + raft::logic_error); + } + EXPECT_THROW( + train_per_dim(res, raft::make_device_matrix_view(nullptr, 0, 2)), + raft::logic_error); +} + +TEST(ScalarDataset, GlobalDatasetStillWorks) +{ + raft::device_resources res; + auto codes = raft::make_device_matrix(res, 1, 2); + device_sq_dataset dataset{std::move(codes), quantizer{-1.f, 1.f}}; + decltype(dataset)::const_view_type view = dataset.view(); + EXPECT_EQ(view.dim(), 2u); + EXPECT_FLOAT_EQ(view.const_manifest_view().min_, -1.f); + EXPECT_FLOAT_EQ(view.const_manifest_view().max_, 1.f); +} + +} // namespace cuvs::preprocessing::quantize::scalar diff --git a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md index a26e57b857..ce75b0abca 100644 --- a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md +++ b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md @@ -13,9 +13,25 @@ _Source header: `cuvs/core/dataset/sq.hpp`_ Scalar-quantized dataset specification: every component of a vector becomes one code, so the -manifest is only the range the codes are mapped back onto and the dimension is still a property of the data. +manifest holds the quantizer and the dimension is still a property of the data. ```cpp -template +template > struct sq_spec; ``` + + +### device_per_dim_sq_dataset + +Owns row-major INT8 codes and their per-dimension offsets/shared scale. + +```cpp +using device_per_dim_sq_dataset = +sq_dataset>, +cuvs::preprocessing::quantize::scalar::per_dim_quantizer>; +``` diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 29638673f1..8f04941378 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -47,6 +47,19 @@ struct quantizer { | `min_` | `T` | Minimum value of the quantization range. | | `max_` | `T` | Maximum value of the quantization range. | + +### preprocessing::quantize::scalar::quantizer::view + +Return a value copy of this quantizer. + +```cpp +[[nodiscard]] auto view() const noexcept -> view_type; +``` + +**Returns** + +`view_type` + ### preprocessing::quantize::scalar::train @@ -517,3 +530,104 @@ Usage example: **Returns** `void` + + +### preprocessing::quantize::scalar::per_dim_quantizer_const_view + +Read-only device offsets and a shared step size. The owner must outlive stream work. + +```cpp +struct per_dim_quantizer_const_view { + raft::device_vector_view offsets; + float scale; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `offsets` | `raft::device_vector_view` | | +| `scale` | `float` | | + + +### preprocessing::quantize::scalar::per_dim_quantizer + +Owns corpus-trained offsets on the device; query batches reuse its view. + +```cpp +struct per_dim_quantizer { + raft::device_vector offsets; + float scale; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `offsets` | `raft::device_vector` | | +| `scale` | `float` | | + + +### preprocessing::quantize::scalar::per_dim_quantizer::view + +Borrow device offsets; this owner must outlive stream work using the view. + +```cpp +[[nodiscard]] auto view() const noexcept -> view_type; +``` + +**Returns** + +`view_type` + +**Additional overload:** `preprocessing::quantize::scalar::transform` + +Encode finite FP32 values as clamp(rintf((x - offset[j]) * (1 / scale)), -127, 127). + +```cpp +void transform(raft::resources const& res, +per_dim_quantizer_const_view quantizer, +raft::device_matrix_view dataset, +raft::device_matrix_view out); +``` + +Rounds ties to even. Use the corpus quantizer for query batches. Custom offsets and scale are supported; scale and its reciprocal must be finite and positive. Input/output shapes must match, with width equal to offsets.size(). Enqueued on the resource stream. + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | | `raft::resources const&` | | +| `quantizer` | | [`per_dim_quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-per-dim-quantizer-const-view) | | +| `dataset` | | `raft::device_matrix_view` | | +| `out` | | `raft::device_matrix_view` | | + +**Returns** + +`void` + +**Additional overload:** `preprocessing::quantize::scalar::inverse_transform` + +Reconstruct offsets[j] + scale * code; same shape/scale requirements as transform. + +```cpp +void inverse_transform(raft::resources const& res, +per_dim_quantizer_const_view quantizer, +raft::device_matrix_view dataset, +raft::device_matrix_view out); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | | `raft::resources const&` | | +| `quantizer` | | [`per_dim_quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-per-dim-quantizer-const-view) | | +| `dataset` | | `raft::device_matrix_view` | | +| `out` | | `raft::device_matrix_view` | | + +**Returns** + +`void` From 4aec0a978008c871c722e137ec1f47240fdc61f2 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 00:02:15 +0000 Subject: [PATCH 02/24] Keep scalar reconstruction finite at FP32 limits --- .../cuvs/preprocessing/quantize/scalar.hpp | 3 ++- .../preprocessing/quantize/detail/scalar.cuh | 10 +++---- cpp/tests/preprocessing/scalar_dataset.cu | 26 ++++++++++--------- .../cpp-api-preprocessing-quantize-scalar.md | 4 ++- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index f52e380feb..748ebbabd0 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -516,7 +516,8 @@ void transform(raft::resources const& res, raft::device_matrix_view dataset, raft::device_matrix_view out); -/** Reconstruct offsets[j] + scale * code; same shape/scale requirements as transform. */ +/** Reconstruct offsets[j] + scale * code, saturating to finite FP32 limits. Same shape/scale + * requirements as transform. */ void inverse_transform(raft::resources const& res, per_dim_quantizer_const_view quantizer, raft::device_matrix_view dataset, diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index e49308c5d5..ebc086ae8b 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -304,12 +304,8 @@ inline per_dim_quantizer train_per_dim(raft::resources const& res, raft::make_host_scalar_view(&half_range), raft::make_device_scalar_view(max_half_range.data_handle())); raft::resource::sync_stream(res); - float scale = + float const scale = half_range > 0.f ? std::max(half_range / 127.f, std::numeric_limits::min()) : 1.f; - // Avoid upward division rounding overflowing the largest reconstructed magnitude. - if (static_cast(scale) * 127.0 > std::numeric_limits::max()) { - scale = std::nextafter(scale, 0.f); - } return {std::move(offsets), scale}; } @@ -354,7 +350,9 @@ inline void inverse_transform(raft::resources const& res, auto const dim = dataset.extent(1); float const scale = quantizer.scale; raft::linalg::map_offset(res, out, [input, offsets, dim, scale] __device__(int64_t i) { - return fmaf(scale, static_cast(input[i]), offsets[i % dim]); + float const value = fmaf(scale, static_cast(input[i]), offsets[i % dim]); + constexpr float limit = std::numeric_limits::max(); + return fminf(fmaxf(value, -limit), limit); }); } diff --git a/cpp/tests/preprocessing/scalar_dataset.cu b/cpp/tests/preprocessing/scalar_dataset.cu index bdcf86d04d..647ef9e061 100644 --- a/cpp/tests/preprocessing/scalar_dataset.cu +++ b/cpp/tests/preprocessing/scalar_dataset.cu @@ -150,18 +150,20 @@ TEST(ScalarDataset, ExtremeFiniteCorpus) { raft::device_resources res; float const max = std::numeric_limits::max(); - std::vector corpus{-max, max}; - auto input = raft::make_device_matrix(res, 2, 1); - raft::update_device( - input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); - EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), - (std::vector{-127, 127})); - inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); - auto decoded = copy_to_host(res, input.data_handle(), 2); - for (int i = 0; i < 2; ++i) { - EXPECT_TRUE(std::isfinite(decoded[i])); - EXPECT_FLOAT_EQ(decoded[i], corpus[i]); + for (float lo : {-max, -std::ldexp(124.f, 104)}) { + std::vector corpus{lo, max}; + auto input = raft::make_device_matrix(res, 2, 1); + raft::update_device( + input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); + auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), + (std::vector{-127, 127})); + inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); + auto decoded = copy_to_host(res, input.data_handle(), 2); + for (int i = 0; i < 2; ++i) { + EXPECT_TRUE(std::isfinite(decoded[i])); + EXPECT_NEAR(decoded[i], corpus[i], encoded.const_manifest_view().scale * 0.5f); + } } } diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 8f04941378..efe8053ae2 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -610,7 +610,7 @@ Rounds ties to even. Use the corpus quantizer for query batches. Custom offsets **Additional overload:** `preprocessing::quantize::scalar::inverse_transform` -Reconstruct offsets[j] + scale * code; same shape/scale requirements as transform. +Reconstruct offsets[j] + scale * code, saturating to finite FP32 limits. Same shape/scale ```cpp void inverse_transform(raft::resources const& res, @@ -619,6 +619,8 @@ raft::device_matrix_view dataset, raft::device_matrix_view out); ``` +requirements as transform. + **Parameters** | Name | Direction | Type | Description | From ce4d9f9116e22e2a445f490e2577576d3177116a Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 00:25:09 +0000 Subject: [PATCH 03/24] Unify scalar quantizer modes behind shared dataset APIs --- c/src/preprocessing/quantize/scalar.cpp | 17 +- cpp/include/cuvs/core/dataset/sq.hpp | 37 +- .../cuvs/preprocessing/quantize/scalar.hpp | 144 ++++---- .../preprocessing/quantize/detail/scalar.cuh | 237 +++++++------ cpp/src/preprocessing/quantize/scalar.cu | 31 +- cpp/tests/preprocessing/scalar_dataset.cu | 110 +++--- .../preprocessing/scalar_quantization.cu | 24 +- fern/pages/cpp_api/cpp-api-core-dataset-sq.md | 18 +- .../cpp-api-preprocessing-quantize-scalar.md | 325 ++++++++++-------- 9 files changed, 481 insertions(+), 462 deletions(-) diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index 99ab090fa8..83b3a29bf5 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -42,8 +42,9 @@ void _train(cuvsResources_t res, RAFT_FAIL("dataset must be accessible on host or device memory"); } - quantizer->min_ = ret.min_; - quantizer->max_ = ret.max_; + auto const& range = std::get(ret.value); + quantizer->min_ = range.min_; + quantizer->max_ = range.max_; } template @@ -54,9 +55,8 @@ void _transform(cuvsResources_t res, { auto res_ptr = reinterpret_cast(res); - cuvs::preprocessing::quantize::scalar::quantizer quantizer; - quantizer.min_ = quantizer_->min_; - quantizer.max_ = quantizer_->max_; + cuvs::preprocessing::quantize::scalar::quantizer quantizer{ + static_cast(quantizer_->min_), static_cast(quantizer_->max_)}; auto dataset = dataset_tensor->dl_tensor; if (cuvs::core::is_dlpack_device_compatible(dataset)) { @@ -91,9 +91,8 @@ void _inverse_transform(cuvsResources_t res, { auto res_ptr = reinterpret_cast(res); - cuvs::preprocessing::quantize::scalar::quantizer quantizer; - quantizer.min_ = quantizer_->min_; - quantizer.max_ = quantizer_->max_; + cuvs::preprocessing::quantize::scalar::quantizer quantizer{ + static_cast(quantizer_->min_), static_cast(quantizer_->max_)}; auto dataset = dataset_tensor->dl_tensor; if (cuvs::core::is_dlpack_device_compatible(dataset)) { diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index ea0d7cccaf..fa9471c2c5 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -25,9 +25,7 @@ namespace CUVS_EXPORT cuvs { * manifest holds the quantizer and the dimension is still a property * of the data. */ -template > +template struct sq_spec { // One code per component, so the codes are laid out exactly like a dense dataset. template @@ -37,7 +35,7 @@ struct sq_spec { using math_type = MathT; // The manifest is either a global interval or owning per-dimension offsets and a shared scale. - using manifest_type = QuantizerT; + using manifest_type = cuvs::preprocessing::quantize::scalar::quantizer; using manifest_view_type = typename manifest_type::view_type; template @@ -53,6 +51,7 @@ struct sq_spec { // NOTE: every accessor is inherited; the dimension is still one code per component, so even // `get_dim` is the dense one. + /** Borrow read-only quantizer parameters from the owning manifest. */ [[nodiscard]] static auto get_manifest_view(const manifest_type& manifest) noexcept -> manifest_view_type { @@ -67,19 +66,14 @@ struct sq_spec { /** Return the read-only view; its owner must outlive stream work using it. */ [[nodiscard]] static auto get_manifest_const_view(manifest_view_type manifest) noexcept -> manifest_view_type - requires(!std::is_same_v) { return manifest; } }; }; -template > -using sq_dataset = dataset>; +template +using sq_dataset = dataset>; template using device_sq_dataset = sq_dataset>>; @@ -87,29 +81,22 @@ template using host_sq_dataset = sq_dataset>>; -/** Owns row-major INT8 codes and their per-dimension offsets/shared scale. */ -using device_per_dim_sq_dataset = - sq_dataset>, - cuvs::preprocessing::quantize::scalar::per_dim_quantizer>; - namespace preprocessing::quantize::scalar { /** - * Train and encode a device FP32 corpus. Reuse result.const_manifest_view() with transform - * for query batches. For separate training/encoding, use train_per_dim and transform. + * Train and encode a device FP32 corpus, using per-dimension ranges by default. + * Reuse result.const_manifest_view() with transform + * for query batches. For separate training/encoding, use train and transform. */ -inline device_per_dim_sq_dataset build_per_dim( +inline device_sq_dataset build( raft::resources const& res, raft::device_matrix_view dataset, - params params = {}) + params params = {.range = range_mode::per_dimension}) { - auto quantizer = train_per_dim(res, dataset, params); + auto quantizer = train(res, params, dataset); auto codes = raft::make_device_matrix(res, dataset.extent(0), dataset.extent(1)); transform(res, quantizer.view(), dataset, codes.view()); - return device_per_dim_sq_dataset{std::move(codes), std::move(quantizer)}; + return device_sq_dataset{std::move(codes), std::move(quantizer)}; } } // namespace preprocessing::quantize::scalar diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 748ebbabd0..681e57d21f 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -14,6 +14,9 @@ #include #include +#include +#include + namespace CUVS_EXPORT cuvs { namespace preprocessing { namespace quantize { @@ -24,6 +27,9 @@ namespace scalar { * @{ */ +/** Select a global interval or per-dimension offsets with a shared scale. */ +enum class range_mode { global, per_dimension }; + /** * @brief quantizer parameters. */ @@ -33,28 +39,61 @@ struct params { * Needs to be within range of (0, 1]. */ float quantile = 0.99; + /** Per-dimension training uses all device FP32 corpus rows, applying quantile independently + * to each dimension. Midpoint offsets share the widest half-range divided by 127. + * Requires nonempty finite input. Constant ranges use scale 1; positive scales are floored + * to the smallest normal float. Training synchronizes to return the scale. */ + range_mode range = range_mode::global; }; -/** - * @brief Defines and stores scalar for quantisation upon training - * - * The quantization is performed by a linear mapping of an interval in the - * float data type to the full range of the quantized int type. - * - * @tparam T data element type - * - */ +/** Read-only scalar quantizer parameters. The owner must outlive stream work using offsets. */ template -struct quantizer { - using view_type = quantizer; - - /** Return a value copy of this quantizer. */ - [[nodiscard]] auto view() const noexcept -> view_type { return *this; } +struct quantizer_const_view { + /** A global interval mapped to [-128, 127]. */ + struct range_type { + T min_; + T max_; + }; + /** Encode clamp(rintf((x - offsets[j]) / scale), -127, 127), with ties to even. + * Decode offsets[j] + scale * code, saturating to finite FP32 limits. */ + struct offsets_type { + raft::device_vector_view offsets; + T scale; + }; + std::variant value; +}; - /** Minimum value of the quantization range. */ - T min_; - /** Maximum value of the quantization range. */ - T max_; +/** Owns either a global interval or device offsets and a shared step size. */ +template +struct quantizer { + using view_type = quantizer_const_view; + using range_type = typename view_type::range_type; + /** Owning per-dimension parameters. */ + struct offsets_type { + raft::device_vector offsets; + T scale; + }; + std::variant value; + + /** Construct an empty global interval. */ + quantizer() : value{range_type{}} {} + /** Construct a global interval. */ + quantizer(T min, T max) : value{range_type{min, max}} {} + /** Take ownership of device offsets and their shared scale. */ + quantizer(raft::device_vector&& offsets, T scale) + : value{offsets_type{std::move(offsets), scale}} + { + } + + /** Borrow read-only parameters; the owner must outlive stream work using offsets. */ + [[nodiscard]] auto view() const noexcept -> view_type + { + if (auto const* range = std::get_if(&value)) { return {*range}; } + auto const& offsets = std::get(value); + return {typename view_type::offsets_type{offsets.offsets.view(), offsets.scale}}; + } + /** An owning quantizer can be passed wherever a read-only view is accepted. */ + operator view_type() const noexcept { return view(); } // NOLINT(google-explicit-constructor) }; /** @@ -117,7 +156,7 @@ quantizer train(raft::resources const& res, * */ void transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -141,7 +180,7 @@ void transform(raft::resources const& res, * */ void transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -168,7 +207,7 @@ void transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -194,7 +233,7 @@ void inverse_transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -258,7 +297,7 @@ quantizer train(raft::resources const& res, * */ void transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -282,7 +321,7 @@ void transform(raft::resources const& res, * */ void transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -308,7 +347,7 @@ void transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -334,7 +373,7 @@ void inverse_transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -398,7 +437,7 @@ quantizer train(raft::resources const& res, * */ void transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -422,7 +461,7 @@ void transform(raft::resources const& res, * */ void transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -448,7 +487,7 @@ void transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -474,55 +513,10 @@ void inverse_transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - const quantizer& quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); -/** Read-only device offsets and a shared step size. The owner must outlive stream work. */ -struct per_dim_quantizer_const_view { - raft::device_vector_view offsets; - float scale; -}; - -/** Owns corpus-trained offsets on the device; query batches reuse its view. */ -struct per_dim_quantizer { - using view_type = per_dim_quantizer_const_view; - raft::device_vector offsets; - float scale; - - /** Borrow device offsets; this owner must outlive stream work using the view. */ - [[nodiscard]] auto view() const noexcept -> view_type { return {offsets.view(), scale}; } -}; - -/** - * Apply params.quantile independently to each dimension using all finite corpus rows. - * The default 0.99 retains the central 99%; 1.0 uses the full min/max range. - * offset[j] = (min[j] + max[j]) / 2; scale = max_j((max[j] - min[j]) / 2) / 127. - * Requires nonempty input. Constant data uses scale = 1; positive scales are floored to the - * smallest normal float. Synchronizes the resource stream to return the scale. - */ -per_dim_quantizer train_per_dim(raft::resources const& res, - raft::device_matrix_view dataset, - params params = {}); - -/** - * Encode finite FP32 values as clamp(rintf((x - offset[j]) * (1 / scale)), -127, 127). - * Rounds ties to even. Use the corpus quantizer for query batches. Custom offsets and scale - * are supported; scale and its reciprocal must be finite and positive. Input/output shapes - * must match, with width equal to offsets.size(). Enqueued on the resource stream. - */ -void transform(raft::resources const& res, - per_dim_quantizer_const_view quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out); - -/** Reconstruct offsets[j] + scale * code, saturating to finite FP32 limits. Same shape/scale - * requirements as transform. */ -void inverse_transform(raft::resources const& res, - per_dim_quantizer_const_view quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out); - /** @} */ // end of group scalar } // namespace scalar diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index ebc086ae8b..3f6b90fd4b 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -133,103 +133,9 @@ std::tuple quantile_min_max( return {minmax_h[0], minmax_h[1]}; } -template -cuvs::preprocessing::quantize::scalar::quantizer train( - raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::params params, - raft::device_matrix_view dataset) -{ - RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, - "quantile for scalar quantization needs to be within (0, 1] but is %f", - params.quantile); - - auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); - - RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); - - return cuvs::preprocessing::quantize::scalar::quantizer{min, max}; -} - -template -cuvs::preprocessing::quantize::scalar::quantizer train( - raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::params params, - raft::host_matrix_view dataset) -{ - RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, - "quantile for scalar quantization needs to be within (0, 1] but is %f", - params.quantile); - - auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); - - RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); - - return cuvs::preprocessing::quantize::scalar::quantizer{min, max}; -} - -template -void transform(raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::quantizer& quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - raft::linalg::map(res, - out, - quantize_op(quantizer.min_, quantizer.max_), - raft::make_const_mdspan(dataset)); -} - -template -void transform(raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::quantizer& quantizer, - raft::host_matrix_view dataset, - raft::host_matrix_view out) -{ - auto main_op = quantize_op(quantizer.min_, quantizer.max_); - size_t n_elements = dataset.extent(0) * dataset.extent(1); - - // In dry-run mode the views may be unbacked or alias the shared probe buffer. - if (raft::resource::get_dry_run_flag(res)) { return; } - -#pragma omp parallel for - for (size_t i = 0; i < n_elements; ++i) { - out.data_handle()[i] = main_op(dataset.data_handle()[i]); - } -} - -template -void inverse_transform(raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::quantizer& quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - raft::linalg::map(res, - out, - quantize_op(quantizer.min_, quantizer.max_), - raft::make_const_mdspan(dataset)); -} - -template -void inverse_transform(raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::quantizer& quantizer, - raft::host_matrix_view dataset, - raft::host_matrix_view out) -{ - auto main_op = quantize_op(quantizer.min_, quantizer.max_); - size_t n_elements = dataset.extent(0) * dataset.extent(1); - - // In dry-run mode the views may be unbacked or alias the shared probe buffer. - if (raft::resource::get_dry_run_flag(res)) { return; } - -#pragma omp parallel for - for (size_t i = 0; i < n_elements; ++i) { - out.data_handle()[i] = main_op(dataset.data_handle()[i]); - } -} - -inline per_dim_quantizer train_per_dim(raft::resources const& res, - raft::device_matrix_view dataset, - params params) +inline quantizer train_offsets(raft::resources const& res, + raft::device_matrix_view dataset, + params params) { RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); auto const dim = dataset.extent(1); @@ -310,7 +216,7 @@ inline per_dim_quantizer train_per_dim(raft::resources const& res, } template -void check_per_dim_shapes(per_dim_quantizer_const_view quantizer, +void check_per_dim_shapes(quantizer_const_view::offsets_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { @@ -324,7 +230,7 @@ void check_per_dim_shapes(per_dim_quantizer_const_view quantizer, } inline void transform(raft::resources const& res, - per_dim_quantizer_const_view quantizer, + quantizer_const_view::offsets_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { @@ -340,7 +246,7 @@ inline void transform(raft::resources const& res, } inline void inverse_transform(raft::resources const& res, - per_dim_quantizer_const_view quantizer, + quantizer_const_view::offsets_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { @@ -356,4 +262,135 @@ inline void inverse_transform(raft::resources const& res, }); } +template +cuvs::preprocessing::quantize::scalar::quantizer train( + raft::resources const& res, + const cuvs::preprocessing::quantize::scalar::params params, + raft::device_matrix_view dataset) +{ + RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, + "quantile for scalar quantization needs to be within (0, 1] but is %f", + params.quantile); + + if (params.range == range_mode::per_dimension) { + if constexpr (std::is_same_v) { + return train_offsets(res, dataset, params); + } else { + RAFT_FAIL("Per-dimension training requires device FP32 input"); + } + } + + auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); + + RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); + + return cuvs::preprocessing::quantize::scalar::quantizer{min, max}; +} + +template +cuvs::preprocessing::quantize::scalar::quantizer train( + raft::resources const& res, + const cuvs::preprocessing::quantize::scalar::params params, + raft::host_matrix_view dataset) +{ + RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, + "quantile for scalar quantization needs to be within (0, 1] but is %f", + params.quantile); + + RAFT_EXPECTS(params.range == range_mode::global, + "Per-dimension training requires device FP32 input"); + + auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); + + RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); + + return cuvs::preprocessing::quantize::scalar::quantizer{min, max}; +} + +template +void transform(raft::resources const& res, + quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) +{ + auto view = quantizer; + if (auto const* offsets = + std::get_if::offsets_type>(&view.value)) { + if constexpr (std::is_same_v) { + detail::transform(res, *offsets, dataset, out); + } else { + RAFT_FAIL("Per-dimension quantization requires device FP32 input"); + } + return; + } + auto const& range = std::get::range_type>(view.value); + raft::linalg::map( + res, out, quantize_op(range.min_, range.max_), raft::make_const_mdspan(dataset)); +} + +template +void transform(raft::resources const& res, + quantizer_const_view quantizer, + raft::host_matrix_view dataset, + raft::host_matrix_view out) +{ + RAFT_EXPECTS( + std::holds_alternative::range_type>(quantizer.value), + "Host quantization requires a global interval"); + auto const& range = std::get::range_type>(quantizer.value); + auto main_op = quantize_op(range.min_, range.max_); + size_t n_elements = dataset.extent(0) * dataset.extent(1); + + // In dry-run mode the views may be unbacked or alias the shared probe buffer. + if (raft::resource::get_dry_run_flag(res)) { return; } + +#pragma omp parallel for + for (size_t i = 0; i < n_elements; ++i) { + out.data_handle()[i] = main_op(dataset.data_handle()[i]); + } +} + +template +void inverse_transform(raft::resources const& res, + quantizer_const_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) +{ + auto view = quantizer; + if (auto const* offsets = + std::get_if::offsets_type>(&view.value)) { + if constexpr (std::is_same_v) { + detail::inverse_transform(res, *offsets, dataset, out); + } else { + RAFT_FAIL("Per-dimension quantization requires device FP32 input"); + } + return; + } + auto const& range = std::get::range_type>(view.value); + raft::linalg::map( + res, out, quantize_op(range.min_, range.max_), raft::make_const_mdspan(dataset)); +} + +template +void inverse_transform(raft::resources const& res, + quantizer_const_view quantizer, + raft::host_matrix_view dataset, + raft::host_matrix_view out) +{ + RAFT_EXPECTS( + std::holds_alternative::range_type>(quantizer.value), + "Host quantization requires a global interval"); + auto const& range = std::get::range_type>(quantizer.value); + auto main_op = quantize_op(range.min_, range.max_); + size_t n_elements = dataset.extent(0) * dataset.extent(1); + + // In dry-run mode the views may be unbacked or alias the shared probe buffer. + if (raft::resource::get_dry_run_flag(res)) { return; } + +#pragma omp parallel for + for (size_t i = 0; i < n_elements; ++i) { + out.data_handle()[i] = main_op(dataset.data_handle()[i]); + } +} + } // namespace cuvs::preprocessing::quantize::scalar::detail diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index 43b9e10e91..6f2c482022 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -23,28 +23,28 @@ namespace cuvs::preprocessing::quantize::scalar { return detail::train(res, params, dataset); \ } \ void transform(raft::resources const& res, \ - const quantizer& quantizer, \ + quantizer_const_view quantizer, \ raft::device_matrix_view dataset, \ raft::device_matrix_view out) \ { \ detail::transform(res, quantizer, dataset, out); \ } \ void transform(raft::resources const& res, \ - const quantizer& quantizer, \ + quantizer_const_view quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ detail::transform(res, quantizer, dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ - const quantizer& quantizer, \ + quantizer_const_view quantizer, \ raft::device_matrix_view dataset, \ raft::device_matrix_view out) \ { \ detail::inverse_transform(res, quantizer, dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ - const quantizer& quantizer, \ + quantizer_const_view quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ @@ -58,27 +58,4 @@ CUVS_INST_QUANTIZATION(half, int8_t); #undef CUVS_INST_QUANTIZATION -per_dim_quantizer train_per_dim(raft::resources const& res, - raft::device_matrix_view dataset, - params params) -{ - return detail::train_per_dim(res, dataset, params); -} - -void transform(raft::resources const& res, - per_dim_quantizer_const_view quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - detail::transform(res, quantizer, dataset, out); -} - -void inverse_transform(raft::resources const& res, - per_dim_quantizer_const_view quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - detail::inverse_transform(res, quantizer, dataset, out); -} - } // namespace cuvs::preprocessing::quantize::scalar diff --git a/cpp/tests/preprocessing/scalar_dataset.cu b/cpp/tests/preprocessing/scalar_dataset.cu index 647ef9e061..12c0c48969 100644 --- a/cpp/tests/preprocessing/scalar_dataset.cu +++ b/cpp/tests/preprocessing/scalar_dataset.cu @@ -8,15 +8,15 @@ #include #include -#include #include #include -#include #include #include namespace cuvs::preprocessing::quantize::scalar { +using offsets_view = quantizer_const_view::offsets_type; + template auto copy_to_host(raft::resources const& res, const T* data, int64_t size) { @@ -34,12 +34,12 @@ TEST(ScalarDataset, CorpusAndQueriesShareQuantizer) std::vector corpus{0.f, 10.f, 9.f, 254.f, 14.f, 9.f}; auto input = raft::make_device_matrix(res, 2, 3); raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); - auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + auto encoded = build(res, raft::make_const_mdspan(input.view())); ASSERT_EQ(encoded.n_rows(), 2); ASSERT_EQ(encoded.dim(), 3u); auto quantizer = encoded.const_manifest_view(); - EXPECT_FLOAT_EQ(quantizer.scale, 1.f); - EXPECT_EQ(copy_to_host(res, quantizer.offsets.data_handle(), 3), + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); + EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).offsets.data_handle(), 3), (std::vector{127.f, 12.f, 9.f})); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 6), (std::vector{-127, -2, 0, 127, 2, 0})); @@ -54,51 +54,6 @@ TEST(ScalarDataset, CorpusAndQueriesShareQuantizer) inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); EXPECT_EQ(copy_to_host(res, input.data_handle(), 6), (std::vector{127.f, 12.f, 136.f, 129.f, 10.f, -118.f})); - - // Ownership moves without copying codes or offsets, and views preserve constness. - static_assert(!std::is_copy_constructible_v); - auto moved = std::move(encoded); - device_per_dim_sq_dataset::const_view_type const_view = moved.view(); - static_assert(std::is_const_v); - EXPECT_EQ(const_view.const_manifest_view().offsets.data_handle(), - quantizer.offsets.data_handle()); - EXPECT_EQ(const_view.n_rows(), 2); - EXPECT_EQ(copy_to_host(res, const_view.data_const_view().data_handle(), 6), - (std::vector{-127, -2, 0, 127, 2, 0})); -} - -TEST(ScalarDataset, FullCorpusExtremaAndChunkedEncoding) -{ - raft::device_resources res; - constexpr int64_t rows = 4097, dim = 5; - std::vector corpus(rows * dim, 4.f); - corpus[0] = -250.f; - corpus[(rows - 1) * dim] = 258.f; - corpus[1] = 2.f; - corpus[(rows - 1) * dim + 1] = 6.f; - auto input = raft::make_device_matrix(res, rows, dim); - raft::update_device( - input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto quantizer = train_per_dim(res, raft::make_const_mdspan(input.view()), params{1.f}); - EXPECT_FLOAT_EQ(quantizer.scale, 2.f); - EXPECT_EQ(copy_to_host(res, quantizer.offsets.data_handle(), dim), - (std::vector(dim, 4.f))); - auto codes = raft::make_device_matrix(res, rows, dim); - for (int64_t start = 0; start < rows; start += 1024) { - auto n = std::min(int64_t{1024}, rows - start); - transform( - res, - quantizer.view(), - raft::make_device_matrix_view( - input.data_handle() + start * dim, n, dim), - raft::make_device_matrix_view(codes.data_handle() + start * dim, n, dim)); - } - auto result = copy_to_host(res, codes.data_handle(), rows * dim); - EXPECT_EQ(result.front(), -127); - EXPECT_EQ(result[(rows - 1) * dim], 127); - EXPECT_EQ(result[1], -1); - EXPECT_EQ(result[(rows - 1) * dim + 1], 1); - EXPECT_EQ(result[1024 * dim + 2], 0); } TEST(ScalarDataset, PerDimensionQuantiles) @@ -114,20 +69,25 @@ TEST(ScalarDataset, PerDimensionQuantiles) auto input = raft::make_device_matrix(res, rows, dim); raft::update_device( input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + auto encoded = build(res, raft::make_const_mdspan(input.view())); auto quantizer = encoded.const_manifest_view(); - EXPECT_EQ(copy_to_host(res, quantizer.offsets.data_handle(), dim), + EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).offsets.data_handle(), dim), (std::vector{499.5f, 10999.f, 9.f})); - EXPECT_FLOAT_EQ(quantizer.scale, 991.f / 127.f); + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 991.f / 127.f); auto codes = copy_to_host(res, encoded.data_const_view().data_handle(), rows * dim); EXPECT_EQ(codes[1], 127); EXPECT_EQ(codes[(rows - 1) * dim + 1], -127); EXPECT_EQ(codes[2], 0); - auto trimmed = train_per_dim(res, raft::make_const_mdspan(input.view()), params{0.5f}); - EXPECT_FLOAT_EQ(trimmed.scale, 499.f / 127.f); + auto full = + train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + EXPECT_FLOAT_EQ(std::get(full.view().value).scale, 999.f / 127.f); + auto trimmed = + train(res, params{0.5f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + EXPECT_FLOAT_EQ(std::get(trimmed.view().value).scale, 499.f / 127.f); for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { - EXPECT_THROW(train_per_dim(res, raft::make_const_mdspan(input.view()), params{q}), - raft::logic_error); + EXPECT_THROW( + train(res, params{q, range_mode::per_dimension}, raft::make_const_mdspan(input.view())), + raft::logic_error); } } @@ -138,8 +98,8 @@ TEST(ScalarDataset, ConstantCorpus) auto input = raft::make_device_matrix(res, 2, 2); raft::update_device( input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(encoded.const_manifest_view().scale, 1.f); + auto encoded = build(res, raft::make_const_mdspan(input.view())); + EXPECT_FLOAT_EQ(std::get(encoded.const_manifest_view().value).scale, 1.f); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 4), (std::vector(4, 0))); inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); @@ -155,14 +115,16 @@ TEST(ScalarDataset, ExtremeFiniteCorpus) auto input = raft::make_device_matrix(res, 2, 1); raft::update_device( input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build_per_dim(res, raft::make_const_mdspan(input.view())); + auto encoded = build(res, raft::make_const_mdspan(input.view())); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), (std::vector{-127, 127})); inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); auto decoded = copy_to_host(res, input.data_handle(), 2); for (int i = 0; i < 2; ++i) { EXPECT_TRUE(std::isfinite(decoded[i])); - EXPECT_NEAR(decoded[i], corpus[i], encoded.const_manifest_view().scale * 0.5f); + EXPECT_NEAR(decoded[i], + corpus[i], + std::get(encoded.const_manifest_view().value).scale * 0.5f); } } } @@ -176,7 +138,7 @@ TEST(ScalarDataset, CustomParametersAndValidation) custom_offsets.data(), custom_offsets.size(), raft::resource::get_cuda_stream(res)); - per_dim_quantizer_const_view quantizer{raft::make_const_mdspan(offsets.view()), 2.f}; + quantizer_const_view quantizer{offsets_view{raft::make_const_mdspan(offsets.view()), 2.f}}; auto input = raft::make_device_matrix(res, 1, 2); std::vector values{104.f, -106.f}; raft::update_device( @@ -188,24 +150,36 @@ TEST(ScalarDataset, CustomParametersAndValidation) EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), raft::logic_error); for (float scale : {0.f, -1.f, std::numeric_limits::infinity()}) { - quantizer.scale = scale; + std::get(quantizer.value).scale = scale; EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()), raft::logic_error); } - EXPECT_THROW( - train_per_dim(res, raft::make_device_matrix_view(nullptr, 0, 2)), - raft::logic_error); + EXPECT_THROW(train(res, + params{.range = range_mode::per_dimension}, + raft::make_device_matrix_view(nullptr, 0, 2)), + raft::logic_error); } TEST(ScalarDataset, GlobalDatasetStillWorks) { raft::device_resources res; + auto input = raft::make_device_matrix(res, 1, 2); + std::vector values{-1.f, 1.f}; + raft::update_device( + input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); + auto encoded = build(res, raft::make_const_mdspan(input.view()), params{1.f}); + EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), + (std::vector{-128, 127})); + inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); + EXPECT_EQ(copy_to_host(res, input.data_handle(), 2), values); auto codes = raft::make_device_matrix(res, 1, 2); device_sq_dataset dataset{std::move(codes), quantizer{-1.f, 1.f}}; decltype(dataset)::const_view_type view = dataset.view(); EXPECT_EQ(view.dim(), 2u); - EXPECT_FLOAT_EQ(view.const_manifest_view().min_, -1.f); - EXPECT_FLOAT_EQ(view.const_manifest_view().max_, 1.f); + EXPECT_FLOAT_EQ( + std::get::range_type>(view.const_manifest_view().value).min_, -1.f); + EXPECT_FLOAT_EQ( + std::get::range_type>(view.const_manifest_view().value).max_, 1.f); } } // namespace cuvs::preprocessing::quantize::scalar diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index e053c99d56..2395ee2daa 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -115,8 +115,12 @@ class QuantizationTest : public ::testing::TestWithParam> }, cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); - std::cerr << "Q1: min = " << (double)quantizer_1.min_ << ", max = " << (double)quantizer_1.max_ - << std::endl; + std::cerr + << "Q1: min = " + << (double)std::get(quantizer_1.value).min_ + << ", max = " + << (double)std::get(quantizer_1.value).max_ + << std::endl; { auto quantized_input_h = raft::make_host_matrix(rows_, cols_); @@ -203,13 +207,21 @@ class QuantizationTest : public ::testing::TestWithParam> }, cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); - std::cerr << "Q2: min = " << (double)quantizer_2.min_ << ", max = " << (double)quantizer_2.max_ - << std::endl; + std::cerr + << "Q2: min = " + << (double)std::get(quantizer_2.value).min_ + << ", max = " + << (double)std::get(quantizer_2.value).max_ + << std::endl; // check both quantizers are the same (valid if sampling is identical) if (input_.size() <= 1000000) { - ASSERT_TRUE((double)quantizer_1.min_ == (double)quantizer_2.min_); - ASSERT_TRUE((double)quantizer_1.max_ == (double)quantizer_2.max_); + ASSERT_TRUE( + (double)std::get(quantizer_1.value).min_ == + (double)std::get(quantizer_2.value).min_); + ASSERT_TRUE( + (double)std::get(quantizer_1.value).max_ == + (double)std::get(quantizer_2.value).max_); } { diff --git a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md index ce75b0abca..d51cd95814 100644 --- a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md +++ b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md @@ -16,22 +16,6 @@ Scalar-quantized dataset specification: every component of a vector becomes one manifest holds the quantizer and the dimension is still a property of the data. ```cpp -template > +template struct sq_spec; ``` - - -### device_per_dim_sq_dataset - -Owns row-major INT8 codes and their per-dimension offsets/shared scale. - -```cpp -using device_per_dim_sq_dataset = -sq_dataset>, -cuvs::preprocessing::quantize::scalar::per_dim_quantizer>; -``` diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index efe8053ae2..a6c167e8ac 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -8,6 +8,25 @@ _Source header: `cuvs/preprocessing/quantize/scalar.hpp`_ ## Scalar quantizer utilities + +### preprocessing::quantize::scalar::range_mode + +Select a global interval or per-dimension offsets with a shared scale. + +```cpp +enum class range_mode { + global, + per_dimension +}; +``` + +**Values** + +| Name | Value | +| --- | --- | +| `global` | `` | +| `per_dimension` | `` | + ### preprocessing::quantize::scalar::params @@ -16,6 +35,7 @@ quantizer parameters. ```cpp struct params { float quantile; + range_mode range; }; ``` @@ -24,19 +44,93 @@ struct params { | Name | Type | Description | | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | +| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses all device FP32 corpus rows, applying quantile independently to each dimension. Midpoint offsets share the widest half-range divided by 127. Requires nonempty finite input. Constant ranges use scale 1; positive scales are floored to the smallest normal float. Training synchronizes to return the scale. | + + +### preprocessing::quantize::scalar::quantizer_const_view + +Read-only scalar quantizer parameters. The owner must outlive stream work using offsets. + +```cpp +template +struct quantizer_const_view { + std::variant value; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | + + +### preprocessing::quantize::scalar::quantizer_const_view::range_type + +A global interval mapped to [-128, 127]. + +```cpp +struct range_type { + T min_; + T max_; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `min_` | `T` | | +| `max_` | `T` | | + + +### preprocessing::quantize::scalar::quantizer_const_view::offsets_type + +Encode clamp(rintf((x - offsets[j]) / scale), -127, 127), with ties to even. + +Decode offsets[j] + scale * code, saturating to finite FP32 limits. + +```cpp +struct offsets_type { + raft::device_vector_view offsets; + T scale; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `offsets` | `raft::device_vector_view` | | +| `scale` | `T` | | ### preprocessing::quantize::scalar::quantizer -Defines and stores scalar for quantisation upon training - -The quantization is performed by a linear mapping of an interval in the float data type to the full range of the quantized int type. +Owns either a global interval or device offsets and a shared step size. ```cpp template struct quantizer { - T min_; - T max_; + std::variant value; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | + + +### preprocessing::quantize::scalar::quantizer::offsets_type + +Owning per-dimension parameters. + +```cpp +struct offsets_type { + raft::device_vector offsets; + T scale; }; ``` @@ -44,13 +138,64 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | -| `min_` | `T` | Minimum value of the quantization range. | -| `max_` | `T` | Maximum value of the quantization range. | +| `offsets` | `raft::device_vector` | | +| `scale` | `T` | | + + +### preprocessing::quantize::scalar::quantizer::quantizer + +Construct an empty global interval. + +```cpp +quantizer(); +``` + +**Returns** + +`void` + +**Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` + +Construct a global interval. + +```cpp +quantizer(T min, T max); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `min` | | `T` | | +| `max` | | `T` | | + +**Returns** + +`void` + +**Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` + +Take ownership of device offsets and their shared scale. + +```cpp +quantizer(raft::device_vector&& offsets, T scale); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `offsets` | | `raft::device_vector&&` | | +| `scale` | | `T` | | + +**Returns** + +`void` ### preprocessing::quantize::scalar::quantizer::view -Return a value copy of this quantizer. +Borrow read-only parameters; the owner must outlive stream work using offsets. ```cpp [[nodiscard]] auto view() const noexcept -> view_type; @@ -60,6 +205,19 @@ Return a value copy of this quantizer. `view_type` + +### preprocessing::quantize::scalar::quantizer::view_type + +An owning quantizer can be passed wherever a read-only view is accepted. + +```cpp +operator view_type() const noexcept; +``` + +**Returns** + +`operator` + ### preprocessing::quantize::scalar::train @@ -116,7 +274,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -128,7 +286,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -142,7 +300,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -154,7 +312,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -169,7 +327,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -183,7 +341,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -197,7 +355,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -211,7 +369,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -273,7 +431,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -285,7 +443,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -299,7 +457,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -311,7 +469,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -325,7 +483,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -339,7 +497,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -353,7 +511,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -367,7 +525,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -429,7 +587,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -441,7 +599,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -455,7 +613,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -467,7 +625,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -481,7 +639,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -495,7 +653,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -509,7 +667,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -const quantizer& quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -523,113 +681,10 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | **Returns** `void` - - -### preprocessing::quantize::scalar::per_dim_quantizer_const_view - -Read-only device offsets and a shared step size. The owner must outlive stream work. - -```cpp -struct per_dim_quantizer_const_view { - raft::device_vector_view offsets; - float scale; -}; -``` - -**Fields** - -| Name | Type | Description | -| --- | --- | --- | -| `offsets` | `raft::device_vector_view` | | -| `scale` | `float` | | - - -### preprocessing::quantize::scalar::per_dim_quantizer - -Owns corpus-trained offsets on the device; query batches reuse its view. - -```cpp -struct per_dim_quantizer { - raft::device_vector offsets; - float scale; -}; -``` - -**Fields** - -| Name | Type | Description | -| --- | --- | --- | -| `offsets` | `raft::device_vector` | | -| `scale` | `float` | | - - -### preprocessing::quantize::scalar::per_dim_quantizer::view - -Borrow device offsets; this owner must outlive stream work using the view. - -```cpp -[[nodiscard]] auto view() const noexcept -> view_type; -``` - -**Returns** - -`view_type` - -**Additional overload:** `preprocessing::quantize::scalar::transform` - -Encode finite FP32 values as clamp(rintf((x - offset[j]) * (1 / scale)), -127, 127). - -```cpp -void transform(raft::resources const& res, -per_dim_quantizer_const_view quantizer, -raft::device_matrix_view dataset, -raft::device_matrix_view out); -``` - -Rounds ties to even. Use the corpus quantizer for query batches. Custom offsets and scale are supported; scale and its reciprocal must be finite and positive. Input/output shapes must match, with width equal to offsets.size(). Enqueued on the resource stream. - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `res` | | `raft::resources const&` | | -| `quantizer` | | [`per_dim_quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-per-dim-quantizer-const-view) | | -| `dataset` | | `raft::device_matrix_view` | | -| `out` | | `raft::device_matrix_view` | | - -**Returns** - -`void` - -**Additional overload:** `preprocessing::quantize::scalar::inverse_transform` - -Reconstruct offsets[j] + scale * code, saturating to finite FP32 limits. Same shape/scale - -```cpp -void inverse_transform(raft::resources const& res, -per_dim_quantizer_const_view quantizer, -raft::device_matrix_view dataset, -raft::device_matrix_view out); -``` - -requirements as transform. - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `res` | | `raft::resources const&` | | -| `quantizer` | | [`per_dim_quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-per-dim-quantizer-const-view) | | -| `dataset` | | `raft::device_matrix_view` | | -| `out` | | `raft::device_matrix_view` | | - -**Returns** - -`void` From 9202b60a0c10b237a24d0419a583270f00d4c1c6 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 00:39:11 +0000 Subject: [PATCH 04/24] Document scalar quantizer field migration --- cpp/include/cuvs/preprocessing/quantize/scalar.hpp | 7 ++++++- .../pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 681e57d21f..d62b5ab5b2 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -63,7 +63,12 @@ struct quantizer_const_view { std::variant value; }; -/** Owns either a global interval or device offsets and a shared step size. */ +/** + * Owns either a global interval or device offsets and a shared step size. + * Move-only; copy view() to borrow the parameters. + * Migration: replace q.min_ with std::get::range_type>(q.value).min_ + * (likewise for max_). + */ template struct quantizer { using view_type = quantizer_const_view; diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index a6c167e8ac..aeff0c9318 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -109,6 +109,8 @@ struct offsets_type { Owns either a global interval or device offsets and a shared step size. +Move-only; copy view() to borrow the parameters. Migration: replace q.min_ with std::get<quantizer<T>::range_type>(q.value).min_ (likewise for max_). + ```cpp template struct quantizer { From 9a356af2a9fe4370ef5cc6323de31e1e7d3b79dd Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 00:51:52 +0000 Subject: [PATCH 05/24] Align SQ builder arguments and generated API documentation --- cpp/include/cuvs/core/dataset/sq.hpp | 10 ++++--- cpp/tests/preprocessing/scalar_dataset.cu | 14 ++++++---- fern/pages/cpp_api/cpp-api-core-dataset-sq.md | 26 +++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index fa9471c2c5..d766bfeafd 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -84,14 +84,14 @@ using host_sq_dataset = namespace preprocessing::quantize::scalar { /** - * Train and encode a device FP32 corpus, using per-dimension ranges by default. + * @defgroup scalar_dataset Scalar-quantized dataset construction + * @{ + * Train and encode a device FP32 corpus with the selected range mode. * Reuse result.const_manifest_view() with transform * for query batches. For separate training/encoding, use train and transform. */ inline device_sq_dataset build( - raft::resources const& res, - raft::device_matrix_view dataset, - params params = {.range = range_mode::per_dimension}) + raft::resources const& res, params params, raft::device_matrix_view dataset) { auto quantizer = train(res, params, dataset); auto codes = raft::make_device_matrix(res, dataset.extent(0), dataset.extent(1)); @@ -99,6 +99,8 @@ inline device_sq_dataset build( return device_sq_dataset{std::move(codes), std::move(quantizer)}; } +/** @} */ + } // namespace preprocessing::quantize::scalar } // namespace CUVS_EXPORT cuvs diff --git a/cpp/tests/preprocessing/scalar_dataset.cu b/cpp/tests/preprocessing/scalar_dataset.cu index 12c0c48969..3c967c2fdc 100644 --- a/cpp/tests/preprocessing/scalar_dataset.cu +++ b/cpp/tests/preprocessing/scalar_dataset.cu @@ -34,7 +34,8 @@ TEST(ScalarDataset, CorpusAndQueriesShareQuantizer) std::vector corpus{0.f, 10.f, 9.f, 254.f, 14.f, 9.f}; auto input = raft::make_device_matrix(res, 2, 3); raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); - auto encoded = build(res, raft::make_const_mdspan(input.view())); + auto encoded = + build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); ASSERT_EQ(encoded.n_rows(), 2); ASSERT_EQ(encoded.dim(), 3u); auto quantizer = encoded.const_manifest_view(); @@ -69,7 +70,8 @@ TEST(ScalarDataset, PerDimensionQuantiles) auto input = raft::make_device_matrix(res, rows, dim); raft::update_device( input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build(res, raft::make_const_mdspan(input.view())); + auto encoded = + build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); auto quantizer = encoded.const_manifest_view(); EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).offsets.data_handle(), dim), (std::vector{499.5f, 10999.f, 9.f})); @@ -98,7 +100,8 @@ TEST(ScalarDataset, ConstantCorpus) auto input = raft::make_device_matrix(res, 2, 2); raft::update_device( input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build(res, raft::make_const_mdspan(input.view())); + auto encoded = + build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); EXPECT_FLOAT_EQ(std::get(encoded.const_manifest_view().value).scale, 1.f); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 4), (std::vector(4, 0))); @@ -115,7 +118,8 @@ TEST(ScalarDataset, ExtremeFiniteCorpus) auto input = raft::make_device_matrix(res, 2, 1); raft::update_device( input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build(res, raft::make_const_mdspan(input.view())); + auto encoded = + build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), (std::vector{-127, 127})); inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); @@ -167,7 +171,7 @@ TEST(ScalarDataset, GlobalDatasetStillWorks) std::vector values{-1.f, 1.f}; raft::update_device( input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build(res, raft::make_const_mdspan(input.view()), params{1.f}); + auto encoded = build(res, params{1.f}, raft::make_const_mdspan(input.view())); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), (std::vector{-128, 127})); inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); diff --git a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md index d51cd95814..47dbd07bc1 100644 --- a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md +++ b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md @@ -19,3 +19,29 @@ manifest holds the quantizer and the dimension is still a property of the data. template struct sq_spec; ``` + +## Scalar-quantized dataset construction + + +### preprocessing::quantize::scalar::build + +Train and encode a device FP32 corpus with the selected range mode. + +```cpp +inline device_sq_dataset build( +raft::resources const& res, params params, raft::device_matrix_view dataset); +``` + +Reuse result.const_manifest_view() with transform for query batches. For separate training/encoding, use train and transform. + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | | `raft::resources const&` | | +| `params` | | `params` | | +| `dataset` | | `raft::device_matrix_view` | | + +**Returns** + +`inline device_sq_dataset` From e73cce0002494723d277134f6fefb70bbfe396db Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 07:40:14 +0000 Subject: [PATCH 06/24] Rename scalar quantizer offsets to shifts --- cpp/include/cuvs/core/dataset/sq.hpp | 2 +- .../cuvs/preprocessing/quantize/scalar.hpp | 36 +++++----- .../preprocessing/quantize/detail/scalar.cuh | 70 +++++++++---------- cpp/tests/preprocessing/scalar_dataset.cu | 32 ++++----- .../cpp-api-preprocessing-quantize-scalar.md | 48 ++++++------- 5 files changed, 94 insertions(+), 94 deletions(-) diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index d766bfeafd..20a1b6aff7 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -34,7 +34,7 @@ struct sq_spec { using math_type = MathT; - // The manifest is either a global interval or owning per-dimension offsets and a shared scale. + // The manifest is either a global interval or owning per-dimension shifts and a shared scale. using manifest_type = cuvs::preprocessing::quantize::scalar::quantizer; using manifest_view_type = typename manifest_type::view_type; diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index d62b5ab5b2..bc8873ddfe 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -27,7 +27,7 @@ namespace scalar { * @{ */ -/** Select a global interval or per-dimension offsets with a shared scale. */ +/** Select a global interval or per-dimension shifts with a shared scale. */ enum class range_mode { global, per_dimension }; /** @@ -40,13 +40,13 @@ struct params { */ float quantile = 0.99; /** Per-dimension training uses all device FP32 corpus rows, applying quantile independently - * to each dimension. Midpoint offsets share the widest half-range divided by 127. + * to each dimension. Midpoint shifts share the widest half-range divided by 127. * Requires nonempty finite input. Constant ranges use scale 1; positive scales are floored * to the smallest normal float. Training synchronizes to return the scale. */ range_mode range = range_mode::global; }; -/** Read-only scalar quantizer parameters. The owner must outlive stream work using offsets. */ +/** Read-only scalar quantizer parameters. The owner must outlive stream work using shifts. */ template struct quantizer_const_view { /** A global interval mapped to [-128, 127]. */ @@ -54,17 +54,17 @@ struct quantizer_const_view { T min_; T max_; }; - /** Encode clamp(rintf((x - offsets[j]) / scale), -127, 127), with ties to even. - * Decode offsets[j] + scale * code, saturating to finite FP32 limits. */ - struct offsets_type { - raft::device_vector_view offsets; + /** Encode clamp(rintf((x - shifts[j]) / scale), -127, 127), with ties to even. + * Decode shifts[j] + scale * code, saturating to finite FP32 limits. */ + struct shifts_type { + raft::device_vector_view shifts; T scale; }; - std::variant value; + std::variant value; }; /** - * Owns either a global interval or device offsets and a shared step size. + * Owns either a global interval or device shifts and a shared step size. * Move-only; copy view() to borrow the parameters. * Migration: replace q.min_ with std::get::range_type>(q.value).min_ * (likewise for max_). @@ -74,28 +74,28 @@ struct quantizer { using view_type = quantizer_const_view; using range_type = typename view_type::range_type; /** Owning per-dimension parameters. */ - struct offsets_type { - raft::device_vector offsets; + struct shifts_type { + raft::device_vector shifts; T scale; }; - std::variant value; + std::variant value; /** Construct an empty global interval. */ quantizer() : value{range_type{}} {} /** Construct a global interval. */ quantizer(T min, T max) : value{range_type{min, max}} {} - /** Take ownership of device offsets and their shared scale. */ - quantizer(raft::device_vector&& offsets, T scale) - : value{offsets_type{std::move(offsets), scale}} + /** Take ownership of device shifts and their shared scale. */ + quantizer(raft::device_vector&& shifts, T scale) + : value{shifts_type{std::move(shifts), scale}} { } - /** Borrow read-only parameters; the owner must outlive stream work using offsets. */ + /** Borrow read-only parameters; the owner must outlive stream work using shifts. */ [[nodiscard]] auto view() const noexcept -> view_type { if (auto const* range = std::get_if(&value)) { return {*range}; } - auto const& offsets = std::get(value); - return {typename view_type::offsets_type{offsets.offsets.view(), offsets.scale}}; + auto const& shifts = std::get(value); + return {typename view_type::shifts_type{shifts.shifts.view(), shifts.scale}}; } /** An owning quantizer can be passed wherever a read-only view is accepted. */ operator view_type() const noexcept { return view(); } // NOLINT(google-explicit-constructor) diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index 3f6b90fd4b..fde0a7c104 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -133,20 +133,20 @@ std::tuple quantile_min_max( return {minmax_h[0], minmax_h[1]}; } -inline quantizer train_offsets(raft::resources const& res, - raft::device_matrix_view dataset, - params params) +inline quantizer train_shifts(raft::resources const& res, + raft::device_matrix_view dataset, + params params) { RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); auto const dim = dataset.extent(1); RAFT_EXPECTS(dataset.extent(0) > 0 && dim > 0, "Training requires nonempty rows and dimensions"); - auto offsets = raft::make_device_vector(res, dim); + auto shifts = raft::make_device_vector(res, dim); auto maxima = raft::make_device_vector(res, dim); auto max_half_range = raft::make_device_vector(res, 1); if (params.quantile == 1.f) { raft::linalg::reduce(res, dataset, - offsets.view(), + shifts.view(), std::numeric_limits::infinity(), false, raft::identity_op{}, @@ -181,20 +181,20 @@ inline quantizer train_offsets(raft::resources const& res, stream)); } raft::copy(res, - raft::make_device_scalar_view(offsets.data_handle() + j), + raft::make_device_scalar_view(shifts.data_handle() + j), raft::make_device_scalar_view(column.data_handle() + lower)); raft::copy(res, raft::make_device_scalar_view(maxima.data_handle() + j), raft::make_device_scalar_view(column.data_handle() + upper)); } } - auto* offset_ptr = offsets.data_handle(); + auto* shift_ptr = shifts.data_handle(); auto const* max_ptr = maxima.data_handle(); - raft::linalg::map_offset(res, maxima.view(), [offset_ptr, max_ptr] __device__(int64_t j) { + raft::linalg::map_offset(res, maxima.view(), [shift_ptr, max_ptr] __device__(int64_t j) { // Halve before adding/subtracting to avoid overflow for large finite inputs. - float const lo = offset_ptr[j] * 0.5f; + float const lo = shift_ptr[j] * 0.5f; float const hi = max_ptr[j] * 0.5f; - offset_ptr[j] = lo + hi; + shift_ptr[j] = lo + hi; return hi - lo; }); raft::linalg::reduce( @@ -212,51 +212,51 @@ inline quantizer train_offsets(raft::resources const& res, raft::resource::sync_stream(res); float const scale = half_range > 0.f ? std::max(half_range / 127.f, std::numeric_limits::min()) : 1.f; - return {std::move(offsets), scale}; + return {std::move(shifts), scale}; } template -void check_per_dim_shapes(quantizer_const_view::offsets_type quantizer, +void check_per_dim_shapes(quantizer_const_view::shifts_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { RAFT_EXPECTS( std::isfinite(quantizer.scale) && quantizer.scale > 0.f && std::isfinite(1.f / quantizer.scale), "Scale and its reciprocal must be finite and positive"); - RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.offsets.extent(0), - "Input width must match the nonempty offsets vector"); + RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.shifts.extent(0), + "Input width must match the nonempty shifts vector"); RAFT_EXPECTS(dataset.extent(0) == out.extent(0) && dataset.extent(1) == out.extent(1), "Input and output shapes must match"); } inline void transform(raft::resources const& res, - quantizer_const_view::offsets_type quantizer, + quantizer_const_view::shifts_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* offsets = quantizer.offsets.data_handle(); - auto const dim = dataset.extent(1); - float const inv = 1.f / quantizer.scale; - raft::linalg::map_offset(res, out, [input, offsets, dim, inv] __device__(int64_t i) { - float const code = rintf((input[i] - offsets[i % dim]) * inv); + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + float const inv = 1.f / quantizer.scale; + raft::linalg::map_offset(res, out, [input, shifts, dim, inv] __device__(int64_t i) { + float const code = rintf((input[i] - shifts[i % dim]) * inv); return static_cast(fminf(fmaxf(code, -127.f), 127.f)); }); } inline void inverse_transform(raft::resources const& res, - quantizer_const_view::offsets_type quantizer, + quantizer_const_view::shifts_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* offsets = quantizer.offsets.data_handle(); - auto const dim = dataset.extent(1); - float const scale = quantizer.scale; - raft::linalg::map_offset(res, out, [input, offsets, dim, scale] __device__(int64_t i) { - float const value = fmaf(scale, static_cast(input[i]), offsets[i % dim]); + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + float const scale = quantizer.scale; + raft::linalg::map_offset(res, out, [input, shifts, dim, scale] __device__(int64_t i) { + float const value = fmaf(scale, static_cast(input[i]), shifts[i % dim]); constexpr float limit = std::numeric_limits::max(); return fminf(fmaxf(value, -limit), limit); }); @@ -274,7 +274,7 @@ cuvs::preprocessing::quantize::scalar::quantizer train( if (params.range == range_mode::per_dimension) { if constexpr (std::is_same_v) { - return train_offsets(res, dataset, params); + return train_shifts(res, dataset, params); } else { RAFT_FAIL("Per-dimension training requires device FP32 input"); } @@ -314,10 +314,10 @@ void transform(raft::resources const& res, raft::device_matrix_view out) { auto view = quantizer; - if (auto const* offsets = - std::get_if::offsets_type>(&view.value)) { + if (auto const* shifts = + std::get_if::shifts_type>(&view.value)) { if constexpr (std::is_same_v) { - detail::transform(res, *offsets, dataset, out); + detail::transform(res, *shifts, dataset, out); } else { RAFT_FAIL("Per-dimension quantization requires device FP32 input"); } @@ -357,10 +357,10 @@ void inverse_transform(raft::resources const& res, raft::device_matrix_view out) { auto view = quantizer; - if (auto const* offsets = - std::get_if::offsets_type>(&view.value)) { + if (auto const* shifts = + std::get_if::shifts_type>(&view.value)) { if constexpr (std::is_same_v) { - detail::inverse_transform(res, *offsets, dataset, out); + detail::inverse_transform(res, *shifts, dataset, out); } else { RAFT_FAIL("Per-dimension quantization requires device FP32 input"); } diff --git a/cpp/tests/preprocessing/scalar_dataset.cu b/cpp/tests/preprocessing/scalar_dataset.cu index 3c967c2fdc..d683f74af7 100644 --- a/cpp/tests/preprocessing/scalar_dataset.cu +++ b/cpp/tests/preprocessing/scalar_dataset.cu @@ -15,7 +15,7 @@ namespace cuvs::preprocessing::quantize::scalar { -using offsets_view = quantizer_const_view::offsets_type; +using shifts_view = quantizer_const_view::shifts_type; template auto copy_to_host(raft::resources const& res, const T* data, int64_t size) @@ -39,8 +39,8 @@ TEST(ScalarDataset, CorpusAndQueriesShareQuantizer) ASSERT_EQ(encoded.n_rows(), 2); ASSERT_EQ(encoded.dim(), 3u); auto quantizer = encoded.const_manifest_view(); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); - EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).offsets.data_handle(), 3), + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); + EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).shifts.data_handle(), 3), (std::vector{127.f, 12.f, 9.f})); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 6), (std::vector{-127, -2, 0, 127, 2, 0})); @@ -73,19 +73,19 @@ TEST(ScalarDataset, PerDimensionQuantiles) auto encoded = build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); auto quantizer = encoded.const_manifest_view(); - EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).offsets.data_handle(), dim), + EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).shifts.data_handle(), dim), (std::vector{499.5f, 10999.f, 9.f})); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 991.f / 127.f); + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 991.f / 127.f); auto codes = copy_to_host(res, encoded.data_const_view().data_handle(), rows * dim); EXPECT_EQ(codes[1], 127); EXPECT_EQ(codes[(rows - 1) * dim + 1], -127); EXPECT_EQ(codes[2], 0); auto full = train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(full.view().value).scale, 999.f / 127.f); + EXPECT_FLOAT_EQ(std::get(full.view().value).scale, 999.f / 127.f); auto trimmed = train(res, params{0.5f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(trimmed.view().value).scale, 499.f / 127.f); + EXPECT_FLOAT_EQ(std::get(trimmed.view().value).scale, 499.f / 127.f); for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { EXPECT_THROW( train(res, params{q, range_mode::per_dimension}, raft::make_const_mdspan(input.view())), @@ -102,7 +102,7 @@ TEST(ScalarDataset, ConstantCorpus) input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); auto encoded = build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(encoded.const_manifest_view().value).scale, 1.f); + EXPECT_FLOAT_EQ(std::get(encoded.const_manifest_view().value).scale, 1.f); EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 4), (std::vector(4, 0))); inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); @@ -128,7 +128,7 @@ TEST(ScalarDataset, ExtremeFiniteCorpus) EXPECT_TRUE(std::isfinite(decoded[i])); EXPECT_NEAR(decoded[i], corpus[i], - std::get(encoded.const_manifest_view().value).scale * 0.5f); + std::get(encoded.const_manifest_view().value).scale * 0.5f); } } } @@ -136,13 +136,13 @@ TEST(ScalarDataset, ExtremeFiniteCorpus) TEST(ScalarDataset, CustomParametersAndValidation) { raft::device_resources res; - auto offsets = raft::make_device_vector(res, 2); - std::vector custom_offsets{100.f, -100.f}; - raft::update_device(offsets.data_handle(), - custom_offsets.data(), - custom_offsets.size(), + auto shifts = raft::make_device_vector(res, 2); + std::vector custom_shifts{100.f, -100.f}; + raft::update_device(shifts.data_handle(), + custom_shifts.data(), + custom_shifts.size(), raft::resource::get_cuda_stream(res)); - quantizer_const_view quantizer{offsets_view{raft::make_const_mdspan(offsets.view()), 2.f}}; + quantizer_const_view quantizer{shifts_view{raft::make_const_mdspan(shifts.view()), 2.f}}; auto input = raft::make_device_matrix(res, 1, 2); std::vector values{104.f, -106.f}; raft::update_device( @@ -154,7 +154,7 @@ TEST(ScalarDataset, CustomParametersAndValidation) EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), raft::logic_error); for (float scale : {0.f, -1.f, std::numeric_limits::infinity()}) { - std::get(quantizer.value).scale = scale; + std::get(quantizer.value).scale = scale; EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()), raft::logic_error); } diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index aeff0c9318..7673ce36df 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -11,7 +11,7 @@ _Source header: `cuvs/preprocessing/quantize/scalar.hpp`_ ### preprocessing::quantize::scalar::range_mode -Select a global interval or per-dimension offsets with a shared scale. +Select a global interval or per-dimension shifts with a shared scale. ```cpp enum class range_mode { @@ -44,17 +44,17 @@ struct params { | Name | Type | Description | | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | -| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses all device FP32 corpus rows, applying quantile independently to each dimension. Midpoint offsets share the widest half-range divided by 127. Requires nonempty finite input. Constant ranges use scale 1; positive scales are floored to the smallest normal float. Training synchronizes to return the scale. | +| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses all device FP32 corpus rows, applying quantile independently to each dimension. Midpoint shifts share the widest half-range divided by 127. Requires nonempty finite input. Constant ranges use scale 1; positive scales are floored to the smallest normal float. Training synchronizes to return the scale. | ### preprocessing::quantize::scalar::quantizer_const_view -Read-only scalar quantizer parameters. The owner must outlive stream work using offsets. +Read-only scalar quantizer parameters. The owner must outlive stream work using shifts. ```cpp template struct quantizer_const_view { - std::variant value; + std::variant value; }; ``` @@ -62,7 +62,7 @@ struct quantizer_const_view { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | ### preprocessing::quantize::scalar::quantizer_const_view::range_type @@ -83,16 +83,16 @@ struct range_type { | `min_` | `T` | | | `max_` | `T` | | - -### preprocessing::quantize::scalar::quantizer_const_view::offsets_type + +### preprocessing::quantize::scalar::quantizer_const_view::shifts_type -Encode clamp(rintf((x - offsets[j]) / scale), -127, 127), with ties to even. +Encode clamp(rintf((x - shifts[j]) / scale), -127, 127), with ties to even. -Decode offsets[j] + scale * code, saturating to finite FP32 limits. +Decode shifts[j] + scale * code, saturating to finite FP32 limits. ```cpp -struct offsets_type { - raft::device_vector_view offsets; +struct shifts_type { + raft::device_vector_view shifts; T scale; }; ``` @@ -101,20 +101,20 @@ struct offsets_type { | Name | Type | Description | | --- | --- | --- | -| `offsets` | `raft::device_vector_view` | | +| `shifts` | `raft::device_vector_view` | | | `scale` | `T` | | ### preprocessing::quantize::scalar::quantizer -Owns either a global interval or device offsets and a shared step size. +Owns either a global interval or device shifts and a shared step size. Move-only; copy view() to borrow the parameters. Migration: replace q.min_ with std::get<quantizer<T>::range_type>(q.value).min_ (likewise for max_). ```cpp template struct quantizer { - std::variant value; + std::variant value; }; ``` @@ -122,16 +122,16 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | - -### preprocessing::quantize::scalar::quantizer::offsets_type + +### preprocessing::quantize::scalar::quantizer::shifts_type Owning per-dimension parameters. ```cpp -struct offsets_type { - raft::device_vector offsets; +struct shifts_type { + raft::device_vector shifts; T scale; }; ``` @@ -140,7 +140,7 @@ struct offsets_type { | Name | Type | Description | | --- | --- | --- | -| `offsets` | `raft::device_vector` | | +| `shifts` | `raft::device_vector` | | | `scale` | `T` | | @@ -177,17 +177,17 @@ quantizer(T min, T max); **Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` -Take ownership of device offsets and their shared scale. +Take ownership of device shifts and their shared scale. ```cpp -quantizer(raft::device_vector&& offsets, T scale); +quantizer(raft::device_vector&& shifts, T scale); ``` **Parameters** | Name | Direction | Type | Description | | --- | --- | --- | --- | -| `offsets` | | `raft::device_vector&&` | | +| `shifts` | | `raft::device_vector&&` | | | `scale` | | `T` | | **Returns** @@ -197,7 +197,7 @@ quantizer(raft::device_vector&& offsets, T scale); ### preprocessing::quantize::scalar::quantizer::view -Borrow read-only parameters; the owner must outlive stream work using offsets. +Borrow read-only parameters; the owner must outlive stream work using shifts. ```cpp [[nodiscard]] auto view() const noexcept -> view_type; From e46013e66d66bc78a675b4fb3e445dcc915dad93 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 17:15:39 +0000 Subject: [PATCH 07/24] Construct scalar quantizers from bounds and align dataset tests --- .../cuvs/preprocessing/quantize/scalar.hpp | 12 +- .../preprocessing/quantize/detail/scalar.cuh | 78 +++--- cpp/src/preprocessing/quantize/scalar.cu | 15 ++ cpp/tests/preprocessing/scalar_dataset.cu | 240 ++++++++++++------ .../cpp-api-preprocessing-quantize-scalar.md | 14 +- 5 files changed, 235 insertions(+), 124 deletions(-) diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index bc8873ddfe..9aa60047f1 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -84,11 +85,12 @@ struct quantizer { quantizer() : value{range_type{}} {} /** Construct a global interval. */ quantizer(T min, T max) : value{range_type{min, max}} {} - /** Take ownership of device shifts and their shared scale. */ - quantizer(raft::device_vector&& shifts, T scale) - : value{shifts_type{std::move(shifts), scale}} - { - } + /** Construct from nonempty device FP32 bounds of equal length, with finite min <= max. + * Derives midpoint shifts and a shared scale; synchronizes before returning. */ + quantizer(raft::resources const& res, + raft::device_vector_view mins, + raft::device_vector_view maxs) + requires std::is_same_v; /** Borrow read-only parameters; the owner must outlive stream work using shifts. */ [[nodiscard]] auto view() const noexcept -> view_type diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index fde0a7c104..e35e8757ed 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -133,16 +133,52 @@ std::tuple quantile_min_max( return {minmax_h[0], minmax_h[1]}; } -inline quantizer train_shifts(raft::resources const& res, - raft::device_matrix_view dataset, - params params) +inline quantizer::shifts_type make_shifts(raft::resources const& res, + raft::device_vector&& shifts, + raft::device_vector&& maxima) +{ + auto const dim = shifts.extent(0); + auto max_half_range = raft::make_device_vector(res, 1); + auto* shift_ptr = shifts.data_handle(); + auto const* max_ptr = maxima.data_handle(); + raft::linalg::map_offset(res, maxima.view(), [shift_ptr, max_ptr] __device__(int64_t j) { + // Halve before adding/subtracting to avoid overflow for large finite inputs. + if (!isfinite(shift_ptr[j]) || !isfinite(max_ptr[j]) || shift_ptr[j] > max_ptr[j]) { + return std::numeric_limits::infinity(); + } + float const lo = shift_ptr[j] * 0.5f; + float const hi = max_ptr[j] * 0.5f; + shift_ptr[j] = lo + hi; + return hi - lo; + }); + raft::linalg::reduce( + res, + raft::make_device_matrix_view(maxima.data_handle(), 1, dim), + max_half_range.view(), + 0.f, + false, + raft::identity_op{}, + raft::max_op{}); + float half_range = 0.f; + raft::copy(res, + raft::make_host_scalar_view(&half_range), + raft::make_device_scalar_view(max_half_range.data_handle())); + raft::resource::sync_stream(res); + RAFT_EXPECTS(std::isfinite(half_range), "Bounds must be finite with min <= max"); + float const scale = + half_range > 0.f ? std::max(half_range / 127.f, std::numeric_limits::min()) : 1.f; + return {std::move(shifts), scale}; +} + +inline quantizer train(raft::resources const& res, + raft::device_matrix_view dataset, + params params) { RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); auto const dim = dataset.extent(1); RAFT_EXPECTS(dataset.extent(0) > 0 && dim > 0, "Training requires nonempty rows and dimensions"); - auto shifts = raft::make_device_vector(res, dim); - auto maxima = raft::make_device_vector(res, dim); - auto max_half_range = raft::make_device_vector(res, 1); + auto shifts = raft::make_device_vector(res, dim); + auto maxima = raft::make_device_vector(res, dim); if (params.quantile == 1.f) { raft::linalg::reduce(res, dataset, @@ -188,31 +224,9 @@ inline quantizer train_shifts(raft::resources const& res, raft::make_device_scalar_view(column.data_handle() + upper)); } } - auto* shift_ptr = shifts.data_handle(); - auto const* max_ptr = maxima.data_handle(); - raft::linalg::map_offset(res, maxima.view(), [shift_ptr, max_ptr] __device__(int64_t j) { - // Halve before adding/subtracting to avoid overflow for large finite inputs. - float const lo = shift_ptr[j] * 0.5f; - float const hi = max_ptr[j] * 0.5f; - shift_ptr[j] = lo + hi; - return hi - lo; - }); - raft::linalg::reduce( - res, - raft::make_device_matrix_view(maxima.data_handle(), 1, dim), - max_half_range.view(), - 0.f, - false, - raft::identity_op{}, - raft::max_op{}); - float half_range = 0.f; - raft::copy(res, - raft::make_host_scalar_view(&half_range), - raft::make_device_scalar_view(max_half_range.data_handle())); - raft::resource::sync_stream(res); - float const scale = - half_range > 0.f ? std::max(half_range / 127.f, std::numeric_limits::min()) : 1.f; - return {std::move(shifts), scale}; + quantizer result; + result.value = make_shifts(res, std::move(shifts), std::move(maxima)); + return result; } template @@ -274,7 +288,7 @@ cuvs::preprocessing::quantize::scalar::quantizer train( if (params.range == range_mode::per_dimension) { if constexpr (std::is_same_v) { - return train_shifts(res, dataset, params); + return train(res, dataset, params); } else { RAFT_FAIL("Per-dimension training requires device FP32 input"); } diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index 6f2c482022..d021200afe 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -9,6 +9,21 @@ namespace cuvs::preprocessing::quantize::scalar { +template +quantizer::quantizer(raft::resources const& res, + raft::device_vector_view mins, + raft::device_vector_view maxs) + requires std::is_same_v +{ + RAFT_EXPECTS(mins.extent(0) > 0 && mins.extent(0) == maxs.extent(0), + "Bounds must be nonempty and have equal lengths"); + auto minima = raft::make_device_vector(res, mins.extent(0)); + auto maxima = raft::make_device_vector(res, maxs.extent(0)); + raft::copy(res, minima.view(), mins); + raft::copy(res, maxima.view(), maxs); + value = detail::make_shifts(res, std::move(minima), std::move(maxima)); +} + #define CUVS_INST_QUANTIZATION(T, QuantI) \ auto train(raft::resources const& res, \ const params params, \ diff --git a/cpp/tests/preprocessing/scalar_dataset.cu b/cpp/tests/preprocessing/scalar_dataset.cu index d683f74af7..fca99621e7 100644 --- a/cpp/tests/preprocessing/scalar_dataset.cu +++ b/cpp/tests/preprocessing/scalar_dataset.cu @@ -3,9 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "../test_utils.cuh" + #include #include -#include +#include +#include #include #include @@ -17,19 +20,71 @@ namespace cuvs::preprocessing::quantize::scalar { using shifts_view = quantizer_const_view::shifts_type; -template -auto copy_to_host(raft::resources const& res, const T* data, int64_t size) +struct ScalarDatasetInputs { + int64_t n_samples; + int64_t n_features; + std::vector dataset; + std::vector codes; + float tolerance; +}; + +std::ostream& operator<<(std::ostream& os, const ScalarDatasetInputs& inputs) { - std::vector result(size); - auto stream = raft::resource::get_cuda_stream(res); - raft::update_host(result.data(), data, size, stream); - raft::resource::sync_stream(res); - return result; + return os << "n_samples:" << inputs.n_samples << " n_features:" << inputs.n_features + << " tolerance:" << inputs.tolerance; } -TEST(ScalarDataset, CorpusAndQueriesShareQuantizer) +class ScalarDatasetTest : public ::testing::TestWithParam { + public: + ScalarDatasetTest() + : params_(GetParam()), + stream(raft::resource::get_cuda_stream(handle).get()), + dataset_( + raft::make_device_matrix(handle, params_.n_samples, params_.n_features)) + { + } + + protected: + void SetUp() override + { + raft::update_device( + dataset_.data_handle(), params_.dataset.data(), params_.dataset.size(), stream); + } + + void testScalarDataset() + { + auto encoded = build( + handle, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(dataset_.view())); + EXPECT_TRUE(devArrMatchHost(params_.codes.data(), + encoded.data_const_view().data_handle(), + params_.codes.size(), + cuvs::Compare(), + stream)); + if (params_.tolerance == 0.f) { + EXPECT_FLOAT_EQ(std::get(encoded.const_manifest_view().value).scale, 1.f); + } + inverse_transform( + handle, encoded.const_manifest_view(), encoded.data_const_view(), dataset_.view()); + auto reconstructed = + raft::make_host_matrix(params_.n_samples, params_.n_features); + raft::update_host(reconstructed.data_handle(), dataset_.data_handle(), dataset_.size(), stream); + raft::resource::sync_stream(handle); + for (size_t i = 0; i < params_.dataset.size(); ++i) { + EXPECT_TRUE(std::isfinite(reconstructed.data_handle()[i])); + EXPECT_NEAR(reconstructed.data_handle()[i], params_.dataset[i], params_.tolerance); + } + } + + private: + raft::resources handle; + ScalarDatasetInputs params_; + cudaStream_t stream; + raft::device_matrix dataset_; +}; + +TEST(ScalarDatasetTest, CorpusAndQueriesShareQuantizer) { - raft::device_resources res; + raft::resources res; auto stream = raft::resource::get_cuda_stream(res); std::vector corpus{0.f, 10.f, 9.f, 254.f, 14.f, 9.f}; auto input = raft::make_device_matrix(res, 2, 3); @@ -40,26 +95,38 @@ TEST(ScalarDataset, CorpusAndQueriesShareQuantizer) ASSERT_EQ(encoded.dim(), 3u); auto quantizer = encoded.const_manifest_view(); EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); - EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).shifts.data_handle(), 3), - (std::vector{127.f, 12.f, 9.f})); - EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 6), - (std::vector{-127, -2, 0, 127, 2, 0})); + EXPECT_TRUE(devArrMatchHost(std::vector{127.f, 12.f, 9.f}.data(), + std::get(quantizer.value).shifts.data_handle(), + 3, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); + EXPECT_TRUE(devArrMatchHost(std::vector{-127, -2, 0, 127, 2, 0}.data(), + encoded.data_const_view().data_handle(), + 6, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); // The corpus manifest, rather than query extrema, controls ties and saturation. std::vector queries{127.5f, 11.5f, 1000.f, 128.5f, 9.5f, -1000.f}; raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); auto codes = raft::make_device_matrix(res, 2, 3); transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_EQ(copy_to_host(res, codes.data_handle(), 6), - (std::vector{0, 0, 127, 2, -2, -127})); + EXPECT_TRUE(devArrMatchHost(std::vector{0, 0, 127, 2, -2, -127}.data(), + codes.data_handle(), + 6, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); - EXPECT_EQ(copy_to_host(res, input.data_handle(), 6), - (std::vector{127.f, 12.f, 136.f, 129.f, 10.f, -118.f})); + EXPECT_TRUE(devArrMatchHost(std::vector{127.f, 12.f, 136.f, 129.f, 10.f, -118.f}.data(), + input.data_handle(), + 6, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); } -TEST(ScalarDataset, PerDimensionQuantiles) +TEST(ScalarDatasetTest, PerDimensionQuantiles) { - raft::device_resources res; + raft::resources res; constexpr int64_t rows = 1000, dim = 3; std::vector corpus(rows * dim); for (int64_t i = 0; i < rows; ++i) { @@ -73,10 +140,18 @@ TEST(ScalarDataset, PerDimensionQuantiles) auto encoded = build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); auto quantizer = encoded.const_manifest_view(); - EXPECT_EQ(copy_to_host(res, std::get(quantizer.value).shifts.data_handle(), dim), - (std::vector{499.5f, 10999.f, 9.f})); + EXPECT_TRUE(devArrMatchHost(std::vector{499.5f, 10999.f, 9.f}.data(), + std::get(quantizer.value).shifts.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 991.f / 127.f); - auto codes = copy_to_host(res, encoded.data_const_view().data_handle(), rows * dim); + std::vector codes(rows * dim); + raft::update_host(codes.data(), + encoded.data_const_view().data_handle(), + codes.size(), + raft::resource::get_cuda_stream(res)); + raft::resource::sync_stream(res); EXPECT_EQ(codes[1], 127); EXPECT_EQ(codes[(rows - 1) * dim + 1], -127); EXPECT_EQ(codes[2], 0); @@ -93,89 +168,71 @@ TEST(ScalarDataset, PerDimensionQuantiles) } } -TEST(ScalarDataset, ConstantCorpus) -{ - raft::device_resources res; - std::vector corpus{10.f, -7.f, 10.f, -7.f}; - auto input = raft::make_device_matrix(res, 2, 2); - raft::update_device( - input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = - build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(encoded.const_manifest_view().value).scale, 1.f); - EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 4), - (std::vector(4, 0))); - inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); - EXPECT_EQ(copy_to_host(res, input.data_handle(), 4), corpus); -} - -TEST(ScalarDataset, ExtremeFiniteCorpus) -{ - raft::device_resources res; - float const max = std::numeric_limits::max(); - for (float lo : {-max, -std::ldexp(124.f, 104)}) { - std::vector corpus{lo, max}; - auto input = raft::make_device_matrix(res, 2, 1); - raft::update_device( - input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = - build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), - (std::vector{-127, 127})); - inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); - auto decoded = copy_to_host(res, input.data_handle(), 2); - for (int i = 0; i < 2; ++i) { - EXPECT_TRUE(std::isfinite(decoded[i])); - EXPECT_NEAR(decoded[i], - corpus[i], - std::get(encoded.const_manifest_view().value).scale * 0.5f); - } - } -} - -TEST(ScalarDataset, CustomParametersAndValidation) +TEST(ScalarDatasetTest, CustomParametersAndValidation) { - raft::device_resources res; - auto shifts = raft::make_device_vector(res, 2); - std::vector custom_shifts{100.f, -100.f}; - raft::update_device(shifts.data_handle(), - custom_shifts.data(), - custom_shifts.size(), - raft::resource::get_cuda_stream(res)); - quantizer_const_view quantizer{shifts_view{raft::make_const_mdspan(shifts.view()), 2.f}}; + raft::resources res; + auto mins = raft::make_device_vector(res, 2); + auto maxs = raft::make_device_vector(res, 2); + std::vector lower{-154.f, -104.f}, upper{354.f, -96.f}; + auto stream = raft::resource::get_cuda_stream(res); + raft::update_device(mins.data_handle(), lower.data(), lower.size(), stream); + raft::update_device(maxs.data_handle(), upper.data(), upper.size(), stream); + scalar::quantizer owner{ + res, raft::make_const_mdspan(mins.view()), raft::make_const_mdspan(maxs.view())}; + auto quantizer = owner.view(); + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 2.f); + EXPECT_TRUE(devArrMatchHost(std::vector{100.f, -100.f}.data(), + std::get(quantizer.value).shifts.data_handle(), + 2, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); auto input = raft::make_device_matrix(res, 1, 2); std::vector values{104.f, -106.f}; raft::update_device( input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); auto codes = raft::make_device_matrix(res, 1, 2); transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_EQ(copy_to_host(res, codes.data_handle(), 2), (std::vector{2, -3})); + EXPECT_TRUE(devArrMatchHost(std::vector{2, -3}.data(), + codes.data_handle(), + 2, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); auto wrong_shape = raft::make_device_matrix(res, 2, 1); EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), raft::logic_error); - for (float scale : {0.f, -1.f, std::numeric_limits::infinity()}) { - std::get(quantizer.value).scale = scale; - EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()), - raft::logic_error); - } + EXPECT_THROW((scalar::quantizer{ + res, + raft::make_const_mdspan(mins.view()), + raft::make_device_vector_view(maxs.data_handle(), 1)}), + raft::logic_error); + EXPECT_THROW((scalar::quantizer{ + res, raft::make_const_mdspan(maxs.view()), raft::make_const_mdspan(mins.view())}), + raft::logic_error); EXPECT_THROW(train(res, params{.range = range_mode::per_dimension}, raft::make_device_matrix_view(nullptr, 0, 2)), raft::logic_error); } -TEST(ScalarDataset, GlobalDatasetStillWorks) +TEST(ScalarDatasetTest, GlobalDatasetStillWorks) { - raft::device_resources res; + raft::resources res; auto input = raft::make_device_matrix(res, 1, 2); std::vector values{-1.f, 1.f}; raft::update_device( input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); auto encoded = build(res, params{1.f}, raft::make_const_mdspan(input.view())); - EXPECT_EQ(copy_to_host(res, encoded.data_const_view().data_handle(), 2), - (std::vector{-128, 127})); + EXPECT_TRUE(devArrMatchHost(std::vector{-128, 127}.data(), + encoded.data_const_view().data_handle(), + 2, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); - EXPECT_EQ(copy_to_host(res, input.data_handle(), 2), values); + EXPECT_TRUE(devArrMatchHost(values.data(), + input.data_handle(), + 2, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); auto codes = raft::make_device_matrix(res, 1, 2); device_sq_dataset dataset{std::move(codes), quantizer{-1.f, 1.f}}; decltype(dataset)::const_view_type view = dataset.view(); @@ -186,4 +243,21 @@ TEST(ScalarDataset, GlobalDatasetStillWorks) std::get::range_type>(view.const_manifest_view().value).max_, 1.f); } +TEST_P(ScalarDatasetTest, Result) { this->testScalarDataset(); } + +const std::vector dataset_inputs = { + {2, 2, {10.f, -7.f, 10.f, -7.f}, {0, 0, 0, 0}, 0.f}, + {2, + 1, + {-std::numeric_limits::max(), std::numeric_limits::max()}, + {-127, 127}, + std::numeric_limits::max() / 254.f}, + {2, + 1, + {-std::ldexp(124.f, 104), std::numeric_limits::max()}, + {-127, 127}, + (std::ldexp(124.f, 103) + std::numeric_limits::max() * 0.5f) / 127.f * 0.5f}}; + +INSTANTIATE_TEST_CASE_P(ScalarDatasetTests, ScalarDatasetTest, ::testing::ValuesIn(dataset_inputs)); + } // namespace cuvs::preprocessing::quantize::scalar diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 7673ce36df..7c5777422f 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -177,18 +177,24 @@ quantizer(T min, T max); **Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` -Take ownership of device shifts and their shared scale. +Construct from nonempty device FP32 bounds of equal length, with finite min <= max. ```cpp -quantizer(raft::device_vector&& shifts, T scale); +quantizer(raft::resources const& res, +raft::device_vector_view mins, +raft::device_vector_view maxs) +requires std::is_same_v; ``` +Derives midpoint shifts and a shared scale; synchronizes before returning. + **Parameters** | Name | Direction | Type | Description | | --- | --- | --- | --- | -| `shifts` | | `raft::device_vector&&` | | -| `scale` | | `T` | | +| `res` | | `raft::resources const&` | | +| `mins` | | `raft::device_vector_view` | | +| `maxs` | | `raft::device_vector_view` | | **Returns** From 31081bd193309b0ef341b2b6b9bbbe94a6126f20 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 18:08:04 +0000 Subject: [PATCH 08/24] Support floating input types and consolidate scalar quantization tests --- cpp/include/cuvs/core/dataset/sq.hpp | 10 +- .../cuvs/preprocessing/quantize/scalar.hpp | 33 +- .../preprocessing/quantize/detail/scalar.cuh | 170 +++++----- cpp/src/preprocessing/quantize/scalar.cu | 3 +- cpp/tests/CMakeLists.txt | 1 - cpp/tests/preprocessing/scalar_dataset.cu | 263 --------------- .../preprocessing/scalar_quantization.cu | 310 +++++++++++++++++- fern/pages/cpp_api/cpp-api-core-dataset-sq.md | 12 +- .../cpp-api-preprocessing-quantize-scalar.md | 21 +- 9 files changed, 433 insertions(+), 390 deletions(-) delete mode 100644 cpp/tests/preprocessing/scalar_dataset.cu diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index 20a1b6aff7..9113bdb817 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -86,17 +86,19 @@ namespace preprocessing::quantize::scalar { /** * @defgroup scalar_dataset Scalar-quantized dataset construction * @{ - * Train and encode a device FP32 corpus with the selected range mode. + * Train and encode a device corpus with the selected range mode. * Reuse result.const_manifest_view() with transform * for query batches. For separate training/encoding, use train and transform. */ -inline device_sq_dataset build( - raft::resources const& res, params params, raft::device_matrix_view dataset) +template +inline device_sq_dataset build(raft::resources const& res, + params params, + raft::device_matrix_view dataset) { auto quantizer = train(res, params, dataset); auto codes = raft::make_device_matrix(res, dataset.extent(0), dataset.extent(1)); transform(res, quantizer.view(), dataset, codes.view()); - return device_sq_dataset{std::move(codes), std::move(quantizer)}; + return device_sq_dataset{std::move(codes), std::move(quantizer)}; } /** @} */ diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 9aa60047f1..c349037418 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -40,26 +40,35 @@ struct params { * Needs to be within range of (0, 1]. */ float quantile = 0.99; - /** Per-dimension training uses all device FP32 corpus rows, applying quantile independently - * to each dimension. Midpoint shifts share the widest half-range divided by 127. - * Requires nonempty finite input. Constant ranges use scale 1; positive scales are floored - * to the smallest normal float. Training synchronizes to return the scale. */ + /** + * Per-dimension training uses every row of a nonempty, finite device dataset. + * It applies the quantile independently to each dimension to select lower and upper bounds. + * Each dimension's shift is the midpoint of its bounds, rounded to the input type. + * A shared scale fits every dimension's bounds around its stored shift into the symmetric + * range of quantized values. + * + * If every interval has zero width, the scale is one. Otherwise, the scale is at least + * the smallest positive normal value of the scale type. Training synchronizes the stream before + * returning the scale to the host. + */ range_mode range = range_mode::global; }; /** Read-only scalar quantizer parameters. The owner must outlive stream work using shifts. */ template struct quantizer_const_view { - /** A global interval mapped to [-128, 127]. */ + using scale_type = std::conditional_t, float, T>; + /** A global interval mapped to the full range of the quantized type. */ struct range_type { T min_; T max_; }; - /** Encode clamp(rintf((x - shifts[j]) / scale), -127, 127), with ties to even. - * Decode shifts[j] + scale * code, saturating to finite FP32 limits. */ + /** Encode (x - shifts[j]) / scale, rounding to nearest with ties to even and + * clamping to the symmetric range of quantized values. + * Decode shifts[j] + scale * code, saturating to finite limits of the input type. */ struct shifts_type { raft::device_vector_view shifts; - T scale; + scale_type scale; }; std::variant value; }; @@ -73,11 +82,12 @@ struct quantizer_const_view { template struct quantizer { using view_type = quantizer_const_view; + using scale_type = typename view_type::scale_type; using range_type = typename view_type::range_type; /** Owning per-dimension parameters. */ struct shifts_type { raft::device_vector shifts; - T scale; + scale_type scale; }; std::variant value; @@ -85,12 +95,11 @@ struct quantizer { quantizer() : value{range_type{}} {} /** Construct a global interval. */ quantizer(T min, T max) : value{range_type{min, max}} {} - /** Construct from nonempty device FP32 bounds of equal length, with finite min <= max. + /** Construct from nonempty device bounds of equal length, with finite min <= max. * Derives midpoint shifts and a shared scale; synchronizes before returning. */ quantizer(raft::resources const& res, raft::device_vector_view mins, - raft::device_vector_view maxs) - requires std::is_same_v; + raft::device_vector_view maxs); /** Borrow read-only parameters; the owner must outlive stream work using shifts. */ [[nodiscard]] auto view() const noexcept -> view_type diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index e35e8757ed..870eb18d58 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -23,6 +23,7 @@ #include #include +#include namespace cuvs::preprocessing::quantize::scalar::detail { @@ -133,74 +134,84 @@ std::tuple quantile_min_max( return {minmax_h[0], minmax_h[1]}; } -inline quantizer::shifts_type make_shifts(raft::resources const& res, - raft::device_vector&& shifts, - raft::device_vector&& maxima) +template +typename quantizer::shifts_type train(raft::resources const& res, + raft::device_vector&& shifts, + raft::device_vector&& maxima) { + using compute_t = typename quantizer_const_view::scale_type; auto const dim = shifts.extent(0); - auto max_half_range = raft::make_device_vector(res, 1); + auto half_ranges = raft::make_device_vector(res, dim); + auto max_half_range = raft::make_device_vector(res, 1); auto* shift_ptr = shifts.data_handle(); auto const* max_ptr = maxima.data_handle(); - raft::linalg::map_offset(res, maxima.view(), [shift_ptr, max_ptr] __device__(int64_t j) { + raft::linalg::map_offset(res, half_ranges.view(), [shift_ptr, max_ptr] __device__(int64_t j) { // Halve before adding/subtracting to avoid overflow for large finite inputs. - if (!isfinite(shift_ptr[j]) || !isfinite(max_ptr[j]) || shift_ptr[j] > max_ptr[j]) { - return std::numeric_limits::infinity(); + if (!isfinite(static_cast(shift_ptr[j])) || + !isfinite(static_cast(max_ptr[j])) || fp_lt(max_ptr[j], shift_ptr[j])) { + return std::numeric_limits::infinity(); } - float const lo = shift_ptr[j] * 0.5f; - float const hi = max_ptr[j] * 0.5f; - shift_ptr[j] = lo + hi; - return hi - lo; + compute_t const lo = static_cast(shift_ptr[j]); + compute_t const hi = static_cast(max_ptr[j]); + shift_ptr[j] = static_cast(lo * compute_t{0.5} + hi * compute_t{0.5}); + compute_t const shift = static_cast(shift_ptr[j]); + return fmax(hi - shift, shift - lo); }); raft::linalg::reduce( res, - raft::make_device_matrix_view(maxima.data_handle(), 1, dim), + raft::make_device_matrix_view(half_ranges.data_handle(), 1, dim), max_half_range.view(), - 0.f, + compute_t{0}, false, raft::identity_op{}, raft::max_op{}); - float half_range = 0.f; + compute_t half_range = 0; raft::copy(res, raft::make_host_scalar_view(&half_range), raft::make_device_scalar_view(max_half_range.data_handle())); raft::resource::sync_stream(res); RAFT_EXPECTS(std::isfinite(half_range), "Bounds must be finite with min <= max"); - float const scale = - half_range > 0.f ? std::max(half_range / 127.f, std::numeric_limits::min()) : 1.f; + compute_t const scale = + half_range > 0 ? std::max(half_range / compute_t{127}, std::numeric_limits::min()) + : compute_t{1}; return {std::move(shifts), scale}; } -inline quantizer train(raft::resources const& res, - raft::device_matrix_view dataset, - params params) +template +quantizer train(raft::resources const& res, + raft::device_matrix_view dataset, + params params) { + using compute_t = typename quantizer_const_view::scale_type; RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); auto const dim = dataset.extent(1); RAFT_EXPECTS(dataset.extent(0) > 0 && dim > 0, "Training requires nonempty rows and dimensions"); - auto shifts = raft::make_device_vector(res, dim); - auto maxima = raft::make_device_vector(res, dim); + auto shifts = raft::make_device_vector(res, dim); + auto maxima = raft::make_device_vector(res, dim); if (params.quantile == 1.f) { - raft::linalg::reduce(res, - dataset, - shifts.view(), - std::numeric_limits::infinity(), - false, - raft::identity_op{}, - raft::min_op{}); - raft::linalg::reduce(res, - dataset, - maxima.view(), - -std::numeric_limits::infinity(), - false, - raft::identity_op{}, - raft::max_op{}); + raft::linalg::reduce( + res, + dataset, + shifts.view(), + static_cast(std::numeric_limits::infinity()), + false, + raft::identity_op{}, + raft::min_op{}); + raft::linalg::reduce( + res, + dataset, + maxima.view(), + static_cast(-std::numeric_limits::infinity()), + false, + raft::identity_op{}, + raft::max_op{}); } else { auto const rows = dataset.extent(0); - auto column = raft::make_device_vector(res, rows); + auto column = raft::make_device_vector(res, rows); auto stream = raft::resource::get_cuda_stream(res).get(); size_t workspace_bytes = 0; RAFT_CUDA_TRY(cub::DeviceMergeSort::SortKeys( - nullptr, workspace_bytes, column.data_handle(), rows, fp_lt_op{}, stream)); + nullptr, workspace_bytes, column.data_handle(), rows, fp_lt_op{}, stream)); auto workspace = raft::make_device_vector(res, workspace_bytes); auto const upper = static_cast(std::ceil((0.5 + 0.5 * params.quantile) * rows)) - 1; auto const lower = rows - upper - 1; @@ -213,7 +224,7 @@ inline quantizer train(raft::resources const& res, workspace_bytes, column.data_handle(), rows, - fp_lt_op{}, + fp_lt_op{}, stream)); } raft::copy(res, @@ -224,13 +235,13 @@ inline quantizer train(raft::resources const& res, raft::make_device_scalar_view(column.data_handle() + upper)); } } - quantizer result; - result.value = make_shifts(res, std::move(shifts), std::move(maxima)); + quantizer result; + result.value = train(res, std::move(shifts), std::move(maxima)); return result; } -template -void check_per_dim_shapes(quantizer_const_view::shifts_type quantizer, +template +void check_per_dim_shapes(typename quantizer_const_view::shifts_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { @@ -243,36 +254,42 @@ void check_per_dim_shapes(quantizer_const_view::shifts_type quantizer, "Input and output shapes must match"); } -inline void transform(raft::resources const& res, - quantizer_const_view::shifts_type quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) +template +void transform(raft::resources const& res, + typename quantizer_const_view::shifts_type quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) { - check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - float const inv = 1.f / quantizer.scale; + using compute_t = typename quantizer_const_view::scale_type; + check_per_dim_shapes(quantizer, dataset, out); + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + compute_t const inv = compute_t{1} / quantizer.scale; raft::linalg::map_offset(res, out, [input, shifts, dim, inv] __device__(int64_t i) { - float const code = rintf((input[i] - shifts[i % dim]) * inv); - return static_cast(fminf(fmaxf(code, -127.f), 127.f)); + compute_t const code = + rint((static_cast(input[i]) - static_cast(shifts[i % dim])) * inv); + return static_cast(fmin(fmax(code, compute_t{-127}), compute_t{127})); }); } -inline void inverse_transform(raft::resources const& res, - quantizer_const_view::shifts_type quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) +template +void inverse_transform(raft::resources const& res, + typename quantizer_const_view::shifts_type quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) { - check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - float const scale = quantizer.scale; + using compute_t = typename quantizer_const_view::scale_type; + check_per_dim_shapes(quantizer, dataset, out); + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + compute_t const scale = quantizer.scale; raft::linalg::map_offset(res, out, [input, shifts, dim, scale] __device__(int64_t i) { - float const value = fmaf(scale, static_cast(input[i]), shifts[i % dim]); - constexpr float limit = std::numeric_limits::max(); - return fminf(fmaxf(value, -limit), limit); + compute_t const value = + fma(scale, static_cast(input[i]), static_cast(shifts[i % dim])); + compute_t const limit = static_cast(cuda::std::numeric_limits::max()); + return static_cast(fmin(fmax(value, -limit), limit)); }); } @@ -286,13 +303,7 @@ cuvs::preprocessing::quantize::scalar::quantizer train( "quantile for scalar quantization needs to be within (0, 1] but is %f", params.quantile); - if (params.range == range_mode::per_dimension) { - if constexpr (std::is_same_v) { - return train(res, dataset, params); - } else { - RAFT_FAIL("Per-dimension training requires device FP32 input"); - } - } + if (params.range == range_mode::per_dimension) { return train(res, dataset, params); } auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); @@ -311,8 +322,7 @@ cuvs::preprocessing::quantize::scalar::quantizer train( "quantile for scalar quantization needs to be within (0, 1] but is %f", params.quantile); - RAFT_EXPECTS(params.range == range_mode::global, - "Per-dimension training requires device FP32 input"); + RAFT_EXPECTS(params.range == range_mode::global, "Per-dimension training requires device input"); auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); @@ -330,11 +340,7 @@ void transform(raft::resources const& res, auto view = quantizer; if (auto const* shifts = std::get_if::shifts_type>(&view.value)) { - if constexpr (std::is_same_v) { - detail::transform(res, *shifts, dataset, out); - } else { - RAFT_FAIL("Per-dimension quantization requires device FP32 input"); - } + detail::transform(res, *shifts, dataset, out); return; } auto const& range = std::get::range_type>(view.value); @@ -373,11 +379,7 @@ void inverse_transform(raft::resources const& res, auto view = quantizer; if (auto const* shifts = std::get_if::shifts_type>(&view.value)) { - if constexpr (std::is_same_v) { - detail::inverse_transform(res, *shifts, dataset, out); - } else { - RAFT_FAIL("Per-dimension quantization requires device FP32 input"); - } + detail::inverse_transform(res, *shifts, dataset, out); return; } auto const& range = std::get::range_type>(view.value); diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index d021200afe..3e5dafc464 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -13,7 +13,6 @@ template quantizer::quantizer(raft::resources const& res, raft::device_vector_view mins, raft::device_vector_view maxs) - requires std::is_same_v { RAFT_EXPECTS(mins.extent(0) > 0 && mins.extent(0) == maxs.extent(0), "Bounds must be nonempty and have equal lengths"); @@ -21,7 +20,7 @@ quantizer::quantizer(raft::resources const& res, auto maxima = raft::make_device_vector(res, maxs.extent(0)); raft::copy(res, minima.view(), mins); raft::copy(res, maxima.view(), maxs); - value = detail::make_shifts(res, std::move(minima), std::move(maxima)); + value = detail::train(res, std::move(minima), std::move(maxima)); } #define CUVS_INST_QUANTIZATION(T, QuantI) \ diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index f707105a09..f132bb21a3 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -477,7 +477,6 @@ ConfigureTest( ConfigureTest( NAME PREPROCESSING_TEST PATH preprocessing/scalar_quantization.cu - preprocessing/scalar_dataset.cu preprocessing/binary_quantization.cu preprocessing/spectral_embedding.cu preprocessing/product_quantization.cu diff --git a/cpp/tests/preprocessing/scalar_dataset.cu b/cpp/tests/preprocessing/scalar_dataset.cu deleted file mode 100644 index fca99621e7..0000000000 --- a/cpp/tests/preprocessing/scalar_dataset.cu +++ /dev/null @@ -1,263 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "../test_utils.cuh" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace cuvs::preprocessing::quantize::scalar { - -using shifts_view = quantizer_const_view::shifts_type; - -struct ScalarDatasetInputs { - int64_t n_samples; - int64_t n_features; - std::vector dataset; - std::vector codes; - float tolerance; -}; - -std::ostream& operator<<(std::ostream& os, const ScalarDatasetInputs& inputs) -{ - return os << "n_samples:" << inputs.n_samples << " n_features:" << inputs.n_features - << " tolerance:" << inputs.tolerance; -} - -class ScalarDatasetTest : public ::testing::TestWithParam { - public: - ScalarDatasetTest() - : params_(GetParam()), - stream(raft::resource::get_cuda_stream(handle).get()), - dataset_( - raft::make_device_matrix(handle, params_.n_samples, params_.n_features)) - { - } - - protected: - void SetUp() override - { - raft::update_device( - dataset_.data_handle(), params_.dataset.data(), params_.dataset.size(), stream); - } - - void testScalarDataset() - { - auto encoded = build( - handle, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(dataset_.view())); - EXPECT_TRUE(devArrMatchHost(params_.codes.data(), - encoded.data_const_view().data_handle(), - params_.codes.size(), - cuvs::Compare(), - stream)); - if (params_.tolerance == 0.f) { - EXPECT_FLOAT_EQ(std::get(encoded.const_manifest_view().value).scale, 1.f); - } - inverse_transform( - handle, encoded.const_manifest_view(), encoded.data_const_view(), dataset_.view()); - auto reconstructed = - raft::make_host_matrix(params_.n_samples, params_.n_features); - raft::update_host(reconstructed.data_handle(), dataset_.data_handle(), dataset_.size(), stream); - raft::resource::sync_stream(handle); - for (size_t i = 0; i < params_.dataset.size(); ++i) { - EXPECT_TRUE(std::isfinite(reconstructed.data_handle()[i])); - EXPECT_NEAR(reconstructed.data_handle()[i], params_.dataset[i], params_.tolerance); - } - } - - private: - raft::resources handle; - ScalarDatasetInputs params_; - cudaStream_t stream; - raft::device_matrix dataset_; -}; - -TEST(ScalarDatasetTest, CorpusAndQueriesShareQuantizer) -{ - raft::resources res; - auto stream = raft::resource::get_cuda_stream(res); - std::vector corpus{0.f, 10.f, 9.f, 254.f, 14.f, 9.f}; - auto input = raft::make_device_matrix(res, 2, 3); - raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); - auto encoded = - build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - ASSERT_EQ(encoded.n_rows(), 2); - ASSERT_EQ(encoded.dim(), 3u); - auto quantizer = encoded.const_manifest_view(); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); - EXPECT_TRUE(devArrMatchHost(std::vector{127.f, 12.f, 9.f}.data(), - std::get(quantizer.value).shifts.data_handle(), - 3, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - EXPECT_TRUE(devArrMatchHost(std::vector{-127, -2, 0, 127, 2, 0}.data(), - encoded.data_const_view().data_handle(), - 6, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - - // The corpus manifest, rather than query extrema, controls ties and saturation. - std::vector queries{127.5f, 11.5f, 1000.f, 128.5f, 9.5f, -1000.f}; - raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); - auto codes = raft::make_device_matrix(res, 2, 3); - transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{0, 0, 127, 2, -2, -127}.data(), - codes.data_handle(), - 6, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{127.f, 12.f, 136.f, 129.f, 10.f, -118.f}.data(), - input.data_handle(), - 6, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); -} - -TEST(ScalarDatasetTest, PerDimensionQuantiles) -{ - raft::resources res; - constexpr int64_t rows = 1000, dim = 3; - std::vector corpus(rows * dim); - for (int64_t i = 0; i < rows; ++i) { - corpus[i * dim] = static_cast(i); - corpus[i * dim + 1] = 10000.f + 2.f * static_cast(rows - i - 1); - corpus[i * dim + 2] = 9.f; - } - auto input = raft::make_device_matrix(res, rows, dim); - raft::update_device( - input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto encoded = - build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - auto quantizer = encoded.const_manifest_view(); - EXPECT_TRUE(devArrMatchHost(std::vector{499.5f, 10999.f, 9.f}.data(), - std::get(quantizer.value).shifts.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 991.f / 127.f); - std::vector codes(rows * dim); - raft::update_host(codes.data(), - encoded.data_const_view().data_handle(), - codes.size(), - raft::resource::get_cuda_stream(res)); - raft::resource::sync_stream(res); - EXPECT_EQ(codes[1], 127); - EXPECT_EQ(codes[(rows - 1) * dim + 1], -127); - EXPECT_EQ(codes[2], 0); - auto full = - train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(full.view().value).scale, 999.f / 127.f); - auto trimmed = - train(res, params{0.5f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(trimmed.view().value).scale, 499.f / 127.f); - for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { - EXPECT_THROW( - train(res, params{q, range_mode::per_dimension}, raft::make_const_mdspan(input.view())), - raft::logic_error); - } -} - -TEST(ScalarDatasetTest, CustomParametersAndValidation) -{ - raft::resources res; - auto mins = raft::make_device_vector(res, 2); - auto maxs = raft::make_device_vector(res, 2); - std::vector lower{-154.f, -104.f}, upper{354.f, -96.f}; - auto stream = raft::resource::get_cuda_stream(res); - raft::update_device(mins.data_handle(), lower.data(), lower.size(), stream); - raft::update_device(maxs.data_handle(), upper.data(), upper.size(), stream); - scalar::quantizer owner{ - res, raft::make_const_mdspan(mins.view()), raft::make_const_mdspan(maxs.view())}; - auto quantizer = owner.view(); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 2.f); - EXPECT_TRUE(devArrMatchHost(std::vector{100.f, -100.f}.data(), - std::get(quantizer.value).shifts.data_handle(), - 2, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - auto input = raft::make_device_matrix(res, 1, 2); - std::vector values{104.f, -106.f}; - raft::update_device( - input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); - auto codes = raft::make_device_matrix(res, 1, 2); - transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{2, -3}.data(), - codes.data_handle(), - 2, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - auto wrong_shape = raft::make_device_matrix(res, 2, 1); - EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), - raft::logic_error); - EXPECT_THROW((scalar::quantizer{ - res, - raft::make_const_mdspan(mins.view()), - raft::make_device_vector_view(maxs.data_handle(), 1)}), - raft::logic_error); - EXPECT_THROW((scalar::quantizer{ - res, raft::make_const_mdspan(maxs.view()), raft::make_const_mdspan(mins.view())}), - raft::logic_error); - EXPECT_THROW(train(res, - params{.range = range_mode::per_dimension}, - raft::make_device_matrix_view(nullptr, 0, 2)), - raft::logic_error); -} - -TEST(ScalarDatasetTest, GlobalDatasetStillWorks) -{ - raft::resources res; - auto input = raft::make_device_matrix(res, 1, 2); - std::vector values{-1.f, 1.f}; - raft::update_device( - input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); - auto encoded = build(res, params{1.f}, raft::make_const_mdspan(input.view())); - EXPECT_TRUE(devArrMatchHost(std::vector{-128, 127}.data(), - encoded.data_const_view().data_handle(), - 2, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - inverse_transform(res, encoded.const_manifest_view(), encoded.data_const_view(), input.view()); - EXPECT_TRUE(devArrMatchHost(values.data(), - input.data_handle(), - 2, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - auto codes = raft::make_device_matrix(res, 1, 2); - device_sq_dataset dataset{std::move(codes), quantizer{-1.f, 1.f}}; - decltype(dataset)::const_view_type view = dataset.view(); - EXPECT_EQ(view.dim(), 2u); - EXPECT_FLOAT_EQ( - std::get::range_type>(view.const_manifest_view().value).min_, -1.f); - EXPECT_FLOAT_EQ( - std::get::range_type>(view.const_manifest_view().value).max_, 1.f); -} - -TEST_P(ScalarDatasetTest, Result) { this->testScalarDataset(); } - -const std::vector dataset_inputs = { - {2, 2, {10.f, -7.f, 10.f, -7.f}, {0, 0, 0, 0}, 0.f}, - {2, - 1, - {-std::numeric_limits::max(), std::numeric_limits::max()}, - {-127, 127}, - std::numeric_limits::max() / 254.f}, - {2, - 1, - {-std::ldexp(124.f, 104), std::numeric_limits::max()}, - {-127, 127}, - (std::ldexp(124.f, 103) + std::numeric_limits::max() * 0.5f) / 127.f * 0.5f}}; - -INSTANTIATE_TEST_CASE_P(ScalarDatasetTests, ScalarDatasetTest, ::testing::ValuesIn(dataset_inputs)); - -} // namespace cuvs::preprocessing::quantize::scalar diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 2395ee2daa..0f84d4ee4c 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -4,6 +4,7 @@ */ #include "../test_utils.cuh" +#include #include #include #include @@ -302,6 +303,291 @@ class QuantizationTest : public ::testing::TestWithParam> std::vector host_input_; }; +using shifts_view = quantizer_const_view::shifts_type; + +struct PerDimensionInputs { + int64_t n_samples; + int64_t n_features; + std::vector dataset; + std::vector codes; + float tolerance; +}; + +std::ostream& operator<<(std::ostream& os, const PerDimensionInputs& inputs) +{ + return os << "n_samples:" << inputs.n_samples << " n_features:" << inputs.n_features + << " tolerance:" << inputs.tolerance; +} + +class PerDimensionRangeTest : public ::testing::TestWithParam { + public: + PerDimensionRangeTest() + : params_(GetParam()), + stream(raft::resource::get_cuda_stream(handle).get()), + dataset_( + raft::make_device_matrix(handle, params_.n_samples, params_.n_features)) + { + } + + protected: + void SetUp() override + { + raft::update_device( + dataset_.data_handle(), params_.dataset.data(), params_.dataset.size(), stream); + } + + void testPerDimensionQuantization() + { + auto quantizer = train( + handle, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(dataset_.view())); + auto codes = + raft::make_device_matrix(handle, params_.n_samples, params_.n_features); + transform(handle, quantizer, raft::make_const_mdspan(dataset_.view()), codes.view()); + EXPECT_TRUE(devArrMatchHost(params_.codes.data(), + codes.data_handle(), + params_.codes.size(), + cuvs::Compare(), + stream)); + if (params_.tolerance == 0.f) { + EXPECT_FLOAT_EQ(std::get(quantizer.view().value).scale, 1.f); + } + inverse_transform(handle, quantizer, raft::make_const_mdspan(codes.view()), dataset_.view()); + auto reconstructed = + raft::make_host_matrix(params_.n_samples, params_.n_features); + raft::update_host(reconstructed.data_handle(), dataset_.data_handle(), dataset_.size(), stream); + raft::resource::sync_stream(handle); + for (size_t i = 0; i < params_.dataset.size(); ++i) { + EXPECT_TRUE(std::isfinite(reconstructed.data_handle()[i])); + EXPECT_NEAR(reconstructed.data_handle()[i], params_.dataset[i], params_.tolerance); + } + } + + private: + raft::resources handle; + PerDimensionInputs params_; + cudaStream_t stream; + raft::device_matrix dataset_; +}; + +template +class PerDimensionQuantizationTest : public ::testing::Test {}; +using PerDimensionTypes = ::testing::Types; +TYPED_TEST_SUITE(PerDimensionQuantizationTest, PerDimensionTypes); + +TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) +{ + using T = TypeParam; + using shifts_view = typename quantizer_const_view::shifts_type; + raft::resources res; + auto stream = raft::resource::get_cuda_stream(res); + std::vector corpus{0.f, 10.f, 9.f, 254.f, 14.f, 9.f}; + auto input = raft::make_device_matrix(res, 2, 3); + raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); + auto owner = + train(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + auto quantizer = owner.view(); + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); + { + std::vector actual(3); + std::vector expected{127.f, 12.f, 9.f}; + raft::update_host(actual.data(), + std::get(quantizer.value).shifts.data_handle(), + actual.size(), + stream); + raft::resource::sync_stream(res); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(static_cast(actual[i]), expected[i]); + } + } + // The corpus manifest, rather than query extrema, controls ties and saturation. + std::vector queries{127.5f, 11.5f, 1000.f, 128.5f, 9.5f, -1000.f}; + raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); + auto codes = raft::make_device_matrix(res, 2, 3); + transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_TRUE(devArrMatchHost(std::vector{0, 0, 127, 2, -2, -127}.data(), + codes.data_handle(), + 6, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); + inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); + { + std::vector actual(6); + std::vector expected{127.f, 12.f, 136.f, 129.f, 10.f, -118.f}; + raft::update_host(actual.data(), input.data_handle(), actual.size(), stream); + raft::resource::sync_stream(res); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(static_cast(actual[i]), expected[i]); + } + } +} + +TEST(PerDimensionRangeTest, AdjacentHalfBounds) +{ + raft::resources res; + auto stream = raft::resource::get_cuda_stream(res); + std::vector values{half{1.f}, half{1.f + 1.f / 1024.f}}; + auto input = raft::make_device_matrix(res, 2, 1); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto quantizer = + train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + auto codes = raft::make_device_matrix(res, 2, 1); + transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); + std::vector reconstructed(2); + raft::update_host(reconstructed.data(), input.data_handle(), reconstructed.size(), stream); + raft::resource::sync_stream(res); + EXPECT_EQ(static_cast(reconstructed[0]), static_cast(values[0])); + EXPECT_EQ(static_cast(reconstructed[1]), static_cast(values[1])); +} + +TEST(PerDimensionRangeTest, PerDimensionQuantiles) +{ + raft::resources res; + constexpr int64_t rows = 1000, dim = 3; + std::vector corpus(rows * dim); + for (int64_t i = 0; i < rows; ++i) { + corpus[i * dim] = static_cast(i); + corpus[i * dim + 1] = 10000.f + 2.f * static_cast(rows - i - 1); + corpus[i * dim + 2] = 9.f; + } + auto input = raft::make_device_matrix(res, rows, dim); + raft::update_device( + input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); + auto owner = + train(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + auto quantizer = owner.view(); + EXPECT_TRUE(devArrMatchHost(std::vector{499.5f, 10999.f, 9.f}.data(), + std::get(quantizer.value).shifts.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 991.f / 127.f); + auto full = + train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + EXPECT_FLOAT_EQ(std::get(full.view().value).scale, 999.f / 127.f); + auto trimmed = + train(res, params{0.5f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + EXPECT_FLOAT_EQ(std::get(trimmed.view().value).scale, 499.f / 127.f); + for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { + EXPECT_THROW( + train(res, params{q, range_mode::per_dimension}, raft::make_const_mdspan(input.view())), + raft::logic_error); + } +} + +TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) +{ + using T = TypeParam; + using shifts_view = typename quantizer_const_view::shifts_type; + raft::resources res; + auto mins = raft::make_device_vector(res, 2); + auto maxs = raft::make_device_vector(res, 2); + std::vector lower{-154.f, -104.f}, upper{354.f, -96.f}; + auto stream = raft::resource::get_cuda_stream(res); + raft::update_device(mins.data_handle(), lower.data(), lower.size(), stream); + raft::update_device(maxs.data_handle(), upper.data(), upper.size(), stream); + scalar::quantizer owner{ + res, raft::make_const_mdspan(mins.view()), raft::make_const_mdspan(maxs.view())}; + auto quantizer = owner.view(); + EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 2.f); + { + std::vector actual(2); + std::vector expected{100.f, -100.f}; + raft::update_host(actual.data(), + std::get(quantizer.value).shifts.data_handle(), + actual.size(), + stream); + raft::resource::sync_stream(res); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(static_cast(actual[i]), expected[i]); + } + } + auto input = raft::make_device_matrix(res, 1, 2); + std::vector values{104.f, -106.f}; + raft::update_device( + input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); + auto codes = raft::make_device_matrix(res, 1, 2); + transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_TRUE(devArrMatchHost(std::vector{2, -3}.data(), + codes.data_handle(), + 2, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); + auto wrong_shape = raft::make_device_matrix(res, 2, 1); + EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), + raft::logic_error); + EXPECT_THROW( + (scalar::quantizer{res, + raft::make_const_mdspan(mins.view()), + raft::make_device_vector_view(maxs.data_handle(), 1)}), + raft::logic_error); + EXPECT_THROW((scalar::quantizer{ + res, raft::make_const_mdspan(maxs.view()), raft::make_const_mdspan(mins.view())}), + raft::logic_error); + EXPECT_THROW(train(res, + params{.range = range_mode::per_dimension}, + raft::make_device_matrix_view(nullptr, 0, 2)), + raft::logic_error); +} + +TEST(QuantizationDatasetTest, GlobalRangeDataset) +{ + raft::resources res; + auto input = raft::make_device_matrix(res, 1, 2); + std::vector values{-1.f, 1.f}; + auto stream = raft::resource::get_cuda_stream(res); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto encoded = build(res, params{1.f}, raft::make_const_mdspan(input.view())); + EXPECT_EQ(encoded.n_rows(), 1); + EXPECT_EQ(encoded.dim(), 2u); + EXPECT_TRUE(devArrMatchHost(std::vector{-128, 127}.data(), + encoded.data_const_view().data_handle(), + 2, + cuvs::Compare(), + stream.get())); + auto range = + std::get::range_type>(encoded.const_manifest_view().value); + EXPECT_FLOAT_EQ(range.min_, -1.f); + EXPECT_FLOAT_EQ(range.max_, 1.f); +} + +TEST(QuantizationDatasetTest, PerDimensionDataset) +{ + raft::resources res; + auto input = raft::make_device_matrix(res, 2, 2); + std::vector values{0.f, 10.f, 254.f, 14.f}; + auto stream = raft::resource::get_cuda_stream(res); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto encoded = + build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + EXPECT_EQ(encoded.n_rows(), 2); + EXPECT_EQ(encoded.dim(), 2u); + EXPECT_TRUE(devArrMatchHost(std::vector{-127, -2, 127, 2}.data(), + encoded.data_const_view().data_handle(), + 4, + cuvs::Compare(), + stream.get())); + auto codes = raft::make_device_matrix(res, 2, 2); + transform( + res, encoded.const_manifest_view(), raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_TRUE(devArrMatch(encoded.data_const_view().data_handle(), + codes.data_handle(), + 4, + cuvs::Compare(), + stream.get())); +} + +TEST_P(PerDimensionRangeTest, Result) { this->testPerDimensionQuantization(); } + +typedef QuantizationTest QuantizationTest_float_int8t; +TEST_P(QuantizationTest_float_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } + +typedef QuantizationTest QuantizationTest_double_int8t; +TEST_P(QuantizationTest_double_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } + +typedef QuantizationTest QuantizationTest_half_int8t; +TEST_P(QuantizationTest_half_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } + template const std::vector> inputs = { {{1.0}, 5, 5, T(0.0), T(1.0)}, @@ -316,14 +602,22 @@ const std::vector> inputs = { {{0.95}, 10, 20, T(5.0), T(5.0)}, }; -typedef QuantizationTest QuantizationTest_float_int8t; -TEST_P(QuantizationTest_float_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } - -typedef QuantizationTest QuantizationTest_double_int8t; -TEST_P(QuantizationTest_double_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } - -typedef QuantizationTest QuantizationTest_half_int8t; -TEST_P(QuantizationTest_half_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } +const std::vector dataset_inputs = { + {2, 2, {10.f, -7.f, 10.f, -7.f}, {0, 0, 0, 0}, 0.f}, + {2, + 1, + {-std::numeric_limits::max(), std::numeric_limits::max()}, + {-127, 127}, + std::numeric_limits::max() / 254.f}, + {2, + 1, + {-std::ldexp(124.f, 104), std::numeric_limits::max()}, + {-127, 127}, + (std::ldexp(124.f, 103) + std::numeric_limits::max() * 0.5f) / 127.f * 0.5f}}; + +INSTANTIATE_TEST_CASE_P(PerDimensionRangeTests, + PerDimensionRangeTest, + ::testing::ValuesIn(dataset_inputs)); INSTANTIATE_TEST_CASE_P(QuantizationTest, QuantizationTest_float_int8t, diff --git a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md index 47dbd07bc1..f2e2e89e58 100644 --- a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md +++ b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md @@ -25,11 +25,13 @@ struct sq_spec; ### preprocessing::quantize::scalar::build -Train and encode a device FP32 corpus with the selected range mode. +Train and encode a device corpus with the selected range mode. ```cpp -inline device_sq_dataset build( -raft::resources const& res, params params, raft::device_matrix_view dataset); +template +inline device_sq_dataset build(raft::resources const& res, +params params, +raft::device_matrix_view dataset); ``` Reuse result.const_manifest_view() with transform for query batches. For separate training/encoding, use train and transform. @@ -40,8 +42,8 @@ Reuse result.const_manifest_view() with transform for query batches. For separat | --- | --- | --- | --- | | `res` | | `raft::resources const&` | | | `params` | | `params` | | -| `dataset` | | `raft::device_matrix_view` | | +| `dataset` | | `raft::device_matrix_view` | | **Returns** -`inline device_sq_dataset` +`inline device_sq_dataset` diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 7c5777422f..41261e9993 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -44,7 +44,7 @@ struct params { | Name | Type | Description | | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | -| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses all device FP32 corpus rows, applying quantile independently to each dimension. Midpoint shifts share the widest half-range divided by 127. Requires nonempty finite input. Constant ranges use scale 1; positive scales are floored to the smallest normal float. Training synchronizes to return the scale. | +| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite device dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. A shared scale fits every dimension's bounds around its stored shift into the symmetric range of quantized values.

If every interval has zero width, the scale is one. Otherwise, the scale is at least the smallest positive normal value of the scale type. Training synchronizes the stream before returning the scale to the host. | ### preprocessing::quantize::scalar::quantizer_const_view @@ -67,7 +67,7 @@ struct quantizer_const_view { ### preprocessing::quantize::scalar::quantizer_const_view::range_type -A global interval mapped to [-128, 127]. +A global interval mapped to the full range of the quantized type. ```cpp struct range_type { @@ -86,14 +86,14 @@ struct range_type { ### preprocessing::quantize::scalar::quantizer_const_view::shifts_type -Encode clamp(rintf((x - shifts[j]) / scale), -127, 127), with ties to even. +Encode (x - shifts[j]) / scale, rounding to nearest with ties to even and -Decode shifts[j] + scale * code, saturating to finite FP32 limits. +clamping to the symmetric range of quantized values. Decode shifts[j] + scale * code, saturating to finite limits of the input type. ```cpp struct shifts_type { raft::device_vector_view shifts; - T scale; + scale_type scale; }; ``` @@ -102,7 +102,7 @@ struct shifts_type { | Name | Type | Description | | --- | --- | --- | | `shifts` | `raft::device_vector_view` | | -| `scale` | `T` | | +| `scale` | `scale_type` | | ### preprocessing::quantize::scalar::quantizer @@ -132,7 +132,7 @@ Owning per-dimension parameters. ```cpp struct shifts_type { raft::device_vector shifts; - T scale; + scale_type scale; }; ``` @@ -141,7 +141,7 @@ struct shifts_type { | Name | Type | Description | | --- | --- | --- | | `shifts` | `raft::device_vector` | | -| `scale` | `T` | | +| `scale` | `scale_type` | | ### preprocessing::quantize::scalar::quantizer::quantizer @@ -177,13 +177,12 @@ quantizer(T min, T max); **Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` -Construct from nonempty device FP32 bounds of equal length, with finite min <= max. +Construct from nonempty device bounds of equal length, with finite min <= max. ```cpp quantizer(raft::resources const& res, raft::device_vector_view mins, -raft::device_vector_view maxs) -requires std::is_same_v; +raft::device_vector_view maxs); ``` Derives midpoint shifts and a shared scale; synchronizes before returning. From a038a48c755c10c7bdf3ee042fd5e444585e1003 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 3 Nov 2026 21:36:21 +0000 Subject: [PATCH 09/24] Fix half scalar quantizer shared-library export --- cpp/include/cuvs/core/dataset/sq.hpp | 9 ++++++++- cpp/src/preprocessing/quantize/scalar.cu | 2 ++ fern/pages/cpp_api/cpp-api-core-dataset-sq.md | 12 ++++++------ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index 9113bdb817..5859f8189d 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -86,9 +86,16 @@ namespace preprocessing::quantize::scalar { /** * @defgroup scalar_dataset Scalar-quantized dataset construction * @{ - * Train and encode a device corpus with the selected range mode. + */ +/** + * @brief Train and encode a device corpus with the selected range mode. * Reuse result.const_manifest_view() with transform * for query batches. For separate training/encoding, use train and transform. + * + * @param[in] res Resources used for training and allocation. + * @param[in] params Scalar quantizer parameters. + * @param[in] dataset Device corpus to train on and encode. + * @return Device dataset owning the quantized codes and quantizer. */ template inline device_sq_dataset build(raft::resources const& res, diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index 3e5dafc464..270c53486c 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + #include "./detail/scalar.cuh" #include diff --git a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md index f2e2e89e58..9f7444ce4e 100644 --- a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md +++ b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md @@ -25,7 +25,7 @@ struct sq_spec; ### preprocessing::quantize::scalar::build -Train and encode a device corpus with the selected range mode. +Train and encode a device corpus with the selected range mode. Reuse result.const_manifest_view() with transform for query batches. For separate training/encoding, use train and transform. ```cpp template @@ -34,16 +34,16 @@ params params, raft::device_matrix_view dataset); ``` -Reuse result.const_manifest_view() with transform for query batches. For separate training/encoding, use train and transform. - **Parameters** | Name | Direction | Type | Description | | --- | --- | --- | --- | -| `res` | | `raft::resources const&` | | -| `params` | | `params` | | -| `dataset` | | `raft::device_matrix_view` | | +| `res` | in | `raft::resources const&` | Resources used for training and allocation. | +| `params` | in | `params` | Scalar quantizer parameters. | +| `dataset` | in | `raft::device_matrix_view` | Device corpus to train on and encode. | **Returns** `inline device_sq_dataset` + +Device dataset owning the quantized codes and quantizer. From 007a55ce299b10d24891166802e63964b4193b1d Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Wed, 4 Nov 2026 20:04:24 +0000 Subject: [PATCH 10/24] Compute scalar quantization reconstruction bounds during training Record the maximum L2 reconstruction error across every training row in quantizer::max_error and carry it into borrowed views and SQ datasets. Include clipped outliers and use a conservative upward rounding margin. Reuse the transform arithmetic when measuring residuals, with bounded GPU scratch and stable norms for extreme finite values. Cover global and per-dimension ranges, host training, all floating input types, dataset manifests, and rows outside the range-fitting sample. Validated 47 scalar quantization tests against a shared library on H100. --- .../cuvs/preprocessing/quantize/scalar.hpp | 17 +- .../preprocessing/quantize/detail/scalar.cuh | 171 +++++++++++++++--- .../preprocessing/scalar_quantization.cu | 122 +++++++++++++ .../cpp-api-preprocessing-quantize-scalar.md | 8 +- 4 files changed, 288 insertions(+), 30 deletions(-) diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index c349037418..db9cf34bca 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -70,7 +71,10 @@ struct quantizer_const_view { raft::device_vector_view shifts; scale_type scale; }; + /** Global interval or borrowed per-dimension shifts and scale. */ std::variant value; + /** Maximum training-row L2 reconstruction error; see quantizer::max_error. */ + scale_type max_error = INFINITY; }; /** @@ -89,7 +93,16 @@ struct quantizer { raft::device_vector shifts; scale_type scale; }; + /** Global interval or owning per-dimension shifts and scale. */ std::variant value; + /** + * Upper bound on the L2 error between a training row and its int8 encode/decode result. + * train measures every input row, including clipped outliers, and rounds the maximum up + * with a relative 1e-6 margin. This bound applies only to the training data, not later queries. + * It is infinity when constructed from parameters alone or when the error is not finite. + * Device training synchronizes before returning this host value. + */ + scale_type max_error = INFINITY; /** Construct an empty global interval. */ quantizer() : value{range_type{}} {} @@ -104,9 +117,9 @@ struct quantizer { /** Borrow read-only parameters; the owner must outlive stream work using shifts. */ [[nodiscard]] auto view() const noexcept -> view_type { - if (auto const* range = std::get_if(&value)) { return {*range}; } + if (auto const* range = std::get_if(&value)) { return {*range, max_error}; } auto const& shifts = std::get(value); - return {typename view_type::shifts_type{shifts.shifts.view(), shifts.scale}}; + return {typename view_type::shifts_type{shifts.shifts.view(), shifts.scale}, max_error}; } /** An owning quantizer can be passed wherever a read-only view is accepted. */ operator view_type() const noexcept { return view(); } // NOLINT(google-explicit-constructor) diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index 870eb18d58..112298eb6d 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -64,19 +64,140 @@ struct quantize_op { { } - constexpr RAFT_INLINE_FUNCTION QuantI operator()(const T& x) const + constexpr RAFT_INLINE_FUNCTION QuantI operator()(const T& x, int64_t = 0) const { if (!fp_lt(min_, x)) return q_type_min_; if (!fp_lt(x, max_)) return q_type_max_; return static_cast(lroundf(scalar_ * static_cast(x) + offset_)); } - constexpr RAFT_INLINE_FUNCTION T operator()(const QuantI& x) const + constexpr RAFT_INLINE_FUNCTION T operator()(const QuantI& x, int64_t = 0) const { return static_cast((static_cast(x) - offset_) / scalar_); } }; +// Share the exact encode/decode arithmetic with the reconstruction-error pass. +template +struct per_dim_quantize_op { + using compute_t = typename quantizer_const_view::scale_type; + T const* shifts; + compute_t scale; + compute_t inv; + + explicit per_dim_quantize_op(typename quantizer_const_view::shifts_type quantizer) + : shifts{quantizer.shifts.data_handle()}, scale{quantizer.scale}, inv{compute_t{1} / scale} + { + } + + RAFT_INLINE_FUNCTION int8_t operator()(T x, int64_t j) const + { + compute_t const code = + rint((static_cast(x) - static_cast(shifts[j])) * inv); + return static_cast(fmin(fmax(code, compute_t{-127}), compute_t{127})); + } + + RAFT_INLINE_FUNCTION T operator()(int8_t code, int64_t j) const + { + compute_t const value = + fma(scale, static_cast(code), static_cast(shifts[j])); + compute_t const limit = static_cast(cuda::std::numeric_limits::max()); + return static_cast(fmin(fmax(value, -limit), limit)); + } +}; + +template +RAFT_INLINE_FUNCTION double reconstruction_error(T x, int64_t j, Op op) +{ + double const value = static_cast(x); + double const decoded = static_cast(op(op(x, j), j)); + return isfinite(value) && isfinite(decoded) ? fabs(value - decoded) + : std::numeric_limits::infinity(); +} + +// One warp per row; each warp keeps its maximum over a grid-stride sequence of rows. +// hypot avoids overflow/underflow of squared residuals for large/tiny finite inputs. +template +__global__ void reconstruction_error_kernel(raft::device_matrix_view dataset, + Op op, + double* errors) +{ + auto const warp = static_cast(blockIdx.x) * (blockDim.x / 32) + threadIdx.x / 32; + auto const lane = threadIdx.x % 32; + auto const stride = static_cast(gridDim.x) * (blockDim.x / 32); + double max_error = 0; + for (int64_t row = warp; row < dataset.extent(0); row += stride) { + double error = 0; + for (int64_t j = lane; j < dataset.extent(1); j += 32) { + error = hypot(error, reconstruction_error(dataset(row, j), j, op)); + } + for (int offset = 16; offset > 0; offset /= 2) { + error = hypot(error, __shfl_down_sync(0xffffffffu, error, offset)); + } + max_error = fmax(max_error, error); + } + if (lane == 0) { errors[warp] = max_error; } +} + +template +auto round_error_up(double error) -> typename quantizer::scale_type +{ + using bound_t = typename quantizer::scale_type; + // Match Sirius's conservative margin, then round upwards in the public result type. + auto const bound = static_cast(error * (1.0 + 1e-6)); + return error > 0 ? std::nextafter(bound, std::numeric_limits::infinity()) : bound; +} + +template +auto max_reconstruction_error(raft::resources const& res, + raft::device_matrix_view dataset, + Op op) -> typename quantizer::scale_type +{ + constexpr int threads = 256; + auto const blocks = static_cast(std::min((dataset.extent(0) + 7) / 8, 65535)); + auto errors = raft::make_device_vector(res, blocks * 8); + auto maximum = raft::make_device_vector(res, 1); + if (!raft::resource::get_dry_run_flag(res)) { + reconstruction_error_kernel<<>>( + dataset, op, errors.data_handle()); + RAFT_CUDA_TRY(cudaPeekAtLastError()); + } + raft::linalg::reduce( + res, + raft::make_device_matrix_view(errors.data_handle(), 1, errors.size()), + maximum.view(), + 0.0, + false, + raft::identity_op{}, + raft::max_op{}); + double result = std::numeric_limits::infinity(); + raft::copy(res, + raft::make_host_scalar_view(&result), + raft::make_device_scalar_view(maximum.data_handle())); + raft::resource::sync_stream(res); + return round_error_up(result); +} + +template +auto max_reconstruction_error(raft::resources const& res, + raft::host_matrix_view dataset, + Op op) -> typename quantizer::scale_type +{ + if (raft::resource::get_dry_run_flag(res)) { + return std::numeric_limits::scale_type>::infinity(); + } + double maximum = 0; +#pragma omp parallel for reduction(max : maximum) + for (int64_t row = 0; row < dataset.extent(0); ++row) { + double error = 0; + for (int64_t j = 0; j < dataset.extent(1); ++j) { + error = std::hypot(error, reconstruction_error(dataset(row, j), j, op)); + } + maximum = std::max(maximum, error); + } + return round_error_up(maximum); +} + template std::tuple quantile_min_max( raft::resources const& res, @@ -236,7 +357,12 @@ quantizer train(raft::resources const& res, } } quantizer result; - result.value = train(res, std::move(shifts), std::move(maxima)); + result.value = train(res, std::move(shifts), std::move(maxima)); + result.max_error = max_reconstruction_error( + res, + dataset, + per_dim_quantize_op{ + std::get::shifts_type>(result.view().value)}); return result; } @@ -260,17 +386,12 @@ void transform(raft::resources const& res, raft::device_matrix_view dataset, raft::device_matrix_view out) { - using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - compute_t const inv = compute_t{1} / quantizer.scale; - raft::linalg::map_offset(res, out, [input, shifts, dim, inv] __device__(int64_t i) { - compute_t const code = - rint((static_cast(input[i]) - static_cast(shifts[i % dim])) * inv); - return static_cast(fmin(fmax(code, compute_t{-127}), compute_t{127})); - }); + auto const* input = dataset.data_handle(); + auto const dim = dataset.extent(1); + auto const op = per_dim_quantize_op{quantizer}; + raft::linalg::map_offset( + res, out, [input, dim, op] __device__(int64_t i) { return op(input[i], i % dim); }); } template @@ -279,18 +400,12 @@ void inverse_transform(raft::resources const& res, raft::device_matrix_view dataset, raft::device_matrix_view out) { - using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - compute_t const scale = quantizer.scale; - raft::linalg::map_offset(res, out, [input, shifts, dim, scale] __device__(int64_t i) { - compute_t const value = - fma(scale, static_cast(input[i]), static_cast(shifts[i % dim])); - compute_t const limit = static_cast(cuda::std::numeric_limits::max()); - return static_cast(fmin(fmax(value, -limit), limit)); - }); + auto const* input = dataset.data_handle(); + auto const dim = dataset.extent(1); + auto const op = per_dim_quantize_op{quantizer}; + raft::linalg::map_offset( + res, out, [input, dim, op] __device__(int64_t i) { return op(input[i], i % dim); }); } template @@ -309,7 +424,9 @@ cuvs::preprocessing::quantize::scalar::quantizer train( RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); - return cuvs::preprocessing::quantize::scalar::quantizer{min, max}; + quantizer result{min, max}; + result.max_error = max_reconstruction_error(res, dataset, quantize_op{min, max}); + return result; } template @@ -328,7 +445,9 @@ cuvs::preprocessing::quantize::scalar::quantizer train( RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); - return cuvs::preprocessing::quantize::scalar::quantizer{min, max}; + quantizer result{min, max}; + result.max_error = max_reconstruction_error(res, dataset, quantize_op{min, max}); + return result; } template diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 0f84d4ee4c..b2691113ef 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -172,6 +172,17 @@ class QuantizationTest : public ::testing::TestWithParam> h, quantizer_1, quantized_input_h_const_view, re_transformed_input_h.view()); }, cuvs::alloc_behavior::NO_ALLOCATIONS); + // Also covers the rows outside the range-fitting sample on the large input. + for (int64_t row = 0; row < rows_; ++row) { + double error = 0; + for (int64_t j = 0; j < cols_; ++j) { + auto const i = row * cols_ + j; + error = std::hypot(error, + static_cast(host_input_[i]) - + static_cast(re_transformed_input_h.data_handle()[i])); + } + EXPECT_GE(static_cast(quantizer_1.max_error), error); + } auto quantized_input_d_const_view = raft::make_device_matrix_view( quantized_input_d.data_handle(), rows_, cols_); @@ -360,6 +371,17 @@ class PerDimensionRangeTest : public ::testing::TestWithParam(params_.dataset[i]) - + static_cast(reconstructed.data_handle()[i])); + } + EXPECT_GE(static_cast(quantizer.max_error), error); + if (params_.tolerance == 0.f) { EXPECT_EQ(quantizer.max_error, 0.f); } + } } private: @@ -374,6 +396,99 @@ class PerDimensionQuantizationTest : public ::testing::Test {}; using PerDimensionTypes = ::testing::Types; TYPED_TEST_SUITE(PerDimensionQuantizationTest, PerDimensionTypes); +TYPED_TEST(PerDimensionQuantizationTest, TrainingReconstructionError) +{ + using T = TypeParam; + raft::resources res; + auto stream = raft::resource::get_cuda_stream(res); + // A partial warp plus a second column per lane, with clipped outliers in different rows. + constexpr int64_t rows = 9, dim = 35; + std::vector values(rows * dim); + for (int64_t row = 0; row < rows; ++row) { + for (int64_t j = 0; j < dim; ++j) { + values[row * dim + j] = static_cast((row - 4) * 0.17 + (j % 5) * 20.0); + } + } + values.front() = T{-300.f}; + values.back() = T{500.f}; + auto input = raft::make_device_matrix(res, rows, dim); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto codes = raft::make_device_matrix(res, rows, dim); + auto decoded = raft::make_device_matrix(res, rows, dim); + std::vector reconstructed(values.size()); + for (auto mode : {range_mode::global, range_mode::per_dimension}) { + for (float quantile : {1.f, 0.5f}) { + SCOPED_TRACE(::testing::Message() + << "range=" << static_cast(mode) << " quantile=" << quantile); + auto owner = train(res, params{quantile, mode}, raft::make_const_mdspan(input.view())); + auto view = owner.view(); + EXPECT_EQ(owner.max_error, view.max_error); + transform(res, view, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, view, raft::make_const_mdspan(codes.view()), decoded.view()); + raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); + raft::resource::sync_stream(res); + long double maximum = 0; + for (int64_t row = 0; row < rows; ++row) { + long double error = 0; + for (int64_t j = 0; j < dim; ++j) { + auto const i = row * dim + j; + auto const delta = static_cast(static_cast(values[i])) - + static_cast(static_cast(reconstructed[i])); + error = std::hypot(error, delta); + } + EXPECT_GE(static_cast(owner.max_error), error); + maximum = std::max(maximum, error); + } + EXPECT_GT(maximum, 0); + EXPECT_NEAR(static_cast(owner.max_error), + static_cast(maximum), + static_cast(maximum) * 2e-6); + if (quantile < 1.f) { EXPECT_GT(owner.max_error, 100); } + if (mode == range_mode::global) { + auto host = train(res, + params{quantile}, + raft::make_host_matrix_view(values.data(), rows, dim)); + EXPECT_GE(static_cast(host.max_error), maximum); + EXPECT_NEAR(static_cast(host.max_error), + static_cast(owner.max_error), + static_cast(maximum) * 1e-6); + } + } + } +} + +TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) +{ + raft::resources res; + auto stream = raft::resource::get_cuda_stream(res); + auto input = raft::make_device_matrix(res, 3, 2); + auto codes = raft::make_device_matrix(res, 3, 2); + auto decoded = raft::make_device_matrix(res, 3, 2); + // Squaring these residuals would underflow or overflow, although their L2 norm is finite. + for (double scale : {1e-300, 1e200}) { + SCOPED_TRACE(scale); + std::vector values{-scale, -scale, scale * 0.3, scale * 0.3, scale, scale}; + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto owner = + train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), decoded.view()); + std::vector reconstructed(values.size()); + raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); + raft::resource::sync_stream(res); + double maximum = 0; + for (int64_t row = 0; row < 3; ++row) { + auto const i = row * 2; + maximum = std::max( + maximum, std::hypot(values[i] - reconstructed[i], values[i + 1] - reconstructed[i + 1])); + } + EXPECT_GT(maximum, 0); + EXPECT_TRUE(std::isfinite(owner.max_error)); + EXPECT_GE(owner.max_error, maximum); + EXPECT_NEAR(owner.max_error, maximum, maximum * 2e-6); + } +} + TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) { using T = TypeParam; @@ -386,6 +501,8 @@ TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) auto owner = train(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); auto quantizer = owner.view(); + EXPECT_EQ(owner.max_error, 0); + EXPECT_EQ(quantizer.max_error, 0); EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); { std::vector actual(3); @@ -489,6 +606,9 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) scalar::quantizer owner{ res, raft::make_const_mdspan(mins.view()), raft::make_const_mdspan(maxs.view())}; auto quantizer = owner.view(); + EXPECT_TRUE(std::isinf(owner.max_error)); + EXPECT_TRUE(std::isinf(quantizer.max_error)); + EXPECT_TRUE(std::isinf((scalar::quantizer{T{-1}, T{1}}.view().max_error))); EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 2.f); { std::vector actual(2); @@ -538,6 +658,7 @@ TEST(QuantizationDatasetTest, GlobalRangeDataset) auto stream = raft::resource::get_cuda_stream(res); raft::update_device(input.data_handle(), values.data(), values.size(), stream); auto encoded = build(res, params{1.f}, raft::make_const_mdspan(input.view())); + EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); EXPECT_EQ(encoded.n_rows(), 1); EXPECT_EQ(encoded.dim(), 2u); EXPECT_TRUE(devArrMatchHost(std::vector{-128, 127}.data(), @@ -560,6 +681,7 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) raft::update_device(input.data_handle(), values.data(), values.size(), stream); auto encoded = build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); EXPECT_EQ(encoded.n_rows(), 2); EXPECT_EQ(encoded.dim(), 2u); EXPECT_TRUE(devArrMatchHost(std::vector{-127, -2, 127, 2}.data(), diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 41261e9993..920bc5f74d 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -55,6 +55,7 @@ Read-only scalar quantizer parameters. The owner must outlive stream work using template struct quantizer_const_view { std::variant value; + scale_type max_error; }; ``` @@ -62,7 +63,8 @@ struct quantizer_const_view { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | Global interval or borrowed per-dimension shifts and scale. | +| `max_error` | `scale_type` | Maximum training-row L2 reconstruction error; see quantizer::max_error. | ### preprocessing::quantize::scalar::quantizer_const_view::range_type @@ -115,6 +117,7 @@ Move-only; copy view() to borrow the parameters. Migration: replace q.min_ with template struct quantizer { std::variant value; + scale_type max_error; }; ``` @@ -122,7 +125,8 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | Global interval or owning per-dimension shifts and scale. | +| `max_error` | `scale_type` | Upper bound on the L2 error between a training row and its int8 encode/decode result. train measures every input row, including clipped outliers, and rounds the maximum up with a relative 1e-6 margin. This bound applies only to the training data, not later queries. It is infinity when constructed from parameters alone or when the error is not finite. Device training synchronizes before returning this host value. | ### preprocessing::quantize::scalar::quantizer::shifts_type From 8e0c620f81652b77ca75dac333a7f798e921fd74 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Wed, 4 Nov 2026 21:38:55 +0000 Subject: [PATCH 11/24] Stream scalar quantized datasets from host memory --- cpp/include/cuvs/core/dataset/sq.hpp | 43 +++ .../cuvs/preprocessing/quantize/scalar.hpp | 15 +- .../cluster/detail/kmeans_batch_loader.cuh | 2 +- .../preprocessing/quantize/detail/scalar.cuh | 341 ++++++++++++------ cpp/src/preprocessing/quantize/scalar.cu | 92 ++--- .../preprocessing/scalar_quantization.cu | 109 ++++++ fern/pages/cpp_api/cpp-api-core-dataset-sq.md | 70 ++++ .../cpp-api-preprocessing-quantize-scalar.md | 6 +- 8 files changed, 524 insertions(+), 154 deletions(-) diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index 5859f8189d..b3afac3f04 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -108,6 +108,49 @@ inline device_sq_dataset build(raft::resources const& res, return device_sq_dataset{std::move(codes), std::move(quantizer)}; } +/** + * @brief Train and encode a host corpus in GPU batches, returning device-resident int8 codes. + * Uses one quantizer across all batches and measures the maximum error over every row. + * Configure params.device_buffer_samples to bound input staging and an auxiliary stream pool + * on res to enable double buffering. All int8 codes must fit in device memory. + * + * @param[in] res Resources used for training, allocation, and transfer streams. + * @param[in] params Scalar quantizer and host batching parameters. + * @param[in] dataset Row-major host corpus; pageable and pinned input are supported. + * @return Device dataset owning the quantized codes and quantizer. + */ +device_sq_dataset build( + raft::resources const& res, params params, raft::host_matrix_view dataset); + +/** + * @brief Train and encode a host corpus in GPU batches, returning device-resident int8 codes. + * Uses one quantizer across all batches and measures the maximum error over every row. + * Configure params.device_buffer_samples to bound input staging and an auxiliary stream pool + * on res to enable double buffering. All int8 codes must fit in device memory. + * + * @param[in] res Resources used for training, allocation, and transfer streams. + * @param[in] params Scalar quantizer and host batching parameters. + * @param[in] dataset Row-major host corpus; pageable and pinned input are supported. + * @return Device dataset owning the quantized codes and quantizer. + */ +device_sq_dataset build( + raft::resources const& res, params params, raft::host_matrix_view dataset); + +/** + * @brief Train and encode a host corpus in GPU batches, returning device-resident int8 codes. + * Uses one quantizer across all batches and measures the maximum error over every row. + * Configure params.device_buffer_samples to bound input staging and an auxiliary stream pool + * on res to enable double buffering. All int8 codes must fit in device memory. + * + * @param[in] res Resources used for training, allocation, and transfer streams. + * @param[in] params Scalar quantizer and host batching parameters. + * @param[in] dataset Row-major host corpus; pageable and pinned input are supported. + * @return Device dataset owning the quantized codes and quantizer. + */ +device_sq_dataset build(raft::resources const& res, + params params, + raft::host_matrix_view dataset); + /** @} */ } // namespace preprocessing::quantize::scalar diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index db9cf34bca..2af04964b9 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -42,7 +42,7 @@ struct params { */ float quantile = 0.99; /** - * Per-dimension training uses every row of a nonempty, finite device dataset. + * Per-dimension training uses every row of a nonempty, finite dataset. * It applies the quantile independently to each dimension to select lower and upper bounds. * Each dimension's shift is the midpoint of its bounds, rounded to the input type. * A shared scale fits every dimension's bounds around its stored shift into the symmetric @@ -53,6 +53,18 @@ struct params { * returning the scale to the host. */ range_mode range = range_mode::global; + /** + * Maximum rows staged on device for host per-dimension training and host dataset build. + * Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. + * Multiple batches use two input buffers when res has an auxiliary stream pool, and one + * otherwise. Pageable input uses batch-sized pinned staging buffers, without registering + * the full input. Each input buffer uses at most device_buffer_samples * dim * sizeof(T) + * bytes, in addition to reduction and encoding workspace. Substitute n_rows when it is zero. + * + * Per-dimension host training with quantile < 1 selects exact quantiles using one column of + * host scratch (n_rows * sizeof(T) bytes). Global training retains its sampled quantiles. + */ + int64_t device_buffer_samples = 1 << 16; }; /** Read-only scalar quantizer parameters. The owner must outlive stream work using shifts. */ @@ -100,7 +112,6 @@ struct quantizer { * train measures every input row, including clipped outliers, and rounds the maximum up * with a relative 1e-6 margin. This bound applies only to the training data, not later queries. * It is infinity when constructed from parameters alone or when the error is not finite. - * Device training synchronizes before returning this host value. */ scale_type max_error = INFINITY; diff --git a/cpp/src/cluster/detail/kmeans_batch_loader.cuh b/cpp/src/cluster/detail/kmeans_batch_loader.cuh index 7c391505ae..cf72c7caf8 100644 --- a/cpp/src/cluster/detail/kmeans_batch_loader.cuh +++ b/cpp/src/cluster/detail/kmeans_batch_loader.cuh @@ -86,7 +86,7 @@ class kmeans_batch { }; /** - * Read-only batch loader used only by KMeans. + * Read-only host/device batch loader. * * The device specialization is a zero-copy view. The host specialization below owns the cyclic * H2D pipeline used by out-of-core KMeans: two buffers when prefetching is enabled, or one buffer diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index 112298eb6d..8bdc4706c1 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -5,6 +5,9 @@ #pragma once +#include "../../../cluster/detail/kmeans_batch_loader.cuh" +#include "../../../neighbors/detail/ann_utils.cuh" + #include #include #include @@ -18,9 +21,12 @@ #include #include #include +#include #include #include #include +#include +#include #include #include @@ -64,81 +70,40 @@ struct quantize_op { { } - constexpr RAFT_INLINE_FUNCTION QuantI operator()(const T& x, int64_t = 0) const + constexpr RAFT_INLINE_FUNCTION QuantI operator()(const T& x) const { if (!fp_lt(min_, x)) return q_type_min_; if (!fp_lt(x, max_)) return q_type_max_; return static_cast(lroundf(scalar_ * static_cast(x) + offset_)); } - constexpr RAFT_INLINE_FUNCTION T operator()(const QuantI& x, int64_t = 0) const + constexpr RAFT_INLINE_FUNCTION T operator()(const QuantI& x) const { return static_cast((static_cast(x) - offset_) / scalar_); } }; -// Share the exact encode/decode arithmetic with the reconstruction-error pass. -template -struct per_dim_quantize_op { - using compute_t = typename quantizer_const_view::scale_type; - T const* shifts; - compute_t scale; - compute_t inv; - - explicit per_dim_quantize_op(typename quantizer_const_view::shifts_type quantizer) - : shifts{quantizer.shifts.data_handle()}, scale{quantizer.scale}, inv{compute_t{1} / scale} - { - } - - RAFT_INLINE_FUNCTION int8_t operator()(T x, int64_t j) const - { - compute_t const code = - rint((static_cast(x) - static_cast(shifts[j])) * inv); - return static_cast(fmin(fmax(code, compute_t{-127}), compute_t{127})); - } - - RAFT_INLINE_FUNCTION T operator()(int8_t code, int64_t j) const - { - compute_t const value = - fma(scale, static_cast(code), static_cast(shifts[j])); - compute_t const limit = static_cast(cuda::std::numeric_limits::max()); - return static_cast(fmin(fmax(value, -limit), limit)); - } -}; - template RAFT_INLINE_FUNCTION double reconstruction_error(T x, int64_t j, Op op) { - double const value = static_cast(x); - double const decoded = static_cast(op(op(x, j), j)); + double const value = static_cast(x); + double decoded; + if constexpr (std::is_same_v::shifts_type>) { + using compute_t = typename quantizer_const_view::scale_type; + compute_t const shift = static_cast(op.shifts.data_handle()[j]); + compute_t const inv = compute_t{1} / op.scale; + compute_t const rounded = rint((static_cast(x) - shift) * inv); + int8_t const code = static_cast(fmin(fmax(rounded, compute_t{-127}), compute_t{127})); + compute_t const reconstructed = fma(op.scale, static_cast(code), shift); + compute_t const limit = static_cast(cuda::std::numeric_limits::max()); + decoded = static_cast(static_cast(fmin(fmax(reconstructed, -limit), limit))); + } else { + decoded = static_cast(op(op(x))); + } return isfinite(value) && isfinite(decoded) ? fabs(value - decoded) : std::numeric_limits::infinity(); } -// One warp per row; each warp keeps its maximum over a grid-stride sequence of rows. -// hypot avoids overflow/underflow of squared residuals for large/tiny finite inputs. -template -__global__ void reconstruction_error_kernel(raft::device_matrix_view dataset, - Op op, - double* errors) -{ - auto const warp = static_cast(blockIdx.x) * (blockDim.x / 32) + threadIdx.x / 32; - auto const lane = threadIdx.x % 32; - auto const stride = static_cast(gridDim.x) * (blockDim.x / 32); - double max_error = 0; - for (int64_t row = warp; row < dataset.extent(0); row += stride) { - double error = 0; - for (int64_t j = lane; j < dataset.extent(1); j += 32) { - error = hypot(error, reconstruction_error(dataset(row, j), j, op)); - } - for (int offset = 16; offset > 0; offset /= 2) { - error = hypot(error, __shfl_down_sync(0xffffffffu, error, offset)); - } - max_error = fmax(max_error, error); - } - if (lane == 0) { errors[warp] = max_error; } -} - template auto round_error_up(double error) -> typename quantizer::scale_type { @@ -149,27 +114,39 @@ auto round_error_up(double error) -> typename quantizer::scale_type } template -auto max_reconstruction_error(raft::resources const& res, +void max_reconstruction_error(raft::resources const& res, raft::device_matrix_view dataset, - Op op) -> typename quantizer::scale_type + Op op, + raft::device_vector_view errors, + raft::device_vector_view maximum, + bool inplace) { - constexpr int threads = 256; - auto const blocks = static_cast(std::min((dataset.extent(0) + 7) / 8, 65535)); - auto errors = raft::make_device_vector(res, blocks * 8); - auto maximum = raft::make_device_vector(res, 1); - if (!raft::resource::get_dry_run_flag(res)) { - reconstruction_error_kernel<<>>( - dataset, op, errors.data_handle()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); - } raft::linalg::reduce( res, - raft::make_device_matrix_view(errors.data_handle(), 1, errors.size()), - maximum.view(), + dataset, + errors, 0.0, false, + [op] __device__(T x, int64_t j) { return reconstruction_error(x, j, op); }, + [] __device__(double a, double b) { return hypot(a, b); }); + raft::linalg::reduce( + res, + raft::make_device_matrix_view(errors.data_handle(), 1, errors.size()), + maximum, + 0.0, + inplace, raft::identity_op{}, raft::max_op{}); +} + +template +auto max_reconstruction_error(raft::resources const& res, + raft::device_matrix_view dataset, + Op op) -> typename quantizer::scale_type +{ + auto errors = raft::make_device_vector(res, dataset.extent(0)); + auto maximum = raft::make_device_vector(res, 1); + max_reconstruction_error(res, dataset, op, errors.view(), maximum.view(), false); double result = std::numeric_limits::infinity(); raft::copy(res, raft::make_host_scalar_view(&result), @@ -272,9 +249,9 @@ typename quantizer::shifts_type train(raft::resources const& res, !isfinite(static_cast(max_ptr[j])) || fp_lt(max_ptr[j], shift_ptr[j])) { return std::numeric_limits::infinity(); } - compute_t const lo = static_cast(shift_ptr[j]); - compute_t const hi = static_cast(max_ptr[j]); - shift_ptr[j] = static_cast(lo * compute_t{0.5} + hi * compute_t{0.5}); + compute_t const lo = static_cast(shift_ptr[j]); + compute_t const hi = static_cast(max_ptr[j]); + shift_ptr[j] = static_cast(lo == hi ? lo : lo * compute_t{0.5} + hi * compute_t{0.5}); compute_t const shift = static_cast(shift_ptr[j]); return fmax(hi - shift, shift - lo); }); @@ -359,10 +336,7 @@ quantizer train(raft::resources const& res, quantizer result; result.value = train(res, std::move(shifts), std::move(maxima)); result.max_error = max_reconstruction_error( - res, - dataset, - per_dim_quantize_op{ - std::get::shifts_type>(result.view().value)}); + res, dataset, std::get::shifts_type>(result.view().value)); return result; } @@ -386,12 +360,17 @@ void transform(raft::resources const& res, raft::device_matrix_view dataset, raft::device_matrix_view out) { + using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const dim = dataset.extent(1); - auto const op = per_dim_quantize_op{quantizer}; - raft::linalg::map_offset( - res, out, [input, dim, op] __device__(int64_t i) { return op(input[i], i % dim); }); + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + compute_t const inv = compute_t{1} / quantizer.scale; + raft::linalg::map_offset(res, out, [input, shifts, dim, inv] __device__(int64_t i) { + compute_t const code = + rint((static_cast(input[i]) - static_cast(shifts[i % dim])) * inv); + return static_cast(fmin(fmax(code, compute_t{-127}), compute_t{127})); + }); } template @@ -400,12 +379,18 @@ void inverse_transform(raft::resources const& res, raft::device_matrix_view dataset, raft::device_matrix_view out) { + using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const dim = dataset.extent(1); - auto const op = per_dim_quantize_op{quantizer}; - raft::linalg::map_offset( - res, out, [input, dim, op] __device__(int64_t i) { return op(input[i], i % dim); }); + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + compute_t const scale = quantizer.scale; + raft::linalg::map_offset(res, out, [input, shifts, dim, scale] __device__(int64_t i) { + compute_t const value = + fma(scale, static_cast(input[i]), static_cast(shifts[i % dim])); + compute_t const limit = static_cast(cuda::std::numeric_limits::max()); + return static_cast(fmin(fmax(value, -limit), limit)); + }); } template @@ -429,27 +414,6 @@ cuvs::preprocessing::quantize::scalar::quantizer train( return result; } -template -cuvs::preprocessing::quantize::scalar::quantizer train( - raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::params params, - raft::host_matrix_view dataset) -{ - RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, - "quantile for scalar quantization needs to be within (0, 1] but is %f", - params.quantile); - - RAFT_EXPECTS(params.range == range_mode::global, "Per-dimension training requires device input"); - - auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); - - RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); - - quantizer result{min, max}; - result.max_error = max_reconstruction_error(res, dataset, quantize_op{min, max}); - return result; -} - template void transform(raft::resources const& res, quantizer_const_view quantizer, @@ -528,4 +492,165 @@ void inverse_transform(raft::resources const& res, } } +// The optional output lets dataset construction encode during the error-measurement pass. +// Standalone host training uses the same shared parameters and corpus-wide error reduction. +template +quantizer train_host(raft::resources const& res, + params params, + raft::host_matrix_view dataset, + std::optional> out = std::nullopt) +{ + using compute_t = typename quantizer::scale_type; + auto const rows = dataset.extent(0); + auto const dim = dataset.extent(1); + RAFT_EXPECTS(rows > 0 && dim > 0 && dataset.data_handle() != nullptr, + "Training requires nonempty rows and dimensions"); + RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); + RAFT_EXPECTS(params.device_buffer_samples >= 0, "device_buffer_samples must be nonnegative"); + auto const batch_rows = + params.device_buffer_samples == 0 ? rows : std::min(rows, params.device_buffer_samples); + auto [copy_stream, prefetch] = cuvs::spatial::knn::detail::utils::get_prefetch_stream(res); + cuvs::cluster::kmeans::detail::kmeans_batch_loader batches( + res, + dataset, + batch_rows, + copy_stream, + raft::resource::get_large_workspace_resource_ref(res), + prefetch); + bool const dry_run = raft::resource::get_dry_run_flag(res); + auto for_each_batch = [&](auto&& consume, bool repeat) { + if (dry_run) { + // Account for the largest batch's workspace without submitting loader copies. + consume(raft::make_device_matrix_view(nullptr, batch_rows, dim), + int64_t{0}); + return; + } + batches.start(); + for (std::size_t pos = 0; pos < batches.num_batches(); ++pos) { + auto batch = batches.acquire(pos); + if (pos + 1 < batches.num_batches() || repeat) { + batches.prefetch((pos + 1) % batches.num_batches()); + } + consume(raft::make_device_matrix_view(batch.data(), batch.size(), dim), + static_cast(batch.offset())); + auto const next = pos + (prefetch ? 2 : 1); + if (next < batches.num_batches() || repeat) { + batches.recycle(batch, next % batches.num_batches()); + } else { + batches.release(batch); + } + } + }; + + quantizer result; + if (params.range == range_mode::global) { + auto [min, max] = quantile_min_max(res, dataset, params.quantile); + result = quantizer{min, max}; + } else { + auto minima = raft::make_device_vector(res, dim); + auto maxima = raft::make_device_vector(res, dim); + if (params.quantile == 1.f) { + for_each_batch( + [&](auto input, int64_t offset) { + raft::linalg::reduce( + res, + input, + minima.view(), + static_cast(std::numeric_limits::infinity()), + offset != 0, + raft::identity_op{}, + raft::min_op{}); + raft::linalg::reduce( + res, + input, + maxima.view(), + static_cast(-std::numeric_limits::infinity()), + offset != 0, + raft::identity_op{}, + raft::max_op{}); + }, + true); + } else { + // Select over the complete column, not independently fitted batch quantiles. + std::vector mins(dim, T{0}), maxs(dim, T{0}); + if (!dry_run) { + std::vector column(rows); + auto const upper = + static_cast(std::ceil((0.5 + 0.5 * params.quantile) * rows)) - 1; + auto const lower = rows - upper - 1; + for (int64_t j = 0; j < dim; ++j) { + for (int64_t i = 0; i < rows; ++i) { + column[i] = dataset(i, j); + } + std::nth_element(column.begin(), column.begin() + upper, column.end(), fp_lt_op{}); + maxs[j] = column[upper]; + std::nth_element( + column.begin(), column.begin() + lower, column.begin() + upper + 1, fp_lt_op{}); + mins[j] = column[lower]; + } + } + raft::copy( + res, minima.view(), raft::make_host_vector_view(mins.data(), dim)); + raft::copy( + res, maxima.view(), raft::make_host_vector_view(maxs.data(), dim)); + raft::resource::sync_stream(res); + } + result.value = train(res, std::move(minima), std::move(maxima)); + } + + auto errors = raft::make_device_vector(res, batch_rows); + auto maximum = raft::make_device_vector(res, 1); + auto view = result.view(); + for_each_batch( + [&](auto input, int64_t offset) { + auto batch_errors = + raft::make_device_vector_view(errors.data_handle(), input.extent(0)); + if (auto const* shifts = + std::get_if::shifts_type>(&view.value)) { + max_reconstruction_error(res, input, *shifts, batch_errors, maximum.view(), offset != 0); + } else { + auto const& range = std::get::range_type>(view.value); + max_reconstruction_error(res, + input, + quantize_op{range.min_, range.max_}, + batch_errors, + maximum.view(), + offset != 0); + } + if (out) { + auto batch_codes = raft::make_device_matrix_view( + out->data_handle() + offset * dim, input.extent(0), dim); + detail::transform(res, view, input, batch_codes); + } + }, + false); + double error = std::numeric_limits::infinity(); + raft::copy( + res, raft::make_host_scalar_view(&error), raft::make_device_scalar_view(maximum.data_handle())); + raft::resource::sync_stream(res); + result.max_error = round_error_up(error); + return result; +} + +template +cuvs::preprocessing::quantize::scalar::quantizer train( + raft::resources const& res, + const cuvs::preprocessing::quantize::scalar::params params, + raft::host_matrix_view dataset) +{ + RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, + "quantile for scalar quantization needs to be within (0, 1] but is %f", + params.quantile); + + if (params.range == range_mode::per_dimension) { return train_host(res, params, dataset); } + + auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); + + RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); + + quantizer result{min, max}; + result.max_error = max_reconstruction_error(res, dataset, quantize_op{min, max}); + return result; +} + } // namespace cuvs::preprocessing::quantize::scalar::detail diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index 270c53486c..d7d967ed51 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -4,6 +4,7 @@ */ #include +#include #include "./detail/scalar.cuh" @@ -25,47 +26,56 @@ quantizer::quantizer(raft::resources const& res, value = detail::train(res, std::move(minima), std::move(maxima)); } -#define CUVS_INST_QUANTIZATION(T, QuantI) \ - auto train(raft::resources const& res, \ - const params params, \ - raft::device_matrix_view dataset) -> quantizer \ - { \ - return detail::train(res, params, dataset); \ - } \ - auto train(raft::resources const& res, \ - const params params, \ - raft::host_matrix_view dataset) -> quantizer \ - { \ - return detail::train(res, params, dataset); \ - } \ - void transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view out) \ - { \ - detail::transform(res, quantizer, dataset, out); \ - } \ - void transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view out) \ - { \ - detail::transform(res, quantizer, dataset, out); \ - } \ - void inverse_transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ - raft::device_matrix_view dataset, \ - raft::device_matrix_view out) \ - { \ - detail::inverse_transform(res, quantizer, dataset, out); \ - } \ - void inverse_transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ - raft::host_matrix_view dataset, \ - raft::host_matrix_view out) \ - { \ - detail::inverse_transform(res, quantizer, dataset, out); \ - } \ +#define CUVS_INST_QUANTIZATION(T, QuantI) \ + auto train(raft::resources const& res, \ + const params params, \ + raft::device_matrix_view dataset) -> quantizer \ + { \ + return detail::train(res, params, dataset); \ + } \ + auto train(raft::resources const& res, \ + const params params, \ + raft::host_matrix_view dataset) -> quantizer \ + { \ + return detail::train(res, params, dataset); \ + } \ + void transform(raft::resources const& res, \ + quantizer_const_view quantizer, \ + raft::device_matrix_view dataset, \ + raft::device_matrix_view out) \ + { \ + detail::transform(res, quantizer, dataset, out); \ + } \ + void transform(raft::resources const& res, \ + quantizer_const_view quantizer, \ + raft::host_matrix_view dataset, \ + raft::host_matrix_view out) \ + { \ + detail::transform(res, quantizer, dataset, out); \ + } \ + void inverse_transform(raft::resources const& res, \ + quantizer_const_view quantizer, \ + raft::device_matrix_view dataset, \ + raft::device_matrix_view out) \ + { \ + detail::inverse_transform(res, quantizer, dataset, out); \ + } \ + void inverse_transform(raft::resources const& res, \ + quantizer_const_view quantizer, \ + raft::host_matrix_view dataset, \ + raft::host_matrix_view out) \ + { \ + detail::inverse_transform(res, quantizer, dataset, out); \ + } \ + auto build( \ + raft::resources const& res, params params, raft::host_matrix_view dataset) \ + -> device_sq_dataset \ + { \ + auto codes = \ + raft::make_device_matrix(res, dataset.extent(0), dataset.extent(1)); \ + auto quantizer = detail::train_host(res, params, dataset, codes.view()); \ + return {std::move(codes), std::move(quantizer)}; \ + } \ template struct quantizer; CUVS_INST_QUANTIZATION(double, int8_t); diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index b2691113ef..0b9ec615b6 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -6,7 +6,9 @@ #include "../test_utils.cuh" #include #include +#include #include +#include #include #include #include @@ -699,6 +701,107 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) stream.get())); } +TYPED_TEST(PerDimensionQuantizationTest, BatchedHostDataset) +{ + using T = TypeParam; + constexpr int64_t rows = 37, dim = 35; + std::vector values(rows * dim); + for (int64_t i = 0; i < rows; ++i) { + for (int64_t j = 0; j < dim; ++j) { + values[i * dim + j] = T((i - 18) * .17 + (j % 5) * 20.0); + } + } + // Put the largest clipping error before the final batch to exercise the running maximum. + values.front() = T{-1200.f}; + values.back() = T{500.f}; + for (bool prefetch : {false, true}) { + raft::resources res; + if (prefetch) { + raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); + } + auto stream = raft::resource::get_cuda_stream(res); + auto input = raft::make_device_matrix(res, rows, dim); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto pinned = raft::make_pinned_matrix(res, rows, dim); + std::copy(values.begin(), values.end(), pinned.data_handle()); + for (auto mode : {range_mode::global, range_mode::per_dimension}) { + for (float quantile : {1.f, .5f, .99f}) { + auto expected = build(res, params{quantile, mode}, raft::make_const_mdspan(input.view())); + for (int64_t batch_rows : {int64_t{1}, int64_t{7}, rows, int64_t{0}}) { + for (bool use_pinned : {false, true}) { + SCOPED_TRACE(::testing::Message() + << "range=" << static_cast(mode) << " quantile=" << quantile + << " batch=" << batch_rows << " prefetch=" << prefetch + << " pinned=" << use_pinned); + auto host_input = raft::make_host_matrix_view( + use_pinned ? pinned.data_handle() : values.data(), rows, dim); + auto actual = build(res, params{quantile, mode, batch_rows}, host_input); + EXPECT_EQ(actual.n_rows(), rows); + EXPECT_EQ(actual.dim(), dim); + EXPECT_TRUE(devArrMatch(expected.data_const_view().data_handle(), + actual.data_const_view().data_handle(), + values.size(), + cuvs::Compare(), + stream.get())); + auto a = actual.const_manifest_view(); + auto e = expected.const_manifest_view(); + EXPECT_NEAR(double(a.max_error), double(e.max_error), double(e.max_error) * 1e-6); + if (mode == range_mode::per_dimension) { + auto actual_shifts = std::get::shifts_type>(a.value); + auto expected_shifts = + std::get::shifts_type>(e.value); + EXPECT_EQ(actual_shifts.scale, expected_shifts.scale); + EXPECT_TRUE(devArrMatch(actual_shifts.shifts.data_handle(), + expected_shifts.shifts.data_handle(), + dim, + cuvs::Compare(), + stream.get())); + } + // Standalone training must give the same manifest without allocating corpus codes. + if (batch_rows == 7 && !use_pinned) { + auto trained = train(res, params{quantile, mode, batch_rows}, host_input); + EXPECT_NEAR( + double(trained.max_error), double(a.max_error), double(a.max_error) * 1e-6); + } + } + } + } + } + } +} + +TEST(QuantizationDatasetTest, HostBuildBatchWorkspace) +{ + constexpr int64_t rows = 4097, dim = 35, batch_rows = 128; + std::vector values(rows * dim, 1.f); + auto input = raft::make_host_matrix_view(values.data(), rows, dim); + for (bool prefetch : {false, true}) { + raft::resources res; + if (prefetch) { + raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); + } + for (float quantile : {1.f, .5f}) { + params p{quantile, range_mode::per_dimension, batch_rows}; + auto action = [&](raft::resources const& h) { + auto encoded = build(h, p, input); + if (!raft::resource::get_dry_run_flag(h)) { + EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); + } + }; + execute_with_dry_run_check(res, action, alloc_behavior::ARGUMENT_DRIVEN); + raft::memory_stats_resources stats(res); + action(stats); + auto peak = stats.get_bytes_peak(); + auto device_bytes = peak.device_workspace + peak.device_large_workspace + peak.device_global + + peak.device_managed; + // Includes all int8 codes; excludes enough memory for even one full FP32 corpus copy. + EXPECT_LT(device_bytes, values.size() * sizeof(float)); + } + } + EXPECT_THROW(build(raft::resources{}, params{1.f, range_mode::per_dimension, -1}, input), + raft::logic_error); +} + TEST_P(PerDimensionRangeTest, Result) { this->testPerDimensionQuantization(); } typedef QuantizationTest QuantizationTest_float_int8t; @@ -726,6 +829,12 @@ const std::vector> inputs = { const std::vector dataset_inputs = { {2, 2, {10.f, -7.f, 10.f, -7.f}, {0, 0, 0, 0}, 0.f}, + // Equal subnormal bounds must retain their value and reconstruct exactly. + {1, + 2, + {std::numeric_limits::denorm_min(), -std::numeric_limits::denorm_min()}, + {0, 0}, + 0.f}, {2, 1, {-std::numeric_limits::max(), std::numeric_limits::max()}, diff --git a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md index 9f7444ce4e..22bfd79bb3 100644 --- a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md +++ b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md @@ -47,3 +47,73 @@ raft::device_matrix_view dataset); `inline device_sq_dataset` Device dataset owning the quantized codes and quantizer. + +**Additional overload:** `preprocessing::quantize::scalar::build` + +Train and encode a host corpus in GPU batches, returning device-resident int8 codes. Uses one quantizer across all batches and measures the maximum error over every row. Configure params.device_buffer_samples to bound input staging and an auxiliary stream pool on res to enable double buffering. All int8 codes must fit in device memory. + +```cpp +device_sq_dataset build( +raft::resources const& res, params params, raft::host_matrix_view dataset); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | in | `raft::resources const&` | Resources used for training, allocation, and transfer streams. | +| `params` | in | `params` | Scalar quantizer and host batching parameters. | +| `dataset` | in | `raft::host_matrix_view` | Row-major host corpus; pageable and pinned input are supported. | + +**Returns** + +`device_sq_dataset` + +Device dataset owning the quantized codes and quantizer. + +**Additional overload:** `preprocessing::quantize::scalar::build` + +Train and encode a host corpus in GPU batches, returning device-resident int8 codes. Uses one quantizer across all batches and measures the maximum error over every row. Configure params.device_buffer_samples to bound input staging and an auxiliary stream pool on res to enable double buffering. All int8 codes must fit in device memory. + +```cpp +device_sq_dataset build( +raft::resources const& res, params params, raft::host_matrix_view dataset); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | in | `raft::resources const&` | Resources used for training, allocation, and transfer streams. | +| `params` | in | `params` | Scalar quantizer and host batching parameters. | +| `dataset` | in | `raft::host_matrix_view` | Row-major host corpus; pageable and pinned input are supported. | + +**Returns** + +`device_sq_dataset` + +Device dataset owning the quantized codes and quantizer. + +**Additional overload:** `preprocessing::quantize::scalar::build` + +Train and encode a host corpus in GPU batches, returning device-resident int8 codes. Uses one quantizer across all batches and measures the maximum error over every row. Configure params.device_buffer_samples to bound input staging and an auxiliary stream pool on res to enable double buffering. All int8 codes must fit in device memory. + +```cpp +device_sq_dataset build(raft::resources const& res, +params params, +raft::host_matrix_view dataset); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | in | `raft::resources const&` | Resources used for training, allocation, and transfer streams. | +| `params` | in | `params` | Scalar quantizer and host batching parameters. | +| `dataset` | in | `raft::host_matrix_view` | Row-major host corpus; pageable and pinned input are supported. | + +**Returns** + +`device_sq_dataset` + +Device dataset owning the quantized codes and quantizer. diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 920bc5f74d..597cf2cb06 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -36,6 +36,7 @@ quantizer parameters. struct params { float quantile; range_mode range; + int64_t device_buffer_samples; }; ``` @@ -44,7 +45,8 @@ struct params { | Name | Type | Description | | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | -| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite device dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. A shared scale fits every dimension's bounds around its stored shift into the symmetric range of quantized values.

If every interval has zero width, the scale is one. Otherwise, the scale is at least the smallest positive normal value of the scale type. Training synchronizes the stream before returning the scale to the host. | +| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. A shared scale fits every dimension's bounds around its stored shift into the symmetric range of quantized values.

If every interval has zero width, the scale is one. Otherwise, the scale is at least the smallest positive normal value of the scale type. Training synchronizes the stream before returning the scale to the host. | +| `device_buffer_samples` | `int64_t` | Maximum rows staged on device for host per-dimension training and host dataset build.
Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. Multiple batches use two input buffers when res has an auxiliary stream pool, and one otherwise. Pageable input uses batch-sized pinned staging buffers, without registering the full input. Each input buffer uses at most device_buffer_samples * dim * sizeof(T) bytes, in addition to reduction and encoding workspace. Substitute n_rows when it is zero.

Per-dimension host training with quantile < 1 selects exact quantiles using one column of host scratch (n_rows * sizeof(T) bytes). Global training retains its sampled quantiles. | ### preprocessing::quantize::scalar::quantizer_const_view @@ -126,7 +128,7 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | | `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | Global interval or owning per-dimension shifts and scale. | -| `max_error` | `scale_type` | Upper bound on the L2 error between a training row and its int8 encode/decode result. train measures every input row, including clipped outliers, and rounds the maximum up with a relative 1e-6 margin. This bound applies only to the training data, not later queries. It is infinity when constructed from parameters alone or when the error is not finite. Device training synchronizes before returning this host value. | +| `max_error` | `scale_type` | Upper bound on the L2 error between a training row and its int8 encode/decode result. train measures every input row, including clipped outliers, and rounds the maximum up with a relative 1e-6 margin. This bound applies only to the training data, not later queries. It is infinity when constructed from parameters alone or when the error is not finite. | ### preprocessing::quantize::scalar::quantizer::shifts_type From 3c10b83571032ba5e8fe34b340b63474bca63bdd Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Wed, 4 Nov 2026 22:12:32 +0000 Subject: [PATCH 12/24] Remove scalar quantizer synchronization notes --- cpp/include/cuvs/preprocessing/quantize/scalar.hpp | 5 ++--- fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 2af04964b9..4ce23f5146 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -49,8 +49,7 @@ struct params { * range of quantized values. * * If every interval has zero width, the scale is one. Otherwise, the scale is at least - * the smallest positive normal value of the scale type. Training synchronizes the stream before - * returning the scale to the host. + * the smallest positive normal value of the scale type. */ range_mode range = range_mode::global; /** @@ -120,7 +119,7 @@ struct quantizer { /** Construct a global interval. */ quantizer(T min, T max) : value{range_type{min, max}} {} /** Construct from nonempty device bounds of equal length, with finite min <= max. - * Derives midpoint shifts and a shared scale; synchronizes before returning. */ + * Derives midpoint shifts and a shared scale. */ quantizer(raft::resources const& res, raft::device_vector_view mins, raft::device_vector_view maxs); diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 597cf2cb06..f13f00bd71 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -45,7 +45,7 @@ struct params { | Name | Type | Description | | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | -| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. A shared scale fits every dimension's bounds around its stored shift into the symmetric range of quantized values.

If every interval has zero width, the scale is one. Otherwise, the scale is at least the smallest positive normal value of the scale type. Training synchronizes the stream before returning the scale to the host. | +| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. A shared scale fits every dimension's bounds around its stored shift into the symmetric range of quantized values.

If every interval has zero width, the scale is one. Otherwise, the scale is at least the smallest positive normal value of the scale type. | | `device_buffer_samples` | `int64_t` | Maximum rows staged on device for host per-dimension training and host dataset build.
Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. Multiple batches use two input buffers when res has an auxiliary stream pool, and one otherwise. Pageable input uses batch-sized pinned staging buffers, without registering the full input. Each input buffer uses at most device_buffer_samples * dim * sizeof(T) bytes, in addition to reduction and encoding workspace. Substitute n_rows when it is zero.

Per-dimension host training with quantile < 1 selects exact quantiles using one column of host scratch (n_rows * sizeof(T) bytes). Global training retains its sampled quantiles. | @@ -191,7 +191,7 @@ raft::device_vector_view mins, raft::device_vector_view maxs); ``` -Derives midpoint shifts and a shared scale; synchronizes before returning. +Derives midpoint shifts and a shared scale. **Parameters** From 053dce752c624e1192f7544bbfd12698a0e638c1 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Wed, 4 Nov 2026 23:04:04 +0000 Subject: [PATCH 13/24] Unify scalar training and derive quantized limits from the code type --- .../cuvs/preprocessing/quantize/scalar.hpp | 2 +- .../preprocessing/quantize/detail/scalar.cuh | 254 ++++++------------ cpp/src/preprocessing/quantize/scalar.cu | 6 +- .../preprocessing/scalar_quantization.cu | 35 ++- .../cpp-api-preprocessing-quantize-scalar.md | 2 +- 5 files changed, 114 insertions(+), 185 deletions(-) diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 4ce23f5146..29df0cc5e0 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -53,7 +53,7 @@ struct params { */ range_mode range = range_mode::global; /** - * Maximum rows staged on device for host per-dimension training and host dataset build. + * Maximum rows staged on device for host training and host dataset build. * Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. * Multiple batches use two input buffers when res has an auxiliary stream pool, and one * otherwise. Pageable input uses batch-sized pinned staging buffers, without registering diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index 8bdc4706c1..e462dea253 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -83,17 +83,19 @@ struct quantize_op { } }; -template +template RAFT_INLINE_FUNCTION double reconstruction_error(T x, int64_t j, Op op) { double const value = static_cast(x); double decoded; if constexpr (std::is_same_v::shifts_type>) { - using compute_t = typename quantizer_const_view::scale_type; - compute_t const shift = static_cast(op.shifts.data_handle()[j]); - compute_t const inv = compute_t{1} / op.scale; - compute_t const rounded = rint((static_cast(x) - shift) * inv); - int8_t const code = static_cast(fmin(fmax(rounded, compute_t{-127}), compute_t{127})); + using compute_t = typename quantizer_const_view::scale_type; + compute_t const shift = static_cast(op.shifts.data_handle()[j]); + compute_t const inv = compute_t{1} / op.scale; + compute_t const rounded = rint((static_cast(x) - shift) * inv); + constexpr compute_t q_max = std::numeric_limits::max(); + constexpr compute_t q_min = std::max(std::numeric_limits::min(), -q_max); + QuantI const code = static_cast(fmin(fmax(rounded, q_min), q_max)); compute_t const reconstructed = fma(op.scale, static_cast(code), shift); compute_t const limit = static_cast(cuda::std::numeric_limits::max()); decoded = static_cast(static_cast(fmin(fmax(reconstructed, -limit), limit))); @@ -113,7 +115,7 @@ auto round_error_up(double error) -> typename quantizer::scale_type return error > 0 ? std::nextafter(bound, std::numeric_limits::infinity()) : bound; } -template +template void max_reconstruction_error(raft::resources const& res, raft::device_matrix_view dataset, Op op, @@ -127,7 +129,7 @@ void max_reconstruction_error(raft::resources const& res, errors, 0.0, false, - [op] __device__(T x, int64_t j) { return reconstruction_error(x, j, op); }, + [op] __device__(T x, int64_t j) { return reconstruction_error(x, j, op); }, [] __device__(double a, double b) { return hypot(a, b); }); raft::linalg::reduce( res, @@ -139,42 +141,6 @@ void max_reconstruction_error(raft::resources const& res, raft::max_op{}); } -template -auto max_reconstruction_error(raft::resources const& res, - raft::device_matrix_view dataset, - Op op) -> typename quantizer::scale_type -{ - auto errors = raft::make_device_vector(res, dataset.extent(0)); - auto maximum = raft::make_device_vector(res, 1); - max_reconstruction_error(res, dataset, op, errors.view(), maximum.view(), false); - double result = std::numeric_limits::infinity(); - raft::copy(res, - raft::make_host_scalar_view(&result), - raft::make_device_scalar_view(maximum.data_handle())); - raft::resource::sync_stream(res); - return round_error_up(result); -} - -template -auto max_reconstruction_error(raft::resources const& res, - raft::host_matrix_view dataset, - Op op) -> typename quantizer::scale_type -{ - if (raft::resource::get_dry_run_flag(res)) { - return std::numeric_limits::scale_type>::infinity(); - } - double maximum = 0; -#pragma omp parallel for reduction(max : maximum) - for (int64_t row = 0; row < dataset.extent(0); ++row) { - double error = 0; - for (int64_t j = 0; j < dataset.extent(1); ++j) { - error = std::hypot(error, reconstruction_error(dataset(row, j), j, op)); - } - maximum = std::max(maximum, error); - } - return round_error_up(maximum); -} - template std::tuple quantile_min_max( raft::resources const& res, @@ -232,7 +198,7 @@ std::tuple quantile_min_max( return {minmax_h[0], minmax_h[1]}; } -template +template typename quantizer::shifts_type train(raft::resources const& res, raft::device_vector&& shifts, raft::device_vector&& maxima) @@ -269,77 +235,13 @@ typename quantizer::shifts_type train(raft::resources const& res, raft::make_device_scalar_view(max_half_range.data_handle())); raft::resource::sync_stream(res); RAFT_EXPECTS(std::isfinite(half_range), "Bounds must be finite with min <= max"); - compute_t const scale = - half_range > 0 ? std::max(half_range / compute_t{127}, std::numeric_limits::min()) - : compute_t{1}; + constexpr compute_t q_max = std::numeric_limits::max(); + compute_t const scale = half_range > 0 + ? std::max(half_range / q_max, std::numeric_limits::min()) + : compute_t{1}; return {std::move(shifts), scale}; } -template -quantizer train(raft::resources const& res, - raft::device_matrix_view dataset, - params params) -{ - using compute_t = typename quantizer_const_view::scale_type; - RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); - auto const dim = dataset.extent(1); - RAFT_EXPECTS(dataset.extent(0) > 0 && dim > 0, "Training requires nonempty rows and dimensions"); - auto shifts = raft::make_device_vector(res, dim); - auto maxima = raft::make_device_vector(res, dim); - if (params.quantile == 1.f) { - raft::linalg::reduce( - res, - dataset, - shifts.view(), - static_cast(std::numeric_limits::infinity()), - false, - raft::identity_op{}, - raft::min_op{}); - raft::linalg::reduce( - res, - dataset, - maxima.view(), - static_cast(-std::numeric_limits::infinity()), - false, - raft::identity_op{}, - raft::max_op{}); - } else { - auto const rows = dataset.extent(0); - auto column = raft::make_device_vector(res, rows); - auto stream = raft::resource::get_cuda_stream(res).get(); - size_t workspace_bytes = 0; - RAFT_CUDA_TRY(cub::DeviceMergeSort::SortKeys( - nullptr, workspace_bytes, column.data_handle(), rows, fp_lt_op{}, stream)); - auto workspace = raft::make_device_vector(res, workspace_bytes); - auto const upper = static_cast(std::ceil((0.5 + 0.5 * params.quantile) * rows)) - 1; - auto const lower = rows - upper - 1; - auto const* input = dataset.data_handle(); - for (int64_t j = 0; j < dim; ++j) { - raft::linalg::map_offset( - res, column.view(), [input, dim, j] __device__(int64_t i) { return input[i * dim + j]; }); - if (!raft::resource::get_dry_run_flag(res)) { - RAFT_CUDA_TRY(cub::DeviceMergeSort::SortKeys(workspace.data_handle(), - workspace_bytes, - column.data_handle(), - rows, - fp_lt_op{}, - stream)); - } - raft::copy(res, - raft::make_device_scalar_view(shifts.data_handle() + j), - raft::make_device_scalar_view(column.data_handle() + lower)); - raft::copy(res, - raft::make_device_scalar_view(maxima.data_handle() + j), - raft::make_device_scalar_view(column.data_handle() + upper)); - } - } - quantizer result; - result.value = train(res, std::move(shifts), std::move(maxima)); - result.max_error = max_reconstruction_error( - res, dataset, std::get::shifts_type>(result.view().value)); - return result; -} - template void check_per_dim_shapes(typename quantizer_const_view::shifts_type quantizer, raft::device_matrix_view dataset, @@ -354,11 +256,11 @@ void check_per_dim_shapes(typename quantizer_const_view::shifts_type quantize "Input and output shapes must match"); } -template +template void transform(raft::resources const& res, typename quantizer_const_view::shifts_type quantizer, raft::device_matrix_view dataset, - raft::device_matrix_view out) + raft::device_matrix_view out) { using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); @@ -369,14 +271,16 @@ void transform(raft::resources const& res, raft::linalg::map_offset(res, out, [input, shifts, dim, inv] __device__(int64_t i) { compute_t const code = rint((static_cast(input[i]) - static_cast(shifts[i % dim])) * inv); - return static_cast(fmin(fmax(code, compute_t{-127}), compute_t{127})); + constexpr compute_t q_max = std::numeric_limits::max(); + constexpr compute_t q_min = std::max(std::numeric_limits::min(), -q_max); + return static_cast(fmin(fmax(code, q_min), q_max)); }); } -template +template void inverse_transform(raft::resources const& res, typename quantizer_const_view::shifts_type quantizer, - raft::device_matrix_view dataset, + raft::device_matrix_view dataset, raft::device_matrix_view out) { using compute_t = typename quantizer_const_view::scale_type; @@ -393,27 +297,6 @@ void inverse_transform(raft::resources const& res, }); } -template -cuvs::preprocessing::quantize::scalar::quantizer train( - raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::params params, - raft::device_matrix_view dataset) -{ - RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, - "quantile for scalar quantization needs to be within (0, 1] but is %f", - params.quantile); - - if (params.range == range_mode::per_dimension) { return train(res, dataset, params); } - - auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); - - RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); - - quantizer result{min, max}; - result.max_error = max_reconstruction_error(res, dataset, quantize_op{min, max}); - return result; -} - template void transform(raft::resources const& res, quantizer_const_view quantizer, @@ -493,12 +376,13 @@ void inverse_transform(raft::resources const& res, } // The optional output lets dataset construction encode during the error-measurement pass. -// Standalone host training uses the same shared parameters and corpus-wide error reduction. -template -quantizer train_host(raft::resources const& res, - params params, - raft::host_matrix_view dataset, - std::optional> out = std::nullopt) +// Host and device inputs share the same batch processing and corpus-wide error reduction. +template +quantizer train( + raft::resources const& res, + params params, + raft::mdspan, raft::row_major, Accessor> dataset, + std::optional> out = std::nullopt) { using compute_t = typename quantizer::scale_type; auto const rows = dataset.extent(0); @@ -506,11 +390,15 @@ quantizer train_host(raft::resources const& res, RAFT_EXPECTS(rows > 0 && dim > 0 && dataset.data_handle() != nullptr, "Training requires nonempty rows and dimensions"); RAFT_EXPECTS(params.quantile > 0.f && params.quantile <= 1.f, "Quantile must be within (0, 1]"); - RAFT_EXPECTS(params.device_buffer_samples >= 0, "device_buffer_samples must be nonnegative"); - auto const batch_rows = - params.device_buffer_samples == 0 ? rows : std::min(rows, params.device_buffer_samples); + constexpr bool data_on_device = raft::is_device_mdspan_v; + if constexpr (!data_on_device) { + RAFT_EXPECTS(params.device_buffer_samples >= 0, "device_buffer_samples must be nonnegative"); + } + auto const batch_rows = data_on_device || params.device_buffer_samples == 0 + ? rows + : std::min(rows, params.device_buffer_samples); auto [copy_stream, prefetch] = cuvs::spatial::knn::detail::utils::get_prefetch_stream(res); - cuvs::cluster::kmeans::detail::kmeans_batch_loader batches( + cuvs::cluster::kmeans::detail::kmeans_batch_loader batches( res, dataset, batch_rows, @@ -570,6 +458,34 @@ quantizer train_host(raft::resources const& res, raft::max_op{}); }, true); + } else if constexpr (data_on_device) { + auto column = raft::make_device_vector(res, rows); + auto stream = raft::resource::get_cuda_stream(res).get(); + size_t workspace_bytes = 0; + RAFT_CUDA_TRY(cub::DeviceMergeSort::SortKeys( + nullptr, workspace_bytes, column.data_handle(), rows, fp_lt_op{}, stream)); + auto workspace = raft::make_device_vector(res, workspace_bytes); + auto const upper = static_cast(std::ceil((0.5 + 0.5 * params.quantile) * rows)) - 1; + auto const lower = rows - upper - 1; + auto const* input = dataset.data_handle(); + for (int64_t j = 0; j < dim; ++j) { + raft::linalg::map_offset( + res, column.view(), [input, dim, j] __device__(int64_t i) { return input[i * dim + j]; }); + if (!raft::resource::get_dry_run_flag(res)) { + RAFT_CUDA_TRY(cub::DeviceMergeSort::SortKeys(workspace.data_handle(), + workspace_bytes, + column.data_handle(), + rows, + fp_lt_op{}, + stream)); + } + raft::copy(res, + raft::make_device_scalar_view(minima.data_handle() + j), + raft::make_device_scalar_view(column.data_handle() + lower)); + raft::copy(res, + raft::make_device_scalar_view(maxima.data_handle() + j), + raft::make_device_scalar_view(column.data_handle() + upper)); + } } else { // Select over the complete column, not independently fitted batch quantiles. std::vector mins(dim, T{0}), maxs(dim, T{0}); @@ -595,7 +511,7 @@ quantizer train_host(raft::resources const& res, res, maxima.view(), raft::make_host_vector_view(maxs.data(), dim)); raft::resource::sync_stream(res); } - result.value = train(res, std::move(minima), std::move(maxima)); + result.value = train(res, std::move(minima), std::move(maxima)); } auto errors = raft::make_device_vector(res, batch_rows); @@ -607,18 +523,19 @@ quantizer train_host(raft::resources const& res, raft::make_device_vector_view(errors.data_handle(), input.extent(0)); if (auto const* shifts = std::get_if::shifts_type>(&view.value)) { - max_reconstruction_error(res, input, *shifts, batch_errors, maximum.view(), offset != 0); + max_reconstruction_error( + res, input, *shifts, batch_errors, maximum.view(), offset != 0); } else { auto const& range = std::get::range_type>(view.value); - max_reconstruction_error(res, - input, - quantize_op{range.min_, range.max_}, - batch_errors, - maximum.view(), - offset != 0); + max_reconstruction_error(res, + input, + quantize_op{range.min_, range.max_}, + batch_errors, + maximum.view(), + offset != 0); } if (out) { - auto batch_codes = raft::make_device_matrix_view( + auto batch_codes = raft::make_device_matrix_view( out->data_handle() + offset * dim, input.extent(0), dim); detail::transform(res, view, input, batch_codes); } @@ -632,25 +549,4 @@ quantizer train_host(raft::resources const& res, return result; } -template -cuvs::preprocessing::quantize::scalar::quantizer train( - raft::resources const& res, - const cuvs::preprocessing::quantize::scalar::params params, - raft::host_matrix_view dataset) -{ - RAFT_EXPECTS(params.quantile > 0.0 && params.quantile <= 1.0, - "quantile for scalar quantization needs to be within (0, 1] but is %f", - params.quantile); - - if (params.range == range_mode::per_dimension) { return train_host(res, params, dataset); } - - auto [min, max] = detail::quantile_min_max(res, dataset, params.quantile); - - RAFT_LOG_DEBUG("quantizer train min=%lf max=%lf.", double(min), double(max)); - - quantizer result{min, max}; - result.max_error = max_reconstruction_error(res, dataset, quantize_op{min, max}); - return result; -} - } // namespace cuvs::preprocessing::quantize::scalar::detail diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index d7d967ed51..cadd12ce82 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -31,13 +31,13 @@ quantizer::quantizer(raft::resources const& res, const params params, \ raft::device_matrix_view dataset) -> quantizer \ { \ - return detail::train(res, params, dataset); \ + return detail::train(res, params, dataset); \ } \ auto train(raft::resources const& res, \ const params params, \ raft::host_matrix_view dataset) -> quantizer \ { \ - return detail::train(res, params, dataset); \ + return detail::train(res, params, dataset); \ } \ void transform(raft::resources const& res, \ quantizer_const_view quantizer, \ @@ -73,7 +73,7 @@ quantizer::quantizer(raft::resources const& res, { \ auto codes = \ raft::make_device_matrix(res, dataset.extent(0), dataset.extent(1)); \ - auto quantizer = detail::train_host(res, params, dataset, codes.view()); \ + auto quantizer = detail::train(res, params, dataset, codes.view()); \ return {std::move(codes), std::move(quantizer)}; \ } \ template struct quantizer; diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 0b9ec615b6..6f5fafa06e 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "../../src/preprocessing/quantize/detail/scalar.cuh" #include "../test_utils.cuh" #include #include @@ -652,6 +653,36 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) raft::logic_error); } +TEST(PerDimensionRangeTest, QuantizedTypeLimits) +{ + using QuantI = int16_t; + raft::resources res; + auto stream = raft::resource::get_cuda_stream(res); + std::vector values{-100000.f, -32767.f, 0.f, 32767.f, 100000.f}; + auto input = raft::make_device_matrix(res, values.size(), 1); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto codes = raft::make_device_matrix(res, values.size(), 1); + auto owner = detail::train(res, + params{.5f, range_mode::per_dimension}, + raft::make_const_mdspan(input.view()), + codes.view()); + EXPECT_FLOAT_EQ(std::get(owner.view().value).scale, 1.f); + std::vector expected{-32767, -32767, 0, 32767, 32767}; + EXPECT_TRUE(devArrMatchHost( + expected.data(), codes.data_handle(), expected.size(), cuvs::Compare(), stream.get())); + detail::inverse_transform(res, owner.view(), raft::make_const_mdspan(codes.view()), input.view()); + std::vector decoded(values.size()); + raft::update_host(decoded.data(), input.data_handle(), decoded.size(), stream); + raft::resource::sync_stream(res); + double maximum = 0; + for (size_t i = 0; i < values.size(); ++i) { + EXPECT_FLOAT_EQ(decoded[i], static_cast(expected[i])); + maximum = std::max(maximum, std::abs(double(values[i]) - decoded[i])); + } + EXPECT_GE(owner.max_error, maximum); + EXPECT_NEAR(owner.max_error, maximum, maximum * 2e-6); +} + TEST(QuantizationDatasetTest, GlobalRangeDataset) { raft::resources res; @@ -726,7 +757,9 @@ TYPED_TEST(PerDimensionQuantizationTest, BatchedHostDataset) std::copy(values.begin(), values.end(), pinned.data_handle()); for (auto mode : {range_mode::global, range_mode::per_dimension}) { for (float quantile : {1.f, .5f, .99f}) { - auto expected = build(res, params{quantile, mode}, raft::make_const_mdspan(input.view())); + // Device input ignores the host staging limit. + auto expected = + build(res, params{quantile, mode, -1}, raft::make_const_mdspan(input.view())); for (int64_t batch_rows : {int64_t{1}, int64_t{7}, rows, int64_t{0}}) { for (bool use_pinned : {false, true}) { SCOPED_TRACE(::testing::Message() diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index f13f00bd71..36bb8b9d5c 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -46,7 +46,7 @@ struct params { | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | | `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. A shared scale fits every dimension's bounds around its stored shift into the symmetric range of quantized values.

If every interval has zero width, the scale is one. Otherwise, the scale is at least the smallest positive normal value of the scale type. | -| `device_buffer_samples` | `int64_t` | Maximum rows staged on device for host per-dimension training and host dataset build.
Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. Multiple batches use two input buffers when res has an auxiliary stream pool, and one otherwise. Pageable input uses batch-sized pinned staging buffers, without registering the full input. Each input buffer uses at most device_buffer_samples * dim * sizeof(T) bytes, in addition to reduction and encoding workspace. Substitute n_rows when it is zero.

Per-dimension host training with quantile < 1 selects exact quantiles using one column of host scratch (n_rows * sizeof(T) bytes). Global training retains its sampled quantiles. | +| `device_buffer_samples` | `int64_t` | Maximum rows staged on device for host training and host dataset build.
Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. Multiple batches use two input buffers when res has an auxiliary stream pool, and one otherwise. Pageable input uses batch-sized pinned staging buffers, without registering the full input. Each input buffer uses at most device_buffer_samples * dim * sizeof(T) bytes, in addition to reduction and encoding workspace. Substitute n_rows when it is zero.

Per-dimension host training with quantile < 1 selects exact quantiles using one column of host scratch (n_rows * sizeof(T) bytes). Global training retains its sampled quantiles. | ### preprocessing::quantize::scalar::quantizer_const_view From 614a84b70f090b23c02128abea58c5b3ab27d91d Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 5 Nov 2026 00:10:52 +0000 Subject: [PATCH 14/24] Store global and per-dimension scalar shifts and scales --- c/src/preprocessing/quantize/scalar.cpp | 13 +- cpp/include/cuvs/core/dataset/sq.hpp | 2 +- .../cuvs/preprocessing/quantize/scalar.hpp | 86 ++++++---- .../preprocessing/quantize/detail/scalar.cuh | 156 +++++++++--------- cpp/src/preprocessing/quantize/scalar.cu | 5 + .../preprocessing/scalar_quantization.cu | 136 +++++++++++---- .../cpp-api-preprocessing-quantize-scalar.md | 72 ++++---- 7 files changed, 301 insertions(+), 169 deletions(-) diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index 83b3a29bf5..49e3093bae 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -3,8 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include +#include #include +#include #include +#include #include #include @@ -42,9 +46,12 @@ void _train(cuvsResources_t res, RAFT_FAIL("dataset must be accessible on host or device memory"); } - auto const& range = std::get(ret.value); - quantizer->min_ = range.min_; - quantizer->max_ = range.max_; + auto const& range = std::get(ret.value); + double const limit = static_cast(cuda::std::numeric_limits::max()); + quantizer->min_ = static_cast(std::clamp( + std::fma(range.scale, std::numeric_limits::min(), range.shift), -limit, limit)); + quantizer->max_ = static_cast(std::clamp( + std::fma(range.scale, std::numeric_limits::max(), range.shift), -limit, limit)); } template diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index b3afac3f04..d2ccadceba 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -34,7 +34,7 @@ struct sq_spec { using math_type = MathT; - // The manifest is either a global interval or owning per-dimension shifts and a shared scale. + // The manifest is either a global shift and scale or owning per-dimension shifts and scales. using manifest_type = cuvs::preprocessing::quantize::scalar::quantizer; using manifest_view_type = typename manifest_type::view_type; diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 29df0cc5e0..01097e4976 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -29,7 +29,7 @@ namespace scalar { * @{ */ -/** Select a global interval or per-dimension shifts with a shared scale. */ +/** Select global or per-dimension shifts and scales. */ enum class range_mode { global, per_dimension }; /** @@ -45,10 +45,10 @@ struct params { * Per-dimension training uses every row of a nonempty, finite dataset. * It applies the quantile independently to each dimension to select lower and upper bounds. * Each dimension's shift is the midpoint of its bounds, rounded to the input type. - * A shared scale fits every dimension's bounds around its stored shift into the symmetric - * range of quantized values. + * Each dimension's scale fits its bounds around its stored shift into the symmetric range + * of quantized values. * - * If every interval has zero width, the scale is one. Otherwise, the scale is at least + * For a zero-width interval, its scale is one. Otherwise, each scale is at least * the smallest positive normal value of the scale type. */ range_mode range = range_mode::global; @@ -66,46 +66,52 @@ struct params { int64_t device_buffer_samples = 1 << 16; }; -/** Read-only scalar quantizer parameters. The owner must outlive stream work using shifts. */ +/** Read-only scalar quantizer parameters. The owner must outlive stream work using shifts and + * scales. */ template struct quantizer_const_view { using scale_type = std::conditional_t, float, T>; - /** A global interval mapped to the full range of the quantized type. */ - struct range_type { - T min_; - T max_; + /** Global affine parameters using double precision for encoding and decoding. + * Encode (x - shift) / scale, rounding to nearest with ties away from zero and clamping + * to the full quantized range. Decode shift + scale * code, saturating to finite input limits. + * A zero scale represents constant training bounds; every code decodes to shift. */ + struct global_type { + /** Shift shared by all dimensions. */ + double shift = 0; + /** Step size shared by all dimensions. */ + double scale = 0; }; - /** Encode (x - shifts[j]) / scale, rounding to nearest with ties to even and + /** Encode (x - shifts[j]) / scales[j], rounding to nearest with ties to even and * clamping to the symmetric range of quantized values. - * Decode shifts[j] + scale * code, saturating to finite limits of the input type. */ + * Decode shifts[j] + scales[j] * code, saturating to finite limits of the input type. */ struct shifts_type { + /** One midpoint shift per dimension. */ raft::device_vector_view shifts; - scale_type scale; + /** One finite, positive scale per dimension, with a finite reciprocal. */ + raft::device_vector_view scales; }; - /** Global interval or borrowed per-dimension shifts and scale. */ - std::variant value; + /** Global parameters or borrowed per-dimension shifts and scales. */ + std::variant value; /** Maximum training-row L2 reconstruction error; see quantizer::max_error. */ scale_type max_error = INFINITY; }; /** - * Owns either a global interval or device shifts and a shared step size. + * Owns a global shift and scale or device shifts and scales for each dimension. * Move-only; copy view() to borrow the parameters. - * Migration: replace q.min_ with std::get::range_type>(q.value).min_ - * (likewise for max_). */ template struct quantizer { - using view_type = quantizer_const_view; - using scale_type = typename view_type::scale_type; - using range_type = typename view_type::range_type; + using view_type = quantizer_const_view; + using scale_type = typename view_type::scale_type; + using global_type = typename view_type::global_type; /** Owning per-dimension parameters. */ struct shifts_type { raft::device_vector shifts; - scale_type scale; + raft::device_vector scales; }; - /** Global interval or owning per-dimension shifts and scale. */ - std::variant value; + /** Global parameters or owning per-dimension shifts and scales. */ + std::variant value; /** * Upper bound on the L2 error between a training row and its int8 encode/decode result. * train measures every input row, including clipped outliers, and rounds the maximum up @@ -114,22 +120,22 @@ struct quantizer { */ scale_type max_error = INFINITY; - /** Construct an empty global interval. */ - quantizer() : value{range_type{}} {} - /** Construct a global interval. */ - quantizer(T min, T max) : value{range_type{min, max}} {} + /** Construct default global parameters. */ + quantizer() : value{global_type{}} {} + /** Derive a global shift and scale from finite bounds with min <= max. */ + quantizer(T min, T max); /** Construct from nonempty device bounds of equal length, with finite min <= max. - * Derives midpoint shifts and a shared scale. */ + * Derives midpoint shifts and a scale for each dimension. */ quantizer(raft::resources const& res, raft::device_vector_view mins, raft::device_vector_view maxs); - /** Borrow read-only parameters; the owner must outlive stream work using shifts. */ + /** Borrow read-only parameters; the owner must outlive stream work using shifts and scales. */ [[nodiscard]] auto view() const noexcept -> view_type { - if (auto const* range = std::get_if(&value)) { return {*range, max_error}; } + if (auto const* range = std::get_if(&value)) { return {*range, max_error}; } auto const& shifts = std::get(value); - return {typename view_type::shifts_type{shifts.shifts.view(), shifts.scale}, max_error}; + return {typename view_type::shifts_type{shifts.shifts.view(), shifts.scales.view()}, max_error}; } /** An owning quantizer can be passed wherever a read-only view is accepted. */ operator view_type() const noexcept { return view(); } // NOLINT(google-explicit-constructor) @@ -202,6 +208,9 @@ void transform(raft::resources const& res, /** * @brief Applies quantization transform to given dataset * + * Host transforms require a global interval and throw raft::logic_error for per-dimension + * parameters. Use device transforms or host build for per-dimension quantization. + * * Usage example: * @code{.cpp} * raft::handle_t handle; @@ -256,6 +265,9 @@ void inverse_transform(raft::resources const& res, * Note that depending on the chosen data types train dataset the conversion is * not lossless. * + * Host transforms require a global interval and throw raft::logic_error for per-dimension + * parameters. Use device transforms or host build for per-dimension quantization. + * * Usage example: * @code{.cpp} * auto quantized_dataset = raft::make_host_matrix(samples, features); @@ -343,6 +355,9 @@ void transform(raft::resources const& res, /** * @brief Applies quantization transform to given dataset * + * Host transforms require a global interval and throw raft::logic_error for per-dimension + * parameters. Use device transforms or host build for per-dimension quantization. + * * Usage example: * @code{.cpp} * raft::handle_t handle; @@ -396,6 +411,9 @@ void inverse_transform(raft::resources const& res, * Note that depending on the chosen data types train dataset the conversion is * not lossless. * + * Host transforms require a global interval and throw raft::logic_error for per-dimension + * parameters. Use device transforms or host build for per-dimension quantization. + * * Usage example: * @code{.cpp} * auto quantized_dataset = raft::make_host_matrix(samples, features); @@ -483,6 +501,9 @@ void transform(raft::resources const& res, /** * @brief Applies quantization transform to given dataset * + * Host transforms require a global interval and throw raft::logic_error for per-dimension + * parameters. Use device transforms or host build for per-dimension quantization. + * * Usage example: * @code{.cpp} * raft::handle_t handle; @@ -536,6 +557,9 @@ void inverse_transform(raft::resources const& res, * Note that depending on the chosen data types train dataset the conversion is * not lossless. * + * Host transforms require a global interval and throw raft::logic_error for per-dimension + * parameters. Use device transforms or host build for per-dimension quantization. + * * Usage example: * @code{.cpp} * auto quantized_dataset = raft::make_host_matrix(samples, features); diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index e462dea253..d2bf55f244 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -50,36 +50,45 @@ struct fp_lt_op { _RAFT_HOST_DEVICE bool operator()(const T& a, const T& b) const { return fp_lt(a, b); } }; -template +template +typename quantizer::global_type train(T min, T max) +{ + double const lo = static_cast(min); + double const hi = static_cast(max); + RAFT_EXPECTS(std::isfinite(lo) && std::isfinite(hi) && lo <= hi, + "Bounds must be finite with min <= max"); + if (lo == hi) { return {lo, 0}; } + constexpr double q_min = std::numeric_limits::min(); + constexpr double q_max = std::numeric_limits::max(); + // Divide the endpoints separately if subtracting them would overflow. + double const width = hi - lo; + double const scale = std::max( + std::isfinite(width) ? width / (q_max - q_min) : hi / (q_max - q_min) - lo / (q_max - q_min), + std::numeric_limits::min()); + double const shift = lo * 0.5 + hi * 0.5 - (q_min + q_max) * 0.5 * scale; + return {shift, scale}; +} + +template struct quantize_op { - const T min_; - const T max_; - const QuantI q_type_min_ = std::numeric_limits::min(); - const QuantI q_type_max_ = std::numeric_limits::max(); - const TempT scalar_; - const TempT offset_; - - constexpr explicit quantize_op(T min, T max) - : min_(min), - max_(max), - scalar_(static_cast(max_) > static_cast(min_) - ? ((static_cast(q_type_max_) - static_cast(q_type_min_)) / - (static_cast(max_) - static_cast(min_))) - : static_cast(1)), - offset_(static_cast(q_type_min_) - static_cast(min_) * scalar_) - { - } + typename quantizer_const_view::global_type params; constexpr RAFT_INLINE_FUNCTION QuantI operator()(const T& x) const { - if (!fp_lt(min_, x)) return q_type_min_; - if (!fp_lt(x, max_)) return q_type_max_; - return static_cast(lroundf(scalar_ * static_cast(x) + offset_)); + constexpr double q_min = std::numeric_limits::min(); + constexpr double q_max = std::numeric_limits::max(); + if (params.scale == 0) { + return static_cast(static_cast(x) <= params.shift ? q_min : q_max); + } + double const code = (static_cast(x) - params.shift) / params.scale; + return static_cast(lroundf(fmin(fmax(code, q_min), q_max))); } constexpr RAFT_INLINE_FUNCTION T operator()(const QuantI& x) const { - return static_cast((static_cast(x) - offset_) / scalar_); + double const value = fma(params.scale, static_cast(x), params.shift); + double const limit = static_cast(cuda::std::numeric_limits::max()); + return static_cast(fmin(fmax(value, -limit), limit)); } }; @@ -91,12 +100,13 @@ RAFT_INLINE_FUNCTION double reconstruction_error(T x, int64_t j, Op op) if constexpr (std::is_same_v::shifts_type>) { using compute_t = typename quantizer_const_view::scale_type; compute_t const shift = static_cast(op.shifts.data_handle()[j]); - compute_t const inv = compute_t{1} / op.scale; + compute_t const scale = op.scales.data_handle()[j]; + compute_t const inv = compute_t{1} / scale; compute_t const rounded = rint((static_cast(x) - shift) * inv); constexpr compute_t q_max = std::numeric_limits::max(); constexpr compute_t q_min = std::max(std::numeric_limits::min(), -q_max); QuantI const code = static_cast(fmin(fmax(rounded, q_min), q_max)); - compute_t const reconstructed = fma(op.scale, static_cast(code), shift); + compute_t const reconstructed = fma(scale, static_cast(code), shift); compute_t const limit = static_cast(cuda::std::numeric_limits::max()); decoded = static_cast(static_cast(fmin(fmax(reconstructed, -limit), limit))); } else { @@ -205,11 +215,11 @@ typename quantizer::shifts_type train(raft::resources const& res, { using compute_t = typename quantizer_const_view::scale_type; auto const dim = shifts.extent(0); - auto half_ranges = raft::make_device_vector(res, dim); - auto max_half_range = raft::make_device_vector(res, 1); + auto scales = raft::make_device_vector(res, dim); + auto max_scale = raft::make_device_vector(res, 1); auto* shift_ptr = shifts.data_handle(); auto const* max_ptr = maxima.data_handle(); - raft::linalg::map_offset(res, half_ranges.view(), [shift_ptr, max_ptr] __device__(int64_t j) { + raft::linalg::map_offset(res, scales.view(), [shift_ptr, max_ptr] __device__(int64_t j) { // Halve before adding/subtracting to avoid overflow for large finite inputs. if (!isfinite(static_cast(shift_ptr[j])) || !isfinite(static_cast(max_ptr[j])) || fp_lt(max_ptr[j], shift_ptr[j])) { @@ -218,28 +228,27 @@ typename quantizer::shifts_type train(raft::resources const& res, compute_t const lo = static_cast(shift_ptr[j]); compute_t const hi = static_cast(max_ptr[j]); shift_ptr[j] = static_cast(lo == hi ? lo : lo * compute_t{0.5} + hi * compute_t{0.5}); - compute_t const shift = static_cast(shift_ptr[j]); - return fmax(hi - shift, shift - lo); + compute_t const shift = static_cast(shift_ptr[j]); + compute_t const half_range = fmax(hi - shift, shift - lo); + constexpr compute_t q_max = std::numeric_limits::max(); + return half_range > 0 ? fmax(half_range / q_max, std::numeric_limits::min()) + : compute_t{1}; }); raft::linalg::reduce( res, - raft::make_device_matrix_view(half_ranges.data_handle(), 1, dim), - max_half_range.view(), + raft::make_device_matrix_view(scales.data_handle(), 1, dim), + max_scale.view(), compute_t{0}, false, raft::identity_op{}, raft::max_op{}); - compute_t half_range = 0; + compute_t maximum_scale = 0; raft::copy(res, - raft::make_host_scalar_view(&half_range), - raft::make_device_scalar_view(max_half_range.data_handle())); + raft::make_host_scalar_view(&maximum_scale), + raft::make_device_scalar_view(max_scale.data_handle())); raft::resource::sync_stream(res); - RAFT_EXPECTS(std::isfinite(half_range), "Bounds must be finite with min <= max"); - constexpr compute_t q_max = std::numeric_limits::max(); - compute_t const scale = half_range > 0 - ? std::max(half_range / q_max, std::numeric_limits::min()) - : compute_t{1}; - return {std::move(shifts), scale}; + RAFT_EXPECTS(std::isfinite(maximum_scale), "Bounds must be finite with min <= max"); + return {std::move(shifts), std::move(scales)}; } template @@ -247,11 +256,9 @@ void check_per_dim_shapes(typename quantizer_const_view::shifts_type quantize raft::device_matrix_view dataset, raft::device_matrix_view out) { - RAFT_EXPECTS( - std::isfinite(quantizer.scale) && quantizer.scale > 0.f && std::isfinite(1.f / quantizer.scale), - "Scale and its reciprocal must be finite and positive"); - RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.shifts.extent(0), - "Input width must match the nonempty shifts vector"); + RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.shifts.extent(0) && + dataset.extent(1) == quantizer.scales.extent(0), + "Input width must match the nonempty shifts and scales vectors"); RAFT_EXPECTS(dataset.extent(0) == out.extent(0) && dataset.extent(1) == out.extent(1), "Input and output shapes must match"); } @@ -264,11 +271,12 @@ void transform(raft::resources const& res, { using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - compute_t const inv = compute_t{1} / quantizer.scale; - raft::linalg::map_offset(res, out, [input, shifts, dim, inv] __device__(int64_t i) { + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + auto const* scales = quantizer.scales.data_handle(); + raft::linalg::map_offset(res, out, [input, shifts, scales, dim] __device__(int64_t i) { + compute_t const inv = compute_t{1} / scales[i % dim]; compute_t const code = rint((static_cast(input[i]) - static_cast(shifts[i % dim])) * inv); constexpr compute_t q_max = std::numeric_limits::max(); @@ -285,13 +293,13 @@ void inverse_transform(raft::resources const& res, { using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - compute_t const scale = quantizer.scale; - raft::linalg::map_offset(res, out, [input, shifts, dim, scale] __device__(int64_t i) { - compute_t const value = - fma(scale, static_cast(input[i]), static_cast(shifts[i % dim])); + auto const* input = dataset.data_handle(); + auto const* shifts = quantizer.shifts.data_handle(); + auto const dim = dataset.extent(1); + auto const* scales = quantizer.scales.data_handle(); + raft::linalg::map_offset(res, out, [input, shifts, scales, dim] __device__(int64_t i) { + compute_t const value = fma( + scales[i % dim], static_cast(input[i]), static_cast(shifts[i % dim])); compute_t const limit = static_cast(cuda::std::numeric_limits::max()); return static_cast(fmin(fmax(value, -limit), limit)); }); @@ -309,9 +317,8 @@ void transform(raft::resources const& res, detail::transform(res, *shifts, dataset, out); return; } - auto const& range = std::get::range_type>(view.value); - raft::linalg::map( - res, out, quantize_op(range.min_, range.max_), raft::make_const_mdspan(dataset)); + auto const& range = std::get::global_type>(view.value); + raft::linalg::map(res, out, quantize_op{range}, raft::make_const_mdspan(dataset)); } template @@ -321,10 +328,10 @@ void transform(raft::resources const& res, raft::host_matrix_view out) { RAFT_EXPECTS( - std::holds_alternative::range_type>(quantizer.value), + std::holds_alternative::global_type>(quantizer.value), "Host quantization requires a global interval"); - auto const& range = std::get::range_type>(quantizer.value); - auto main_op = quantize_op(range.min_, range.max_); + auto const& range = std::get::global_type>(quantizer.value); + auto main_op = quantize_op{range}; size_t n_elements = dataset.extent(0) * dataset.extent(1); // In dry-run mode the views may be unbacked or alias the shared probe buffer. @@ -348,9 +355,8 @@ void inverse_transform(raft::resources const& res, detail::inverse_transform(res, *shifts, dataset, out); return; } - auto const& range = std::get::range_type>(view.value); - raft::linalg::map( - res, out, quantize_op(range.min_, range.max_), raft::make_const_mdspan(dataset)); + auto const& range = std::get::global_type>(view.value); + raft::linalg::map(res, out, quantize_op{range}, raft::make_const_mdspan(dataset)); } template @@ -360,10 +366,10 @@ void inverse_transform(raft::resources const& res, raft::host_matrix_view out) { RAFT_EXPECTS( - std::holds_alternative::range_type>(quantizer.value), + std::holds_alternative::global_type>(quantizer.value), "Host quantization requires a global interval"); - auto const& range = std::get::range_type>(quantizer.value); - auto main_op = quantize_op(range.min_, range.max_); + auto const& range = std::get::global_type>(quantizer.value); + auto main_op = quantize_op{range}; size_t n_elements = dataset.extent(0) * dataset.extent(1); // In dry-run mode the views may be unbacked or alias the shared probe buffer. @@ -433,7 +439,7 @@ quantizer train( quantizer result; if (params.range == range_mode::global) { auto [min, max] = quantile_min_max(res, dataset, params.quantile); - result = quantizer{min, max}; + result.value = detail::train(min, max); } else { auto minima = raft::make_device_vector(res, dim); auto maxima = raft::make_device_vector(res, dim); @@ -526,13 +532,9 @@ quantizer train( max_reconstruction_error( res, input, *shifts, batch_errors, maximum.view(), offset != 0); } else { - auto const& range = std::get::range_type>(view.value); - max_reconstruction_error(res, - input, - quantize_op{range.min_, range.max_}, - batch_errors, - maximum.view(), - offset != 0); + auto const& range = std::get::global_type>(view.value); + max_reconstruction_error( + res, input, quantize_op{range}, batch_errors, maximum.view(), offset != 0); } if (out) { auto batch_codes = raft::make_device_matrix_view( diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index cadd12ce82..a7f2645b91 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -12,6 +12,11 @@ namespace cuvs::preprocessing::quantize::scalar { +template +quantizer::quantizer(T min, T max) : value{detail::train(min, max)} +{ +} + template quantizer::quantizer(raft::resources const& res, raft::device_vector_view mins, diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 6f5fafa06e..50c9604062 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -120,10 +120,10 @@ class QuantizationTest : public ::testing::TestWithParam> cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); std::cerr - << "Q1: min = " - << (double)std::get(quantizer_1.value).min_ - << ", max = " - << (double)std::get(quantizer_1.value).max_ + << "Q1: shift = " + << (double)std::get(quantizer_1.value).shift + << ", scale = " + << (double)std::get(quantizer_1.value).scale << std::endl; { @@ -223,20 +223,20 @@ class QuantizationTest : public ::testing::TestWithParam> cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); std::cerr - << "Q2: min = " - << (double)std::get(quantizer_2.value).min_ - << ", max = " - << (double)std::get(quantizer_2.value).max_ + << "Q2: shift = " + << (double)std::get(quantizer_2.value).shift + << ", scale = " + << (double)std::get(quantizer_2.value).scale << std::endl; // check both quantizers are the same (valid if sampling is identical) if (input_.size() <= 1000000) { ASSERT_TRUE( - (double)std::get(quantizer_1.value).min_ == - (double)std::get(quantizer_2.value).min_); + (double)std::get(quantizer_1.value).shift == + (double)std::get(quantizer_2.value).shift); ASSERT_TRUE( - (double)std::get(quantizer_1.value).max_ == - (double)std::get(quantizer_2.value).max_); + (double)std::get(quantizer_1.value).scale == + (double)std::get(quantizer_2.value).scale); } { @@ -363,7 +363,12 @@ class PerDimensionRangeTest : public ::testing::TestWithParam(), stream)); if (params_.tolerance == 0.f) { - EXPECT_FLOAT_EQ(std::get(quantizer.view().value).scale, 1.f); + EXPECT_TRUE( + devArrMatchHost(std::vector(params_.n_features, 1.f).data(), + std::get(quantizer.view().value).scales.data_handle(), + params_.n_features, + cuvs::Compare(), + stream)); } inverse_transform(handle, quantizer, raft::make_const_mdspan(codes.view()), dataset_.view()); auto reconstructed = @@ -498,7 +503,7 @@ TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) using shifts_view = typename quantizer_const_view::shifts_type; raft::resources res; auto stream = raft::resource::get_cuda_stream(res); - std::vector corpus{0.f, 10.f, 9.f, 254.f, 14.f, 9.f}; + std::vector corpus{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; auto input = raft::make_device_matrix(res, 2, 3); raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); auto owner = @@ -506,10 +511,15 @@ TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) auto quantizer = owner.view(); EXPECT_EQ(owner.max_error, 0); EXPECT_EQ(quantizer.max_error, 0); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 1.f); + using scale_t = typename quantizer_const_view::scale_type; + EXPECT_TRUE(devArrMatchHost(std::vector{1, .5, 1}.data(), + std::get(quantizer.value).scales.data_handle(), + 3, + cuvs::Compare(), + stream.get())); { std::vector actual(3); - std::vector expected{127.f, 12.f, 9.f}; + std::vector expected{127.f, 73.5f, 9.f}; raft::update_host(actual.data(), std::get(quantizer.value).shifts.data_handle(), actual.size(), @@ -520,7 +530,7 @@ TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) } } // The corpus manifest, rather than query extrema, controls ties and saturation. - std::vector queries{127.5f, 11.5f, 1000.f, 128.5f, 9.5f, -1000.f}; + std::vector queries{127.5f, 73.75f, 1000.f, 128.5f, 72.75f, -1000.f}; raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); auto codes = raft::make_device_matrix(res, 2, 3); transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); @@ -532,7 +542,7 @@ TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); { std::vector actual(6); - std::vector expected{127.f, 12.f, 136.f, 129.f, 10.f, -118.f}; + std::vector expected{127.f, 73.5f, 136.f, 129.f, 72.5f, -118.f}; raft::update_host(actual.data(), input.data_handle(), actual.size(), stream); raft::resource::sync_stream(res); for (size_t i = 0; i < actual.size(); ++i) { @@ -581,13 +591,25 @@ TEST(PerDimensionRangeTest, PerDimensionQuantiles) dim, cuvs::Compare(), raft::resource::get_cuda_stream(res).get())); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 991.f / 127.f); + EXPECT_TRUE(devArrMatchHost(std::vector{495.5f / 127.f, 991.f / 127.f, 1.f}.data(), + std::get(quantizer.value).scales.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); auto full = train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(full.view().value).scale, 999.f / 127.f); + EXPECT_TRUE(devArrMatchHost(std::vector{499.5f / 127.f, 999.f / 127.f, 1.f}.data(), + std::get(full.view().value).scales.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); auto trimmed = train(res, params{0.5f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_FLOAT_EQ(std::get(trimmed.view().value).scale, 499.f / 127.f); + EXPECT_TRUE(devArrMatchHost(std::vector{249.5f / 127.f, 499.f / 127.f, 1.f}.data(), + std::get(trimmed.view().value).scales.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { EXPECT_THROW( train(res, params{q, range_mode::per_dimension}, raft::make_const_mdspan(input.view())), @@ -612,7 +634,12 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) EXPECT_TRUE(std::isinf(owner.max_error)); EXPECT_TRUE(std::isinf(quantizer.max_error)); EXPECT_TRUE(std::isinf((scalar::quantizer{T{-1}, T{1}}.view().max_error))); - EXPECT_FLOAT_EQ(std::get(quantizer.value).scale, 2.f); + using scale_t = typename quantizer_const_view::scale_type; + EXPECT_TRUE(devArrMatchHost(std::vector{2, scale_t{4} / scale_t{127}}.data(), + std::get(quantizer.value).scales.data_handle(), + 2, + cuvs::Compare(), + stream.get())); { std::vector actual(2); std::vector expected{100.f, -100.f}; @@ -631,11 +658,17 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); auto codes = raft::make_device_matrix(res, 1, 2); transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{2, -3}.data(), + EXPECT_TRUE(devArrMatchHost(std::vector{2, -127}.data(), codes.data_handle(), 2, cuvs::Compare(), raft::resource::get_cuda_stream(res).get())); + auto wrong_scales = quantizer; + std::get(wrong_scales.value).scales = + raft::make_device_vector_view( + std::get(quantizer.value).scales.data_handle(), 1); + EXPECT_THROW(transform(res, wrong_scales, raft::make_const_mdspan(input.view()), codes.view()), + raft::logic_error); auto wrong_shape = raft::make_device_matrix(res, 2, 1); EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), raft::logic_error); @@ -666,7 +699,11 @@ TEST(PerDimensionRangeTest, QuantizedTypeLimits) params{.5f, range_mode::per_dimension}, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_FLOAT_EQ(std::get(owner.view().value).scale, 1.f); + EXPECT_TRUE(devArrMatchHost(std::vector{1.f}.data(), + std::get(owner.view().value).scales.data_handle(), + 1, + cuvs::Compare(), + stream.get())); std::vector expected{-32767, -32767, 0, 32767, 32767}; EXPECT_TRUE(devArrMatchHost( expected.data(), codes.data_handle(), expected.size(), cuvs::Compare(), stream.get())); @@ -683,6 +720,37 @@ TEST(PerDimensionRangeTest, QuantizedTypeLimits) EXPECT_NEAR(owner.max_error, maximum, maximum * 2e-6); } +TYPED_TEST(PerDimensionQuantizationTest, GlobalShiftScale) +{ + using T = TypeParam; + raft::resources res; + auto stream = raft::resource::get_cuda_stream(res); + for (auto bounds : {std::pair{T{-3}, T{7}}, std::pair{T{5}, T{5}}}) { + auto owner = quantizer{bounds.first, bounds.second}; + auto parameters = std::get::global_type>(owner.value); + EXPECT_EQ(static_cast( + std::fma(parameters.scale, std::numeric_limits::min(), parameters.shift)), + bounds.first); + EXPECT_EQ(static_cast( + std::fma(parameters.scale, std::numeric_limits::max(), parameters.shift)), + bounds.second); + if (bounds.first == bounds.second) { EXPECT_EQ(parameters.scale, 0); } + std::vector values{T{-100}, bounds.first, bounds.second, T{100}}; + auto input = raft::make_device_matrix(res, 1, values.size()); + auto codes = raft::make_device_matrix(res, 1, values.size()); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); + std::vector decoded(values.size()); + raft::update_host(decoded.data(), input.data_handle(), decoded.size(), stream); + raft::resource::sync_stream(res); + EXPECT_EQ(decoded.front(), bounds.first); + EXPECT_EQ(decoded.back(), bounds.second); + EXPECT_EQ(decoded[1], bounds.first); + EXPECT_EQ(decoded[2], bounds.second); + } +} + TEST(QuantizationDatasetTest, GlobalRangeDataset) { raft::resources res; @@ -700,9 +768,9 @@ TEST(QuantizationDatasetTest, GlobalRangeDataset) cuvs::Compare(), stream.get())); auto range = - std::get::range_type>(encoded.const_manifest_view().value); - EXPECT_FLOAT_EQ(range.min_, -1.f); - EXPECT_FLOAT_EQ(range.max_, 1.f); + std::get::global_type>(encoded.const_manifest_view().value); + EXPECT_DOUBLE_EQ(range.shift, 1.0 / 255.0); + EXPECT_DOUBLE_EQ(range.scale, 2.0 / 255.0); } TEST(QuantizationDatasetTest, PerDimensionDataset) @@ -717,7 +785,7 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); EXPECT_EQ(encoded.n_rows(), 2); EXPECT_EQ(encoded.dim(), 2u); - EXPECT_TRUE(devArrMatchHost(std::vector{-127, -2, 127, 2}.data(), + EXPECT_TRUE(devArrMatchHost(std::vector{-127, -127, 127, 127}.data(), encoded.data_const_view().data_handle(), 4, cuvs::Compare(), @@ -783,7 +851,11 @@ TYPED_TEST(PerDimensionQuantizationTest, BatchedHostDataset) auto actual_shifts = std::get::shifts_type>(a.value); auto expected_shifts = std::get::shifts_type>(e.value); - EXPECT_EQ(actual_shifts.scale, expected_shifts.scale); + EXPECT_TRUE(devArrMatch(actual_shifts.scales.data_handle(), + expected_shifts.scales.data_handle(), + dim, + cuvs::Compare::scale_type>(), + stream.get())); EXPECT_TRUE(devArrMatch(actual_shifts.shifts.data_handle(), expected_shifts.shifts.data_handle(), dim, @@ -861,6 +933,12 @@ const std::vector> inputs = { }; const std::vector dataset_inputs = { + // Each nonconstant dimension uses the full code range despite different input ranges. + {3, + 3, + {-127.f, -127.f / 128.f, 9.f, 1.f, 1.f / 128.f, 9.f, 127.f, 127.f / 128.f, 9.f}, + {-127, -127, 0, 1, 1, 0, 127, 127, 0}, + 1e-6f}, {2, 2, {10.f, -7.f, 10.f, -7.f}, {0, 0, 0, 0}, 0.f}, // Equal subnormal bounds must retain their value and reconstruct exactly. {1, diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 36bb8b9d5c..a22bcc6392 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -11,7 +11,7 @@ _Source header: `cuvs/preprocessing/quantize/scalar.hpp`_ ### preprocessing::quantize::scalar::range_mode -Select a global interval or per-dimension shifts with a shared scale. +Select global or per-dimension shifts and scales. ```cpp enum class range_mode { @@ -45,18 +45,20 @@ struct params { | Name | Type | Description | | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | -| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. A shared scale fits every dimension's bounds around its stored shift into the symmetric range of quantized values.

If every interval has zero width, the scale is one. Otherwise, the scale is at least the smallest positive normal value of the scale type. | +| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. Each dimension's scale fits its bounds around its stored shift into the symmetric range of quantized values.

For a zero-width interval, its scale is one. Otherwise, each scale is at least the smallest positive normal value of the scale type. | | `device_buffer_samples` | `int64_t` | Maximum rows staged on device for host training and host dataset build.
Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. Multiple batches use two input buffers when res has an auxiliary stream pool, and one otherwise. Pageable input uses batch-sized pinned staging buffers, without registering the full input. Each input buffer uses at most device_buffer_samples * dim * sizeof(T) bytes, in addition to reduction and encoding workspace. Substitute n_rows when it is zero.

Per-dimension host training with quantile < 1 selects exact quantiles using one column of host scratch (n_rows * sizeof(T) bytes). Global training retains its sampled quantiles. | ### preprocessing::quantize::scalar::quantizer_const_view -Read-only scalar quantizer parameters. The owner must outlive stream work using shifts. +Read-only scalar quantizer parameters. The owner must outlive stream work using shifts and + +scales. ```cpp template struct quantizer_const_view { - std::variant value; + std::variant value; scale_type max_error; }; ``` @@ -65,18 +67,20 @@ struct quantizer_const_view { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | Global interval or borrowed per-dimension shifts and scale. | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or borrowed per-dimension shifts and scales. | | `max_error` | `scale_type` | Maximum training-row L2 reconstruction error; see quantizer::max_error. | - -### preprocessing::quantize::scalar::quantizer_const_view::range_type + +### preprocessing::quantize::scalar::quantizer_const_view::global_type + +Global affine parameters using double precision for encoding and decoding. -A global interval mapped to the full range of the quantized type. +Encode (x - shift) / scale, rounding to nearest with ties away from zero and clamping to the full quantized range. Decode shift + scale * code, saturating to finite input limits. A zero scale represents constant training bounds; every code decodes to shift. ```cpp -struct range_type { - T min_; - T max_; +struct global_type { + double shift; + double scale; }; ``` @@ -84,20 +88,20 @@ struct range_type { | Name | Type | Description | | --- | --- | --- | -| `min_` | `T` | | -| `max_` | `T` | | +| `shift` | `double` | Shift shared by all dimensions. | +| `scale` | `double` | Step size shared by all dimensions. | ### preprocessing::quantize::scalar::quantizer_const_view::shifts_type -Encode (x - shifts[j]) / scale, rounding to nearest with ties to even and +Encode (x - shifts[j]) / scales[j], rounding to nearest with ties to even and -clamping to the symmetric range of quantized values. Decode shifts[j] + scale * code, saturating to finite limits of the input type. +clamping to the symmetric range of quantized values. Decode shifts[j] + scales[j] * code, saturating to finite limits of the input type. ```cpp struct shifts_type { raft::device_vector_view shifts; - scale_type scale; + raft::device_vector_view scales; }; ``` @@ -105,20 +109,20 @@ struct shifts_type { | Name | Type | Description | | --- | --- | --- | -| `shifts` | `raft::device_vector_view` | | -| `scale` | `scale_type` | | +| `shifts` | `raft::device_vector_view` | One midpoint shift per dimension. | +| `scales` | `raft::device_vector_view` | One finite, positive scale per dimension, with a finite reciprocal. | ### preprocessing::quantize::scalar::quantizer -Owns either a global interval or device shifts and a shared step size. +Owns a global shift and scale or device shifts and scales for each dimension. -Move-only; copy view() to borrow the parameters. Migration: replace q.min_ with std::get<quantizer<T>::range_type>(q.value).min_ (likewise for max_). +Move-only; copy view() to borrow the parameters. ```cpp template struct quantizer { - std::variant value; + std::variant value; scale_type max_error; }; ``` @@ -127,7 +131,7 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-range-type) | Global interval or owning per-dimension shifts and scale. | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or owning per-dimension shifts and scales. | | `max_error` | `scale_type` | Upper bound on the L2 error between a training row and its int8 encode/decode result. train measures every input row, including clipped outliers, and rounds the maximum up with a relative 1e-6 margin. This bound applies only to the training data, not later queries. It is infinity when constructed from parameters alone or when the error is not finite. | @@ -138,7 +142,7 @@ Owning per-dimension parameters. ```cpp struct shifts_type { raft::device_vector shifts; - scale_type scale; + raft::device_vector scales; }; ``` @@ -147,12 +151,12 @@ struct shifts_type { | Name | Type | Description | | --- | --- | --- | | `shifts` | `raft::device_vector` | | -| `scale` | `scale_type` | | +| `scales` | `raft::device_vector` | | ### preprocessing::quantize::scalar::quantizer::quantizer -Construct an empty global interval. +Construct default global parameters. ```cpp quantizer(); @@ -164,7 +168,7 @@ quantizer(); **Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` -Construct a global interval. +Derive a global shift and scale from finite bounds with min <= max. ```cpp quantizer(T min, T max); @@ -191,7 +195,7 @@ raft::device_vector_view mins, raft::device_vector_view maxs); ``` -Derives midpoint shifts and a shared scale. +Derives midpoint shifts and a scale for each dimension. **Parameters** @@ -208,7 +212,7 @@ Derives midpoint shifts and a shared scale. ### preprocessing::quantize::scalar::quantizer::view -Borrow read-only parameters; the owner must outlive stream work using shifts. +Borrow read-only parameters; the owner must outlive stream work using shifts and scales. ```cpp [[nodiscard]] auto view() const noexcept -> view_type; @@ -318,6 +322,8 @@ raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -375,6 +381,8 @@ raft::host_matrix_view out); Note that depending on the chosen data types train dataset the conversion is not lossless. +Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -475,6 +483,8 @@ raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -531,6 +541,8 @@ raft::host_matrix_view out); Note that depending on the chosen data types train dataset the conversion is not lossless. +Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -631,6 +643,8 @@ raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -687,6 +701,8 @@ raft::host_matrix_view out); Note that depending on the chosen data types train dataset the conversion is not lossless. +Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** From 823726d3b78f099eee5f6e080c79cf2cc99576a9 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 5 Nov 2026 01:22:49 +0000 Subject: [PATCH 15/24] Restrict host scalar transforms to global parameters --- c/src/preprocessing/quantize/scalar.cpp | 4 +- .../cuvs/preprocessing/quantize/scalar.hpp | 90 ++++++++----------- .../preprocessing/quantize/detail/scalar.cuh | 16 +--- cpp/src/preprocessing/quantize/scalar.cu | 4 +- .../preprocessing/scalar_quantization.cu | 15 +++- .../cpp-api-preprocessing-quantize-scalar.md | 48 ++++------ 6 files changed, 77 insertions(+), 100 deletions(-) diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index 49e3093bae..ba544ff21e 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -82,7 +82,7 @@ void _transform(cuvsResources_t res, cuvs::preprocessing::quantize::scalar::transform( *res_ptr, - quantizer, + std::get(quantizer.value), cuvs::core::from_dlpack(dataset_tensor), cuvs::core::from_dlpack(out_tensor)); } else { @@ -118,7 +118,7 @@ void _inverse_transform(cuvsResources_t res, cuvs::preprocessing::quantize::scalar::inverse_transform( *res_ptr, - quantizer, + std::get(quantizer.value), cuvs::core::from_dlpack(dataset_tensor), cuvs::core::from_dlpack(out_tensor)); } else { diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 01097e4976..58e168c487 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -206,29 +206,27 @@ void transform(raft::resources const& res, raft::device_matrix_view out); /** - * @brief Applies quantization transform to given dataset - * - * Host transforms require a global interval and throw raft::logic_error for per-dimension - * parameters. Use device transforms or host build for per-dimension quantization. + * @brief Applies global quantization to a host dataset * * Usage example: * @code{.cpp} * raft::handle_t handle; * cuvs::preprocessing::quantize::scalar::params params; - * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, + * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, * dataset); auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, + * auto global = std::get(quantizer.value); + * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, * quantized_dataset.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer a scalar quantizer + * @param[in] quantizer global shift and scale * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -260,31 +258,29 @@ void inverse_transform(raft::resources const& res, raft::device_matrix_view out); /** - * @brief Perform inverse quantization step on previously quantized dataset + * @brief Applies inverse global quantization to a host dataset * * Note that depending on the chosen data types train dataset the conversion is * not lossless. * - * Host transforms require a global interval and throw raft::logic_error for per-dimension - * parameters. Use device transforms or host build for per-dimension quantization. - * * Usage example: * @code{.cpp} + * auto global = std::get(quantizer.value); * auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, * quantized_dataset.view()); auto dataset_revert = raft::make_host_matrix(samples, - * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, quantizer, - * dataset_revert.view()); + * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, global, + * raft::make_const_mdspan(quantized_dataset.view()), dataset_revert.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer a scalar quantizer + * @param[in] quantizer global shift and scale * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -353,29 +349,27 @@ void transform(raft::resources const& res, raft::device_matrix_view out); /** - * @brief Applies quantization transform to given dataset - * - * Host transforms require a global interval and throw raft::logic_error for per-dimension - * parameters. Use device transforms or host build for per-dimension quantization. + * @brief Applies global quantization to a host dataset * * Usage example: * @code{.cpp} * raft::handle_t handle; * cuvs::preprocessing::quantize::scalar::params params; - * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, + * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, * dataset); auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, + * auto global = std::get(quantizer.value); + * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, * quantized_dataset.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer a scalar quantizer + * @param[in] quantizer global shift and scale * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -406,31 +400,29 @@ void inverse_transform(raft::resources const& res, raft::device_matrix_view out); /** - * @brief Perform inverse quantization step on previously quantized dataset + * @brief Applies inverse global quantization to a host dataset * * Note that depending on the chosen data types train dataset the conversion is * not lossless. * - * Host transforms require a global interval and throw raft::logic_error for per-dimension - * parameters. Use device transforms or host build for per-dimension quantization. - * * Usage example: * @code{.cpp} + * auto global = std::get(quantizer.value); * auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, * quantized_dataset.view()); auto dataset_revert = raft::make_host_matrix(samples, - * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, quantizer, - * dataset_revert.view()); + * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, global, + * raft::make_const_mdspan(quantized_dataset.view()), dataset_revert.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer a scalar quantizer + * @param[in] quantizer global shift and scale * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -499,29 +491,27 @@ void transform(raft::resources const& res, raft::device_matrix_view out); /** - * @brief Applies quantization transform to given dataset - * - * Host transforms require a global interval and throw raft::logic_error for per-dimension - * parameters. Use device transforms or host build for per-dimension quantization. + * @brief Applies global quantization to a host dataset * * Usage example: * @code{.cpp} * raft::handle_t handle; * cuvs::preprocessing::quantize::scalar::params params; - * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, + * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, * dataset); auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, + * auto global = std::get(quantizer.value); + * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, * quantized_dataset.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer a scalar quantizer + * @param[in] quantizer global shift and scale * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -552,31 +542,29 @@ void inverse_transform(raft::resources const& res, raft::device_matrix_view out); /** - * @brief Perform inverse quantization step on previously quantized dataset + * @brief Applies inverse global quantization to a host dataset * * Note that depending on the chosen data types train dataset the conversion is * not lossless. * - * Host transforms require a global interval and throw raft::logic_error for per-dimension - * parameters. Use device transforms or host build for per-dimension quantization. - * * Usage example: * @code{.cpp} + * auto global = std::get(quantizer.value); * auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, * quantized_dataset.view()); auto dataset_revert = raft::make_host_matrix(samples, - * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, quantizer, - * dataset_revert.view()); + * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, global, + * raft::make_const_mdspan(quantized_dataset.view()), dataset_revert.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer a scalar quantizer + * @param[in] quantizer global shift and scale * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index d2bf55f244..490edff934 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -323,15 +323,11 @@ void transform(raft::resources const& res, template void transform(raft::resources const& res, - quantizer_const_view quantizer, + typename quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out) { - RAFT_EXPECTS( - std::holds_alternative::global_type>(quantizer.value), - "Host quantization requires a global interval"); - auto const& range = std::get::global_type>(quantizer.value); - auto main_op = quantize_op{range}; + auto main_op = quantize_op{quantizer}; size_t n_elements = dataset.extent(0) * dataset.extent(1); // In dry-run mode the views may be unbacked or alias the shared probe buffer. @@ -361,15 +357,11 @@ void inverse_transform(raft::resources const& res, template void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + typename quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out) { - RAFT_EXPECTS( - std::holds_alternative::global_type>(quantizer.value), - "Host quantization requires a global interval"); - auto const& range = std::get::global_type>(quantizer.value); - auto main_op = quantize_op{range}; + auto main_op = quantize_op{quantizer}; size_t n_elements = dataset.extent(0) * dataset.extent(1); // In dry-run mode the views may be unbacked or alias the shared probe buffer. diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index a7f2645b91..da58776e2a 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -52,7 +52,7 @@ quantizer::quantizer(raft::resources const& res, detail::transform(res, quantizer, dataset, out); \ } \ void transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ + typename quantizer_const_view::global_type quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ @@ -66,7 +66,7 @@ quantizer::quantizer(raft::resources const& res, detail::inverse_transform(res, quantizer, dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ + typename quantizer_const_view::global_type quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 50c9604062..e87a09a5fe 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -140,7 +140,10 @@ class QuantizationTest : public ::testing::TestWithParam> handle, [&](raft::resources const& h) { cuvs::preprocessing::quantize::scalar::transform( - h, quantizer_1, dataset_h, quantized_input_h.view()); + h, + std::get(quantizer_1.value), + dataset_h, + quantized_input_h.view()); }, cuvs::alloc_behavior::NO_ALLOCATIONS); @@ -172,7 +175,10 @@ class QuantizationTest : public ::testing::TestWithParam> handle, [&](raft::resources const& h) { cuvs::preprocessing::quantize::scalar::inverse_transform( - h, quantizer_1, quantized_input_h_const_view, re_transformed_input_h.view()); + h, + std::get(quantizer_1.value), + quantized_input_h_const_view, + re_transformed_input_h.view()); }, cuvs::alloc_behavior::NO_ALLOCATIONS); // Also covers the rows outside the range-fitting sample on the large input. @@ -246,7 +252,10 @@ class QuantizationTest : public ::testing::TestWithParam> cuvs::preprocessing::quantize::scalar::transform( handle, quantizer_2, dataset, quantized_input_d.view()); cuvs::preprocessing::quantize::scalar::transform( - handle, quantizer_2, dataset_h, quantized_input_h.view()); + handle, + std::get(quantizer_2.value), + dataset_h, + quantized_input_h.view()); { rmm::device_uvector quantization_for_print(print_size, stream); diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index a22bcc6392..53d092715c 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -313,17 +313,15 @@ Usage example: **Additional overload:** `preprocessing::quantize::scalar::transform` -Applies quantization transform to given dataset +Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` -Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. - Usage example: **Parameters** @@ -331,7 +329,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -370,19 +368,17 @@ Usage example: **Additional overload:** `preprocessing::quantize::scalar::inverse_transform` -Perform inverse quantization step on previously quantized dataset +Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` Note that depending on the chosen data types train dataset the conversion is not lossless. -Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. - Usage example: **Parameters** @@ -390,7 +386,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -474,17 +470,15 @@ Usage example: **Additional overload:** `preprocessing::quantize::scalar::transform` -Applies quantization transform to given dataset +Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` -Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. - Usage example: **Parameters** @@ -492,7 +486,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -530,19 +524,17 @@ Usage example: **Additional overload:** `preprocessing::quantize::scalar::inverse_transform` -Perform inverse quantization step on previously quantized dataset +Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` Note that depending on the chosen data types train dataset the conversion is not lossless. -Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. - Usage example: **Parameters** @@ -550,7 +542,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -634,17 +626,15 @@ Usage example: **Additional overload:** `preprocessing::quantize::scalar::transform` -Applies quantization transform to given dataset +Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` -Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. - Usage example: **Parameters** @@ -652,7 +642,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -690,19 +680,17 @@ Usage example: **Additional overload:** `preprocessing::quantize::scalar::inverse_transform` -Perform inverse quantization step on previously quantized dataset +Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +quantizer_const_view::global_type quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` Note that depending on the chosen data types train dataset the conversion is not lossless. -Host transforms require a global interval and throw raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. - Usage example: **Parameters** @@ -710,7 +698,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | From 6e0f0b08ace11c3ca7727977def1a1c283d0d3e0 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 5 Nov 2026 17:37:52 +0000 Subject: [PATCH 16/24] Fix HNSW ACE example for global scalar transform API --- examples/cpp/src/hnsw_ace_layered_example.cu | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/cpp/src/hnsw_ace_layered_example.cu b/examples/cpp/src/hnsw_ace_layered_example.cu index 14de426605..c12a2daccd 100644 --- a/examples/cpp/src/hnsw_ace_layered_example.cu +++ b/examples/cpp/src/hnsw_ace_layered_example.cu @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -86,7 +87,10 @@ auto quantize_dataset(raft::device_resources const& dev_resources, auto dataset_i8 = raft::make_host_matrix(dataset_float.extent(0), dataset_float.extent(1)); cuvs::preprocessing::quantize::scalar::transform( - dev_resources, quantizer, dataset_float, dataset_i8.view()); + dev_resources, + std::get(quantizer.value), + dataset_float, + dataset_i8.view()); return dataset_i8; } From 4368cdcdbf65965d6b940214c3e6ca93dda4123b Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 6 Nov 2026 17:59:36 +0000 Subject: [PATCH 17/24] Simplify scalar quantization tests into focused cases --- .../preprocessing/scalar_quantization.cu | 321 ++++++++++-------- 1 file changed, 186 insertions(+), 135 deletions(-) diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index e87a09a5fe..df3c1038f9 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -18,6 +18,37 @@ namespace cuvs::preprocessing::quantize::scalar { +// Compute row norms in extended precision without squaring extreme residuals. +template +long double expectReconstructionErrorBound( + const T* original, const T* reconstructed, int64_t rows, int64_t dim, long double bound) +{ + long double maximum = 0; + for (int64_t row = 0; row < rows; ++row) { + SCOPED_TRACE(row); + long double error = 0; + for (int64_t col = 0; col < dim; ++col) { + auto i = row * dim + col; + auto delta = static_cast(static_cast(original[i])) - + static_cast(static_cast(reconstructed[i])); + error = std::hypot(error, delta); + } + EXPECT_GE(bound, error); + maximum = std::max(maximum, error); + } + return maximum; +} + +template +std::vector makeShiftedDataset(int64_t rows, int64_t dim) +{ + std::vector values(rows * dim); + for (int64_t i = 0; i < rows * dim; ++i) { + values[i] = static_cast((i / dim - rows / 2) * .17 + (i % dim % 5) * 20.0); + } + return values; +} + template struct QuantizationInputs { cuvs::preprocessing::quantize::scalar::params quantization_params; @@ -182,16 +213,11 @@ class QuantizationTest : public ::testing::TestWithParam> }, cuvs::alloc_behavior::NO_ALLOCATIONS); // Also covers the rows outside the range-fitting sample on the large input. - for (int64_t row = 0; row < rows_; ++row) { - double error = 0; - for (int64_t j = 0; j < cols_; ++j) { - auto const i = row * cols_ + j; - error = std::hypot(error, - static_cast(host_input_[i]) - - static_cast(re_transformed_input_h.data_handle()[i])); - } - EXPECT_GE(static_cast(quantizer_1.max_error), error); - } + expectReconstructionErrorBound(host_input_.data(), + re_transformed_input_h.data_handle(), + rows_, + cols_, + quantizer_1.max_error); auto quantized_input_d_const_view = raft::make_device_matrix_view( quantized_input_d.data_handle(), rows_, cols_); @@ -388,17 +414,12 @@ class PerDimensionRangeTest : public ::testing::TestWithParam(params_.dataset[i]) - - static_cast(reconstructed.data_handle()[i])); - } - EXPECT_GE(static_cast(quantizer.max_error), error); - if (params_.tolerance == 0.f) { EXPECT_EQ(quantizer.max_error, 0.f); } - } + expectReconstructionErrorBound(params_.dataset.data(), + reconstructed.data_handle(), + params_.n_samples, + params_.n_features, + quantizer.max_error); + if (params_.tolerance == 0.f) { EXPECT_EQ(quantizer.max_error, 0.f); } } private: @@ -413,19 +434,14 @@ class PerDimensionQuantizationTest : public ::testing::Test {}; using PerDimensionTypes = ::testing::Types; TYPED_TEST_SUITE(PerDimensionQuantizationTest, PerDimensionTypes); -TYPED_TEST(PerDimensionQuantizationTest, TrainingReconstructionError) +template +void testTrainingReconstructionError(params p) { - using T = TypeParam; raft::resources res; auto stream = raft::resource::get_cuda_stream(res); // A partial warp plus a second column per lane, with clipped outliers in different rows. constexpr int64_t rows = 9, dim = 35; - std::vector values(rows * dim); - for (int64_t row = 0; row < rows; ++row) { - for (int64_t j = 0; j < dim; ++j) { - values[row * dim + j] = static_cast((row - 4) * 0.17 + (j % 5) * 20.0); - } - } + auto values = makeShiftedDataset(rows, dim); values.front() = T{-300.f}; values.back() = T{500.f}; auto input = raft::make_device_matrix(res, rows, dim); @@ -433,46 +449,49 @@ TYPED_TEST(PerDimensionQuantizationTest, TrainingReconstructionError) auto codes = raft::make_device_matrix(res, rows, dim); auto decoded = raft::make_device_matrix(res, rows, dim); std::vector reconstructed(values.size()); - for (auto mode : {range_mode::global, range_mode::per_dimension}) { - for (float quantile : {1.f, 0.5f}) { - SCOPED_TRACE(::testing::Message() - << "range=" << static_cast(mode) << " quantile=" << quantile); - auto owner = train(res, params{quantile, mode}, raft::make_const_mdspan(input.view())); - auto view = owner.view(); - EXPECT_EQ(owner.max_error, view.max_error); - transform(res, view, raft::make_const_mdspan(input.view()), codes.view()); - inverse_transform(res, view, raft::make_const_mdspan(codes.view()), decoded.view()); - raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); - raft::resource::sync_stream(res); - long double maximum = 0; - for (int64_t row = 0; row < rows; ++row) { - long double error = 0; - for (int64_t j = 0; j < dim; ++j) { - auto const i = row * dim + j; - auto const delta = static_cast(static_cast(values[i])) - - static_cast(static_cast(reconstructed[i])); - error = std::hypot(error, delta); - } - EXPECT_GE(static_cast(owner.max_error), error); - maximum = std::max(maximum, error); - } - EXPECT_GT(maximum, 0); - EXPECT_NEAR(static_cast(owner.max_error), - static_cast(maximum), - static_cast(maximum) * 2e-6); - if (quantile < 1.f) { EXPECT_GT(owner.max_error, 100); } - if (mode == range_mode::global) { - auto host = train(res, - params{quantile}, - raft::make_host_matrix_view(values.data(), rows, dim)); - EXPECT_GE(static_cast(host.max_error), maximum); - EXPECT_NEAR(static_cast(host.max_error), - static_cast(owner.max_error), - static_cast(maximum) * 1e-6); - } - } + auto owner = train(res, p, raft::make_const_mdspan(input.view())); + auto view = owner.view(); + EXPECT_EQ(owner.max_error, view.max_error); + transform(res, view, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, view, raft::make_const_mdspan(codes.view()), decoded.view()); + raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); + raft::resource::sync_stream(res); + auto maximum = + expectReconstructionErrorBound(values.data(), reconstructed.data(), rows, dim, owner.max_error); + EXPECT_GT(maximum, 0); + EXPECT_NEAR(static_cast(owner.max_error), + static_cast(maximum), + static_cast(maximum) * 2e-6); + if (p.quantile < 1.f) { EXPECT_GT(owner.max_error, 100); } + if (p.range == range_mode::global) { + auto host = train(res, + params{p.quantile}, + raft::make_host_matrix_view(values.data(), rows, dim)); + EXPECT_GE(static_cast(host.max_error), maximum); + EXPECT_NEAR(static_cast(host.max_error), + static_cast(owner.max_error), + static_cast(maximum) * 1e-6); } } +TYPED_TEST(PerDimensionQuantizationTest, GlobalTrainingReconstructionError) +{ + testTrainingReconstructionError(params{1.f, range_mode::global}); +} + +TYPED_TEST(PerDimensionQuantizationTest, GlobalClippedReconstructionError) +{ + testTrainingReconstructionError(params{.5f, range_mode::global}); +} + +TYPED_TEST(PerDimensionQuantizationTest, PerDimensionTrainingReconstructionError) +{ + testTrainingReconstructionError(params{1.f, range_mode::per_dimension}); +} + +TYPED_TEST(PerDimensionQuantizationTest, PerDimensionClippedReconstructionError) +{ + testTrainingReconstructionError(params{.5f, range_mode::per_dimension}); +} TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) { @@ -809,79 +828,111 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) stream.get())); } -TYPED_TEST(PerDimensionQuantizationTest, BatchedHostDataset) +template +void testHostDataset(params p, bool prefetch = false, bool use_pinned = false) { - using T = TypeParam; constexpr int64_t rows = 37, dim = 35; - std::vector values(rows * dim); - for (int64_t i = 0; i < rows; ++i) { - for (int64_t j = 0; j < dim; ++j) { - values[i * dim + j] = T((i - 18) * .17 + (j % 5) * 20.0); - } - } - // Put the largest clipping error before the final batch to exercise the running maximum. + auto values = makeShiftedDataset(rows, dim); + // The largest clipping error precedes the final batch to check the running maximum. values.front() = T{-1200.f}; values.back() = T{500.f}; - for (bool prefetch : {false, true}) { - raft::resources res; - if (prefetch) { - raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); - } - auto stream = raft::resource::get_cuda_stream(res); - auto input = raft::make_device_matrix(res, rows, dim); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); - auto pinned = raft::make_pinned_matrix(res, rows, dim); - std::copy(values.begin(), values.end(), pinned.data_handle()); - for (auto mode : {range_mode::global, range_mode::per_dimension}) { - for (float quantile : {1.f, .5f, .99f}) { - // Device input ignores the host staging limit. - auto expected = - build(res, params{quantile, mode, -1}, raft::make_const_mdspan(input.view())); - for (int64_t batch_rows : {int64_t{1}, int64_t{7}, rows, int64_t{0}}) { - for (bool use_pinned : {false, true}) { - SCOPED_TRACE(::testing::Message() - << "range=" << static_cast(mode) << " quantile=" << quantile - << " batch=" << batch_rows << " prefetch=" << prefetch - << " pinned=" << use_pinned); - auto host_input = raft::make_host_matrix_view( - use_pinned ? pinned.data_handle() : values.data(), rows, dim); - auto actual = build(res, params{quantile, mode, batch_rows}, host_input); - EXPECT_EQ(actual.n_rows(), rows); - EXPECT_EQ(actual.dim(), dim); - EXPECT_TRUE(devArrMatch(expected.data_const_view().data_handle(), - actual.data_const_view().data_handle(), - values.size(), - cuvs::Compare(), - stream.get())); - auto a = actual.const_manifest_view(); - auto e = expected.const_manifest_view(); - EXPECT_NEAR(double(a.max_error), double(e.max_error), double(e.max_error) * 1e-6); - if (mode == range_mode::per_dimension) { - auto actual_shifts = std::get::shifts_type>(a.value); - auto expected_shifts = - std::get::shifts_type>(e.value); - EXPECT_TRUE(devArrMatch(actual_shifts.scales.data_handle(), - expected_shifts.scales.data_handle(), - dim, - cuvs::Compare::scale_type>(), - stream.get())); - EXPECT_TRUE(devArrMatch(actual_shifts.shifts.data_handle(), - expected_shifts.shifts.data_handle(), - dim, - cuvs::Compare(), - stream.get())); - } - // Standalone training must give the same manifest without allocating corpus codes. - if (batch_rows == 7 && !use_pinned) { - auto trained = train(res, params{quantile, mode, batch_rows}, host_input); - EXPECT_NEAR( - double(trained.max_error), double(a.max_error), double(a.max_error) * 1e-6); - } - } - } - } - } + raft::resources res; + if (prefetch) { + raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); } + auto stream = raft::resource::get_cuda_stream(res); + auto input = raft::make_device_matrix(res, rows, dim); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto pinned = raft::make_pinned_matrix(res, rows, dim); + if (use_pinned) { std::copy(values.begin(), values.end(), pinned.data_handle()); } + // A negative staging limit is ignored for device input. + auto expected = + build(res, params{p.quantile, p.range, -1}, raft::make_const_mdspan(input.view())); + auto host_input = raft::make_host_matrix_view( + use_pinned ? pinned.data_handle() : values.data(), rows, dim); + auto actual = build(res, p, host_input); + EXPECT_EQ(actual.n_rows(), rows); + EXPECT_EQ(actual.dim(), dim); + EXPECT_TRUE(devArrMatch(expected.data_const_view().data_handle(), + actual.data_const_view().data_handle(), + values.size(), + cuvs::Compare(), + stream.get())); + auto a = actual.const_manifest_view(); + auto e = expected.const_manifest_view(); + EXPECT_NEAR(double(a.max_error), double(e.max_error), double(e.max_error) * 1e-6); + if (p.range == range_mode::per_dimension) { + auto actual_shifts = std::get::shifts_type>(a.value); + auto expected_shifts = std::get::shifts_type>(e.value); + EXPECT_TRUE(devArrMatch(actual_shifts.scales.data_handle(), + expected_shifts.scales.data_handle(), + dim, + cuvs::Compare::scale_type>(), + stream.get())); + EXPECT_TRUE(devArrMatch(actual_shifts.shifts.data_handle(), + expected_shifts.shifts.data_handle(), + dim, + cuvs::Compare(), + stream.get())); + } + // Standalone training must agree without allocating corpus codes. + auto trained = train(res, p, host_input); + EXPECT_NEAR(double(trained.max_error), double(a.max_error), double(a.max_error) * 1e-6); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPartialBatch) +{ + testHostDataset(params{1.f, range_mode::per_dimension, 7}); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetSingleRowBatch) +{ + testHostDataset(params{1.f, range_mode::per_dimension, 1}); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetFullBatch) +{ + testHostDataset(params{1.f, range_mode::per_dimension, 37}); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetUnlimitedBatch) +{ + testHostDataset(params{1.f, range_mode::per_dimension, 0}); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPinned) +{ + testHostDataset(params{1.f, range_mode::per_dimension, 7}, false, true); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPrefetch) +{ + testHostDataset(params{1.f, range_mode::per_dimension, 7}, true); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPinnedPrefetch) +{ + testHostDataset(params{1.f, range_mode::per_dimension, 7}, true, true); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetClippedQuantile) +{ + testHostDataset(params{.5f, range_mode::per_dimension, 7}); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetDefaultQuantile) +{ + testHostDataset(params{.99f, range_mode::per_dimension, 7}); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetGlobalRange) +{ + testHostDataset(params{1.f, range_mode::global, 7}); +} + +TYPED_TEST(PerDimensionQuantizationTest, HostDatasetGlobalClippedQuantile) +{ + testHostDataset(params{.5f, range_mode::global, 7}); } TEST(QuantizationDatasetTest, HostBuildBatchWorkspace) From 1a0de8df8922bc3463f684c2cd3507c3c56155be Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 6 Nov 2026 18:33:33 +0000 Subject: [PATCH 18/24] Restore quantizer arguments for host scalar transforms --- c/src/preprocessing/quantize/scalar.cpp | 4 +- .../cuvs/preprocessing/quantize/scalar.hpp | 66 +++++++++++-------- .../preprocessing/quantize/detail/scalar.cuh | 16 +++-- cpp/src/preprocessing/quantize/scalar.cu | 4 +- .../preprocessing/scalar_quantization.cu | 31 +++++---- examples/cpp/src/hnsw_ace_layered_example.cu | 5 +- .../cpp-api-preprocessing-quantize-scalar.md | 36 ++++++---- 7 files changed, 99 insertions(+), 63 deletions(-) diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index ba544ff21e..49e3093bae 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -82,7 +82,7 @@ void _transform(cuvsResources_t res, cuvs::preprocessing::quantize::scalar::transform( *res_ptr, - std::get(quantizer.value), + quantizer, cuvs::core::from_dlpack(dataset_tensor), cuvs::core::from_dlpack(out_tensor)); } else { @@ -118,7 +118,7 @@ void _inverse_transform(cuvsResources_t res, cuvs::preprocessing::quantize::scalar::inverse_transform( *res_ptr, - std::get(quantizer.value), + quantizer, cuvs::core::from_dlpack(dataset_tensor), cuvs::core::from_dlpack(out_tensor)); } else { diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 58e168c487..a4003966cf 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -208,25 +208,27 @@ void transform(raft::resources const& res, /** * @brief Applies global quantization to a host dataset * + * Throws raft::logic_error for per-dimension parameters. Use device transforms or host build + * for per-dimension quantization. + * * Usage example: * @code{.cpp} * raft::handle_t handle; * cuvs::preprocessing::quantize::scalar::params params; * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, * dataset); auto quantized_dataset = raft::make_host_matrix(samples, features); - * auto global = std::get(quantizer.value); - * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, * quantized_dataset.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer global shift and scale + * @param[in] quantizer a scalar quantizer with global parameters * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void transform(raft::resources const& res, - quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -260,27 +262,29 @@ void inverse_transform(raft::resources const& res, /** * @brief Applies inverse global quantization to a host dataset * + * Throws raft::logic_error for per-dimension parameters. Use device inverse_transform + * for per-dimension quantization. + * * Note that depending on the chosen data types train dataset the conversion is * not lossless. * * Usage example: * @code{.cpp} - * auto global = std::get(quantizer.value); * auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, * quantized_dataset.view()); auto dataset_revert = raft::make_host_matrix(samples, - * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, global, + * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, quantizer, * raft::make_const_mdspan(quantized_dataset.view()), dataset_revert.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer global shift and scale + * @param[in] quantizer a scalar quantizer with global parameters * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void inverse_transform(raft::resources const& res, - quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -351,25 +355,27 @@ void transform(raft::resources const& res, /** * @brief Applies global quantization to a host dataset * + * Throws raft::logic_error for per-dimension parameters. Use device transforms or host build + * for per-dimension quantization. + * * Usage example: * @code{.cpp} * raft::handle_t handle; * cuvs::preprocessing::quantize::scalar::params params; * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, * dataset); auto quantized_dataset = raft::make_host_matrix(samples, features); - * auto global = std::get(quantizer.value); - * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, * quantized_dataset.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer global shift and scale + * @param[in] quantizer a scalar quantizer with global parameters * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void transform(raft::resources const& res, - quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -402,27 +408,29 @@ void inverse_transform(raft::resources const& res, /** * @brief Applies inverse global quantization to a host dataset * + * Throws raft::logic_error for per-dimension parameters. Use device inverse_transform + * for per-dimension quantization. + * * Note that depending on the chosen data types train dataset the conversion is * not lossless. * * Usage example: * @code{.cpp} - * auto global = std::get(quantizer.value); * auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, * quantized_dataset.view()); auto dataset_revert = raft::make_host_matrix(samples, - * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, global, + * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, quantizer, * raft::make_const_mdspan(quantized_dataset.view()), dataset_revert.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer global shift and scale + * @param[in] quantizer a scalar quantizer with global parameters * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void inverse_transform(raft::resources const& res, - quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -493,25 +501,27 @@ void transform(raft::resources const& res, /** * @brief Applies global quantization to a host dataset * + * Throws raft::logic_error for per-dimension parameters. Use device transforms or host build + * for per-dimension quantization. + * * Usage example: * @code{.cpp} * raft::handle_t handle; * cuvs::preprocessing::quantize::scalar::params params; * auto quantizer = cuvs::preprocessing::quantize::scalar::train(handle, params, * dataset); auto quantized_dataset = raft::make_host_matrix(samples, features); - * auto global = std::get(quantizer.value); - * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, * quantized_dataset.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer global shift and scale + * @param[in] quantizer a scalar quantizer with global parameters * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void transform(raft::resources const& res, - quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -544,27 +554,29 @@ void inverse_transform(raft::resources const& res, /** * @brief Applies inverse global quantization to a host dataset * + * Throws raft::logic_error for per-dimension parameters. Use device inverse_transform + * for per-dimension quantization. + * * Note that depending on the chosen data types train dataset the conversion is * not lossless. * * Usage example: * @code{.cpp} - * auto global = std::get(quantizer.value); * auto quantized_dataset = raft::make_host_matrix(samples, features); - * cuvs::preprocessing::quantize::scalar::transform(handle, global, dataset, + * cuvs::preprocessing::quantize::scalar::transform(handle, quantizer, dataset, * quantized_dataset.view()); auto dataset_revert = raft::make_host_matrix(samples, - * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, global, + * features); cuvs::preprocessing::quantize::scalar::inverse_transform(handle, quantizer, * raft::make_const_mdspan(quantized_dataset.view()), dataset_revert.view()); * @endcode * * @param[in] res raft resource - * @param[in] quantizer global shift and scale + * @param[in] quantizer a scalar quantizer with global parameters * @param[in] dataset a row-major matrix view on host * @param[out] out a row-major matrix view on host * */ void inverse_transform(raft::resources const& res, - quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index 490edff934..b16a018974 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -323,11 +323,15 @@ void transform(raft::resources const& res, template void transform(raft::resources const& res, - typename quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out) { - auto main_op = quantize_op{quantizer}; + RAFT_EXPECTS( + std::holds_alternative::global_type>(quantizer.value), + "Host quantization requires global parameters"); + auto const& range = std::get::global_type>(quantizer.value); + auto main_op = quantize_op{range}; size_t n_elements = dataset.extent(0) * dataset.extent(1); // In dry-run mode the views may be unbacked or alias the shared probe buffer. @@ -357,11 +361,15 @@ void inverse_transform(raft::resources const& res, template void inverse_transform(raft::resources const& res, - typename quantizer_const_view::global_type quantizer, + quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out) { - auto main_op = quantize_op{quantizer}; + RAFT_EXPECTS( + std::holds_alternative::global_type>(quantizer.value), + "Host quantization requires global parameters"); + auto const& range = std::get::global_type>(quantizer.value); + auto main_op = quantize_op{range}; size_t n_elements = dataset.extent(0) * dataset.extent(1); // In dry-run mode the views may be unbacked or alias the shared probe buffer. diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index da58776e2a..a7f2645b91 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -52,7 +52,7 @@ quantizer::quantizer(raft::resources const& res, detail::transform(res, quantizer, dataset, out); \ } \ void transform(raft::resources const& res, \ - typename quantizer_const_view::global_type quantizer, \ + quantizer_const_view quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ @@ -66,7 +66,7 @@ quantizer::quantizer(raft::resources const& res, detail::inverse_transform(res, quantizer, dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ - typename quantizer_const_view::global_type quantizer, \ + quantizer_const_view quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index df3c1038f9..6e4a754d46 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -171,10 +171,7 @@ class QuantizationTest : public ::testing::TestWithParam> handle, [&](raft::resources const& h) { cuvs::preprocessing::quantize::scalar::transform( - h, - std::get(quantizer_1.value), - dataset_h, - quantized_input_h.view()); + h, quantizer_1, dataset_h, quantized_input_h.view()); }, cuvs::alloc_behavior::NO_ALLOCATIONS); @@ -206,10 +203,7 @@ class QuantizationTest : public ::testing::TestWithParam> handle, [&](raft::resources const& h) { cuvs::preprocessing::quantize::scalar::inverse_transform( - h, - std::get(quantizer_1.value), - quantized_input_h_const_view, - re_transformed_input_h.view()); + h, quantizer_1, quantized_input_h_const_view, re_transformed_input_h.view()); }, cuvs::alloc_behavior::NO_ALLOCATIONS); // Also covers the rows outside the range-fitting sample on the large input. @@ -278,10 +272,7 @@ class QuantizationTest : public ::testing::TestWithParam> cuvs::preprocessing::quantize::scalar::transform( handle, quantizer_2, dataset, quantized_input_d.view()); cuvs::preprocessing::quantize::scalar::transform( - handle, - std::get(quantizer_2.value), - dataset_h, - quantized_input_h.view()); + handle, quantizer_2, dataset_h, quantized_input_h.view()); { rmm::device_uvector quantization_for_print(print_size, stream); @@ -714,6 +705,22 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) raft::logic_error); } +TYPED_TEST(PerDimensionQuantizationTest, HostTransformsRejectPerDimensionQuantizer) +{ + using T = TypeParam; + raft::resources res; + auto values = raft::make_host_matrix(1, 1); + values(0, 0) = T{1}; + auto quantizer = + train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(values.view())); + auto codes = raft::make_host_matrix(1, 1); + EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(values.view()), codes.view()), + raft::logic_error); + EXPECT_THROW( + inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), values.view()), + raft::logic_error); +} + TEST(PerDimensionRangeTest, QuantizedTypeLimits) { using QuantI = int16_t; diff --git a/examples/cpp/src/hnsw_ace_layered_example.cu b/examples/cpp/src/hnsw_ace_layered_example.cu index c12a2daccd..d712644dbc 100644 --- a/examples/cpp/src/hnsw_ace_layered_example.cu +++ b/examples/cpp/src/hnsw_ace_layered_example.cu @@ -87,10 +87,7 @@ auto quantize_dataset(raft::device_resources const& dev_resources, auto dataset_i8 = raft::make_host_matrix(dataset_float.extent(0), dataset_float.extent(1)); cuvs::preprocessing::quantize::scalar::transform( - dev_resources, - std::get(quantizer.value), - dataset_float, - dataset_i8.view()); + dev_resources, quantizer, dataset_float, dataset_i8.view()); return dataset_i8; } diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index 53d092715c..f7f24c2f86 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -317,11 +317,13 @@ Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view::global_type quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Throws raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -329,7 +331,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -372,11 +374,13 @@ Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view::global_type quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Throws raft::logic_error for per-dimension parameters. Use device inverse_transform for per-dimension quantization. + Note that depending on the chosen data types train dataset the conversion is not lossless. Usage example: @@ -386,7 +390,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -474,11 +478,13 @@ Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view::global_type quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Throws raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -486,7 +492,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -528,11 +534,13 @@ Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view::global_type quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Throws raft::logic_error for per-dimension parameters. Use device inverse_transform for per-dimension quantization. + Note that depending on the chosen data types train dataset the conversion is not lossless. Usage example: @@ -542,7 +550,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -630,11 +638,13 @@ Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view::global_type quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Throws raft::logic_error for per-dimension parameters. Use device transforms or host build for per-dimension quantization. + Usage example: **Parameters** @@ -642,7 +652,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -684,11 +694,13 @@ Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view::global_type quantizer, +quantizer_const_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` +Throws raft::logic_error for per-dimension parameters. Use device inverse_transform for per-dimension quantization. + Note that depending on the chosen data types train dataset the conversion is not lossless. Usage example: @@ -698,7 +710,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view::global_type`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | global shift and scale | +| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | From 3f9984c05e3e8ede01f5f9194a7082967427752a Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 6 Nov 2026 18:57:27 +0000 Subject: [PATCH 19/24] Rename scalar quantizer shifts_type to per_dim_type --- .../cuvs/preprocessing/quantize/scalar.hpp | 12 +++++------ .../preprocessing/quantize/detail/scalar.cuh | 16 +++++++-------- .../preprocessing/scalar_quantization.cu | 10 +++++----- .../cpp-api-preprocessing-quantize-scalar.md | 20 +++++++++---------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index a4003966cf..07232901fa 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -84,14 +84,14 @@ struct quantizer_const_view { /** Encode (x - shifts[j]) / scales[j], rounding to nearest with ties to even and * clamping to the symmetric range of quantized values. * Decode shifts[j] + scales[j] * code, saturating to finite limits of the input type. */ - struct shifts_type { + struct per_dim_type { /** One midpoint shift per dimension. */ raft::device_vector_view shifts; /** One finite, positive scale per dimension, with a finite reciprocal. */ raft::device_vector_view scales; }; /** Global parameters or borrowed per-dimension shifts and scales. */ - std::variant value; + std::variant value; /** Maximum training-row L2 reconstruction error; see quantizer::max_error. */ scale_type max_error = INFINITY; }; @@ -106,12 +106,12 @@ struct quantizer { using scale_type = typename view_type::scale_type; using global_type = typename view_type::global_type; /** Owning per-dimension parameters. */ - struct shifts_type { + struct per_dim_type { raft::device_vector shifts; raft::device_vector scales; }; /** Global parameters or owning per-dimension shifts and scales. */ - std::variant value; + std::variant value; /** * Upper bound on the L2 error between a training row and its int8 encode/decode result. * train measures every input row, including clipped outliers, and rounds the maximum up @@ -134,8 +134,8 @@ struct quantizer { [[nodiscard]] auto view() const noexcept -> view_type { if (auto const* range = std::get_if(&value)) { return {*range, max_error}; } - auto const& shifts = std::get(value); - return {typename view_type::shifts_type{shifts.shifts.view(), shifts.scales.view()}, max_error}; + auto const& shifts = std::get(value); + return {typename view_type::per_dim_type{shifts.shifts.view(), shifts.scales.view()}, max_error}; } /** An owning quantizer can be passed wherever a read-only view is accepted. */ operator view_type() const noexcept { return view(); } // NOLINT(google-explicit-constructor) diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index b16a018974..b794d12689 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -97,7 +97,7 @@ RAFT_INLINE_FUNCTION double reconstruction_error(T x, int64_t j, Op op) { double const value = static_cast(x); double decoded; - if constexpr (std::is_same_v::shifts_type>) { + if constexpr (std::is_same_v::per_dim_type>) { using compute_t = typename quantizer_const_view::scale_type; compute_t const shift = static_cast(op.shifts.data_handle()[j]); compute_t const scale = op.scales.data_handle()[j]; @@ -209,7 +209,7 @@ std::tuple quantile_min_max( } template -typename quantizer::shifts_type train(raft::resources const& res, +typename quantizer::per_dim_type train(raft::resources const& res, raft::device_vector&& shifts, raft::device_vector&& maxima) { @@ -252,7 +252,7 @@ typename quantizer::shifts_type train(raft::resources const& res, } template -void check_per_dim_shapes(typename quantizer_const_view::shifts_type quantizer, +void check_per_dim_shapes(typename quantizer_const_view::per_dim_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { @@ -265,7 +265,7 @@ void check_per_dim_shapes(typename quantizer_const_view::shifts_type quantize template void transform(raft::resources const& res, - typename quantizer_const_view::shifts_type quantizer, + typename quantizer_const_view::per_dim_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { @@ -287,7 +287,7 @@ void transform(raft::resources const& res, template void inverse_transform(raft::resources const& res, - typename quantizer_const_view::shifts_type quantizer, + typename quantizer_const_view::per_dim_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { @@ -313,7 +313,7 @@ void transform(raft::resources const& res, { auto view = quantizer; if (auto const* shifts = - std::get_if::shifts_type>(&view.value)) { + std::get_if::per_dim_type>(&view.value)) { detail::transform(res, *shifts, dataset, out); return; } @@ -351,7 +351,7 @@ void inverse_transform(raft::resources const& res, { auto view = quantizer; if (auto const* shifts = - std::get_if::shifts_type>(&view.value)) { + std::get_if::per_dim_type>(&view.value)) { detail::inverse_transform(res, *shifts, dataset, out); return; } @@ -528,7 +528,7 @@ quantizer train( auto batch_errors = raft::make_device_vector_view(errors.data_handle(), input.extent(0)); if (auto const* shifts = - std::get_if::shifts_type>(&view.value)) { + std::get_if::per_dim_type>(&view.value)) { max_reconstruction_error( res, input, *shifts, batch_errors, maximum.view(), offset != 0); } else { diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 6e4a754d46..2ba4391064 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -343,7 +343,7 @@ class QuantizationTest : public ::testing::TestWithParam> std::vector host_input_; }; -using shifts_view = quantizer_const_view::shifts_type; +using shifts_view = quantizer_const_view::per_dim_type; struct PerDimensionInputs { int64_t n_samples; @@ -519,7 +519,7 @@ TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) { using T = TypeParam; - using shifts_view = typename quantizer_const_view::shifts_type; + using shifts_view = typename quantizer_const_view::per_dim_type; raft::resources res; auto stream = raft::resource::get_cuda_stream(res); std::vector corpus{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; @@ -639,7 +639,7 @@ TEST(PerDimensionRangeTest, PerDimensionQuantiles) TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) { using T = TypeParam; - using shifts_view = typename quantizer_const_view::shifts_type; + using shifts_view = typename quantizer_const_view::per_dim_type; raft::resources res; auto mins = raft::make_device_vector(res, 2); auto maxs = raft::make_device_vector(res, 2); @@ -869,8 +869,8 @@ void testHostDataset(params p, bool prefetch = false, bool use_pinned = false) auto e = expected.const_manifest_view(); EXPECT_NEAR(double(a.max_error), double(e.max_error), double(e.max_error) * 1e-6); if (p.range == range_mode::per_dimension) { - auto actual_shifts = std::get::shifts_type>(a.value); - auto expected_shifts = std::get::shifts_type>(e.value); + auto actual_shifts = std::get::per_dim_type>(a.value); + auto expected_shifts = std::get::per_dim_type>(e.value); EXPECT_TRUE(devArrMatch(actual_shifts.scales.data_handle(), expected_shifts.scales.data_handle(), dim, diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index f7f24c2f86..f2c220586c 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -58,7 +58,7 @@ scales. ```cpp template struct quantizer_const_view { - std::variant value; + std::variant value; scale_type max_error; }; ``` @@ -67,7 +67,7 @@ struct quantizer_const_view { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or borrowed per-dimension shifts and scales. | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or borrowed per-dimension shifts and scales. | | `max_error` | `scale_type` | Maximum training-row L2 reconstruction error; see quantizer::max_error. | @@ -91,15 +91,15 @@ struct global_type { | `shift` | `double` | Shift shared by all dimensions. | | `scale` | `double` | Step size shared by all dimensions. | - -### preprocessing::quantize::scalar::quantizer_const_view::shifts_type + +### preprocessing::quantize::scalar::quantizer_const_view::per_dim_type Encode (x - shifts[j]) / scales[j], rounding to nearest with ties to even and clamping to the symmetric range of quantized values. Decode shifts[j] + scales[j] * code, saturating to finite limits of the input type. ```cpp -struct shifts_type { +struct per_dim_type { raft::device_vector_view shifts; raft::device_vector_view scales; }; @@ -122,7 +122,7 @@ Move-only; copy view() to borrow the parameters. ```cpp template struct quantizer { - std::variant value; + std::variant value; scale_type max_error; }; ``` @@ -131,16 +131,16 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or owning per-dimension shifts and scales. | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or owning per-dimension shifts and scales. | | `max_error` | `scale_type` | Upper bound on the L2 error between a training row and its int8 encode/decode result. train measures every input row, including clipped outliers, and rounds the maximum up with a relative 1e-6 margin. This bound applies only to the training data, not later queries. It is infinity when constructed from parameters alone or when the error is not finite. | - -### preprocessing::quantize::scalar::quantizer::shifts_type + +### preprocessing::quantize::scalar::quantizer::per_dim_type Owning per-dimension parameters. ```cpp -struct shifts_type { +struct per_dim_type { raft::device_vector shifts; raft::device_vector scales; }; From 10728a714b9c15761429d42633fa09a599d8bfd1 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 6 Nov 2026 20:17:13 +0000 Subject: [PATCH 20/24] Preserve original global scalar quantizer bounds --- c/src/preprocessing/quantize/scalar.cpp | 12 ++---------- cpp/include/cuvs/preprocessing/quantize/scalar.hpp | 7 ++++++- cpp/src/preprocessing/quantize/detail/scalar.cuh | 6 ++++-- cpp/src/preprocessing/quantize/scalar.cu | 2 +- cpp/tests/preprocessing/scalar_quantization.cu | 8 ++++++++ .../cpp_api/cpp-api-preprocessing-quantize-scalar.md | 4 ++++ 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index 49e3093bae..bb069ff3a4 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -3,12 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include -#include #include -#include #include -#include #include #include @@ -46,12 +42,8 @@ void _train(cuvsResources_t res, RAFT_FAIL("dataset must be accessible on host or device memory"); } - auto const& range = std::get(ret.value); - double const limit = static_cast(cuda::std::numeric_limits::max()); - quantizer->min_ = static_cast(std::clamp( - std::fma(range.scale, std::numeric_limits::min(), range.shift), -limit, limit)); - quantizer->max_ = static_cast(std::clamp( - std::fma(range.scale, std::numeric_limits::max(), range.shift), -limit, limit)); + quantizer->min_ = ret.min_; + quantizer->max_ = ret.max_; } template diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 07232901fa..0e528a362c 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -112,6 +112,10 @@ struct quantizer { }; /** Global parameters or owning per-dimension shifts and scales. */ std::variant value; + /** Original lower global bound; unused in per-dimension mode. Does not track changes to value. */ + T min_ = T{0}; + /** Original upper global bound; unused in per-dimension mode. Does not track changes to value. */ + T max_ = T{0}; /** * Upper bound on the L2 error between a training row and its int8 encode/decode result. * train measures every input row, including clipped outliers, and rounds the maximum up @@ -135,7 +139,8 @@ struct quantizer { { if (auto const* range = std::get_if(&value)) { return {*range, max_error}; } auto const& shifts = std::get(value); - return {typename view_type::per_dim_type{shifts.shifts.view(), shifts.scales.view()}, max_error}; + return {typename view_type::per_dim_type{shifts.shifts.view(), shifts.scales.view()}, + max_error}; } /** An owning quantizer can be passed wherever a read-only view is accepted. */ operator view_type() const noexcept { return view(); } // NOLINT(google-explicit-constructor) diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index b794d12689..c3c45e3e1e 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -210,8 +210,8 @@ std::tuple quantile_min_max( template typename quantizer::per_dim_type train(raft::resources const& res, - raft::device_vector&& shifts, - raft::device_vector&& maxima) + raft::device_vector&& shifts, + raft::device_vector&& maxima) { using compute_t = typename quantizer_const_view::scale_type; auto const dim = shifts.extent(0); @@ -440,6 +440,8 @@ quantizer train( if (params.range == range_mode::global) { auto [min, max] = quantile_min_max(res, dataset, params.quantile); result.value = detail::train(min, max); + result.min_ = min; + result.max_ = max; } else { auto minima = raft::make_device_vector(res, dim); auto maxima = raft::make_device_vector(res, dim); diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index a7f2645b91..ca8f5176e5 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -13,7 +13,7 @@ namespace cuvs::preprocessing::quantize::scalar { template -quantizer::quantizer(T min, T max) : value{detail::train(min, max)} +quantizer::quantizer(T min, T max) : value{detail::train(min, max)}, min_{min}, max_{max} { } diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 2ba4391064..42cc6b67ff 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -458,6 +458,12 @@ void testTrainingReconstructionError(params p) auto host = train(res, params{p.quantile}, raft::make_host_matrix_view(values.data(), rows, dim)); + EXPECT_EQ(host.min_, owner.min_); + EXPECT_EQ(host.max_, owner.max_); + if (p.quantile == 1.f) { + EXPECT_EQ(owner.min_, values.front()); + EXPECT_EQ(owner.max_, values.back()); + } EXPECT_GE(static_cast(host.max_error), maximum); EXPECT_NEAR(static_cast(host.max_error), static_cast(owner.max_error), @@ -763,6 +769,8 @@ TYPED_TEST(PerDimensionQuantizationTest, GlobalShiftScale) for (auto bounds : {std::pair{T{-3}, T{7}}, std::pair{T{5}, T{5}}}) { auto owner = quantizer{bounds.first, bounds.second}; auto parameters = std::get::global_type>(owner.value); + EXPECT_EQ(owner.min_, bounds.first); + EXPECT_EQ(owner.max_, bounds.second); EXPECT_EQ(static_cast( std::fma(parameters.scale, std::numeric_limits::min(), parameters.shift)), bounds.first); diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index f2c220586c..e70c67454d 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -123,6 +123,8 @@ Move-only; copy view() to borrow the parameters. template struct quantizer { std::variant value; + T min_; + T max_; scale_type max_error; }; ``` @@ -132,6 +134,8 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | | `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or owning per-dimension shifts and scales. | +| `min_` | `T` | Original lower global bound; unused in per-dimension mode. Does not track changes to value. | +| `max_` | `T` | Original upper global bound; unused in per-dimension mode. Does not track changes to value. | | `max_error` | `scale_type` | Upper bound on the L2 error between a training row and its int8 encode/decode result. train measures every input row, including clipped outliers, and rounds the maximum up with a relative 1e-6 margin. This bound applies only to the training data, not later queries. It is infinity when constructed from parameters alone or when the error is not finite. | From ea9e9223b155f1747f96c2af26e07d171ca23ac3 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 6 Nov 2026 22:24:56 +0000 Subject: [PATCH 21/24] Store scalar quantizer bounds and align preprocessing views --- c/src/preprocessing/quantize/scalar.cpp | 5 +- cpp/include/cuvs/core/dataset/sq.hpp | 45 +- .../cuvs/preprocessing/quantize/scalar.hpp | 140 +++--- .../preprocessing/quantize/detail/scalar.cuh | 245 ++++++----- cpp/src/preprocessing/quantize/scalar.cu | 20 +- .../preprocessing/scalar_quantization.cu | 414 +++++++++++------- fern/pages/cpp_api/cpp-api-core-dataset-sq.md | 26 +- .../cpp-api-preprocessing-quantize-scalar.md | 172 ++++---- 8 files changed, 597 insertions(+), 470 deletions(-) diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index bb069ff3a4..9d00da2959 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -42,8 +42,9 @@ void _train(cuvsResources_t res, RAFT_FAIL("dataset must be accessible on host or device memory"); } - quantizer->min_ = ret.min_; - quantizer->max_ = ret.max_; + auto const& range = std::get(ret.value); + quantizer->min_ = range.min; + quantizer->max_ = range.max; } template diff --git a/cpp/include/cuvs/core/dataset/sq.hpp b/cpp/include/cuvs/core/dataset/sq.hpp index d2ccadceba..7f636c33d8 100644 --- a/cpp/include/cuvs/core/dataset/sq.hpp +++ b/cpp/include/cuvs/core/dataset/sq.hpp @@ -34,9 +34,10 @@ struct sq_spec { using math_type = MathT; - // The manifest is either a global shift and scale or owning per-dimension shifts and scales. - using manifest_type = cuvs::preprocessing::quantize::scalar::quantizer; - using manifest_view_type = typename manifest_type::view_type; + // The manifest owns either global bounds or per-dimension bounds. + using manifest_type = cuvs::preprocessing::quantize::scalar::quantizer; + using manifest_view_type = typename manifest_type::view_type; + using const_manifest_view_type = typename manifest_type::const_view_type; template struct apply : storage_spec { @@ -45,27 +46,28 @@ struct sq_spec { using code_type = int8_t; // NOTE: re-exported rather than declared here, so that they do not belong to one instantiation // of `apply`; qualified lookup into `apply` does not reach the enclosing spec on its own. - using math_type = sq_spec::math_type; - using manifest_type = sq_spec::manifest_type; - using manifest_view_type = sq_spec::manifest_view_type; + using math_type = sq_spec::math_type; + using manifest_type = sq_spec::manifest_type; + using manifest_view_type = sq_spec::manifest_view_type; + using const_manifest_view_type = sq_spec::const_manifest_view_type; // NOTE: every accessor is inherited; the dimension is still one code per component, so even // `get_dim` is the dense one. - /** Borrow read-only quantizer parameters from the owning manifest. */ - [[nodiscard]] static auto get_manifest_view(const manifest_type& manifest) noexcept + /** Borrow mutable per-dimension bounds from the owning manifest. */ + [[nodiscard]] static auto get_manifest_view(manifest_type& manifest) noexcept -> manifest_view_type { return manifest.view(); } /** Borrow read-only quantizer parameters; the manifest must outlive use of the view. */ [[nodiscard]] static auto get_manifest_const_view(const manifest_type& manifest) noexcept - -> manifest_view_type + -> const_manifest_view_type { return manifest.view(); } - /** Return the read-only view; its owner must outlive stream work using it. */ + /** Convert a mutable view to read-only; its owner must outlive stream work using it. */ [[nodiscard]] static auto get_manifest_const_view(manifest_view_type manifest) noexcept - -> manifest_view_type + -> const_manifest_view_type { return manifest; } @@ -88,9 +90,8 @@ namespace preprocessing::quantize::scalar { * @{ */ /** - * @brief Train and encode a device corpus with the selected range mode. - * Reuse result.const_manifest_view() with transform - * for query batches. For separate training/encoding, use train and transform. + * @brief Train and encode a device corpus with the selected quantization mode. + * For separate training and encoding, use train and transform. * * @param[in] res Resources used for training and allocation. * @param[in] params Scalar quantizer parameters. @@ -98,13 +99,12 @@ namespace preprocessing::quantize::scalar { * @return Device dataset owning the quantized codes and quantizer. */ template -inline device_sq_dataset build(raft::resources const& res, - params params, - raft::device_matrix_view dataset) +inline device_sq_dataset make_sq_dataset( + raft::resources const& res, params params, raft::device_matrix_view dataset) { auto quantizer = train(res, params, dataset); auto codes = raft::make_device_matrix(res, dataset.extent(0), dataset.extent(1)); - transform(res, quantizer.view(), dataset, codes.view()); + transform(res, quantizer, dataset, codes.view()); return device_sq_dataset{std::move(codes), std::move(quantizer)}; } @@ -119,7 +119,7 @@ inline device_sq_dataset build(raft::resources const& res, * @param[in] dataset Row-major host corpus; pageable and pinned input are supported. * @return Device dataset owning the quantized codes and quantizer. */ -device_sq_dataset build( +device_sq_dataset make_sq_dataset( raft::resources const& res, params params, raft::host_matrix_view dataset); /** @@ -133,7 +133,7 @@ device_sq_dataset build( * @param[in] dataset Row-major host corpus; pageable and pinned input are supported. * @return Device dataset owning the quantized codes and quantizer. */ -device_sq_dataset build( +device_sq_dataset make_sq_dataset( raft::resources const& res, params params, raft::host_matrix_view dataset); /** @@ -147,9 +147,8 @@ device_sq_dataset build( * @param[in] dataset Row-major host corpus; pageable and pinned input are supported. * @return Device dataset owning the quantized codes and quantizer. */ -device_sq_dataset build(raft::resources const& res, - params params, - raft::host_matrix_view dataset); +device_sq_dataset make_sq_dataset( + raft::resources const& res, params params, raft::host_matrix_view dataset); /** @} */ diff --git a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp index 0e528a362c..1fc452132b 100644 --- a/cpp/include/cuvs/preprocessing/quantize/scalar.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/scalar.hpp @@ -29,8 +29,8 @@ namespace scalar { * @{ */ -/** Select global or per-dimension shifts and scales. */ -enum class range_mode { global, per_dimension }; +/** Select global or per-dimension quantization bounds. */ +enum class quantization_mode { GLOBAL, PER_DIMENSION }; /** * @brief quantizer parameters. @@ -44,14 +44,14 @@ struct params { /** * Per-dimension training uses every row of a nonempty, finite dataset. * It applies the quantile independently to each dimension to select lower and upper bounds. - * Each dimension's shift is the midpoint of its bounds, rounded to the input type. - * Each dimension's scale fits its bounds around its stored shift into the symmetric range - * of quantized values. + * Transforms derive each dimension's shift from the midpoint of its bounds, rounded to the input + * type. Each dimension's scale fits its bounds around that shift into the symmetric range of + * quantized values. * * For a zero-width interval, its scale is one. Otherwise, each scale is at least * the smallest positive normal value of the scale type. */ - range_mode range = range_mode::global; + quantization_mode range = quantization_mode::GLOBAL; /** * Maximum rows staged on device for host training and host dataset build. * Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. @@ -66,56 +66,76 @@ struct params { int64_t device_buffer_samples = 1 << 16; }; -/** Read-only scalar quantizer parameters. The owner must outlive stream work using shifts and - * scales. */ +/** + * Non-owning scalar quantizer bounds, with the constness semantics of an mdspan: + * quantizer_view can modify per-dimension bounds; quantizer_view cannot. + * Global bounds and max_error are copied. The owner must outlive stream work using the vectors. + */ template -struct quantizer_const_view { - using scale_type = std::conditional_t, float, T>; - /** Global affine parameters using double precision for encoding and decoding. +struct quantizer_view { + using value_type = std::remove_const_t; + using scale_type = std::conditional_t, float, value_type>; + using vector_view_type = raft::device_vector_view; + /** Global bounds. Transforms derive an affine mapping in double precision. * Encode (x - shift) / scale, rounding to nearest with ties away from zero and clamping * to the full quantized range. Decode shift + scale * code, saturating to finite input limits. - * A zero scale represents constant training bounds; every code decodes to shift. */ + * Equal bounds represent a constant range; every code decodes to that value. */ struct global_type { - /** Shift shared by all dimensions. */ - double shift = 0; - /** Step size shared by all dimensions. */ - double scale = 0; + /** Lower bound shared by all dimensions. */ + value_type min = value_type{0}; + /** Upper bound shared by all dimensions. */ + value_type max = value_type{0}; }; - /** Encode (x - shifts[j]) / scales[j], rounding to nearest with ties to even and + /** Per-dimension bounds. Transforms derive midpoint shifts and scales from these bounds. + * Encode (x - shift) / scale, rounding to nearest with ties to even and * clamping to the symmetric range of quantized values. - * Decode shifts[j] + scales[j] * code, saturating to finite limits of the input type. */ + * Decode shift + scale * code, saturating to finite limits of the input type. */ struct per_dim_type { - /** One midpoint shift per dimension. */ - raft::device_vector_view shifts; - /** One finite, positive scale per dimension, with a finite reciprocal. */ - raft::device_vector_view scales; + /** One finite lower bound per dimension, with mins[j] <= maxs[j]. */ + vector_view_type mins; + /** One finite upper bound per dimension; must have the same extent as mins. */ + vector_view_type maxs; }; - /** Global parameters or borrowed per-dimension shifts and scales. */ + /** Global bounds or borrowed per-dimension bounds. */ std::variant value; /** Maximum training-row L2 reconstruction error; see quantizer::max_error. */ scale_type max_error = INFINITY; + + /** A mutable view decays to the read-only one, the way an mdspan does. */ + template + requires(!std::is_const_v && std::is_same_v) + operator quantizer_view() const noexcept // NOLINT(google-explicit-constructor) + { + using const_view_type = quantizer_view; + if (auto const* range = std::get_if(&value)) { + return {typename const_view_type::global_type{range->min, range->max}, max_error}; + } + auto const& bounds = std::get(value); + return {typename const_view_type::per_dim_type{raft::make_const_mdspan(bounds.mins), + raft::make_const_mdspan(bounds.maxs)}, + max_error}; + } }; /** - * Owns a global shift and scale or device shifts and scales for each dimension. + * Owns global bounds or device bounds for each dimension. * Move-only; copy view() to borrow the parameters. */ template struct quantizer { - using view_type = quantizer_const_view; - using scale_type = typename view_type::scale_type; - using global_type = typename view_type::global_type; - /** Owning per-dimension parameters. */ + static_assert(!std::is_const_v, "An owning quantizer cannot hold const elements."); + + using view_type = quantizer_view; + using const_view_type = quantizer_view; + using scale_type = typename view_type::scale_type; + using global_type = typename view_type::global_type; + /** Owning per-dimension bounds. */ struct per_dim_type { - raft::device_vector shifts; - raft::device_vector scales; + raft::device_vector mins; + raft::device_vector maxs; }; - /** Global parameters or owning per-dimension shifts and scales. */ + /** Global bounds or owning per-dimension bounds. */ std::variant value; - /** Original lower global bound; unused in per-dimension mode. Does not track changes to value. */ - T min_ = T{0}; - /** Original upper global bound; unused in per-dimension mode. Does not track changes to value. */ - T max_ = T{0}; /** * Upper bound on the L2 error between a training row and its int8 encode/decode result. * train measures every input row, including clipped outliers, and rounds the maximum up @@ -124,26 +144,32 @@ struct quantizer { */ scale_type max_error = INFINITY; - /** Construct default global parameters. */ + /** Construct default global bounds. */ quantizer() : value{global_type{}} {} - /** Derive a global shift and scale from finite bounds with min <= max. */ + /** Construct from finite global bounds with min <= max. */ quantizer(T min, T max); - /** Construct from nonempty device bounds of equal length, with finite min <= max. - * Derives midpoint shifts and a scale for each dimension. */ + /** Copy nonempty device bounds of equal length, with finite min <= max. */ quantizer(raft::resources const& res, raft::device_vector_view mins, raft::device_vector_view maxs); - /** Borrow read-only parameters; the owner must outlive stream work using shifts and scales. */ - [[nodiscard]] auto view() const noexcept -> view_type + /** Borrow mutable per-dimension bounds; the owner must outlive stream work using them. */ + [[nodiscard]] auto view() noexcept -> view_type { if (auto const* range = std::get_if(&value)) { return {*range, max_error}; } - auto const& shifts = std::get(value); - return {typename view_type::per_dim_type{shifts.shifts.view(), shifts.scales.view()}, + auto& bounds = std::get(value); + return {typename view_type::per_dim_type{bounds.mins.view(), bounds.maxs.view()}, max_error}; + } + /** Borrow read-only bounds; the owner must outlive stream work using per-dimension bounds. */ + [[nodiscard]] auto view() const noexcept -> const_view_type + { + if (auto const* range = std::get_if(&value)) { + return {typename const_view_type::global_type{range->min, range->max}, max_error}; + } + auto const& bounds = std::get(value); + return {typename const_view_type::per_dim_type{bounds.mins.view(), bounds.maxs.view()}, max_error}; } - /** An owning quantizer can be passed wherever a read-only view is accepted. */ - operator view_type() const noexcept { return view(); } // NOLINT(google-explicit-constructor) }; /** @@ -206,7 +232,7 @@ quantizer train(raft::resources const& res, * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -233,7 +259,7 @@ void transform(raft::resources const& res, * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -260,7 +286,7 @@ void transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -289,7 +315,7 @@ void inverse_transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -353,7 +379,7 @@ quantizer train(raft::resources const& res, * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -380,7 +406,7 @@ void transform(raft::resources const& res, * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -406,7 +432,7 @@ void transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -435,7 +461,7 @@ void inverse_transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -499,7 +525,7 @@ quantizer train(raft::resources const& res, * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -526,7 +552,7 @@ void transform(raft::resources const& res, * */ void transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); @@ -552,7 +578,7 @@ void transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); @@ -581,7 +607,7 @@ void inverse_transform(raft::resources const& res, * */ void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index c3c45e3e1e..ba4fc844ca 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -50,65 +50,103 @@ struct fp_lt_op { _RAFT_HOST_DEVICE bool operator()(const T& a, const T& b) const { return fp_lt(a, b); } }; -template +template typename quantizer::global_type train(T min, T max) { double const lo = static_cast(min); double const hi = static_cast(max); RAFT_EXPECTS(std::isfinite(lo) && std::isfinite(hi) && lo <= hi, "Bounds must be finite with min <= max"); - if (lo == hi) { return {lo, 0}; } - constexpr double q_min = std::numeric_limits::min(); - constexpr double q_max = std::numeric_limits::max(); - // Divide the endpoints separately if subtracting them would overflow. - double const width = hi - lo; - double const scale = std::max( - std::isfinite(width) ? width / (q_max - q_min) : hi / (q_max - q_min) - lo / (q_max - q_min), - std::numeric_limits::min()); - double const shift = lo * 0.5 + hi * 0.5 - (q_min + q_max) * 0.5 * scale; - return {shift, scale}; + return {min, max}; } template struct quantize_op { - typename quantizer_const_view::global_type params; + double shift; + double scale; + + explicit quantize_op(typename quantizer_view::global_type bounds) + { + auto const range = detail::train(bounds.min, bounds.max); + double const lo = static_cast(range.min); + double const hi = static_cast(range.max); + if (lo == hi) { + shift = lo; + scale = 0; + return; + } + constexpr double q_min = std::numeric_limits::min(); + constexpr double q_max = std::numeric_limits::max(); + // Divide the endpoints separately if subtracting them would overflow. + double const width = hi - lo; + scale = std::max( + std::isfinite(width) ? width / (q_max - q_min) : hi / (q_max - q_min) - lo / (q_max - q_min), + std::numeric_limits::min()); + shift = lo * 0.5 + hi * 0.5 - (q_min + q_max) * 0.5 * scale; + } constexpr RAFT_INLINE_FUNCTION QuantI operator()(const T& x) const { constexpr double q_min = std::numeric_limits::min(); constexpr double q_max = std::numeric_limits::max(); - if (params.scale == 0) { - return static_cast(static_cast(x) <= params.shift ? q_min : q_max); - } - double const code = (static_cast(x) - params.shift) / params.scale; + if (scale == 0) { return static_cast(static_cast(x) <= shift ? q_min : q_max); } + double const code = (static_cast(x) - shift) / scale; return static_cast(lroundf(fmin(fmax(code, q_min), q_max))); } constexpr RAFT_INLINE_FUNCTION T operator()(const QuantI& x) const { - double const value = fma(params.scale, static_cast(x), params.shift); + double const value = fma(scale, static_cast(x), shift); double const limit = static_cast(cuda::std::numeric_limits::max()); return static_cast(fmin(fmax(value, -limit), limit)); } }; +template +struct per_dim_quantize_op { + using compute_t = typename quantizer_view::scale_type; + compute_t shift; + compute_t scale; + + RAFT_INLINE_FUNCTION per_dim_quantize_op(T min, T max) + { + compute_t const lo = static_cast(min); + compute_t const hi = static_cast(max); + // Halve before adding to avoid overflow, and round the midpoint to the input type. + shift = static_cast( + static_cast(lo == hi ? lo : lo * compute_t{0.5} + hi * compute_t{0.5})); + compute_t const half_range = fmax(hi - shift, shift - lo); + constexpr compute_t q_max = std::numeric_limits::max(); + scale = half_range > 0 ? fmax(half_range / q_max, std::numeric_limits::min()) + : compute_t{1}; + } + + RAFT_INLINE_FUNCTION QuantI operator()(const T& x) const + { + compute_t const inv = compute_t{1} / scale; + compute_t const code = rint((static_cast(x) - shift) * inv); + constexpr compute_t q_max = std::numeric_limits::max(); + constexpr compute_t q_min = std::max(std::numeric_limits::min(), -q_max); + return static_cast(fmin(fmax(code, q_min), q_max)); + } + + RAFT_INLINE_FUNCTION T operator()(const QuantI& x) const + { + compute_t const value = fma(scale, static_cast(x), shift); + compute_t const limit = static_cast(cuda::std::numeric_limits::max()); + return static_cast(fmin(fmax(value, -limit), limit)); + } +}; + template RAFT_INLINE_FUNCTION double reconstruction_error(T x, int64_t j, Op op) { double const value = static_cast(x); double decoded; - if constexpr (std::is_same_v::per_dim_type>) { - using compute_t = typename quantizer_const_view::scale_type; - compute_t const shift = static_cast(op.shifts.data_handle()[j]); - compute_t const scale = op.scales.data_handle()[j]; - compute_t const inv = compute_t{1} / scale; - compute_t const rounded = rint((static_cast(x) - shift) * inv); - constexpr compute_t q_max = std::numeric_limits::max(); - constexpr compute_t q_min = std::max(std::numeric_limits::min(), -q_max); - QuantI const code = static_cast(fmin(fmax(rounded, q_min), q_max)); - compute_t const reconstructed = fma(scale, static_cast(code), shift); - compute_t const limit = static_cast(cuda::std::numeric_limits::max()); - decoded = static_cast(static_cast(fmin(fmax(reconstructed, -limit), limit))); + if constexpr (std::is_same_v::per_dim_type>) { + auto const quantize = + per_dim_quantize_op{op.mins.data_handle()[j], op.maxs.data_handle()[j]}; + decoded = static_cast(quantize(quantize(x))); } else { decoded = static_cast(op(op(x))); } @@ -208,129 +246,106 @@ std::tuple quantile_min_max( return {minmax_h[0], minmax_h[1]}; } -template +template typename quantizer::per_dim_type train(raft::resources const& res, - raft::device_vector&& shifts, + raft::device_vector&& minima, raft::device_vector&& maxima) { - using compute_t = typename quantizer_const_view::scale_type; - auto const dim = shifts.extent(0); - auto scales = raft::make_device_vector(res, dim); - auto max_scale = raft::make_device_vector(res, 1); - auto* shift_ptr = shifts.data_handle(); + using compute_t = typename quantizer_view::scale_type; + auto const dim = minima.extent(0); + auto invalid = raft::make_device_vector(res, 1); auto const* max_ptr = maxima.data_handle(); - raft::linalg::map_offset(res, scales.view(), [shift_ptr, max_ptr] __device__(int64_t j) { - // Halve before adding/subtracting to avoid overflow for large finite inputs. - if (!isfinite(static_cast(shift_ptr[j])) || - !isfinite(static_cast(max_ptr[j])) || fp_lt(max_ptr[j], shift_ptr[j])) { - return std::numeric_limits::infinity(); - } - compute_t const lo = static_cast(shift_ptr[j]); - compute_t const hi = static_cast(max_ptr[j]); - shift_ptr[j] = static_cast(lo == hi ? lo : lo * compute_t{0.5} + hi * compute_t{0.5}); - compute_t const shift = static_cast(shift_ptr[j]); - compute_t const half_range = fmax(hi - shift, shift - lo); - constexpr compute_t q_max = std::numeric_limits::max(); - return half_range > 0 ? fmax(half_range / q_max, std::numeric_limits::min()) - : compute_t{1}; - }); raft::linalg::reduce( res, - raft::make_device_matrix_view(scales.data_handle(), 1, dim), - max_scale.view(), - compute_t{0}, + raft::make_device_matrix_view(minima.data_handle(), 1, dim), + invalid.view(), + 0, false, - raft::identity_op{}, + [max_ptr] __device__(T min, int64_t j) { + compute_t const lo = static_cast(min); + compute_t const hi = static_cast(max_ptr[j]); + return static_cast(!isfinite(lo) || !isfinite(hi) || lo > hi); + }, raft::max_op{}); - compute_t maximum_scale = 0; + int invalid_bounds = 0; raft::copy(res, - raft::make_host_scalar_view(&maximum_scale), - raft::make_device_scalar_view(max_scale.data_handle())); + raft::make_host_scalar_view(&invalid_bounds), + raft::make_device_scalar_view(invalid.data_handle())); raft::resource::sync_stream(res); - RAFT_EXPECTS(std::isfinite(maximum_scale), "Bounds must be finite with min <= max"); - return {std::move(shifts), std::move(scales)}; + RAFT_EXPECTS(invalid_bounds == 0, "Bounds must be finite with min <= max"); + return {std::move(minima), std::move(maxima)}; } template -void check_per_dim_shapes(typename quantizer_const_view::per_dim_type quantizer, +void check_per_dim_shapes(typename quantizer_view::per_dim_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { - RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.shifts.extent(0) && - dataset.extent(1) == quantizer.scales.extent(0), - "Input width must match the nonempty shifts and scales vectors"); + RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.mins.extent(0) && + dataset.extent(1) == quantizer.maxs.extent(0), + "Input width must match the nonempty min and max vectors"); RAFT_EXPECTS(dataset.extent(0) == out.extent(0) && dataset.extent(1) == out.extent(1), "Input and output shapes must match"); } template void transform(raft::resources const& res, - typename quantizer_const_view::per_dim_type quantizer, + typename quantizer_view::per_dim_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { - using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - auto const* scales = quantizer.scales.data_handle(); - raft::linalg::map_offset(res, out, [input, shifts, scales, dim] __device__(int64_t i) { - compute_t const inv = compute_t{1} / scales[i % dim]; - compute_t const code = - rint((static_cast(input[i]) - static_cast(shifts[i % dim])) * inv); - constexpr compute_t q_max = std::numeric_limits::max(); - constexpr compute_t q_min = std::max(std::numeric_limits::min(), -q_max); - return static_cast(fmin(fmax(code, q_min), q_max)); + auto const* input = dataset.data_handle(); + auto const* mins = quantizer.mins.data_handle(); + auto const* maxs = quantizer.maxs.data_handle(); + auto const dim = dataset.extent(1); + raft::linalg::map_offset(res, out, [input, mins, maxs, dim] __device__(int64_t i) { + return per_dim_quantize_op{mins[i % dim], maxs[i % dim]}(input[i]); }); } template void inverse_transform(raft::resources const& res, - typename quantizer_const_view::per_dim_type quantizer, + typename quantizer_view::per_dim_type quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { - using compute_t = typename quantizer_const_view::scale_type; check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* shifts = quantizer.shifts.data_handle(); - auto const dim = dataset.extent(1); - auto const* scales = quantizer.scales.data_handle(); - raft::linalg::map_offset(res, out, [input, shifts, scales, dim] __device__(int64_t i) { - compute_t const value = fma( - scales[i % dim], static_cast(input[i]), static_cast(shifts[i % dim])); - compute_t const limit = static_cast(cuda::std::numeric_limits::max()); - return static_cast(fmin(fmax(value, -limit), limit)); + auto const* input = dataset.data_handle(); + auto const* mins = quantizer.mins.data_handle(); + auto const* maxs = quantizer.maxs.data_handle(); + auto const dim = dataset.extent(1); + raft::linalg::map_offset(res, out, [input, mins, maxs, dim] __device__(int64_t i) { + return per_dim_quantize_op{mins[i % dim], maxs[i % dim]}(input[i]); }); } template void transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { auto view = quantizer; - if (auto const* shifts = - std::get_if::per_dim_type>(&view.value)) { - detail::transform(res, *shifts, dataset, out); + if (auto const* bounds = + std::get_if::per_dim_type>(&view.value)) { + detail::transform(res, *bounds, dataset, out); return; } - auto const& range = std::get::global_type>(view.value); + auto const& range = std::get::global_type>(view.value); raft::linalg::map(res, out, quantize_op{range}, raft::make_const_mdspan(dataset)); } template void transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out) { RAFT_EXPECTS( - std::holds_alternative::global_type>(quantizer.value), + std::holds_alternative::global_type>(quantizer.value), "Host quantization requires global parameters"); - auto const& range = std::get::global_type>(quantizer.value); + auto const& range = std::get::global_type>(quantizer.value); auto main_op = quantize_op{range}; size_t n_elements = dataset.extent(0) * dataset.extent(1); @@ -345,30 +360,30 @@ void transform(raft::resources const& res, template void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_view quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out) { auto view = quantizer; - if (auto const* shifts = - std::get_if::per_dim_type>(&view.value)) { - detail::inverse_transform(res, *shifts, dataset, out); + if (auto const* bounds = + std::get_if::per_dim_type>(&view.value)) { + detail::inverse_transform(res, *bounds, dataset, out); return; } - auto const& range = std::get::global_type>(view.value); + auto const& range = std::get::global_type>(view.value); raft::linalg::map(res, out, quantize_op{range}, raft::make_const_mdspan(dataset)); } template void inverse_transform(raft::resources const& res, - quantizer_const_view quantizer, + quantizer_view quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out) { RAFT_EXPECTS( - std::holds_alternative::global_type>(quantizer.value), + std::holds_alternative::global_type>(quantizer.value), "Host quantization requires global parameters"); - auto const& range = std::get::global_type>(quantizer.value); + auto const& range = std::get::global_type>(quantizer.value); auto main_op = quantize_op{range}; size_t n_elements = dataset.extent(0) * dataset.extent(1); @@ -437,11 +452,9 @@ quantizer train( }; quantizer result; - if (params.range == range_mode::global) { + if (params.range == quantization_mode::GLOBAL) { auto [min, max] = quantile_min_max(res, dataset, params.quantile); - result.value = detail::train(min, max); - result.min_ = min; - result.max_ = max; + result.value = detail::train(min, max); } else { auto minima = raft::make_device_vector(res, dim); auto maxima = raft::make_device_vector(res, dim); @@ -519,22 +532,22 @@ quantizer train( res, maxima.view(), raft::make_host_vector_view(maxs.data(), dim)); raft::resource::sync_stream(res); } - result.value = train(res, std::move(minima), std::move(maxima)); + result.value = train(res, std::move(minima), std::move(maxima)); } - auto errors = raft::make_device_vector(res, batch_rows); - auto maximum = raft::make_device_vector(res, 1); - auto view = result.view(); + auto errors = raft::make_device_vector(res, batch_rows); + auto maximum = raft::make_device_vector(res, 1); + quantizer_view view = result.view(); for_each_batch( [&](auto input, int64_t offset) { auto batch_errors = raft::make_device_vector_view(errors.data_handle(), input.extent(0)); - if (auto const* shifts = - std::get_if::per_dim_type>(&view.value)) { + if (auto const* bounds = + std::get_if::per_dim_type>(&view.value)) { max_reconstruction_error( - res, input, *shifts, batch_errors, maximum.view(), offset != 0); + res, input, *bounds, batch_errors, maximum.view(), offset != 0); } else { - auto const& range = std::get::global_type>(view.value); + auto const& range = std::get::global_type>(view.value); max_reconstruction_error( res, input, quantize_op{range}, batch_errors, maximum.view(), offset != 0); } diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index ca8f5176e5..b16d2d1afa 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -13,7 +13,7 @@ namespace cuvs::preprocessing::quantize::scalar { template -quantizer::quantizer(T min, T max) : value{detail::train(min, max)}, min_{min}, max_{max} +quantizer::quantizer(T min, T max) : value{detail::train(min, max)} { } @@ -45,34 +45,34 @@ quantizer::quantizer(raft::resources const& res, return detail::train(res, params, dataset); \ } \ void transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ + const quantizer& quantizer, \ raft::device_matrix_view dataset, \ raft::device_matrix_view out) \ { \ - detail::transform(res, quantizer, dataset, out); \ + detail::transform(res, quantizer.view(), dataset, out); \ } \ void transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ + const quantizer& quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ - detail::transform(res, quantizer, dataset, out); \ + detail::transform(res, quantizer.view(), dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ + const quantizer& quantizer, \ raft::device_matrix_view dataset, \ raft::device_matrix_view out) \ { \ - detail::inverse_transform(res, quantizer, dataset, out); \ + detail::inverse_transform(res, quantizer.view(), dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ - quantizer_const_view quantizer, \ + const quantizer& quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ - detail::inverse_transform(res, quantizer, dataset, out); \ + detail::inverse_transform(res, quantizer.view(), dataset, out); \ } \ - auto build( \ + auto make_sq_dataset( \ raft::resources const& res, params params, raft::host_matrix_view dataset) \ -> device_sq_dataset \ { \ diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 42cc6b67ff..48b2b9d6fe 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -151,10 +151,10 @@ class QuantizationTest : public ::testing::TestWithParam> cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); std::cerr - << "Q1: shift = " - << (double)std::get(quantizer_1.value).shift - << ", scale = " - << (double)std::get(quantizer_1.value).scale + << "Q1: min = " + << (double)std::get(quantizer_1.value).min + << ", max = " + << (double)std::get(quantizer_1.value).max << std::endl; { @@ -249,20 +249,20 @@ class QuantizationTest : public ::testing::TestWithParam> cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); std::cerr - << "Q2: shift = " - << (double)std::get(quantizer_2.value).shift - << ", scale = " - << (double)std::get(quantizer_2.value).scale + << "Q2: min = " + << (double)std::get(quantizer_2.value).min + << ", max = " + << (double)std::get(quantizer_2.value).max << std::endl; // check both quantizers are the same (valid if sampling is identical) if (input_.size() <= 1000000) { ASSERT_TRUE( - (double)std::get(quantizer_1.value).shift == - (double)std::get(quantizer_2.value).shift); + (double)std::get(quantizer_1.value).min == + (double)std::get(quantizer_2.value).min); ASSERT_TRUE( - (double)std::get(quantizer_1.value).scale == - (double)std::get(quantizer_2.value).scale); + (double)std::get(quantizer_1.value).max == + (double)std::get(quantizer_2.value).max); } { @@ -343,7 +343,7 @@ class QuantizationTest : public ::testing::TestWithParam> std::vector host_input_; }; -using shifts_view = quantizer_const_view::per_dim_type; +using bounds_view = quantizer_view::per_dim_type; struct PerDimensionInputs { int64_t n_samples; @@ -378,8 +378,9 @@ class PerDimensionRangeTest : public ::testing::TestWithParam(handle, params_.n_samples, params_.n_features); transform(handle, quantizer, raft::make_const_mdspan(dataset_.view()), codes.view()); @@ -389,12 +390,12 @@ class PerDimensionRangeTest : public ::testing::TestWithParam(), stream)); if (params_.tolerance == 0.f) { - EXPECT_TRUE( - devArrMatchHost(std::vector(params_.n_features, 1.f).data(), - std::get(quantizer.view().value).scales.data_handle(), - params_.n_features, - cuvs::Compare(), - stream)); + auto bounds = std::get(std::as_const(quantizer).view().value); + EXPECT_TRUE(devArrMatch(bounds.mins.data_handle(), + bounds.maxs.data_handle(), + params_.n_features, + cuvs::Compare(), + stream)); } inverse_transform(handle, quantizer, raft::make_const_mdspan(codes.view()), dataset_.view()); auto reconstructed = @@ -425,6 +426,20 @@ class PerDimensionQuantizationTest : public ::testing::Test {}; using PerDimensionTypes = ::testing::Types; TYPED_TEST_SUITE(PerDimensionQuantizationTest, PerDimensionTypes); +template +void expectDeviceValues(raft::resources const& res, + const T* device_values, + std::vector const& expected) +{ + std::vector actual(expected.size()); + raft::update_host( + actual.data(), device_values, actual.size(), raft::resource::get_cuda_stream(res)); + raft::resource::sync_stream(res); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(static_cast(actual[i]), static_cast(expected[i])) << i; + } +} + template void testTrainingReconstructionError(params p) { @@ -441,10 +456,10 @@ void testTrainingReconstructionError(params p) auto decoded = raft::make_device_matrix(res, rows, dim); std::vector reconstructed(values.size()); auto owner = train(res, p, raft::make_const_mdspan(input.view())); - auto view = owner.view(); + auto view = std::as_const(owner).view(); EXPECT_EQ(owner.max_error, view.max_error); - transform(res, view, raft::make_const_mdspan(input.view()), codes.view()); - inverse_transform(res, view, raft::make_const_mdspan(codes.view()), decoded.view()); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), decoded.view()); raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); raft::resource::sync_stream(res); auto maximum = @@ -454,15 +469,17 @@ void testTrainingReconstructionError(params p) static_cast(maximum), static_cast(maximum) * 2e-6); if (p.quantile < 1.f) { EXPECT_GT(owner.max_error, 100); } - if (p.range == range_mode::global) { - auto host = train(res, + if (p.range == quantization_mode::GLOBAL) { + auto host = train(res, params{p.quantile}, raft::make_host_matrix_view(values.data(), rows, dim)); - EXPECT_EQ(host.min_, owner.min_); - EXPECT_EQ(host.max_, owner.max_); + auto const host_bounds = std::get::global_type>(host.value); + auto const bounds = std::get::global_type>(owner.value); + EXPECT_EQ(host_bounds.min, bounds.min); + EXPECT_EQ(host_bounds.max, bounds.max); if (p.quantile == 1.f) { - EXPECT_EQ(owner.min_, values.front()); - EXPECT_EQ(owner.max_, values.back()); + EXPECT_EQ(bounds.min, values.front()); + EXPECT_EQ(bounds.max, values.back()); } EXPECT_GE(static_cast(host.max_error), maximum); EXPECT_NEAR(static_cast(host.max_error), @@ -472,22 +489,22 @@ void testTrainingReconstructionError(params p) } TYPED_TEST(PerDimensionQuantizationTest, GlobalTrainingReconstructionError) { - testTrainingReconstructionError(params{1.f, range_mode::global}); + testTrainingReconstructionError(params{1.f, quantization_mode::GLOBAL}); } TYPED_TEST(PerDimensionQuantizationTest, GlobalClippedReconstructionError) { - testTrainingReconstructionError(params{.5f, range_mode::global}); + testTrainingReconstructionError(params{.5f, quantization_mode::GLOBAL}); } TYPED_TEST(PerDimensionQuantizationTest, PerDimensionTrainingReconstructionError) { - testTrainingReconstructionError(params{1.f, range_mode::per_dimension}); + testTrainingReconstructionError(params{1.f, quantization_mode::PER_DIMENSION}); } TYPED_TEST(PerDimensionQuantizationTest, PerDimensionClippedReconstructionError) { - testTrainingReconstructionError(params{.5f, range_mode::per_dimension}); + testTrainingReconstructionError(params{.5f, quantization_mode::PER_DIMENSION}); } TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) @@ -502,8 +519,8 @@ TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) SCOPED_TRACE(scale); std::vector values{-scale, -scale, scale * 0.3, scale * 0.3, scale, scale}; raft::update_device(input.data_handle(), values.data(), values.size(), stream); - auto owner = - train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + auto owner = train( + res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), decoded.view()); std::vector reconstructed(values.size()); @@ -525,46 +542,33 @@ TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) { using T = TypeParam; - using shifts_view = typename quantizer_const_view::per_dim_type; + using bounds_view = typename quantizer_view::per_dim_type; raft::resources res; auto stream = raft::resource::get_cuda_stream(res); std::vector corpus{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; auto input = raft::make_device_matrix(res, 2, 3); raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); - auto owner = - train(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - auto quantizer = owner.view(); + auto owner = train( + res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + auto quantizer = std::as_const(owner).view(); EXPECT_EQ(owner.max_error, 0); EXPECT_EQ(quantizer.max_error, 0); - using scale_t = typename quantizer_const_view::scale_type; - EXPECT_TRUE(devArrMatchHost(std::vector{1, .5, 1}.data(), - std::get(quantizer.value).scales.data_handle(), - 3, - cuvs::Compare(), - stream.get())); - { - std::vector actual(3); - std::vector expected{127.f, 73.5f, 9.f}; - raft::update_host(actual.data(), - std::get(quantizer.value).shifts.data_handle(), - actual.size(), - stream); - raft::resource::sync_stream(res); - for (size_t i = 0; i < actual.size(); ++i) { - EXPECT_EQ(static_cast(actual[i]), expected[i]); - } - } + expectDeviceValues( + res, std::get(quantizer.value).mins.data_handle(), std::vector{0.f, 10.f, 9.f}); + expectDeviceValues(res, + std::get(quantizer.value).maxs.data_handle(), + std::vector{254.f, 137.f, 9.f}); // The corpus manifest, rather than query extrema, controls ties and saturation. std::vector queries{127.5f, 73.75f, 1000.f, 128.5f, 72.75f, -1000.f}; raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); auto codes = raft::make_device_matrix(res, 2, 3); - transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); EXPECT_TRUE(devArrMatchHost(std::vector{0, 0, 127, 2, -2, -127}.data(), codes.data_handle(), 6, cuvs::Compare(), raft::resource::get_cuda_stream(res).get())); - inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); { std::vector actual(6); std::vector expected{127.f, 73.5f, 136.f, 129.f, 72.5f, -118.f}; @@ -583,8 +587,8 @@ TEST(PerDimensionRangeTest, AdjacentHalfBounds) std::vector values{half{1.f}, half{1.f + 1.f / 1024.f}}; auto input = raft::make_device_matrix(res, 2, 1); raft::update_device(input.data_handle(), values.data(), values.size(), stream); - auto quantizer = - train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); + auto quantizer = train( + res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); auto codes = raft::make_device_matrix(res, 2, 1); transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); @@ -608,36 +612,51 @@ TEST(PerDimensionRangeTest, PerDimensionQuantiles) auto input = raft::make_device_matrix(res, rows, dim); raft::update_device( input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto owner = - train(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - auto quantizer = owner.view(); - EXPECT_TRUE(devArrMatchHost(std::vector{499.5f, 10999.f, 9.f}.data(), - std::get(quantizer.value).shifts.data_handle(), + auto owner = train( + res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + auto quantizer = std::as_const(owner).view(); + EXPECT_TRUE(devArrMatchHost(std::vector{4.f, 10008.f, 9.f}.data(), + std::get(quantizer.value).mins.data_handle(), dim, cuvs::Compare(), raft::resource::get_cuda_stream(res).get())); - EXPECT_TRUE(devArrMatchHost(std::vector{495.5f / 127.f, 991.f / 127.f, 1.f}.data(), - std::get(quantizer.value).scales.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - auto full = - train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_TRUE(devArrMatchHost(std::vector{499.5f / 127.f, 999.f / 127.f, 1.f}.data(), - std::get(full.view().value).scales.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - auto trimmed = - train(res, params{0.5f, range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_TRUE(devArrMatchHost(std::vector{249.5f / 127.f, 499.f / 127.f, 1.f}.data(), - std::get(trimmed.view().value).scales.data_handle(), + EXPECT_TRUE(devArrMatchHost(std::vector{995.f, 11990.f, 9.f}.data(), + std::get(quantizer.value).maxs.data_handle(), dim, cuvs::Compare(), raft::resource::get_cuda_stream(res).get())); + auto full = train( + res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + EXPECT_TRUE( + devArrMatchHost(std::vector{999.f, 11998.f, 9.f}.data(), + std::get(std::as_const(full).view().value).maxs.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); + EXPECT_TRUE( + devArrMatchHost(std::vector{0.f, 10000.f, 9.f}.data(), + std::get(std::as_const(full).view().value).mins.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); + auto trimmed = train( + res, params{0.5f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + EXPECT_TRUE( + devArrMatchHost(std::vector{749.f, 11498.f, 9.f}.data(), + std::get(std::as_const(trimmed).view().value).maxs.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); + EXPECT_TRUE( + devArrMatchHost(std::vector{250.f, 10500.f, 9.f}.data(), + std::get(std::as_const(trimmed).view().value).mins.data_handle(), + dim, + cuvs::Compare(), + raft::resource::get_cuda_stream(res).get())); for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { EXPECT_THROW( - train(res, params{q, range_mode::per_dimension}, raft::make_const_mdspan(input.view())), + train( + res, params{q, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())), raft::logic_error); } } @@ -645,7 +664,7 @@ TEST(PerDimensionRangeTest, PerDimensionQuantiles) TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) { using T = TypeParam; - using shifts_view = typename quantizer_const_view::per_dim_type; + using bounds_view = typename quantizer_view::per_dim_type; raft::resources res; auto mins = raft::make_device_vector(res, 2); auto maxs = raft::make_device_vector(res, 2); @@ -655,47 +674,30 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) raft::update_device(maxs.data_handle(), upper.data(), upper.size(), stream); scalar::quantizer owner{ res, raft::make_const_mdspan(mins.view()), raft::make_const_mdspan(maxs.view())}; - auto quantizer = owner.view(); + auto quantizer = std::as_const(owner).view(); EXPECT_TRUE(std::isinf(owner.max_error)); EXPECT_TRUE(std::isinf(quantizer.max_error)); EXPECT_TRUE(std::isinf((scalar::quantizer{T{-1}, T{1}}.view().max_error))); - using scale_t = typename quantizer_const_view::scale_type; - EXPECT_TRUE(devArrMatchHost(std::vector{2, scale_t{4} / scale_t{127}}.data(), - std::get(quantizer.value).scales.data_handle(), - 2, - cuvs::Compare(), - stream.get())); - { - std::vector actual(2); - std::vector expected{100.f, -100.f}; - raft::update_host(actual.data(), - std::get(quantizer.value).shifts.data_handle(), - actual.size(), - stream); - raft::resource::sync_stream(res); - for (size_t i = 0; i < actual.size(); ++i) { - EXPECT_EQ(static_cast(actual[i]), expected[i]); - } - } + expectDeviceValues(res, std::get(quantizer.value).mins.data_handle(), lower); + expectDeviceValues(res, std::get(quantizer.value).maxs.data_handle(), upper); auto input = raft::make_device_matrix(res, 1, 2); std::vector values{104.f, -106.f}; raft::update_device( input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); auto codes = raft::make_device_matrix(res, 1, 2); - transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); EXPECT_TRUE(devArrMatchHost(std::vector{2, -127}.data(), codes.data_handle(), 2, cuvs::Compare(), raft::resource::get_cuda_stream(res).get())); - auto wrong_scales = quantizer; - std::get(wrong_scales.value).scales = - raft::make_device_vector_view( - std::get(quantizer.value).scales.data_handle(), 1); - EXPECT_THROW(transform(res, wrong_scales, raft::make_const_mdspan(input.view()), codes.view()), + scalar::quantizer wrong_bounds; + wrong_bounds.value = typename scalar::quantizer::per_dim_type{ + raft::make_device_vector(res, 2), raft::make_device_vector(res, 1)}; + EXPECT_THROW(transform(res, wrong_bounds, raft::make_const_mdspan(input.view()), codes.view()), raft::logic_error); auto wrong_shape = raft::make_device_matrix(res, 2, 1); - EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(input.view()), wrong_shape.view()), + EXPECT_THROW(transform(res, owner, raft::make_const_mdspan(input.view()), wrong_shape.view()), raft::logic_error); EXPECT_THROW( (scalar::quantizer{res, @@ -706,19 +708,94 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) res, raft::make_const_mdspan(maxs.view()), raft::make_const_mdspan(mins.view())}), raft::logic_error); EXPECT_THROW(train(res, - params{.range = range_mode::per_dimension}, + params{.range = quantization_mode::PER_DIMENSION}, raft::make_device_matrix_view(nullptr, 0, 2)), raft::logic_error); } +TYPED_TEST(PerDimensionQuantizationTest, DirectPerDimensionBounds) +{ + using T = TypeParam; + raft::resources res; + auto stream = raft::resource::get_cuda_stream(res); + auto mins = raft::make_device_vector(res, 2); + auto maxs = raft::make_device_vector(res, 2); + std::vector lower{-127.f, -254.f}, upper{127.f, 254.f}, values{1.f, 2.f}; + raft::update_device(mins.data_handle(), lower.data(), lower.size(), stream); + raft::update_device(maxs.data_handle(), upper.data(), upper.size(), stream); + quantizer owner; + owner.value = typename quantizer::per_dim_type{std::move(mins), std::move(maxs)}; + auto input = raft::make_device_matrix(res, 1, 2); + auto codes = raft::make_device_matrix(res, 1, 2); + raft::update_device(input.data_handle(), values.data(), values.size(), stream); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_TRUE(devArrMatchHost(std::vector{1, 1}.data(), + codes.data_handle(), + 2, + cuvs::Compare(), + stream.get())); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); + expectDeviceValues(res, input.data_handle(), values); + + // Transforms use the owner's current bounds. + lower = {T{-254.f}, T{-508.f}}; + upper = {T{254.f}, T{508.f}}; + auto mutable_view = owner.view(); + quantizer_view read_only = mutable_view; + auto bounds = std::get::per_dim_type>(mutable_view.value); + auto const_bounds = std::get::per_dim_type>(read_only.value); + static_assert(std::is_same_v, T*>); + static_assert( + std::is_same_v, const T*>); + static_assert(!std::is_convertible_v, quantizer_view>); + EXPECT_EQ(bounds.mins.data_handle(), const_bounds.mins.data_handle()); + EXPECT_EQ(bounds.maxs.data_handle(), const_bounds.maxs.data_handle()); + raft::update_device(bounds.mins.data_handle(), lower.data(), lower.size(), stream); + raft::update_device(bounds.maxs.data_handle(), upper.data(), upper.size(), stream); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_TRUE(devArrMatchHost(std::vector{0, 0}.data(), + codes.data_handle(), + 2, + cuvs::Compare(), + stream.get())); +} + +TYPED_TEST(PerDimensionQuantizationTest, DirectGlobalBounds) +{ + using T = TypeParam; + raft::resources res; + auto input = raft::make_host_matrix(1, 2); + input(0, 0) = T{1}; + input(0, 1) = T{2}; + auto codes = raft::make_host_matrix(1, 2); + quantizer owner; + owner.value = typename quantizer::global_type{T{-128}, T{127}}; + quantizer_view read_only = owner.view(); + auto copied_bounds = std::get::global_type>(read_only.value); + EXPECT_EQ(double(copied_bounds.min), -128.0); + EXPECT_EQ(double(copied_bounds.max), 127.0); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_EQ(codes(0, 0), 1); + EXPECT_EQ(codes(0, 1), 2); + auto& bounds = std::get::global_type>(owner.value); + bounds.min = T{-256}; + bounds.max = T{254}; + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + EXPECT_EQ(codes(0, 0), 1); + EXPECT_EQ(codes(0, 1), 1); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); + EXPECT_EQ(static_cast(input(0, 0)), 2); + EXPECT_EQ(static_cast(input(0, 1)), 2); +} + TYPED_TEST(PerDimensionQuantizationTest, HostTransformsRejectPerDimensionQuantizer) { using T = TypeParam; raft::resources res; - auto values = raft::make_host_matrix(1, 1); - values(0, 0) = T{1}; - auto quantizer = - train(res, params{1.f, range_mode::per_dimension}, raft::make_const_mdspan(values.view())); + auto values = raft::make_host_matrix(1, 1); + values(0, 0) = T{1}; + auto quantizer = train( + res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(values.view())); auto codes = raft::make_host_matrix(1, 1); EXPECT_THROW(transform(res, quantizer, raft::make_const_mdspan(values.view()), codes.view()), raft::logic_error); @@ -737,18 +814,20 @@ TEST(PerDimensionRangeTest, QuantizedTypeLimits) raft::update_device(input.data_handle(), values.data(), values.size(), stream); auto codes = raft::make_device_matrix(res, values.size(), 1); auto owner = detail::train(res, - params{.5f, range_mode::per_dimension}, + params{.5f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{1.f}.data(), - std::get(owner.view().value).scales.data_handle(), - 1, - cuvs::Compare(), - stream.get())); + EXPECT_TRUE( + devArrMatchHost(std::vector{32767.f}.data(), + std::get(std::as_const(owner).view().value).maxs.data_handle(), + 1, + cuvs::Compare(), + stream.get())); std::vector expected{-32767, -32767, 0, 32767, 32767}; EXPECT_TRUE(devArrMatchHost( expected.data(), codes.data_handle(), expected.size(), cuvs::Compare(), stream.get())); - detail::inverse_transform(res, owner.view(), raft::make_const_mdspan(codes.view()), input.view()); + detail::inverse_transform( + res, std::as_const(owner).view(), raft::make_const_mdspan(codes.view()), input.view()); std::vector decoded(values.size()); raft::update_host(decoded.data(), input.data_handle(), decoded.size(), stream); raft::resource::sync_stream(res); @@ -761,7 +840,7 @@ TEST(PerDimensionRangeTest, QuantizedTypeLimits) EXPECT_NEAR(owner.max_error, maximum, maximum * 2e-6); } -TYPED_TEST(PerDimensionQuantizationTest, GlobalShiftScale) +TYPED_TEST(PerDimensionQuantizationTest, GlobalBounds) { using T = TypeParam; raft::resources res; @@ -769,15 +848,8 @@ TYPED_TEST(PerDimensionQuantizationTest, GlobalShiftScale) for (auto bounds : {std::pair{T{-3}, T{7}}, std::pair{T{5}, T{5}}}) { auto owner = quantizer{bounds.first, bounds.second}; auto parameters = std::get::global_type>(owner.value); - EXPECT_EQ(owner.min_, bounds.first); - EXPECT_EQ(owner.max_, bounds.second); - EXPECT_EQ(static_cast( - std::fma(parameters.scale, std::numeric_limits::min(), parameters.shift)), - bounds.first); - EXPECT_EQ(static_cast( - std::fma(parameters.scale, std::numeric_limits::max(), parameters.shift)), - bounds.second); - if (bounds.first == bounds.second) { EXPECT_EQ(parameters.scale, 0); } + EXPECT_EQ(parameters.min, bounds.first); + EXPECT_EQ(parameters.max, bounds.second); std::vector values{T{-100}, bounds.first, bounds.second, T{100}}; auto input = raft::make_device_matrix(res, 1, values.size()); auto codes = raft::make_device_matrix(res, 1, values.size()); @@ -801,8 +873,9 @@ TEST(QuantizationDatasetTest, GlobalRangeDataset) std::vector values{-1.f, 1.f}; auto stream = raft::resource::get_cuda_stream(res); raft::update_device(input.data_handle(), values.data(), values.size(), stream); - auto encoded = build(res, params{1.f}, raft::make_const_mdspan(input.view())); - EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); + auto encoded = make_sq_dataset(res, params{1.f}, raft::make_const_mdspan(input.view())); + typename decltype(encoded)::const_view_type read_only = encoded.view(); + EXPECT_EQ(read_only.const_manifest_view().max_error, 0.f); EXPECT_EQ(encoded.n_rows(), 1); EXPECT_EQ(encoded.dim(), 2u); EXPECT_TRUE(devArrMatchHost(std::vector{-128, 127}.data(), @@ -811,9 +884,9 @@ TEST(QuantizationDatasetTest, GlobalRangeDataset) cuvs::Compare(), stream.get())); auto range = - std::get::global_type>(encoded.const_manifest_view().value); - EXPECT_DOUBLE_EQ(range.shift, 1.0 / 255.0); - EXPECT_DOUBLE_EQ(range.scale, 2.0 / 255.0); + std::get::global_type>(encoded.const_manifest_view().value); + EXPECT_EQ(range.min, -1.f); + EXPECT_EQ(range.max, 1.f); } TEST(QuantizationDatasetTest, PerDimensionDataset) @@ -823,9 +896,10 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) std::vector values{0.f, 10.f, 254.f, 14.f}; auto stream = raft::resource::get_cuda_stream(res); raft::update_device(input.data_handle(), values.data(), values.size(), stream); - auto encoded = - build(res, params{.range = range_mode::per_dimension}, raft::make_const_mdspan(input.view())); - EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); + auto encoded = make_sq_dataset( + res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + typename decltype(encoded)::const_view_type read_only = encoded.view(); + EXPECT_EQ(read_only.const_manifest_view().max_error, 0.f); EXPECT_EQ(encoded.n_rows(), 2); EXPECT_EQ(encoded.dim(), 2u); EXPECT_TRUE(devArrMatchHost(std::vector{-127, -127, 127, 127}.data(), @@ -833,9 +907,10 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) 4, cuvs::Compare(), stream.get())); - auto codes = raft::make_device_matrix(res, 2, 2); - transform( - res, encoded.const_manifest_view(), raft::make_const_mdspan(input.view()), codes.view()); + auto codes = raft::make_device_matrix(res, 2, 2); + auto quantizer = train( + res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); EXPECT_TRUE(devArrMatch(encoded.data_const_view().data_handle(), codes.data_handle(), 4, @@ -862,10 +937,10 @@ void testHostDataset(params p, bool prefetch = false, bool use_pinned = false) if (use_pinned) { std::copy(values.begin(), values.end(), pinned.data_handle()); } // A negative staging limit is ignored for device input. auto expected = - build(res, params{p.quantile, p.range, -1}, raft::make_const_mdspan(input.view())); + make_sq_dataset(res, params{p.quantile, p.range, -1}, raft::make_const_mdspan(input.view())); auto host_input = raft::make_host_matrix_view( use_pinned ? pinned.data_handle() : values.data(), rows, dim); - auto actual = build(res, p, host_input); + auto actual = make_sq_dataset(res, p, host_input); EXPECT_EQ(actual.n_rows(), rows); EXPECT_EQ(actual.dim(), dim); EXPECT_TRUE(devArrMatch(expected.data_const_view().data_handle(), @@ -876,16 +951,16 @@ void testHostDataset(params p, bool prefetch = false, bool use_pinned = false) auto a = actual.const_manifest_view(); auto e = expected.const_manifest_view(); EXPECT_NEAR(double(a.max_error), double(e.max_error), double(e.max_error) * 1e-6); - if (p.range == range_mode::per_dimension) { - auto actual_shifts = std::get::per_dim_type>(a.value); - auto expected_shifts = std::get::per_dim_type>(e.value); - EXPECT_TRUE(devArrMatch(actual_shifts.scales.data_handle(), - expected_shifts.scales.data_handle(), + if (p.range == quantization_mode::PER_DIMENSION) { + auto actual_bounds = std::get::per_dim_type>(a.value); + auto expected_bounds = std::get::per_dim_type>(e.value); + EXPECT_TRUE(devArrMatch(actual_bounds.maxs.data_handle(), + expected_bounds.maxs.data_handle(), dim, - cuvs::Compare::scale_type>(), + cuvs::Compare(), stream.get())); - EXPECT_TRUE(devArrMatch(actual_shifts.shifts.data_handle(), - expected_shifts.shifts.data_handle(), + EXPECT_TRUE(devArrMatch(actual_bounds.mins.data_handle(), + expected_bounds.mins.data_handle(), dim, cuvs::Compare(), stream.get())); @@ -897,57 +972,57 @@ void testHostDataset(params p, bool prefetch = false, bool use_pinned = false) TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPartialBatch) { - testHostDataset(params{1.f, range_mode::per_dimension, 7}); + testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetSingleRowBatch) { - testHostDataset(params{1.f, range_mode::per_dimension, 1}); + testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 1}); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetFullBatch) { - testHostDataset(params{1.f, range_mode::per_dimension, 37}); + testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 37}); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetUnlimitedBatch) { - testHostDataset(params{1.f, range_mode::per_dimension, 0}); + testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 0}); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPinned) { - testHostDataset(params{1.f, range_mode::per_dimension, 7}, false, true); + testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}, false, true); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPrefetch) { - testHostDataset(params{1.f, range_mode::per_dimension, 7}, true); + testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}, true); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPinnedPrefetch) { - testHostDataset(params{1.f, range_mode::per_dimension, 7}, true, true); + testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}, true, true); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetClippedQuantile) { - testHostDataset(params{.5f, range_mode::per_dimension, 7}); + testHostDataset(params{.5f, quantization_mode::PER_DIMENSION, 7}); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetDefaultQuantile) { - testHostDataset(params{.99f, range_mode::per_dimension, 7}); + testHostDataset(params{.99f, quantization_mode::PER_DIMENSION, 7}); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetGlobalRange) { - testHostDataset(params{1.f, range_mode::global, 7}); + testHostDataset(params{1.f, quantization_mode::GLOBAL, 7}); } TYPED_TEST(PerDimensionQuantizationTest, HostDatasetGlobalClippedQuantile) { - testHostDataset(params{.5f, range_mode::global, 7}); + testHostDataset(params{.5f, quantization_mode::GLOBAL, 7}); } TEST(QuantizationDatasetTest, HostBuildBatchWorkspace) @@ -961,9 +1036,9 @@ TEST(QuantizationDatasetTest, HostBuildBatchWorkspace) raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); } for (float quantile : {1.f, .5f}) { - params p{quantile, range_mode::per_dimension, batch_rows}; + params p{quantile, quantization_mode::PER_DIMENSION, batch_rows}; auto action = [&](raft::resources const& h) { - auto encoded = build(h, p, input); + auto encoded = make_sq_dataset(h, p, input); if (!raft::resource::get_dry_run_flag(h)) { EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); } @@ -978,8 +1053,9 @@ TEST(QuantizationDatasetTest, HostBuildBatchWorkspace) EXPECT_LT(device_bytes, values.size() * sizeof(float)); } } - EXPECT_THROW(build(raft::resources{}, params{1.f, range_mode::per_dimension, -1}, input), - raft::logic_error); + EXPECT_THROW( + make_sq_dataset(raft::resources{}, params{1.f, quantization_mode::PER_DIMENSION, -1}, input), + raft::logic_error); } TEST_P(PerDimensionRangeTest, Result) { this->testPerDimensionQuantization(); } diff --git a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md index 22bfd79bb3..9a5a308033 100644 --- a/fern/pages/cpp_api/cpp-api-core-dataset-sq.md +++ b/fern/pages/cpp_api/cpp-api-core-dataset-sq.md @@ -22,16 +22,15 @@ struct sq_spec; ## Scalar-quantized dataset construction - -### preprocessing::quantize::scalar::build + +### preprocessing::quantize::scalar::make_sq_dataset -Train and encode a device corpus with the selected range mode. Reuse result.const_manifest_view() with transform for query batches. For separate training/encoding, use train and transform. +Train and encode a device corpus with the selected quantization mode. For separate training and encoding, use train and transform. ```cpp template -inline device_sq_dataset build(raft::resources const& res, -params params, -raft::device_matrix_view dataset); +inline device_sq_dataset make_sq_dataset( +raft::resources const& res, params params, raft::device_matrix_view dataset); ``` **Parameters** @@ -48,12 +47,12 @@ raft::device_matrix_view dataset); Device dataset owning the quantized codes and quantizer. -**Additional overload:** `preprocessing::quantize::scalar::build` +**Additional overload:** `preprocessing::quantize::scalar::make_sq_dataset` Train and encode a host corpus in GPU batches, returning device-resident int8 codes. Uses one quantizer across all batches and measures the maximum error over every row. Configure params.device_buffer_samples to bound input staging and an auxiliary stream pool on res to enable double buffering. All int8 codes must fit in device memory. ```cpp -device_sq_dataset build( +device_sq_dataset make_sq_dataset( raft::resources const& res, params params, raft::host_matrix_view dataset); ``` @@ -71,12 +70,12 @@ raft::resources const& res, params params, raft::host_matrix_view build( +device_sq_dataset make_sq_dataset( raft::resources const& res, params params, raft::host_matrix_view dataset); ``` @@ -94,14 +93,13 @@ raft::resources const& res, params params, raft::host_matrix_view build(raft::resources const& res, -params params, -raft::host_matrix_view dataset); +device_sq_dataset make_sq_dataset( +raft::resources const& res, params params, raft::host_matrix_view dataset); ``` **Parameters** diff --git a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md index e70c67454d..a4ad8c4ae0 100644 --- a/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md +++ b/fern/pages/cpp_api/cpp-api-preprocessing-quantize-scalar.md @@ -8,15 +8,15 @@ _Source header: `cuvs/preprocessing/quantize/scalar.hpp`_ ## Scalar quantizer utilities - -### preprocessing::quantize::scalar::range_mode + +### preprocessing::quantize::scalar::quantization_mode -Select global or per-dimension shifts and scales. +Select global or per-dimension quantization bounds. ```cpp -enum class range_mode { - global, - per_dimension +enum class quantization_mode { + GLOBAL, + PER_DIMENSION }; ``` @@ -24,8 +24,8 @@ enum class range_mode { | Name | Value | | --- | --- | -| `global` | `` | -| `per_dimension` | `` | +| `GLOBAL` | `` | +| `PER_DIMENSION` | `` | ### preprocessing::quantize::scalar::params @@ -35,7 +35,7 @@ quantizer parameters. ```cpp struct params { float quantile; - range_mode range; + quantization_mode range; int64_t device_buffer_samples; }; ``` @@ -45,19 +45,19 @@ struct params { | Name | Type | Description | | --- | --- | --- | | `quantile` | `float` | Specifies how many outliers at top & bottom will be ignored. Needs to be within range of (0, 1]. | -| `range` | [`range_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-range-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Each dimension's shift is the midpoint of its bounds, rounded to the input type. Each dimension's scale fits its bounds around its stored shift into the symmetric range of quantized values.

For a zero-width interval, its scale is one. Otherwise, each scale is at least the smallest positive normal value of the scale type. | +| `range` | [`quantization_mode`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantization-mode) | Per-dimension training uses every row of a nonempty, finite dataset. It applies the quantile independently to each dimension to select lower and upper bounds. Transforms derive each dimension's shift from the midpoint of its bounds, rounded to the input type. Each dimension's scale fits its bounds around that shift into the symmetric range of quantized values.

For a zero-width interval, its scale is one. Otherwise, each scale is at least the smallest positive normal value of the scale type. | | `device_buffer_samples` | `int64_t` | Maximum rows staged on device for host training and host dataset build.
Default: 65536. Zero processes the entire host input in one batch. Ignored for device input. Multiple batches use two input buffers when res has an auxiliary stream pool, and one otherwise. Pageable input uses batch-sized pinned staging buffers, without registering the full input. Each input buffer uses at most device_buffer_samples * dim * sizeof(T) bytes, in addition to reduction and encoding workspace. Substitute n_rows when it is zero.

Per-dimension host training with quantile < 1 selects exact quantiles using one column of host scratch (n_rows * sizeof(T) bytes). Global training retains its sampled quantiles. | - -### preprocessing::quantize::scalar::quantizer_const_view + +### preprocessing::quantize::scalar::quantizer_view -Read-only scalar quantizer parameters. The owner must outlive stream work using shifts and +Non-owning scalar quantizer bounds, with the constness semantics of an mdspan: -scales. +quantizer_view<T> can modify per-dimension bounds; quantizer_view<const T> cannot. Global bounds and max_error are copied. The owner must outlive stream work using the vectors. ```cpp template -struct quantizer_const_view { +struct quantizer_view { std::variant value; scale_type max_error; }; @@ -67,20 +67,20 @@ struct quantizer_const_view { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or borrowed per-dimension shifts and scales. | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-view-global-type) | Global bounds or borrowed per-dimension bounds. | | `max_error` | `scale_type` | Maximum training-row L2 reconstruction error; see quantizer::max_error. | - -### preprocessing::quantize::scalar::quantizer_const_view::global_type + +### preprocessing::quantize::scalar::quantizer_view::global_type -Global affine parameters using double precision for encoding and decoding. +Global bounds. Transforms derive an affine mapping in double precision. -Encode (x - shift) / scale, rounding to nearest with ties away from zero and clamping to the full quantized range. Decode shift + scale * code, saturating to finite input limits. A zero scale represents constant training bounds; every code decodes to shift. +Encode (x - shift) / scale, rounding to nearest with ties away from zero and clamping to the full quantized range. Decode shift + scale * code, saturating to finite input limits. Equal bounds represent a constant range; every code decodes to that value. ```cpp struct global_type { - double shift; - double scale; + value_type min; + value_type max; }; ``` @@ -88,20 +88,20 @@ struct global_type { | Name | Type | Description | | --- | --- | --- | -| `shift` | `double` | Shift shared by all dimensions. | -| `scale` | `double` | Step size shared by all dimensions. | +| `min` | `value_type` | Lower bound shared by all dimensions. | +| `max` | `value_type` | Upper bound shared by all dimensions. | - -### preprocessing::quantize::scalar::quantizer_const_view::per_dim_type + +### preprocessing::quantize::scalar::quantizer_view::per_dim_type -Encode (x - shifts[j]) / scales[j], rounding to nearest with ties to even and +Per-dimension bounds. Transforms derive midpoint shifts and scales from these bounds. -clamping to the symmetric range of quantized values. Decode shifts[j] + scales[j] * code, saturating to finite limits of the input type. +Encode (x - shift) / scale, rounding to nearest with ties to even and clamping to the symmetric range of quantized values. Decode shift + scale * code, saturating to finite limits of the input type. ```cpp struct per_dim_type { - raft::device_vector_view shifts; - raft::device_vector_view scales; + vector_view_type mins; + vector_view_type maxs; }; ``` @@ -109,13 +109,34 @@ struct per_dim_type { | Name | Type | Description | | --- | --- | --- | -| `shifts` | `raft::device_vector_view` | One midpoint shift per dimension. | -| `scales` | `raft::device_vector_view` | One finite, positive scale per dimension, with a finite reciprocal. | +| `mins` | `vector_view_type` | One finite lower bound per dimension, with mins[j] <= maxs[j]. | +| `maxs` | `vector_view_type` | One finite upper bound per dimension; must have the same extent as mins. | + + +### preprocessing::quantize::scalar::quantizer_view::requires + +A mutable view decays to the read-only one, the way an mdspan does. + +```cpp +template +requires(!std::is_const_v && std::is_same_v) +operator quantizer_view() const noexcept // NOLINT(google-explicit-constructor); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `arg1` | | `!std::is_const_v && std::is_same_v` | | + +**Returns** + +`void` ### preprocessing::quantize::scalar::quantizer -Owns a global shift and scale or device shifts and scales for each dimension. +Owns global bounds or device bounds for each dimension. Move-only; copy view() to borrow the parameters. @@ -123,8 +144,6 @@ Move-only; copy view() to borrow the parameters. template struct quantizer { std::variant value; - T min_; - T max_; scale_type max_error; }; ``` @@ -133,20 +152,18 @@ struct quantizer { | Name | Type | Description | | --- | --- | --- | -| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view-global-type) | Global parameters or owning per-dimension shifts and scales. | -| `min_` | `T` | Original lower global bound; unused in per-dimension mode. Does not track changes to value. | -| `max_` | `T` | Original upper global bound; unused in per-dimension mode. Does not track changes to value. | +| `value` | [`std::variant`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-view-global-type) | Global bounds or owning per-dimension bounds. | | `max_error` | `scale_type` | Upper bound on the L2 error between a training row and its int8 encode/decode result. train measures every input row, including clipped outliers, and rounds the maximum up with a relative 1e-6 margin. This bound applies only to the training data, not later queries. It is infinity when constructed from parameters alone or when the error is not finite. | ### preprocessing::quantize::scalar::quantizer::per_dim_type -Owning per-dimension parameters. +Owning per-dimension bounds. ```cpp struct per_dim_type { - raft::device_vector shifts; - raft::device_vector scales; + raft::device_vector mins; + raft::device_vector maxs; }; ``` @@ -154,13 +171,13 @@ struct per_dim_type { | Name | Type | Description | | --- | --- | --- | -| `shifts` | `raft::device_vector` | | -| `scales` | `raft::device_vector` | | +| `mins` | `raft::device_vector` | | +| `maxs` | `raft::device_vector` | | ### preprocessing::quantize::scalar::quantizer::quantizer -Construct default global parameters. +Construct default global bounds. ```cpp quantizer(); @@ -172,7 +189,7 @@ quantizer(); **Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` -Derive a global shift and scale from finite bounds with min <= max. +Construct from finite global bounds with min <= max. ```cpp quantizer(T min, T max); @@ -191,7 +208,7 @@ quantizer(T min, T max); **Additional overload:** `preprocessing::quantize::scalar::quantizer::quantizer` -Construct from nonempty device bounds of equal length, with finite min <= max. +Copy nonempty device bounds of equal length, with finite min <= max. ```cpp quantizer(raft::resources const& res, @@ -199,8 +216,6 @@ raft::device_vector_view mins, raft::device_vector_view maxs); ``` -Derives midpoint shifts and a scale for each dimension. - **Parameters** | Name | Direction | Type | Description | @@ -216,28 +231,27 @@ Derives midpoint shifts and a scale for each dimension. ### preprocessing::quantize::scalar::quantizer::view -Borrow read-only parameters; the owner must outlive stream work using shifts and scales. +Borrow mutable per-dimension bounds; the owner must outlive stream work using them. ```cpp -[[nodiscard]] auto view() const noexcept -> view_type; +[[nodiscard]] auto view() noexcept -> view_type; ``` **Returns** `view_type` - -### preprocessing::quantize::scalar::quantizer::view_type +**Additional overload:** `preprocessing::quantize::scalar::quantizer::view` -An owning quantizer can be passed wherever a read-only view is accepted. +Borrow read-only bounds; the owner must outlive stream work using per-dimension bounds. ```cpp -operator view_type() const noexcept; +[[nodiscard]] auto view() const noexcept -> const_view_type; ``` **Returns** -`operator` +`const_view_type` ### preprocessing::quantize::scalar::train @@ -295,7 +309,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -307,7 +321,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -321,7 +335,7 @@ Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -335,7 +349,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -350,7 +364,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -364,7 +378,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -378,7 +392,7 @@ Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -394,7 +408,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -456,7 +470,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -468,7 +482,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -482,7 +496,7 @@ Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -496,7 +510,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -510,7 +524,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -524,7 +538,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -538,7 +552,7 @@ Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -554,7 +568,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -616,7 +630,7 @@ Applies quantization transform to given dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -628,7 +642,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -642,7 +656,7 @@ Applies global quantization to a host dataset ```cpp void transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -656,7 +670,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | @@ -670,7 +684,7 @@ Perform inverse quantization step on previously quantized dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::device_matrix_view dataset, raft::device_matrix_view out); ``` @@ -684,7 +698,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer | | `dataset` | in | `raft::device_matrix_view` | a row-major matrix view on device | | `out` | out | `raft::device_matrix_view` | a row-major matrix view on device | @@ -698,7 +712,7 @@ Applies inverse global quantization to a host dataset ```cpp void inverse_transform(raft::resources const& res, -quantizer_const_view quantizer, +const quantizer& quantizer, raft::host_matrix_view dataset, raft::host_matrix_view out); ``` @@ -714,7 +728,7 @@ Usage example: | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | `raft::resources const&` | raft resource | -| `quantizer` | in | [`quantizer_const_view`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer-const-view) | a scalar quantizer with global parameters | +| `quantizer` | in | [`const quantizer&`](/api-reference/cpp-api-preprocessing-quantize-scalar#preprocessing-quantize-scalar-quantizer) | a scalar quantizer with global parameters | | `dataset` | in | `raft::host_matrix_view` | a row-major matrix view on host | | `out` | out | `raft::host_matrix_view` | a row-major matrix view on host | From 9caac96912a92d4723a3b979d50804a0c072a4a5 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 6 Nov 2026 23:18:55 +0000 Subject: [PATCH 22/24] Simplify scalar quantization and add per-dimension C API --- .../cuvs/preprocessing/quantize/scalar.h | 77 ++++++- c/src/preprocessing/quantize/scalar.cpp | 196 ++++++++++++------ c/tests/CMakeLists.txt | 1 + c/tests/preprocessing/scalar_c.cu | 142 +++++++++++++ .../cluster/detail/kmeans_batch_loader.cuh | 2 +- .../preprocessing/quantize/detail/scalar.cuh | 126 +++-------- cpp/src/preprocessing/quantize/scalar.cu | 8 +- .../preprocessing/scalar_quantization.cu | 157 ++++---------- examples/cpp/src/hnsw_ace_layered_example.cu | 1 - .../c-api-preprocessing-quantize-scalar.md | 133 ++++++++++++ 10 files changed, 567 insertions(+), 276 deletions(-) create mode 100644 c/tests/preprocessing/scalar_c.cu diff --git a/c/include/cuvs/preprocessing/quantize/scalar.h b/c/include/cuvs/preprocessing/quantize/scalar.h index 9724ddbfa8..de0cc8f0da 100644 --- a/c/include/cuvs/preprocessing/quantize/scalar.h +++ b/c/include/cuvs/preprocessing/quantize/scalar.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -120,6 +120,81 @@ CUVS_EXPORT cuvsError_t cuvsScalarQuantizerInverseTransform(cuvsResources_t res, DLManagedTensor* dataset, DLManagedTensor* out); +/** + * @brief Owns a scalar quantizer with separate min/max bounds for each dimension. + * The fields are managed by the lifecycle and training functions. + */ +typedef struct { + uintptr_t addr; + DLDataType dtype; +} cuvsScalarQuantizerPerDim; + +typedef cuvsScalarQuantizerPerDim* cuvsScalarQuantizerPerDim_t; + +/** + * @brief Allocate an untrained per-dimension scalar quantizer. + * @param[out] quantizer Handle to allocate. + * @return cuvsError_t + */ +CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimCreate(cuvsScalarQuantizerPerDim_t* quantizer); + +/** + * @brief Release the quantizer and its device bounds after outstanding stream work completes. + * @param[in] quantizer Handle to destroy. + * @return cuvsError_t + */ +CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimDestroy(cuvsScalarQuantizerPerDim_t quantizer); + +/** + * @brief Train per-dimension bounds from every row of a host or device matrix. + * Supports float16, float32, and float64. The quantile is applied independently per dimension. + * Successful retraining replaces the previous bounds; failed training preserves them. + * Complete outstanding stream work using the quantizer before retraining. + * + * @param[in] res Resources used for training and allocation. + * @param[in] params Scalar quantizer parameters. + * @param[in] dataset Row-major host or device matrix. + * @param[in,out] quantizer Handle created by cuvsScalarQuantizerPerDimCreate. + * @return cuvsError_t + */ +CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimTrain(cuvsResources_t res, + cuvsScalarQuantizerParams_t params, + DLManagedTensor* dataset, + cuvsScalarQuantizerPerDim_t quantizer); + +/** + * @brief Quantize a device matrix using previously trained per-dimension bounds. + * The input type and width must match training. Output codes are int8, in [-127, 127]. + * The quantizer must outlive stream work using its bounds. + * + * @param[in] res Resources used for execution. + * @param[in] quantizer Trained per-dimension scalar quantizer. + * @param[in] dataset Row-major device matrix to encode. + * @param[out] out Row-major device int8 matrix with the same shape. + * @return cuvsError_t + */ +CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimTransform(cuvsResources_t res, + cuvsScalarQuantizerPerDim_t quantizer, + DLManagedTensor* dataset, + DLManagedTensor* out); + +/** + * @brief Reconstruct a device matrix from per-dimension scalar codes. + * The output type and width must match training. + * The quantizer must outlive stream work using its bounds. + * + * @param[in] res Resources used for execution. + * @param[in] quantizer Trained per-dimension scalar quantizer. + * @param[in] dataset Row-major device int8 matrix to decode. + * @param[out] out Row-major device matrix with the same shape, in the training data type. + * @return cuvsError_t + */ +CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimInverseTransform( + cuvsResources_t res, + cuvsScalarQuantizerPerDim_t quantizer, + DLManagedTensor* dataset, + DLManagedTensor* out); + /** * @} */ diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index 9d00da2959..23e70dc930 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -15,49 +16,46 @@ namespace { +namespace scalar = cuvs::preprocessing::quantize::scalar; + template -void _train(cuvsResources_t res, - cuvsScalarQuantizerParams params, - DLManagedTensor* dataset_tensor, - cuvsScalarQuantizer_t quantizer) +scalar::quantizer _train(cuvsResources_t res, + scalar::params params, + DLManagedTensor* dataset_tensor) { auto dataset = dataset_tensor->dl_tensor; - auto res_ptr = reinterpret_cast(res); - - auto quantizer_params = cuvs::preprocessing::quantize::scalar::params(); - quantizer_params.quantile = params.quantile; - - cuvs::preprocessing::quantize::scalar::quantizer ret; - if (cuvs::core::is_dlpack_device_compatible(dataset)) { using mdspan_type = raft::device_matrix_view; - auto mds = cuvs::core::from_dlpack(dataset_tensor); - ret = cuvs::preprocessing::quantize::scalar::train(*res_ptr, quantizer_params, mds); + return scalar::train(*res_ptr, params, cuvs::core::from_dlpack(dataset_tensor)); } else if (cuvs::core::is_dlpack_host_compatible(dataset)) { using mdspan_type = raft::host_matrix_view; - auto mds = cuvs::core::from_dlpack(dataset_tensor); - ret = cuvs::preprocessing::quantize::scalar::train(*res_ptr, quantizer_params, mds); + return scalar::train(*res_ptr, params, cuvs::core::from_dlpack(dataset_tensor)); } else { RAFT_FAIL("dataset must be accessible on host or device memory"); } +} - auto const& range = std::get(ret.value); +template +void _train(cuvsResources_t res, + cuvsScalarQuantizerParams params, + DLManagedTensor* dataset_tensor, + cuvsScalarQuantizer_t quantizer) +{ + auto trained = _train(res, scalar::params{params.quantile}, dataset_tensor); + auto const& range = std::get::global_type>(trained.value); quantizer->min_ = range.min; quantizer->max_ = range.max; } template void _transform(cuvsResources_t res, - cuvsScalarQuantizer_t quantizer_, + const scalar::quantizer& quantizer, DLManagedTensor* dataset_tensor, DLManagedTensor* out_tensor) { auto res_ptr = reinterpret_cast(res); - cuvs::preprocessing::quantize::scalar::quantizer quantizer{ - static_cast(quantizer_->min_), static_cast(quantizer_->max_)}; - auto dataset = dataset_tensor->dl_tensor; if (cuvs::core::is_dlpack_device_compatible(dataset)) { using mdspan_type = raft::device_matrix_view; @@ -85,15 +83,12 @@ void _transform(cuvsResources_t res, template void _inverse_transform(cuvsResources_t res, - cuvsScalarQuantizer_t quantizer_, + const scalar::quantizer& quantizer, DLManagedTensor* dataset_tensor, DLManagedTensor* out_tensor) { auto res_ptr = reinterpret_cast(res); - cuvs::preprocessing::quantize::scalar::quantizer quantizer{ - static_cast(quantizer_->min_), static_cast(quantizer_->max_)}; - auto dataset = dataset_tensor->dl_tensor; if (cuvs::core::is_dlpack_device_compatible(dataset)) { using mdspan_type = raft::device_matrix_view; @@ -118,6 +113,53 @@ void _inverse_transform(cuvsResources_t res, RAFT_FAIL("dataset must be accessible on host or device memory"); } } + +template +void _transform(cuvsResources_t res, + cuvsScalarQuantizer_t quantizer, + DLManagedTensor* dataset, + DLManagedTensor* out) +{ + _transform(res, + scalar::quantizer{static_cast(quantizer->min_), static_cast(quantizer->max_)}, + dataset, + out); +} + +template +void _inverse_transform(cuvsResources_t res, + cuvsScalarQuantizer_t quantizer, + DLManagedTensor* dataset, + DLManagedTensor* out) +{ + _inverse_transform(res, + scalar::quantizer{static_cast(quantizer->min_), static_cast(quantizer->max_)}, + dataset, + out); +} + +template +void dispatch_float(DLDataType dtype, Fn&& fn) +{ + RAFT_EXPECTS(dtype.code == kDLFloat && dtype.lanes == 1, "Expected a scalar floating-point dtype"); + switch (dtype.bits) { + case 16: fn.template operator()(); break; + case 32: fn.template operator()(); break; + case 64: fn.template operator()(); break; + default: RAFT_FAIL("Unsupported floating-point dtype bits: %d", dtype.bits); + } +} + +void destroy_per_dim(cuvsScalarQuantizerPerDim_t quantizer) +{ + if (quantizer->addr) { + dispatch_float(quantizer->dtype, [&]() { + delete reinterpret_cast*>(quantizer->addr); + }); + quantizer->addr = 0; + } +} + } // namespace extern "C" cuvsError_t cuvsScalarQuantizerParamsCreate(cuvsScalarQuantizerParams_t* params) @@ -147,18 +189,9 @@ extern "C" cuvsError_t cuvsScalarQuantizerTrain(cuvsResources_t res, cuvsScalarQuantizer_t quantizer) { return cuvs::core::translate_exceptions([=] { - auto dataset = dataset_tensor->dl_tensor; - if (dataset.dtype.code == kDLFloat && dataset.dtype.bits == 32) { - _train(res, *params, dataset_tensor, quantizer); - } else if (dataset.dtype.code == kDLFloat && dataset.dtype.bits == 16) { - _train(res, *params, dataset_tensor, quantizer); - } else if (dataset.dtype.code == kDLFloat && dataset.dtype.bits == 64) { - _train(res, *params, dataset_tensor, quantizer); - } else { - RAFT_FAIL("Unsupported dataset DLtensor dtype: %d and bits: %d", - dataset.dtype.code, - dataset.dtype.bits); - } + dispatch_float(dataset_tensor->dl_tensor.dtype, [&]() { + _train(res, *params, dataset_tensor, quantizer); + }); }); } @@ -168,18 +201,9 @@ extern "C" cuvsError_t cuvsScalarQuantizerTransform(cuvsResources_t res, DLManagedTensor* out_tensor) { return cuvs::core::translate_exceptions([=] { - auto dataset = dataset_tensor->dl_tensor; - if (dataset.dtype.code == kDLFloat && dataset.dtype.bits == 32) { - _transform(res, quantizer, dataset_tensor, out_tensor); - } else if (dataset.dtype.code == kDLFloat && dataset.dtype.bits == 16) { - _transform(res, quantizer, dataset_tensor, out_tensor); - } else if (dataset.dtype.code == kDLFloat && dataset.dtype.bits == 64) { - _transform(res, quantizer, dataset_tensor, out_tensor); - } else { - RAFT_FAIL("Unsupported dataset DLtensor dtype: %d and bits: %d", - dataset.dtype.code, - dataset.dtype.bits); - } + dispatch_float(dataset_tensor->dl_tensor.dtype, [&]() { + _transform(res, quantizer, dataset_tensor, out_tensor); + }); }); } @@ -189,16 +213,72 @@ cuvsError_t cuvsScalarQuantizerInverseTransform(cuvsResources_t res, DLManagedTensor* out) { return cuvs::core::translate_exceptions([=] { - auto dtype = out->dl_tensor.dtype; - if (dtype.code == kDLFloat && dtype.bits == 32) { - _inverse_transform(res, quantizer, dataset, out); - } else if (dtype.code == kDLFloat && dtype.bits == 16) { - _inverse_transform(res, quantizer, dataset, out); - } else if (dtype.code == kDLFloat && dtype.bits == 64) { - _inverse_transform(res, quantizer, dataset, out); - } else { - RAFT_FAIL( - "Unsupported output dataset DLtensor dtype: %d and bits: %d", dtype.code, dtype.bits); + dispatch_float(out->dl_tensor.dtype, [&]() { + _inverse_transform(res, quantizer, dataset, out); + }); + }); +} + +extern "C" cuvsError_t cuvsScalarQuantizerPerDimCreate(cuvsScalarQuantizerPerDim_t* quantizer) +{ + return cuvs::core::translate_exceptions([=] { + RAFT_EXPECTS(quantizer != nullptr, "quantizer must not be null"); + *quantizer = new cuvsScalarQuantizerPerDim{}; + }); +} + +extern "C" cuvsError_t cuvsScalarQuantizerPerDimDestroy(cuvsScalarQuantizerPerDim_t quantizer) +{ + return cuvs::core::translate_exceptions([=] { + if (quantizer) { + destroy_per_dim(quantizer); + delete quantizer; } }); } + +extern "C" cuvsError_t cuvsScalarQuantizerPerDimTrain(cuvsResources_t res, + cuvsScalarQuantizerParams_t params, + DLManagedTensor* dataset, + cuvsScalarQuantizerPerDim_t quantizer) +{ + return cuvs::core::translate_exceptions([=] { + RAFT_EXPECTS(params && dataset && quantizer, "params, dataset, and quantizer must not be null"); + dispatch_float(dataset->dl_tensor.dtype, [&]() { + auto trained = std::make_unique>( + _train(res, scalar::params{params->quantile, scalar::quantization_mode::PER_DIMENSION}, dataset)); + destroy_per_dim(quantizer); + quantizer->addr = reinterpret_cast(trained.release()); + quantizer->dtype = dataset->dl_tensor.dtype; + }); + }); +} + +extern "C" cuvsError_t cuvsScalarQuantizerPerDimTransform(cuvsResources_t res, + cuvsScalarQuantizerPerDim_t quantizer, + DLManagedTensor* dataset, + DLManagedTensor* out) +{ + return cuvs::core::translate_exceptions([=] { + RAFT_EXPECTS(quantizer && quantizer->addr, "quantizer must be trained before transforming"); + RAFT_EXPECTS(dataset && out, "dataset and out must not be null"); + dispatch_float(quantizer->dtype, [&]() { + _transform(res, *reinterpret_cast*>(quantizer->addr), dataset, out); + }); + }); +} + +extern "C" cuvsError_t cuvsScalarQuantizerPerDimInverseTransform( + cuvsResources_t res, + cuvsScalarQuantizerPerDim_t quantizer, + DLManagedTensor* dataset, + DLManagedTensor* out) +{ + return cuvs::core::translate_exceptions([=] { + RAFT_EXPECTS(quantizer && quantizer->addr, "quantizer must be trained before transforming"); + RAFT_EXPECTS(dataset && out, "dataset and out must not be null"); + dispatch_float(quantizer->dtype, [&]() { + _inverse_transform(res, *reinterpret_cast*>(quantizer->addr), dataset, out); + }); + }); +} diff --git a/c/tests/CMakeLists.txt b/c/tests/CMakeLists.txt index cfceb58f33..e52d1f2208 100644 --- a/c/tests/CMakeLists.txt +++ b/c/tests/CMakeLists.txt @@ -95,6 +95,7 @@ ConfigureTest( ) ConfigureTest(NAME PCA_C_TEST PATH preprocessing/pca_c.cu) +ConfigureTest(NAME SCALAR_QUANTIZATION_C_TEST PATH preprocessing/scalar_c.cu) if(BUILD_CAGRA_HNSWLIB) ConfigureTest(NAME HNSW_C_TEST PATH neighbors/ann_hnsw_c.cu) diff --git a/c/tests/preprocessing/scalar_c.cu b/c/tests/preprocessing/scalar_c.cu new file mode 100644 index 0000000000..12cd7c1a6d --- /dev/null +++ b/c/tests/preprocessing/scalar_c.cu @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include +#include +#include + +#include +#include + +namespace { + +template +DLManagedTensor matrix_tensor(T* data, DLDeviceType device, int64_t* shape) +{ + DLManagedTensor tensor{}; + tensor.dl_tensor.data = data; + tensor.dl_tensor.device = {device, 0}; + tensor.dl_tensor.ndim = 2; + tensor.dl_tensor.dtype = {std::is_same_v ? uint8_t{kDLInt} : uint8_t{kDLFloat}, + uint8_t{sizeof(T) * 8}, 1}; + tensor.dl_tensor.shape = shape; + return tensor; +} + +template +class ScalarQuantizerC : public ::testing::Test { + protected: + cuvsResources_t res{}; + cudaStream_t stream{}; + cuvsScalarQuantizerParams params{1.f}; + cuvsScalarQuantizerPerDim_t quantizer{}; + + void SetUp() override + { + ASSERT_EQ(cuvsResourcesCreate(&res), CUVS_SUCCESS); + ASSERT_EQ(cuvsStreamGet(res, &stream), CUVS_SUCCESS); + ASSERT_EQ(cuvsScalarQuantizerPerDimCreate(&quantizer), CUVS_SUCCESS); + } + + void TearDown() override + { + EXPECT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); + EXPECT_EQ(cuvsScalarQuantizerPerDimDestroy(quantizer), CUVS_SUCCESS); + EXPECT_EQ(cuvsResourcesDestroy(res), CUVS_SUCCESS); + } + + void check_round_trip(bool host_training) + { + int64_t shape[]{2, 3}; + std::vector values{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; + rmm::device_uvector input(values.size(), stream); + rmm::device_uvector codes(values.size(), stream); + raft::update_device(input.data(), values.data(), values.size(), stream); + auto host = matrix_tensor(values.data(), kDLCPU, shape); + auto device = matrix_tensor(input.data(), kDLCUDA, shape); + auto encoded = matrix_tensor(codes.data(), kDLCUDA, shape); + ASSERT_EQ(cuvsScalarQuantizerPerDimTrain(res, ¶ms, host_training ? &host : &device, quantizer), + CUVS_SUCCESS); + ASSERT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); + ASSERT_EQ(cuvsScalarQuantizerPerDimInverseTransform(res, quantizer, &encoded, &device), CUVS_SUCCESS); + std::vector actual_codes(values.size()); + std::vector decoded(values.size()); + raft::update_host(actual_codes.data(), codes.data(), actual_codes.size(), stream); + raft::update_host(decoded.data(), input.data(), decoded.size(), stream); + ASSERT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); + EXPECT_EQ(actual_codes, (std::vector{-127, -127, 0, 127, 127, 0})); + for (size_t i = 0; i < values.size(); ++i) { + EXPECT_EQ(double(decoded[i]), double(values[i])) << i; + } + } +}; + +} // namespace + +using ScalarTypes = ::testing::Types; +TYPED_TEST_SUITE(ScalarQuantizerC, ScalarTypes); + +TYPED_TEST(ScalarQuantizerC, PerDimensionTrainAndRetrain) +{ + this->check_round_trip(false); + this->check_round_trip(true); +} + +TYPED_TEST(ScalarQuantizerC, GlobalBoundsAndRoundTrip) +{ + using T = TypeParam; + int64_t shape[]{2, 1}; + std::vector values{T{-3}, T{7}}, decoded(2); + std::vector codes(2); + auto input = matrix_tensor(values.data(), kDLCPU, shape); + auto encoded = matrix_tensor(codes.data(), kDLCPU, shape); + auto output = matrix_tensor(decoded.data(), kDLCPU, shape); + cuvsScalarQuantizer global{}; + ASSERT_EQ(cuvsScalarQuantizerTrain(this->res, &this->params, &input, &global), CUVS_SUCCESS); + EXPECT_EQ(global.min_, -3); + EXPECT_EQ(global.max_, 7); + ASSERT_EQ(cuvsScalarQuantizerTransform(this->res, &global, &input, &encoded), CUVS_SUCCESS); + ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, &global, &encoded, &output), CUVS_SUCCESS); + EXPECT_EQ(codes, (std::vector{-128, 127})); + EXPECT_EQ(double(decoded[0]), -3); + EXPECT_EQ(double(decoded[1]), 7); +} + +using ScalarQuantizerCFloat = ScalarQuantizerC; +TEST_F(ScalarQuantizerCFloat, InvalidArgumentsPreserveTrainedQuantizer) +{ + int64_t shape[]{2, 3}; + std::vector values{0, 10, 9, 254, 137, 9}; + rmm::device_uvector input(values.size(), stream); + rmm::device_uvector codes(values.size(), stream); + raft::update_device(input.data(), values.data(), values.size(), stream); + auto host = matrix_tensor(values.data(), kDLCPU, shape); + auto device = matrix_tensor(input.data(), kDLCUDA, shape); + auto encoded = matrix_tensor(codes.data(), kDLCUDA, shape); + EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); + ASSERT_EQ(cuvsScalarQuantizerPerDimTrain(res, ¶ms, &host, quantizer), CUVS_SUCCESS); + auto address = quantizer->addr; + params.quantile = 0; + EXPECT_EQ(cuvsScalarQuantizerPerDimTrain(res, ¶ms, &host, quantizer), CUVS_ERROR); + EXPECT_EQ(quantizer->addr, address); + EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); + device.dl_tensor.dtype.bits = 64; + EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); + device.dl_tensor.dtype.bits = 32; + encoded.dl_tensor.dtype.code = kDLUInt; + EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); + encoded.dl_tensor.dtype.code = kDLInt; + int64_t wrong_shape[]{2, 2}; + device.dl_tensor.shape = wrong_shape; + EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); + device.dl_tensor.shape = shape; + std::vector host_codes(values.size()); + auto host_encoded = matrix_tensor(host_codes.data(), kDLCPU, shape); + EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &host, &host_encoded), CUVS_ERROR); + EXPECT_EQ(cuvsScalarQuantizerPerDimInverseTransform(res, quantizer, &host_encoded, &host), CUVS_ERROR); +} diff --git a/cpp/src/cluster/detail/kmeans_batch_loader.cuh b/cpp/src/cluster/detail/kmeans_batch_loader.cuh index cf72c7caf8..7c391505ae 100644 --- a/cpp/src/cluster/detail/kmeans_batch_loader.cuh +++ b/cpp/src/cluster/detail/kmeans_batch_loader.cuh @@ -86,7 +86,7 @@ class kmeans_batch { }; /** - * Read-only host/device batch loader. + * Read-only batch loader used only by KMeans. * * The device specialization is a zero-copy view. The host specialization below owns the cyclic * H2D pipeline used by out-of-core KMeans: two buffers when prefetching is enabled, or one buffer diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index ba4fc844ca..703fcb0dd8 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -276,109 +276,37 @@ typename quantizer::per_dim_type train(raft::resources const& res, return {std::move(minima), std::move(maxima)}; } -template -void check_per_dim_shapes(typename quantizer_view::per_dim_type quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) +// Encoding and decoding share the traversal; the input type selects the operator overload. +template +void apply_quantizer(raft::resources const& res, + quantizer_view quantizer, + raft::device_matrix_view dataset, + raft::device_matrix_view out) { - RAFT_EXPECTS(dataset.extent(1) > 0 && dataset.extent(1) == quantizer.mins.extent(0) && - dataset.extent(1) == quantizer.maxs.extent(0), - "Input width must match the nonempty min and max vectors"); - RAFT_EXPECTS(dataset.extent(0) == out.extent(0) && dataset.extent(1) == out.extent(1), - "Input and output shapes must match"); -} - -template -void transform(raft::resources const& res, - typename quantizer_view::per_dim_type quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* mins = quantizer.mins.data_handle(); - auto const* maxs = quantizer.maxs.data_handle(); - auto const dim = dataset.extent(1); - raft::linalg::map_offset(res, out, [input, mins, maxs, dim] __device__(int64_t i) { - return per_dim_quantize_op{mins[i % dim], maxs[i % dim]}(input[i]); - }); -} - -template -void inverse_transform(raft::resources const& res, - typename quantizer_view::per_dim_type quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - check_per_dim_shapes(quantizer, dataset, out); - auto const* input = dataset.data_handle(); - auto const* mins = quantizer.mins.data_handle(); - auto const* maxs = quantizer.maxs.data_handle(); - auto const dim = dataset.extent(1); - raft::linalg::map_offset(res, out, [input, mins, maxs, dim] __device__(int64_t i) { - return per_dim_quantize_op{mins[i % dim], maxs[i % dim]}(input[i]); - }); -} - -template -void transform(raft::resources const& res, - quantizer_view quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - auto view = quantizer; - if (auto const* bounds = - std::get_if::per_dim_type>(&view.value)) { - detail::transform(res, *bounds, dataset, out); - return; - } - auto const& range = std::get::global_type>(view.value); - raft::linalg::map(res, out, quantize_op{range}, raft::make_const_mdspan(dataset)); -} - -template -void transform(raft::resources const& res, - quantizer_view quantizer, - raft::host_matrix_view dataset, - raft::host_matrix_view out) -{ - RAFT_EXPECTS( - std::holds_alternative::global_type>(quantizer.value), - "Host quantization requires global parameters"); - auto const& range = std::get::global_type>(quantizer.value); - auto main_op = quantize_op{range}; - size_t n_elements = dataset.extent(0) * dataset.extent(1); - - // In dry-run mode the views may be unbacked or alias the shared probe buffer. - if (raft::resource::get_dry_run_flag(res)) { return; } - -#pragma omp parallel for - for (size_t i = 0; i < n_elements; ++i) { - out.data_handle()[i] = main_op(dataset.data_handle()[i]); - } -} - -template -void inverse_transform(raft::resources const& res, - quantizer_view quantizer, - raft::device_matrix_view dataset, - raft::device_matrix_view out) -{ - auto view = quantizer; if (auto const* bounds = - std::get_if::per_dim_type>(&view.value)) { - detail::inverse_transform(res, *bounds, dataset, out); - return; + std::get_if::per_dim_type>(&quantizer.value)) { + auto const dim = dataset.extent(1); + RAFT_EXPECTS(dim > 0 && dim == bounds->mins.extent(0) && dim == bounds->maxs.extent(0), + "Input width must match the nonempty min and max vectors"); + RAFT_EXPECTS(dataset.extent(0) == out.extent(0) && dim == out.extent(1), + "Input and output shapes must match"); + auto const* input = dataset.data_handle(); + auto const* mins = bounds->mins.data_handle(); + auto const* maxs = bounds->maxs.data_handle(); + raft::linalg::map_offset(res, out, [input, mins, maxs, dim] __device__(int64_t i) { + return per_dim_quantize_op{mins[i % dim], maxs[i % dim]}(input[i]); + }); + } else { + auto const& range = std::get::global_type>(quantizer.value); + raft::linalg::map(res, out, quantize_op{range}, dataset); } - auto const& range = std::get::global_type>(view.value); - raft::linalg::map(res, out, quantize_op{range}, raft::make_const_mdspan(dataset)); } -template -void inverse_transform(raft::resources const& res, - quantizer_view quantizer, - raft::host_matrix_view dataset, - raft::host_matrix_view out) +template +void apply_quantizer(raft::resources const& res, + quantizer_view quantizer, + raft::host_matrix_view dataset, + raft::host_matrix_view out) { RAFT_EXPECTS( std::holds_alternative::global_type>(quantizer.value), @@ -554,7 +482,7 @@ quantizer train( if (out) { auto batch_codes = raft::make_device_matrix_view( out->data_handle() + offset * dim, input.extent(0), dim); - detail::transform(res, view, input, batch_codes); + apply_quantizer(res, view, input, batch_codes); } }, false); diff --git a/cpp/src/preprocessing/quantize/scalar.cu b/cpp/src/preprocessing/quantize/scalar.cu index b16d2d1afa..e4efcf16ea 100644 --- a/cpp/src/preprocessing/quantize/scalar.cu +++ b/cpp/src/preprocessing/quantize/scalar.cu @@ -49,28 +49,28 @@ quantizer::quantizer(raft::resources const& res, raft::device_matrix_view dataset, \ raft::device_matrix_view out) \ { \ - detail::transform(res, quantizer.view(), dataset, out); \ + detail::apply_quantizer(res, quantizer.view(), dataset, out); \ } \ void transform(raft::resources const& res, \ const quantizer& quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ - detail::transform(res, quantizer.view(), dataset, out); \ + detail::apply_quantizer(res, quantizer.view(), dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ const quantizer& quantizer, \ raft::device_matrix_view dataset, \ raft::device_matrix_view out) \ { \ - detail::inverse_transform(res, quantizer.view(), dataset, out); \ + detail::apply_quantizer(res, quantizer.view(), dataset, out); \ } \ void inverse_transform(raft::resources const& res, \ const quantizer& quantizer, \ raft::host_matrix_view dataset, \ raft::host_matrix_view out) \ { \ - detail::inverse_transform(res, quantizer.view(), dataset, out); \ + detail::apply_quantizer(res, quantizer.view(), dataset, out); \ } \ auto make_sq_dataset( \ raft::resources const& res, params params, raft::host_matrix_view dataset) \ diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index 48b2b9d6fe..ff942647c4 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -39,6 +39,20 @@ long double expectReconstructionErrorBound( return maximum; } +template +void expectDeviceValues(raft::resources const& res, + const T* device_values, + std::vector const& expected) +{ + std::vector actual(expected.size()); + raft::update_host( + actual.data(), device_values, actual.size(), raft::resource::get_cuda_stream(res)); + raft::resource::sync_stream(res); + for (size_t i = 0; i < actual.size(); ++i) { + EXPECT_EQ(static_cast(actual[i]), static_cast(expected[i])) << i; + } +} + template std::vector makeShiftedDataset(int64_t rows, int64_t dim) { @@ -384,11 +398,7 @@ class PerDimensionRangeTest : public ::testing::TestWithParam(handle, params_.n_samples, params_.n_features); transform(handle, quantizer, raft::make_const_mdspan(dataset_.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(params_.codes.data(), - codes.data_handle(), - params_.codes.size(), - cuvs::Compare(), - stream)); + expectDeviceValues(handle, codes.data_handle(), params_.codes); if (params_.tolerance == 0.f) { auto bounds = std::get(std::as_const(quantizer).view().value); EXPECT_TRUE(devArrMatch(bounds.mins.data_handle(), @@ -426,20 +436,6 @@ class PerDimensionQuantizationTest : public ::testing::Test {}; using PerDimensionTypes = ::testing::Types; TYPED_TEST_SUITE(PerDimensionQuantizationTest, PerDimensionTypes); -template -void expectDeviceValues(raft::resources const& res, - const T* device_values, - std::vector const& expected) -{ - std::vector actual(expected.size()); - raft::update_host( - actual.data(), device_values, actual.size(), raft::resource::get_cuda_stream(res)); - raft::resource::sync_stream(res); - for (size_t i = 0; i < actual.size(); ++i) { - EXPECT_EQ(static_cast(actual[i]), static_cast(expected[i])) << i; - } -} - template void testTrainingReconstructionError(params p) { @@ -526,12 +522,8 @@ TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) std::vector reconstructed(values.size()); raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); raft::resource::sync_stream(res); - double maximum = 0; - for (int64_t row = 0; row < 3; ++row) { - auto const i = row * 2; - maximum = std::max( - maximum, std::hypot(values[i] - reconstructed[i], values[i + 1] - reconstructed[i + 1])); - } + auto maximum = + expectReconstructionErrorBound(values.data(), reconstructed.data(), 3, 2, owner.max_error); EXPECT_GT(maximum, 0); EXPECT_TRUE(std::isfinite(owner.max_error)); EXPECT_GE(owner.max_error, maximum); @@ -563,21 +555,9 @@ TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); auto codes = raft::make_device_matrix(res, 2, 3); transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{0, 0, 127, 2, -2, -127}.data(), - codes.data_handle(), - 6, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); + expectDeviceValues(res, codes.data_handle(), {0, 0, 127, 2, -2, -127}); inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); - { - std::vector actual(6); - std::vector expected{127.f, 73.5f, 136.f, 129.f, 72.5f, -118.f}; - raft::update_host(actual.data(), input.data_handle(), actual.size(), stream); - raft::resource::sync_stream(res); - for (size_t i = 0; i < actual.size(); ++i) { - EXPECT_EQ(static_cast(actual[i]), expected[i]); - } - } + expectDeviceValues(res, input.data_handle(), {127.f, 73.5f, 136.f, 129.f, 72.5f, -118.f}); } TEST(PerDimensionRangeTest, AdjacentHalfBounds) @@ -592,11 +572,7 @@ TEST(PerDimensionRangeTest, AdjacentHalfBounds) auto codes = raft::make_device_matrix(res, 2, 1); transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); inverse_transform(res, quantizer, raft::make_const_mdspan(codes.view()), input.view()); - std::vector reconstructed(2); - raft::update_host(reconstructed.data(), input.data_handle(), reconstructed.size(), stream); - raft::resource::sync_stream(res); - EXPECT_EQ(static_cast(reconstructed[0]), static_cast(values[0])); - EXPECT_EQ(static_cast(reconstructed[1]), static_cast(values[1])); + expectDeviceValues(res, input.data_handle(), values); } TEST(PerDimensionRangeTest, PerDimensionQuantiles) @@ -615,44 +591,26 @@ TEST(PerDimensionRangeTest, PerDimensionQuantiles) auto owner = train( res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); auto quantizer = std::as_const(owner).view(); - EXPECT_TRUE(devArrMatchHost(std::vector{4.f, 10008.f, 9.f}.data(), - std::get(quantizer.value).mins.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - EXPECT_TRUE(devArrMatchHost(std::vector{995.f, 11990.f, 9.f}.data(), - std::get(quantizer.value).maxs.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); + expectDeviceValues( + res, std::get(quantizer.value).mins.data_handle(), {4.f, 10008.f, 9.f}); + expectDeviceValues( + res, std::get(quantizer.value).maxs.data_handle(), {995.f, 11990.f, 9.f}); auto full = train( res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); - EXPECT_TRUE( - devArrMatchHost(std::vector{999.f, 11998.f, 9.f}.data(), - std::get(std::as_const(full).view().value).maxs.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - EXPECT_TRUE( - devArrMatchHost(std::vector{0.f, 10000.f, 9.f}.data(), - std::get(std::as_const(full).view().value).mins.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); + expectDeviceValues(res, + std::get(std::as_const(full).view().value).maxs.data_handle(), + {999.f, 11998.f, 9.f}); + expectDeviceValues(res, + std::get(std::as_const(full).view().value).mins.data_handle(), + {0.f, 10000.f, 9.f}); auto trimmed = train( res, params{0.5f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); - EXPECT_TRUE( - devArrMatchHost(std::vector{749.f, 11498.f, 9.f}.data(), - std::get(std::as_const(trimmed).view().value).maxs.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); - EXPECT_TRUE( - devArrMatchHost(std::vector{250.f, 10500.f, 9.f}.data(), - std::get(std::as_const(trimmed).view().value).mins.data_handle(), - dim, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); + expectDeviceValues(res, + std::get(std::as_const(trimmed).view().value).maxs.data_handle(), + {749.f, 11498.f, 9.f}); + expectDeviceValues(res, + std::get(std::as_const(trimmed).view().value).mins.data_handle(), + {250.f, 10500.f, 9.f}); for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { EXPECT_THROW( train( @@ -686,11 +644,7 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); auto codes = raft::make_device_matrix(res, 1, 2); transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{2, -127}.data(), - codes.data_handle(), - 2, - cuvs::Compare(), - raft::resource::get_cuda_stream(res).get())); + expectDeviceValues(res, codes.data_handle(), {2, -127}); scalar::quantizer wrong_bounds; wrong_bounds.value = typename scalar::quantizer::per_dim_type{ raft::make_device_vector(res, 2), raft::make_device_vector(res, 1)}; @@ -729,11 +683,7 @@ TYPED_TEST(PerDimensionQuantizationTest, DirectPerDimensionBounds) auto codes = raft::make_device_matrix(res, 1, 2); raft::update_device(input.data_handle(), values.data(), values.size(), stream); transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{1, 1}.data(), - codes.data_handle(), - 2, - cuvs::Compare(), - stream.get())); + expectDeviceValues(res, codes.data_handle(), {1, 1}); inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); expectDeviceValues(res, input.data_handle(), values); @@ -753,11 +703,7 @@ TYPED_TEST(PerDimensionQuantizationTest, DirectPerDimensionBounds) raft::update_device(bounds.mins.data_handle(), lower.data(), lower.size(), stream); raft::update_device(bounds.maxs.data_handle(), upper.data(), upper.size(), stream); transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatchHost(std::vector{0, 0}.data(), - codes.data_handle(), - 2, - cuvs::Compare(), - stream.get())); + expectDeviceValues(res, codes.data_handle(), {0, 0}); } TYPED_TEST(PerDimensionQuantizationTest, DirectGlobalBounds) @@ -817,16 +763,11 @@ TEST(PerDimensionRangeTest, QuantizedTypeLimits) params{.5f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE( - devArrMatchHost(std::vector{32767.f}.data(), - std::get(std::as_const(owner).view().value).maxs.data_handle(), - 1, - cuvs::Compare(), - stream.get())); + expectDeviceValues( + res, std::get(std::as_const(owner).view().value).maxs.data_handle(), {32767.f}); std::vector expected{-32767, -32767, 0, 32767, 32767}; - EXPECT_TRUE(devArrMatchHost( - expected.data(), codes.data_handle(), expected.size(), cuvs::Compare(), stream.get())); - detail::inverse_transform( + expectDeviceValues(res, codes.data_handle(), expected); + detail::apply_quantizer( res, std::as_const(owner).view(), raft::make_const_mdspan(codes.view()), input.view()); std::vector decoded(values.size()); raft::update_host(decoded.data(), input.data_handle(), decoded.size(), stream); @@ -878,11 +819,7 @@ TEST(QuantizationDatasetTest, GlobalRangeDataset) EXPECT_EQ(read_only.const_manifest_view().max_error, 0.f); EXPECT_EQ(encoded.n_rows(), 1); EXPECT_EQ(encoded.dim(), 2u); - EXPECT_TRUE(devArrMatchHost(std::vector{-128, 127}.data(), - encoded.data_const_view().data_handle(), - 2, - cuvs::Compare(), - stream.get())); + expectDeviceValues(res, encoded.data_const_view().data_handle(), {-128, 127}); auto range = std::get::global_type>(encoded.const_manifest_view().value); EXPECT_EQ(range.min, -1.f); @@ -902,11 +839,7 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) EXPECT_EQ(read_only.const_manifest_view().max_error, 0.f); EXPECT_EQ(encoded.n_rows(), 2); EXPECT_EQ(encoded.dim(), 2u); - EXPECT_TRUE(devArrMatchHost(std::vector{-127, -127, 127, 127}.data(), - encoded.data_const_view().data_handle(), - 4, - cuvs::Compare(), - stream.get())); + expectDeviceValues(res, encoded.data_const_view().data_handle(), {-127, -127, 127, 127}); auto codes = raft::make_device_matrix(res, 2, 2); auto quantizer = train( res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); diff --git a/examples/cpp/src/hnsw_ace_layered_example.cu b/examples/cpp/src/hnsw_ace_layered_example.cu index d712644dbc..14de426605 100644 --- a/examples/cpp/src/hnsw_ace_layered_example.cu +++ b/examples/cpp/src/hnsw_ace_layered_example.cu @@ -42,7 +42,6 @@ #include #include #include -#include #include #include diff --git a/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md b/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md index 86aacd7bce..29ac8ff15f 100644 --- a/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md +++ b/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md @@ -190,3 +190,136 @@ Note that depending on the chosen data types train dataset the conversion is not **Returns** [`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) + + +### cuvsScalarQuantizerPerDim + +Owns a scalar quantizer with separate min/max bounds for each dimension. The fields are managed by the lifecycle and training functions. + +```c +typedef struct { + uintptr_t addr; + DLDataType dtype; +} cuvsScalarQuantizerPerDim; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `addr` | `uintptr_t` | | +| `dtype` | `DLDataType` | | + + +### cuvsScalarQuantizerPerDimCreate + +Allocate an untrained per-dimension scalar quantizer. + +```c +cuvsError_t cuvsScalarQuantizerPerDimCreate(cuvsScalarQuantizerPerDim_t* quantizer); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `quantizer` | out | [`cuvsScalarQuantizerPerDim_t*`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Handle to allocate. | + +**Returns** + +[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) + + +### cuvsScalarQuantizerPerDimDestroy + +Release the quantizer and its device bounds after outstanding stream work completes. + +```c +cuvsError_t cuvsScalarQuantizerPerDimDestroy(cuvsScalarQuantizerPerDim_t quantizer); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `quantizer` | in | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Handle to destroy. | + +**Returns** + +[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) + + +### cuvsScalarQuantizerPerDimTrain + +Train per-dimension bounds from every row of a host or device matrix. Supports float16, float32, and float64. The quantile is applied independently per dimension. Successful retraining replaces the previous bounds; failed training preserves them. Complete outstanding stream work using the quantizer before retraining. + +```c +cuvsError_t cuvsScalarQuantizerPerDimTrain(cuvsResources_t res, +cuvsScalarQuantizerParams_t params, +DLManagedTensor* dataset, +cuvsScalarQuantizerPerDim_t quantizer); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | Resources used for training and allocation. | +| `params` | in | [`cuvsScalarQuantizerParams_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerparams) | Scalar quantizer parameters. | +| `dataset` | in | `DLManagedTensor*` | Row-major host or device matrix. | +| `quantizer` | in,out | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Handle created by cuvsScalarQuantizerPerDimCreate. | + +**Returns** + +[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) + + +### cuvsScalarQuantizerPerDimTransform + +Quantize a device matrix using previously trained per-dimension bounds. The input type and width must match training. Output codes are int8, in [-127, 127]. The quantizer must outlive stream work using its bounds. + +```c +cuvsError_t cuvsScalarQuantizerPerDimTransform(cuvsResources_t res, +cuvsScalarQuantizerPerDim_t quantizer, +DLManagedTensor* dataset, +DLManagedTensor* out); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | Resources used for execution. | +| `quantizer` | in | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Trained per-dimension scalar quantizer. | +| `dataset` | in | `DLManagedTensor*` | Row-major device matrix to encode. | +| `out` | out | `DLManagedTensor*` | Row-major device int8 matrix with the same shape. | + +**Returns** + +[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) + + +### cuvsScalarQuantizerPerDimInverseTransform + +Reconstruct a device matrix from per-dimension scalar codes. The output type and width must match training. The quantizer must outlive stream work using its bounds. + +```c +cuvsError_t cuvsScalarQuantizerPerDimInverseTransform( +cuvsResources_t res, +cuvsScalarQuantizerPerDim_t quantizer, +DLManagedTensor* dataset, +DLManagedTensor* out); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | Resources used for execution. | +| `quantizer` | in | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Trained per-dimension scalar quantizer. | +| `dataset` | in | `DLManagedTensor*` | Row-major device int8 matrix to decode. | +| `out` | out | `DLManagedTensor*` | Row-major device matrix with the same shape, in the training data type. | + +**Returns** + +[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) From 1f3237ed9bad506a84123882fa5e406ddec9d441 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 6 Nov 2026 23:53:29 +0000 Subject: [PATCH 23/24] Use existing scalar C APIs for per-dimension quantization --- .../cuvs/preprocessing/quantize/scalar.h | 108 +++-------- c/src/preprocessing/quantize/scalar.cpp | 148 +++++++-------- c/tests/preprocessing/scalar_c.cu | 72 ++++--- .../preprocessing/quantize/detail/scalar.cuh | 2 +- .../c-api-preprocessing-quantize-scalar.md | 178 ++++-------------- 5 files changed, 178 insertions(+), 330 deletions(-) diff --git a/c/include/cuvs/preprocessing/quantize/scalar.h b/c/include/cuvs/preprocessing/quantize/scalar.h index de0cc8f0da..68175d98ca 100644 --- a/c/include/cuvs/preprocessing/quantize/scalar.h +++ b/c/include/cuvs/preprocessing/quantize/scalar.h @@ -19,15 +19,24 @@ extern "C" { * @defgroup preprocessing_c_scalar C API for Scalar Quantizer * @{ */ +/** Select global or per-dimension scalar quantization bounds. */ +enum cuvsScalarQuantizationMode { + CUVS_SCALAR_GLOBAL = 0, + CUVS_SCALAR_PER_DIMENSION = 1 +}; + /** - * @brief Scalar quantizer parameters. + * @brief Scalar quantizer parameters. Allocate with cuvsScalarQuantizerParamsCreate. */ struct cuvsScalarQuantizerParams { - /* + /** * specifies how many outliers at top & bottom will be ignored * needs to be within range of (0, 1] */ float quantile; + + /** Quantization mode. Defaults to CUVS_SCALAR_GLOBAL. */ + enum cuvsScalarQuantizationMode mode; }; typedef struct cuvsScalarQuantizerParams* cuvsScalarQuantizerParams_t; @@ -53,6 +62,11 @@ CUVS_EXPORT cuvsError_t cuvsScalarQuantizerParamsDestroy(cuvsScalarQuantizerPara * * The quantization is performed by a linear mapping of an interval in the * float data type to the full range of the quantized int type. + * Allocate with cuvsScalarQuantizerCreate and release with cuvsScalarQuantizerDestroy; + * do not copy the handle's contents. The implementation also owns any per-dimension bounds. + * In global mode, min_ and max_ are the trained bounds. In per-dimension mode, they are + * numeric_limits::min() and numeric_limits::max() for the training data type T, + * and are not used for transformation. */ typedef struct { double min_; @@ -70,7 +84,7 @@ typedef cuvsScalarQuantizer* cuvsScalarQuantizer_t; CUVS_EXPORT cuvsError_t cuvsScalarQuantizerCreate(cuvsScalarQuantizer_t* quantizer); /** - * @brief De-allocate Scalar Quantizer + * @brief De-allocate Scalar Quantizer after outstanding stream work completes. * * @param[in] quantizer * @return cuvsError_t @@ -80,10 +94,15 @@ CUVS_EXPORT cuvsError_t cuvsScalarQuantizerDestroy(cuvsScalarQuantizer_t quantiz /** * @brief Trains a scalar quantizer to be used later for quantizing the dataset. * + * Supports float16, float32, and float64 in global or per-dimension mode. + * Per-dimension mode applies the quantile independently to each dimension using every row. + * Successful retraining replaces the previous bounds; failed training preserves them. + * Complete outstanding stream work using the quantizer before retraining. + * * @param[in] res raft resource * @param[in] params configure scalar quantizer, e.g. quantile * @param[in] dataset a row-major host or device matrix - * @param[out] quantizer trained scalar quantizer + * @param[in,out] quantizer scalar quantizer allocated by cuvsScalarQuantizerCreate */ CUVS_EXPORT cuvsError_t cuvsScalarQuantizerTrain(cuvsResources_t res, cuvsScalarQuantizerParams_t params, @@ -93,6 +112,9 @@ CUVS_EXPORT cuvsError_t cuvsScalarQuantizerTrain(cuvsResources_t res, /** * @brief Applies quantization transform to given dataset * + * Per-dimension mode requires device matrices with the same input type and width as training. + * Its output codes are int8 in [-127, 127]. The quantizer must outlive stream work using its bounds. + * * @param[in] res raft resource * @param[in] quantizer a scalar quantizer * @param[in] dataset a row-major host or device matrix to transform @@ -107,7 +129,8 @@ CUVS_EXPORT cuvsError_t cuvsScalarQuantizerTransform(cuvsResources_t res, * @brief Perform inverse quantization step on previously quantized dataset * * Note that depending on the chosen data types train dataset the conversion is - * not lossless. + * not lossless. Per-dimension mode requires device matrices with the same output type and + * width as training. The quantizer must outlive stream work using its bounds. * * @param[in] res raft resource * @param[in] quantizer a scalar quantizer @@ -120,81 +143,6 @@ CUVS_EXPORT cuvsError_t cuvsScalarQuantizerInverseTransform(cuvsResources_t res, DLManagedTensor* dataset, DLManagedTensor* out); -/** - * @brief Owns a scalar quantizer with separate min/max bounds for each dimension. - * The fields are managed by the lifecycle and training functions. - */ -typedef struct { - uintptr_t addr; - DLDataType dtype; -} cuvsScalarQuantizerPerDim; - -typedef cuvsScalarQuantizerPerDim* cuvsScalarQuantizerPerDim_t; - -/** - * @brief Allocate an untrained per-dimension scalar quantizer. - * @param[out] quantizer Handle to allocate. - * @return cuvsError_t - */ -CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimCreate(cuvsScalarQuantizerPerDim_t* quantizer); - -/** - * @brief Release the quantizer and its device bounds after outstanding stream work completes. - * @param[in] quantizer Handle to destroy. - * @return cuvsError_t - */ -CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimDestroy(cuvsScalarQuantizerPerDim_t quantizer); - -/** - * @brief Train per-dimension bounds from every row of a host or device matrix. - * Supports float16, float32, and float64. The quantile is applied independently per dimension. - * Successful retraining replaces the previous bounds; failed training preserves them. - * Complete outstanding stream work using the quantizer before retraining. - * - * @param[in] res Resources used for training and allocation. - * @param[in] params Scalar quantizer parameters. - * @param[in] dataset Row-major host or device matrix. - * @param[in,out] quantizer Handle created by cuvsScalarQuantizerPerDimCreate. - * @return cuvsError_t - */ -CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimTrain(cuvsResources_t res, - cuvsScalarQuantizerParams_t params, - DLManagedTensor* dataset, - cuvsScalarQuantizerPerDim_t quantizer); - -/** - * @brief Quantize a device matrix using previously trained per-dimension bounds. - * The input type and width must match training. Output codes are int8, in [-127, 127]. - * The quantizer must outlive stream work using its bounds. - * - * @param[in] res Resources used for execution. - * @param[in] quantizer Trained per-dimension scalar quantizer. - * @param[in] dataset Row-major device matrix to encode. - * @param[out] out Row-major device int8 matrix with the same shape. - * @return cuvsError_t - */ -CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimTransform(cuvsResources_t res, - cuvsScalarQuantizerPerDim_t quantizer, - DLManagedTensor* dataset, - DLManagedTensor* out); - -/** - * @brief Reconstruct a device matrix from per-dimension scalar codes. - * The output type and width must match training. - * The quantizer must outlive stream work using its bounds. - * - * @param[in] res Resources used for execution. - * @param[in] quantizer Trained per-dimension scalar quantizer. - * @param[in] dataset Row-major device int8 matrix to decode. - * @param[out] out Row-major device matrix with the same shape, in the training data type. - * @return cuvsError_t - */ -CUVS_EXPORT cuvsError_t cuvsScalarQuantizerPerDimInverseTransform( - cuvsResources_t res, - cuvsScalarQuantizerPerDim_t quantizer, - DLManagedTensor* dataset, - DLManagedTensor* out); - /** * @} */ diff --git a/c/src/preprocessing/quantize/scalar.cpp b/c/src/preprocessing/quantize/scalar.cpp index 23e70dc930..7b7eca81bd 100644 --- a/c/src/preprocessing/quantize/scalar.cpp +++ b/c/src/preprocessing/quantize/scalar.cpp @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -18,6 +19,12 @@ namespace { namespace scalar = cuvs::preprocessing::quantize::scalar; +// Keep the public min/max layout while owning per-dimension bounds behind the factory handle. +struct scalar_quantizer : cuvsScalarQuantizer { + uintptr_t addr = 0; + DLDataType dtype{}; +}; + template scalar::quantizer _train(cuvsResources_t res, scalar::params params, @@ -36,18 +43,6 @@ scalar::quantizer _train(cuvsResources_t res, } } -template -void _train(cuvsResources_t res, - cuvsScalarQuantizerParams params, - DLManagedTensor* dataset_tensor, - cuvsScalarQuantizer_t quantizer) -{ - auto trained = _train(res, scalar::params{params.quantile}, dataset_tensor); - auto const& range = std::get::global_type>(trained.value); - quantizer->min_ = range.min; - quantizer->max_ = range.max; -} - template void _transform(cuvsResources_t res, const scalar::quantizer& quantizer, @@ -150,7 +145,7 @@ void dispatch_float(DLDataType dtype, Fn&& fn) } } -void destroy_per_dim(cuvsScalarQuantizerPerDim_t quantizer) +void destroy_per_dim(scalar_quantizer* quantizer) { if (quantizer->addr) { dispatch_float(quantizer->dtype, [&]() { @@ -165,7 +160,7 @@ void destroy_per_dim(cuvsScalarQuantizerPerDim_t quantizer) extern "C" cuvsError_t cuvsScalarQuantizerParamsCreate(cuvsScalarQuantizerParams_t* params) { return cuvs::core::translate_exceptions( - [=] { *params = new cuvsScalarQuantizerParams{.quantile = 0.99}; }); + [=] { *params = new cuvsScalarQuantizerParams{.quantile = 0.99, .mode = CUVS_SCALAR_GLOBAL}; }); } extern "C" cuvsError_t cuvsScalarQuantizerParamsDestroy(cuvsScalarQuantizerParams_t params) @@ -175,12 +170,21 @@ extern "C" cuvsError_t cuvsScalarQuantizerParamsDestroy(cuvsScalarQuantizerParam extern "C" cuvsError_t cuvsScalarQuantizerCreate(cuvsScalarQuantizer_t* quantizer) { - return cuvs::core::translate_exceptions([=] { *quantizer = new cuvsScalarQuantizer{}; }); + return cuvs::core::translate_exceptions([=] { + RAFT_EXPECTS(quantizer != nullptr, "quantizer must not be null"); + *quantizer = new scalar_quantizer{}; + }); } extern "C" cuvsError_t cuvsScalarQuantizerDestroy(cuvsScalarQuantizer_t quantizer) { - return cuvs::core::translate_exceptions([=] { delete quantizer; }); + return cuvs::core::translate_exceptions([=] { + if (quantizer) { + auto q = static_cast(quantizer); + destroy_per_dim(q); + delete q; + } + }); } extern "C" cuvsError_t cuvsScalarQuantizerTrain(cuvsResources_t res, @@ -189,8 +193,33 @@ extern "C" cuvsError_t cuvsScalarQuantizerTrain(cuvsResources_t res, cuvsScalarQuantizer_t quantizer) { return cuvs::core::translate_exceptions([=] { + RAFT_EXPECTS(params && dataset_tensor && quantizer, + "params, dataset, and quantizer must not be null"); + scalar::params cpp_params; + cpp_params.quantile = params->quantile; + switch (params->mode) { + case CUVS_SCALAR_GLOBAL: cpp_params.range = scalar::quantization_mode::GLOBAL; break; + case CUVS_SCALAR_PER_DIMENSION: + cpp_params.range = scalar::quantization_mode::PER_DIMENSION; + break; + default: RAFT_FAIL("Unsupported scalar quantization mode"); + } + auto q = static_cast(quantizer); dispatch_float(dataset_tensor->dl_tensor.dtype, [&]() { - _train(res, *params, dataset_tensor, quantizer); + auto trained = _train(res, cpp_params, dataset_tensor); + if (auto const* bounds = + std::get_if::global_type>(&trained.value)) { + destroy_per_dim(q); + q->min_ = bounds->min; + q->max_ = bounds->max; + } else { + auto owned = std::make_unique>(std::move(trained)); + destroy_per_dim(q); + q->addr = reinterpret_cast(owned.release()); + q->dtype = dataset_tensor->dl_tensor.dtype; + q->min_ = static_cast(cuda::std::numeric_limits::min()); + q->max_ = static_cast(cuda::std::numeric_limits::max()); + } }); }); } @@ -201,8 +230,15 @@ extern "C" cuvsError_t cuvsScalarQuantizerTransform(cuvsResources_t res, DLManagedTensor* out_tensor) { return cuvs::core::translate_exceptions([=] { - dispatch_float(dataset_tensor->dl_tensor.dtype, [&]() { - _transform(res, quantizer, dataset_tensor, out_tensor); + RAFT_EXPECTS(quantizer && dataset_tensor && out_tensor, + "quantizer, dataset, and out must not be null"); + auto q = static_cast(quantizer); + dispatch_float(q->addr ? q->dtype : dataset_tensor->dl_tensor.dtype, [&]() { + if (q->addr) { + _transform(res, *reinterpret_cast*>(q->addr), dataset_tensor, out_tensor); + } else { + _transform(res, quantizer, dataset_tensor, out_tensor); + } }); }); } @@ -213,72 +249,14 @@ cuvsError_t cuvsScalarQuantizerInverseTransform(cuvsResources_t res, DLManagedTensor* out) { return cuvs::core::translate_exceptions([=] { - dispatch_float(out->dl_tensor.dtype, [&]() { - _inverse_transform(res, quantizer, dataset, out); - }); - }); -} - -extern "C" cuvsError_t cuvsScalarQuantizerPerDimCreate(cuvsScalarQuantizerPerDim_t* quantizer) -{ - return cuvs::core::translate_exceptions([=] { - RAFT_EXPECTS(quantizer != nullptr, "quantizer must not be null"); - *quantizer = new cuvsScalarQuantizerPerDim{}; - }); -} - -extern "C" cuvsError_t cuvsScalarQuantizerPerDimDestroy(cuvsScalarQuantizerPerDim_t quantizer) -{ - return cuvs::core::translate_exceptions([=] { - if (quantizer) { - destroy_per_dim(quantizer); - delete quantizer; - } - }); -} - -extern "C" cuvsError_t cuvsScalarQuantizerPerDimTrain(cuvsResources_t res, - cuvsScalarQuantizerParams_t params, - DLManagedTensor* dataset, - cuvsScalarQuantizerPerDim_t quantizer) -{ - return cuvs::core::translate_exceptions([=] { - RAFT_EXPECTS(params && dataset && quantizer, "params, dataset, and quantizer must not be null"); - dispatch_float(dataset->dl_tensor.dtype, [&]() { - auto trained = std::make_unique>( - _train(res, scalar::params{params->quantile, scalar::quantization_mode::PER_DIMENSION}, dataset)); - destroy_per_dim(quantizer); - quantizer->addr = reinterpret_cast(trained.release()); - quantizer->dtype = dataset->dl_tensor.dtype; - }); - }); -} - -extern "C" cuvsError_t cuvsScalarQuantizerPerDimTransform(cuvsResources_t res, - cuvsScalarQuantizerPerDim_t quantizer, - DLManagedTensor* dataset, - DLManagedTensor* out) -{ - return cuvs::core::translate_exceptions([=] { - RAFT_EXPECTS(quantizer && quantizer->addr, "quantizer must be trained before transforming"); - RAFT_EXPECTS(dataset && out, "dataset and out must not be null"); - dispatch_float(quantizer->dtype, [&]() { - _transform(res, *reinterpret_cast*>(quantizer->addr), dataset, out); - }); - }); -} - -extern "C" cuvsError_t cuvsScalarQuantizerPerDimInverseTransform( - cuvsResources_t res, - cuvsScalarQuantizerPerDim_t quantizer, - DLManagedTensor* dataset, - DLManagedTensor* out) -{ - return cuvs::core::translate_exceptions([=] { - RAFT_EXPECTS(quantizer && quantizer->addr, "quantizer must be trained before transforming"); - RAFT_EXPECTS(dataset && out, "dataset and out must not be null"); - dispatch_float(quantizer->dtype, [&]() { - _inverse_transform(res, *reinterpret_cast*>(quantizer->addr), dataset, out); + RAFT_EXPECTS(quantizer && dataset && out, "quantizer, dataset, and out must not be null"); + auto q = static_cast(quantizer); + dispatch_float(q->addr ? q->dtype : out->dl_tensor.dtype, [&]() { + if (q->addr) { + _inverse_transform(res, *reinterpret_cast*>(q->addr), dataset, out); + } else { + _inverse_transform(res, quantizer, dataset, out); + } }); }); } diff --git a/c/tests/preprocessing/scalar_c.cu b/c/tests/preprocessing/scalar_c.cu index 12cd7c1a6d..133df87a27 100644 --- a/c/tests/preprocessing/scalar_c.cu +++ b/c/tests/preprocessing/scalar_c.cu @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -33,25 +34,30 @@ class ScalarQuantizerC : public ::testing::Test { protected: cuvsResources_t res{}; cudaStream_t stream{}; - cuvsScalarQuantizerParams params{1.f}; - cuvsScalarQuantizerPerDim_t quantizer{}; + cuvsScalarQuantizerParams_t params{}; + cuvsScalarQuantizer_t quantizer{}; void SetUp() override { ASSERT_EQ(cuvsResourcesCreate(&res), CUVS_SUCCESS); ASSERT_EQ(cuvsStreamGet(res, &stream), CUVS_SUCCESS); - ASSERT_EQ(cuvsScalarQuantizerPerDimCreate(&quantizer), CUVS_SUCCESS); + ASSERT_EQ(cuvsScalarQuantizerParamsCreate(¶ms), CUVS_SUCCESS); + EXPECT_EQ(params->mode, CUVS_SCALAR_GLOBAL); + params->quantile = 1.f; + ASSERT_EQ(cuvsScalarQuantizerCreate(&quantizer), CUVS_SUCCESS); } void TearDown() override { EXPECT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); - EXPECT_EQ(cuvsScalarQuantizerPerDimDestroy(quantizer), CUVS_SUCCESS); + EXPECT_EQ(cuvsScalarQuantizerDestroy(quantizer), CUVS_SUCCESS); + EXPECT_EQ(cuvsScalarQuantizerParamsDestroy(params), CUVS_SUCCESS); EXPECT_EQ(cuvsResourcesDestroy(res), CUVS_SUCCESS); } void check_round_trip(bool host_training) { + params->mode = CUVS_SCALAR_PER_DIMENSION; int64_t shape[]{2, 3}; std::vector values{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; rmm::device_uvector input(values.size(), stream); @@ -60,10 +66,12 @@ class ScalarQuantizerC : public ::testing::Test { auto host = matrix_tensor(values.data(), kDLCPU, shape); auto device = matrix_tensor(input.data(), kDLCUDA, shape); auto encoded = matrix_tensor(codes.data(), kDLCUDA, shape); - ASSERT_EQ(cuvsScalarQuantizerPerDimTrain(res, ¶ms, host_training ? &host : &device, quantizer), + ASSERT_EQ(cuvsScalarQuantizerTrain(res, params, host_training ? &host : &device, quantizer), CUVS_SUCCESS); - ASSERT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); - ASSERT_EQ(cuvsScalarQuantizerPerDimInverseTransform(res, quantizer, &encoded, &device), CUVS_SUCCESS); + EXPECT_EQ(quantizer->min_, double(cuda::std::numeric_limits::min())); + EXPECT_EQ(quantizer->max_, double(cuda::std::numeric_limits::max())); + ASSERT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); + ASSERT_EQ(cuvsScalarQuantizerInverseTransform(res, quantizer, &encoded, &device), CUVS_SUCCESS); std::vector actual_codes(values.size()); std::vector decoded(values.size()); raft::update_host(actual_codes.data(), codes.data(), actual_codes.size(), stream); @@ -96,15 +104,24 @@ TYPED_TEST(ScalarQuantizerC, GlobalBoundsAndRoundTrip) auto input = matrix_tensor(values.data(), kDLCPU, shape); auto encoded = matrix_tensor(codes.data(), kDLCPU, shape); auto output = matrix_tensor(decoded.data(), kDLCPU, shape); - cuvsScalarQuantizer global{}; - ASSERT_EQ(cuvsScalarQuantizerTrain(this->res, &this->params, &input, &global), CUVS_SUCCESS); - EXPECT_EQ(global.min_, -3); - EXPECT_EQ(global.max_, 7); - ASSERT_EQ(cuvsScalarQuantizerTransform(this->res, &global, &input, &encoded), CUVS_SUCCESS); - ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, &global, &encoded, &output), CUVS_SUCCESS); + this->check_round_trip(false); + this->params->mode = CUVS_SCALAR_GLOBAL; + auto global = this->quantizer; + ASSERT_EQ(cuvsScalarQuantizerTrain(this->res, this->params, &input, global), CUVS_SUCCESS); + EXPECT_EQ(global->min_, -3); + EXPECT_EQ(global->max_, 7); + ASSERT_EQ(cuvsScalarQuantizerTransform(this->res, global, &input, &encoded), CUVS_SUCCESS); + ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, global, &encoded, &output), CUVS_SUCCESS); EXPECT_EQ(codes, (std::vector{-128, 127})); EXPECT_EQ(double(decoded[0]), -3); EXPECT_EQ(double(decoded[1]), 7); + // Global bounds remain directly editable, as in the original C API. + global->min_ = 1; + global->max_ = 5; + ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, global, &encoded, &output), CUVS_SUCCESS); + EXPECT_EQ(double(decoded[0]), 1); + EXPECT_EQ(double(decoded[1]), 5); + this->check_round_trip(true); } using ScalarQuantizerCFloat = ScalarQuantizerC; @@ -118,25 +135,30 @@ TEST_F(ScalarQuantizerCFloat, InvalidArgumentsPreserveTrainedQuantizer) auto host = matrix_tensor(values.data(), kDLCPU, shape); auto device = matrix_tensor(input.data(), kDLCUDA, shape); auto encoded = matrix_tensor(codes.data(), kDLCUDA, shape); - EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); - ASSERT_EQ(cuvsScalarQuantizerPerDimTrain(res, ¶ms, &host, quantizer), CUVS_SUCCESS); - auto address = quantizer->addr; - params.quantile = 0; - EXPECT_EQ(cuvsScalarQuantizerPerDimTrain(res, ¶ms, &host, quantizer), CUVS_ERROR); - EXPECT_EQ(quantizer->addr, address); - EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); + params->mode = CUVS_SCALAR_PER_DIMENSION; + ASSERT_EQ(cuvsScalarQuantizerTrain(res, params, &host, quantizer), CUVS_SUCCESS); + params->quantile = 0; + EXPECT_EQ(cuvsScalarQuantizerTrain(res, params, &host, quantizer), CUVS_ERROR); + params->quantile = 1; + params->mode = static_cast(2); + EXPECT_EQ(cuvsScalarQuantizerTrain(res, params, &host, quantizer), CUVS_ERROR); + EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); + std::vector actual_codes(values.size()); + raft::update_host(actual_codes.data(), codes.data(), actual_codes.size(), stream); + ASSERT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); + EXPECT_EQ(actual_codes, (std::vector{-127, -127, 0, 127, 127, 0})); device.dl_tensor.dtype.bits = 64; - EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); + EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_ERROR); device.dl_tensor.dtype.bits = 32; encoded.dl_tensor.dtype.code = kDLUInt; - EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); + EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_ERROR); encoded.dl_tensor.dtype.code = kDLInt; int64_t wrong_shape[]{2, 2}; device.dl_tensor.shape = wrong_shape; - EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &device, &encoded), CUVS_ERROR); + EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_ERROR); device.dl_tensor.shape = shape; std::vector host_codes(values.size()); auto host_encoded = matrix_tensor(host_codes.data(), kDLCPU, shape); - EXPECT_EQ(cuvsScalarQuantizerPerDimTransform(res, quantizer, &host, &host_encoded), CUVS_ERROR); - EXPECT_EQ(cuvsScalarQuantizerPerDimInverseTransform(res, quantizer, &host_encoded, &host), CUVS_ERROR); + EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &host, &host_encoded), CUVS_ERROR); + EXPECT_EQ(cuvsScalarQuantizerInverseTransform(res, quantizer, &host_encoded, &host), CUVS_ERROR); } diff --git a/cpp/src/preprocessing/quantize/detail/scalar.cuh b/cpp/src/preprocessing/quantize/detail/scalar.cuh index 703fcb0dd8..d4549eecd9 100644 --- a/cpp/src/preprocessing/quantize/detail/scalar.cuh +++ b/cpp/src/preprocessing/quantize/detail/scalar.cuh @@ -158,7 +158,7 @@ template auto round_error_up(double error) -> typename quantizer::scale_type { using bound_t = typename quantizer::scale_type; - // Match Sirius's conservative margin, then round upwards in the public result type. + // Apply a conservative margin, then round upwards in the public result type. auto const bound = static_cast(error * (1.0 + 1e-6)); return error > 0 ? std::nextafter(bound, std::numeric_limits::infinity()) : bound; } diff --git a/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md b/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md index 29ac8ff15f..efc3e4e5e7 100644 --- a/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md +++ b/fern/pages/c_api/c-api-preprocessing-quantize-scalar.md @@ -8,15 +8,44 @@ _Source header: `cuvs/preprocessing/quantize/scalar.h`_ ## C API for Scalar Quantizer + +### cuvsScalarQuantizationMode + +Select global or per-dimension scalar quantization bounds. + +```c +enum cuvsScalarQuantizationMode { + CUVS_SCALAR_GLOBAL = 0, + CUVS_SCALAR_PER_DIMENSION = 1 +}; +``` + +**Values** + +| Name | Value | +| --- | --- | +| `CUVS_SCALAR_GLOBAL` | `0` | +| `CUVS_SCALAR_PER_DIMENSION` | `1` | + ### cuvsScalarQuantizerParams -Scalar quantizer parameters. +Scalar quantizer parameters. Allocate with cuvsScalarQuantizerParamsCreate. ```c -struct cuvsScalarQuantizerParams; +struct cuvsScalarQuantizerParams { + float quantile; + enum cuvsScalarQuantizationMode mode; +}; ``` +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `quantile` | `float` | specifies how many outliers at top & bottom will be ignored needs to be within range of (0, 1] | +| `mode` | [`enum cuvsScalarQuantizationMode`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizationmode) | Quantization mode.
Defaults to CUVS_SCALAR_GLOBAL. | + ### cuvsScalarQuantizerParamsCreate @@ -60,7 +89,7 @@ cuvsError_t cuvsScalarQuantizerParamsDestroy(cuvsScalarQuantizerParams_t params) Defines and stores scalar for quantisation upon training -The quantization is performed by a linear mapping of an interval in the float data type to the full range of the quantized int type. +The quantization is performed by a linear mapping of an interval in the float data type to the full range of the quantized int type. Allocate with cuvsScalarQuantizerCreate and release with cuvsScalarQuantizerDestroy; do not copy the handle's contents. The implementation also owns any per-dimension bounds. In global mode, min_ and max_ are the trained bounds. In per-dimension mode, they are numeric_limits<T>::min() and numeric_limits<T>::max() for the training data type T, and are not used for transformation. ```c typedef struct { @@ -98,7 +127,7 @@ cuvsError_t cuvsScalarQuantizerCreate(cuvsScalarQuantizer_t* quantizer); ### cuvsScalarQuantizerDestroy -De-allocate Scalar Quantizer +De-allocate Scalar Quantizer after outstanding stream work completes. ```c cuvsError_t cuvsScalarQuantizerDestroy(cuvsScalarQuantizer_t quantizer); @@ -126,6 +155,8 @@ DLManagedTensor* dataset, cuvsScalarQuantizer_t quantizer); ``` +Supports float16, float32, and float64 in global or per-dimension mode. Per-dimension mode applies the quantile independently to each dimension using every row. Successful retraining replaces the previous bounds; failed training preserves them. Complete outstanding stream work using the quantizer before retraining. + **Parameters** | Name | Direction | Type | Description | @@ -133,7 +164,7 @@ cuvsScalarQuantizer_t quantizer); | `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | raft resource | | `params` | in | [`cuvsScalarQuantizerParams_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerparams) | configure scalar quantizer, e.g. quantile | | `dataset` | in | `DLManagedTensor*` | a row-major host or device matrix | -| `quantizer` | out | [`cuvsScalarQuantizer_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizer) | trained scalar quantizer | +| `quantizer` | in,out | [`cuvsScalarQuantizer_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizer) | scalar quantizer allocated by cuvsScalarQuantizerCreate | **Returns** @@ -151,6 +182,8 @@ DLManagedTensor* dataset, DLManagedTensor* out); ``` +Per-dimension mode requires device matrices with the same input type and width as training. Its output codes are int8 in [-127, 127]. The quantizer must outlive stream work using its bounds. + **Parameters** | Name | Direction | Type | Description | @@ -176,7 +209,7 @@ DLManagedTensor* dataset, DLManagedTensor* out); ``` -Note that depending on the chosen data types train dataset the conversion is not lossless. +Note that depending on the chosen data types train dataset the conversion is not lossless. Per-dimension mode requires device matrices with the same output type and width as training. The quantizer must outlive stream work using its bounds. **Parameters** @@ -190,136 +223,3 @@ Note that depending on the chosen data types train dataset the conversion is not **Returns** [`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) - - -### cuvsScalarQuantizerPerDim - -Owns a scalar quantizer with separate min/max bounds for each dimension. The fields are managed by the lifecycle and training functions. - -```c -typedef struct { - uintptr_t addr; - DLDataType dtype; -} cuvsScalarQuantizerPerDim; -``` - -**Fields** - -| Name | Type | Description | -| --- | --- | --- | -| `addr` | `uintptr_t` | | -| `dtype` | `DLDataType` | | - - -### cuvsScalarQuantizerPerDimCreate - -Allocate an untrained per-dimension scalar quantizer. - -```c -cuvsError_t cuvsScalarQuantizerPerDimCreate(cuvsScalarQuantizerPerDim_t* quantizer); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `quantizer` | out | [`cuvsScalarQuantizerPerDim_t*`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Handle to allocate. | - -**Returns** - -[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) - - -### cuvsScalarQuantizerPerDimDestroy - -Release the quantizer and its device bounds after outstanding stream work completes. - -```c -cuvsError_t cuvsScalarQuantizerPerDimDestroy(cuvsScalarQuantizerPerDim_t quantizer); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `quantizer` | in | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Handle to destroy. | - -**Returns** - -[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) - - -### cuvsScalarQuantizerPerDimTrain - -Train per-dimension bounds from every row of a host or device matrix. Supports float16, float32, and float64. The quantile is applied independently per dimension. Successful retraining replaces the previous bounds; failed training preserves them. Complete outstanding stream work using the quantizer before retraining. - -```c -cuvsError_t cuvsScalarQuantizerPerDimTrain(cuvsResources_t res, -cuvsScalarQuantizerParams_t params, -DLManagedTensor* dataset, -cuvsScalarQuantizerPerDim_t quantizer); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | Resources used for training and allocation. | -| `params` | in | [`cuvsScalarQuantizerParams_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerparams) | Scalar quantizer parameters. | -| `dataset` | in | `DLManagedTensor*` | Row-major host or device matrix. | -| `quantizer` | in,out | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Handle created by cuvsScalarQuantizerPerDimCreate. | - -**Returns** - -[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) - - -### cuvsScalarQuantizerPerDimTransform - -Quantize a device matrix using previously trained per-dimension bounds. The input type and width must match training. Output codes are int8, in [-127, 127]. The quantizer must outlive stream work using its bounds. - -```c -cuvsError_t cuvsScalarQuantizerPerDimTransform(cuvsResources_t res, -cuvsScalarQuantizerPerDim_t quantizer, -DLManagedTensor* dataset, -DLManagedTensor* out); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | Resources used for execution. | -| `quantizer` | in | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Trained per-dimension scalar quantizer. | -| `dataset` | in | `DLManagedTensor*` | Row-major device matrix to encode. | -| `out` | out | `DLManagedTensor*` | Row-major device int8 matrix with the same shape. | - -**Returns** - -[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) - - -### cuvsScalarQuantizerPerDimInverseTransform - -Reconstruct a device matrix from per-dimension scalar codes. The output type and width must match training. The quantizer must outlive stream work using its bounds. - -```c -cuvsError_t cuvsScalarQuantizerPerDimInverseTransform( -cuvsResources_t res, -cuvsScalarQuantizerPerDim_t quantizer, -DLManagedTensor* dataset, -DLManagedTensor* out); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | Resources used for execution. | -| `quantizer` | in | [`cuvsScalarQuantizerPerDim_t`](/api-reference/c-api-preprocessing-quantize-scalar#cuvsscalarquantizerperdim) | Trained per-dimension scalar quantizer. | -| `dataset` | in | `DLManagedTensor*` | Row-major device int8 matrix to decode. | -| `out` | out | `DLManagedTensor*` | Row-major device matrix with the same shape, in the training data type. | - -**Returns** - -[`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) From 643b31a5f5350cfe17a7f4e4525f63d76560e855 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Sat, 7 Nov 2026 00:13:47 +0000 Subject: [PATCH 24/24] Simplify scalar quantization tests and reuse DLPack conversion --- c/tests/preprocessing/scalar_c.cu | 173 +++--- .../preprocessing/scalar_quantization.cu | 588 +++++++----------- 2 files changed, 327 insertions(+), 434 deletions(-) diff --git a/c/tests/preprocessing/scalar_c.cu b/c/tests/preprocessing/scalar_c.cu index 133df87a27..3ed1d52d51 100644 --- a/c/tests/preprocessing/scalar_c.cu +++ b/c/tests/preprocessing/scalar_c.cu @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "../../src/core/interop.hpp" #include #include @@ -11,24 +12,11 @@ #include #include -#include +#include #include namespace { -template -DLManagedTensor matrix_tensor(T* data, DLDeviceType device, int64_t* shape) -{ - DLManagedTensor tensor{}; - tensor.dl_tensor.data = data; - tensor.dl_tensor.device = {device, 0}; - tensor.dl_tensor.ndim = 2; - tensor.dl_tensor.dtype = {std::is_same_v ? uint8_t{kDLInt} : uint8_t{kDLFloat}, - uint8_t{sizeof(T) * 8}, 1}; - tensor.dl_tensor.shape = shape; - return tensor; -} - template class ScalarQuantizerC : public ::testing::Test { protected: @@ -36,6 +24,11 @@ class ScalarQuantizerC : public ::testing::Test { cudaStream_t stream{}; cuvsScalarQuantizerParams_t params{}; cuvsScalarQuantizer_t quantizer{}; + std::vector values{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; + std::vector host_codes = std::vector(values.size()); + rmm::device_uvector input{0, stream}; + rmm::device_uvector codes{0, stream}; + DLManagedTensor host{}, device{}, encoded{}, host_encoded{}; void SetUp() override { @@ -45,6 +38,14 @@ class ScalarQuantizerC : public ::testing::Test { EXPECT_EQ(params->mode, CUVS_SCALAR_GLOBAL); params->quantile = 1.f; ASSERT_EQ(cuvsScalarQuantizerCreate(&quantizer), CUVS_SUCCESS); + input.resize(values.size(), stream); + codes.resize(values.size(), stream); + raft::update_device(input.data(), values.data(), values.size(), stream); + cuvs::core::to_dlpack(raft::make_host_matrix_view(values.data(), 2, 3), &host); + cuvs::core::to_dlpack(raft::make_device_matrix_view(input.data(), 2, 3), &device); + cuvs::core::to_dlpack(raft::make_device_matrix_view(codes.data(), 2, 3), &encoded); + cuvs::core::to_dlpack(raft::make_host_matrix_view(host_codes.data(), 2, 3), &host_encoded); + ASSERT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); } void TearDown() override @@ -52,35 +53,28 @@ class ScalarQuantizerC : public ::testing::Test { EXPECT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); EXPECT_EQ(cuvsScalarQuantizerDestroy(quantizer), CUVS_SUCCESS); EXPECT_EQ(cuvsScalarQuantizerParamsDestroy(params), CUVS_SUCCESS); + for (auto tensor : {&host, &device, &encoded, &host_encoded}) { + if (tensor->deleter) { tensor->deleter(tensor); } + } EXPECT_EQ(cuvsResourcesDestroy(res), CUVS_SUCCESS); } - void check_round_trip(bool host_training) + cuvsError_t train(cuvsScalarQuantizationMode mode, DLManagedTensor& data) + { + params->mode = mode; + return cuvsScalarQuantizerTrain(res, params, &data, quantizer); + } + + void check_per_dimension_round_trip() { - params->mode = CUVS_SCALAR_PER_DIMENSION; - int64_t shape[]{2, 3}; - std::vector values{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; - rmm::device_uvector input(values.size(), stream); - rmm::device_uvector codes(values.size(), stream); - raft::update_device(input.data(), values.data(), values.size(), stream); - auto host = matrix_tensor(values.data(), kDLCPU, shape); - auto device = matrix_tensor(input.data(), kDLCUDA, shape); - auto encoded = matrix_tensor(codes.data(), kDLCUDA, shape); - ASSERT_EQ(cuvsScalarQuantizerTrain(res, params, host_training ? &host : &device, quantizer), - CUVS_SUCCESS); - EXPECT_EQ(quantizer->min_, double(cuda::std::numeric_limits::min())); - EXPECT_EQ(quantizer->max_, double(cuda::std::numeric_limits::max())); ASSERT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); ASSERT_EQ(cuvsScalarQuantizerInverseTransform(res, quantizer, &encoded, &device), CUVS_SUCCESS); - std::vector actual_codes(values.size()); std::vector decoded(values.size()); - raft::update_host(actual_codes.data(), codes.data(), actual_codes.size(), stream); + raft::update_host(host_codes.data(), codes.data(), host_codes.size(), stream); raft::update_host(decoded.data(), input.data(), decoded.size(), stream); ASSERT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); - EXPECT_EQ(actual_codes, (std::vector{-127, -127, 0, 127, 127, 0})); - for (size_t i = 0; i < values.size(); ++i) { - EXPECT_EQ(double(decoded[i]), double(values[i])) << i; - } + EXPECT_EQ(host_codes, (std::vector{-127, -127, 0, 127, 127, 0})); + EXPECT_EQ(decoded, values); } }; @@ -89,76 +83,81 @@ class ScalarQuantizerC : public ::testing::Test { using ScalarTypes = ::testing::Types; TYPED_TEST_SUITE(ScalarQuantizerC, ScalarTypes); -TYPED_TEST(ScalarQuantizerC, PerDimensionTrainAndRetrain) +TYPED_TEST(ScalarQuantizerC, PerDimensionRoundTrip) { - this->check_round_trip(false); - this->check_round_trip(true); + ASSERT_EQ(this->train(CUVS_SCALAR_PER_DIMENSION, this->device), CUVS_SUCCESS); + EXPECT_EQ(this->quantizer->min_, double(cuda::std::numeric_limits::min())); + EXPECT_EQ(this->quantizer->max_, double(cuda::std::numeric_limits::max())); + this->check_per_dimension_round_trip(); } TYPED_TEST(ScalarQuantizerC, GlobalBoundsAndRoundTrip) { - using T = TypeParam; - int64_t shape[]{2, 1}; - std::vector values{T{-3}, T{7}}, decoded(2); - std::vector codes(2); - auto input = matrix_tensor(values.data(), kDLCPU, shape); - auto encoded = matrix_tensor(codes.data(), kDLCPU, shape); - auto output = matrix_tensor(decoded.data(), kDLCPU, shape); - this->check_round_trip(false); - this->params->mode = CUVS_SCALAR_GLOBAL; - auto global = this->quantizer; - ASSERT_EQ(cuvsScalarQuantizerTrain(this->res, this->params, &input, global), CUVS_SUCCESS); - EXPECT_EQ(global->min_, -3); - EXPECT_EQ(global->max_, 7); - ASSERT_EQ(cuvsScalarQuantizerTransform(this->res, global, &input, &encoded), CUVS_SUCCESS); - ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, global, &encoded, &output), CUVS_SUCCESS); - EXPECT_EQ(codes, (std::vector{-128, 127})); - EXPECT_EQ(double(decoded[0]), -3); - EXPECT_EQ(double(decoded[1]), 7); - // Global bounds remain directly editable, as in the original C API. - global->min_ = 1; - global->max_ = 5; - ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, global, &encoded, &output), CUVS_SUCCESS); - EXPECT_EQ(double(decoded[0]), 1); - EXPECT_EQ(double(decoded[1]), 5); - this->check_round_trip(true); + std::fill_n(this->values.begin(), 3, TypeParam{-3}); + std::fill_n(this->values.begin() + 3, 3, TypeParam{7}); + auto original = this->values; + ASSERT_EQ(this->train(CUVS_SCALAR_GLOBAL, this->host), CUVS_SUCCESS); + EXPECT_EQ(this->quantizer->min_, -3); + EXPECT_EQ(this->quantizer->max_, 7); + ASSERT_EQ(cuvsScalarQuantizerTransform(this->res, this->quantizer, &this->host, &this->host_encoded), CUVS_SUCCESS); + EXPECT_EQ(this->host_codes, (std::vector{-128, -128, -128, 127, 127, 127})); + ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, this->quantizer, &this->host_encoded, &this->host), CUVS_SUCCESS); + EXPECT_EQ(this->values, original); + this->quantizer->min_ = 1; + this->quantizer->max_ = 5; + ASSERT_EQ(cuvsScalarQuantizerInverseTransform(this->res, this->quantizer, &this->host_encoded, &this->host), CUVS_SUCCESS); + EXPECT_EQ(this->values, (std::vector{1.f, 1.f, 1.f, 5.f, 5.f, 5.f})); } using ScalarQuantizerCFloat = ScalarQuantizerC; -TEST_F(ScalarQuantizerCFloat, InvalidArgumentsPreserveTrainedQuantizer) +TEST_F(ScalarQuantizerCFloat, HostRetraining) { - int64_t shape[]{2, 3}; - std::vector values{0, 10, 9, 254, 137, 9}; - rmm::device_uvector input(values.size(), stream); - rmm::device_uvector codes(values.size(), stream); - raft::update_device(input.data(), values.data(), values.size(), stream); - auto host = matrix_tensor(values.data(), kDLCPU, shape); - auto device = matrix_tensor(input.data(), kDLCUDA, shape); - auto encoded = matrix_tensor(codes.data(), kDLCUDA, shape); - params->mode = CUVS_SCALAR_PER_DIMENSION; - ASSERT_EQ(cuvsScalarQuantizerTrain(res, params, &host, quantizer), CUVS_SUCCESS); + ASSERT_EQ(train(CUVS_SCALAR_PER_DIMENSION, device), CUVS_SUCCESS); + ASSERT_EQ(train(CUVS_SCALAR_PER_DIMENSION, host), CUVS_SUCCESS); + check_per_dimension_round_trip(); +} + +TEST_F(ScalarQuantizerCFloat, RetrainingChangesMode) +{ + ASSERT_EQ(train(CUVS_SCALAR_GLOBAL, host), CUVS_SUCCESS); + ASSERT_EQ(train(CUVS_SCALAR_PER_DIMENSION, host), CUVS_SUCCESS); + check_per_dimension_round_trip(); + ASSERT_EQ(train(CUVS_SCALAR_GLOBAL, host), CUVS_SUCCESS); + EXPECT_EQ(quantizer->min_, 0); + EXPECT_EQ(quantizer->max_, 254); + EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &host, &host_encoded), CUVS_SUCCESS); +} + +TEST_F(ScalarQuantizerCFloat, InvalidTrainingPreservesQuantizer) +{ + ASSERT_EQ(train(CUVS_SCALAR_PER_DIMENSION, host), CUVS_SUCCESS); params->quantile = 0; - EXPECT_EQ(cuvsScalarQuantizerTrain(res, params, &host, quantizer), CUVS_ERROR); + EXPECT_EQ(train(CUVS_SCALAR_PER_DIMENSION, host), CUVS_ERROR); params->quantile = 1; - params->mode = static_cast(2); - EXPECT_EQ(cuvsScalarQuantizerTrain(res, params, &host, quantizer), CUVS_ERROR); - EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_SUCCESS); - std::vector actual_codes(values.size()); - raft::update_host(actual_codes.data(), codes.data(), actual_codes.size(), stream); - ASSERT_EQ(cuvsStreamSync(res), CUVS_SUCCESS); - EXPECT_EQ(actual_codes, (std::vector{-127, -127, 0, 127, 127, 0})); + EXPECT_EQ(train(static_cast(2), host), CUVS_ERROR); + check_per_dimension_round_trip(); +} + +TEST_F(ScalarQuantizerCFloat, RejectsMismatchedTypes) +{ + ASSERT_EQ(train(CUVS_SCALAR_PER_DIMENSION, host), CUVS_SUCCESS); device.dl_tensor.dtype.bits = 64; EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_ERROR); device.dl_tensor.dtype.bits = 32; encoded.dl_tensor.dtype.code = kDLUInt; EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_ERROR); - encoded.dl_tensor.dtype.code = kDLInt; - int64_t wrong_shape[]{2, 2}; - device.dl_tensor.shape = wrong_shape; +} + +TEST_F(ScalarQuantizerCFloat, RejectsMismatchedWidth) +{ + ASSERT_EQ(train(CUVS_SCALAR_PER_DIMENSION, host), CUVS_SUCCESS); + device.dl_tensor.shape[1] = 2; EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &device, &encoded), CUVS_ERROR); - device.dl_tensor.shape = shape; - std::vector host_codes(values.size()); - auto host_encoded = matrix_tensor(host_codes.data(), kDLCPU, shape); +} + +TEST_F(ScalarQuantizerCFloat, RejectsHostPerDimensionTransforms) +{ + ASSERT_EQ(train(CUVS_SCALAR_PER_DIMENSION, host), CUVS_SUCCESS); EXPECT_EQ(cuvsScalarQuantizerTransform(res, quantizer, &host, &host_encoded), CUVS_ERROR); EXPECT_EQ(cuvsScalarQuantizerInverseTransform(res, quantizer, &host_encoded, &host), CUVS_ERROR); } diff --git a/cpp/tests/preprocessing/scalar_quantization.cu b/cpp/tests/preprocessing/scalar_quantization.cu index ff942647c4..03022edbab 100644 --- a/cpp/tests/preprocessing/scalar_quantization.cu +++ b/cpp/tests/preprocessing/scalar_quantization.cu @@ -39,18 +39,32 @@ long double expectReconstructionErrorBound( return maximum; } +template +auto makeDeviceInput(raft::resources const& res, std::vector const& values, int64_t dim) +{ + auto input = raft::make_device_matrix(res, values.size() / dim, dim); + raft::update_device( + input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); + // The caller may pass a temporary host vector. + raft::resource::sync_stream(res); + return input; +} + +template +std::vector copyToHost(raft::resources const& res, const T* data, size_t size) +{ + std::vector values(size); + raft::update_host(values.data(), data, size, raft::resource::get_cuda_stream(res)); + raft::resource::sync_stream(res); + return values; +} + template void expectDeviceValues(raft::resources const& res, const T* device_values, std::vector const& expected) { - std::vector actual(expected.size()); - raft::update_host( - actual.data(), device_values, actual.size(), raft::resource::get_cuda_stream(res)); - raft::resource::sync_stream(res); - for (size_t i = 0; i < actual.size(); ++i) { - EXPECT_EQ(static_cast(actual[i]), static_cast(expected[i])) << i; - } + EXPECT_EQ(copyToHost(res, device_values, expected.size()), expected); } template @@ -164,12 +178,9 @@ class QuantizationTest : public ::testing::TestWithParam> }, cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); - std::cerr - << "Q1: min = " - << (double)std::get(quantizer_1.value).min - << ", max = " - << (double)std::get(quantizer_1.value).max - << std::endl; + auto bounds_1 = std::get::global_type>(quantizer_1.value); + std::cerr << "Q1: min = " << (double)bounds_1.min << ", max = " << (double)bounds_1.max + << std::endl; { auto quantized_input_h = raft::make_host_matrix(rows_, cols_); @@ -262,21 +273,14 @@ class QuantizationTest : public ::testing::TestWithParam> }, cuvs::alloc_behavior::DATA_DRIVEN, min_train_alloc); - std::cerr - << "Q2: min = " - << (double)std::get(quantizer_2.value).min - << ", max = " - << (double)std::get(quantizer_2.value).max - << std::endl; + auto bounds_2 = std::get::global_type>(quantizer_2.value); + std::cerr << "Q2: min = " << (double)bounds_2.min << ", max = " << (double)bounds_2.max + << std::endl; // check both quantizers are the same (valid if sampling is identical) if (input_.size() <= 1000000) { - ASSERT_TRUE( - (double)std::get(quantizer_1.value).min == - (double)std::get(quantizer_2.value).min); - ASSERT_TRUE( - (double)std::get(quantizer_1.value).max == - (double)std::get(quantizer_2.value).max); + ASSERT_TRUE((double)bounds_1.min == (double)bounds_2.min); + ASSERT_TRUE((double)bounds_1.max == (double)bounds_2.max); } { @@ -359,77 +363,79 @@ class QuantizationTest : public ::testing::TestWithParam> using bounds_view = quantizer_view::per_dim_type; -struct PerDimensionInputs { - int64_t n_samples; - int64_t n_features; - std::vector dataset; - std::vector codes; - float tolerance; -}; - -std::ostream& operator<<(std::ostream& os, const PerDimensionInputs& inputs) +template +void expectPerDimensionBounds(raft::resources const& res, + quantizer const& owner, + std::vector const& mins, + std::vector const& maxs) { - return os << "n_samples:" << inputs.n_samples << " n_features:" << inputs.n_features - << " tolerance:" << inputs.tolerance; + auto bounds = std::get::per_dim_type>(owner.view().value); + expectDeviceValues(res, bounds.mins.data_handle(), mins); + expectDeviceValues(res, bounds.maxs.data_handle(), maxs); } -class PerDimensionRangeTest : public ::testing::TestWithParam { - public: - PerDimensionRangeTest() - : params_(GetParam()), - stream(raft::resource::get_cuda_stream(handle).get()), - dataset_( - raft::make_device_matrix(handle, params_.n_samples, params_.n_features)) - { +void testPerDimensionRange(std::vector const& values, + int64_t dim, + std::vector const& expected, + float tolerance = 0.f) +{ + raft::resources res; + auto input = makeDeviceInput(res, values, dim); + auto owner = train( + res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + auto codes = raft::make_device_matrix(res, input.extent(0), dim); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + expectDeviceValues(res, codes.data_handle(), expected); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); + auto decoded = copyToHost(res, input.data_handle(), values.size()); + for (size_t i = 0; i < values.size(); ++i) { + EXPECT_TRUE(std::isfinite(decoded[i])); + EXPECT_NEAR(decoded[i], values[i], tolerance) << i; } - - protected: - void SetUp() override - { - raft::update_device( - dataset_.data_handle(), params_.dataset.data(), params_.dataset.size(), stream); + expectReconstructionErrorBound( + values.data(), decoded.data(), input.extent(0), dim, owner.max_error); + if (tolerance == 0.f) { + auto bounds = std::get(std::as_const(owner).view().value); + EXPECT_EQ(copyToHost(res, bounds.mins.data_handle(), dim), + copyToHost(res, bounds.maxs.data_handle(), dim)); + EXPECT_EQ(owner.max_error, 0.f); } +} - void testPerDimensionQuantization() - { - auto quantizer = train(handle, - params{.range = quantization_mode::PER_DIMENSION}, - raft::make_const_mdspan(dataset_.view())); - auto codes = - raft::make_device_matrix(handle, params_.n_samples, params_.n_features); - transform(handle, quantizer, raft::make_const_mdspan(dataset_.view()), codes.view()); - expectDeviceValues(handle, codes.data_handle(), params_.codes); - if (params_.tolerance == 0.f) { - auto bounds = std::get(std::as_const(quantizer).view().value); - EXPECT_TRUE(devArrMatch(bounds.mins.data_handle(), - bounds.maxs.data_handle(), - params_.n_features, - cuvs::Compare(), - stream)); - } - inverse_transform(handle, quantizer, raft::make_const_mdspan(codes.view()), dataset_.view()); - auto reconstructed = - raft::make_host_matrix(params_.n_samples, params_.n_features); - raft::update_host(reconstructed.data_handle(), dataset_.data_handle(), dataset_.size(), stream); - raft::resource::sync_stream(handle); - for (size_t i = 0; i < params_.dataset.size(); ++i) { - EXPECT_TRUE(std::isfinite(reconstructed.data_handle()[i])); - EXPECT_NEAR(reconstructed.data_handle()[i], params_.dataset[i], params_.tolerance); - } - expectReconstructionErrorBound(params_.dataset.data(), - reconstructed.data_handle(), - params_.n_samples, - params_.n_features, - quantizer.max_error); - if (params_.tolerance == 0.f) { EXPECT_EQ(quantizer.max_error, 0.f); } - } +TEST(PerDimensionRangeTest, IndependentRanges) +{ + testPerDimensionRange( + {-127.f, -127.f / 128.f, 9.f, 1.f, 1.f / 128.f, 9.f, 127.f, 127.f / 128.f, 9.f}, + 3, + {-127, -127, 0, 1, 1, 0, 127, 127, 0}, + 1e-6f); +} - private: - raft::resources handle; - PerDimensionInputs params_; - cudaStream_t stream; - raft::device_matrix dataset_; -}; +TEST(PerDimensionRangeTest, ConstantDimensions) +{ + testPerDimensionRange({10.f, -7.f, 10.f, -7.f}, 2, {0, 0, 0, 0}); +} + +TEST(PerDimensionRangeTest, ConstantSubnormalDimensions) +{ + auto tiny = std::numeric_limits::denorm_min(); + testPerDimensionRange({tiny, -tiny}, 2, {0, 0}); +} + +TEST(PerDimensionRangeTest, SymmetricExtremeBounds) +{ + auto limit = std::numeric_limits::max(); + testPerDimensionRange({-limit, limit}, 1, {-127, 127}, limit / 254.f); +} + +TEST(PerDimensionRangeTest, AsymmetricExtremeBounds) +{ + auto limit = std::numeric_limits::max(); + testPerDimensionRange({-std::ldexp(124.f, 104), limit}, + 1, + {-127, 127}, + (std::ldexp(124.f, 103) + limit * 0.5f) / 254.f); +} template class PerDimensionQuantizationTest : public ::testing::Test {}; @@ -440,24 +446,20 @@ template void testTrainingReconstructionError(params p) { raft::resources res; - auto stream = raft::resource::get_cuda_stream(res); // A partial warp plus a second column per lane, with clipped outliers in different rows. constexpr int64_t rows = 9, dim = 35; auto values = makeShiftedDataset(rows, dim); values.front() = T{-300.f}; values.back() = T{500.f}; - auto input = raft::make_device_matrix(res, rows, dim); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); - auto codes = raft::make_device_matrix(res, rows, dim); - auto decoded = raft::make_device_matrix(res, rows, dim); - std::vector reconstructed(values.size()); - auto owner = train(res, p, raft::make_const_mdspan(input.view())); - auto view = std::as_const(owner).view(); + auto input = makeDeviceInput(res, values, dim); + auto codes = raft::make_device_matrix(res, rows, dim); + auto decoded = raft::make_device_matrix(res, rows, dim); + auto owner = train(res, p, raft::make_const_mdspan(input.view())); + auto view = std::as_const(owner).view(); EXPECT_EQ(owner.max_error, view.max_error); transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), decoded.view()); - raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); - raft::resource::sync_stream(res); + auto reconstructed = copyToHost(res, decoded.data_handle(), values.size()); auto maximum = expectReconstructionErrorBound(values.data(), reconstructed.data(), rows, dim, owner.max_error); EXPECT_GT(maximum, 0); @@ -503,53 +505,41 @@ TYPED_TEST(PerDimensionQuantizationTest, PerDimensionClippedReconstructionError) testTrainingReconstructionError(params{.5f, quantization_mode::PER_DIMENSION}); } -TEST(PerDimensionRangeTest, DoubleReconstructionErrorExtremes) +void testDoubleReconstructionError(double scale) { raft::resources res; - auto stream = raft::resource::get_cuda_stream(res); - auto input = raft::make_device_matrix(res, 3, 2); - auto codes = raft::make_device_matrix(res, 3, 2); - auto decoded = raft::make_device_matrix(res, 3, 2); - // Squaring these residuals would underflow or overflow, although their L2 norm is finite. - for (double scale : {1e-300, 1e200}) { - SCOPED_TRACE(scale); - std::vector values{-scale, -scale, scale * 0.3, scale * 0.3, scale, scale}; - raft::update_device(input.data_handle(), values.data(), values.size(), stream); - auto owner = train( - res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); - transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); - inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), decoded.view()); - std::vector reconstructed(values.size()); - raft::update_host(reconstructed.data(), decoded.data_handle(), reconstructed.size(), stream); - raft::resource::sync_stream(res); - auto maximum = - expectReconstructionErrorBound(values.data(), reconstructed.data(), 3, 2, owner.max_error); - EXPECT_GT(maximum, 0); - EXPECT_TRUE(std::isfinite(owner.max_error)); - EXPECT_GE(owner.max_error, maximum); - EXPECT_NEAR(owner.max_error, maximum, maximum * 2e-6); - } + std::vector values{-scale, -scale, scale * 0.3, scale * 0.3, scale, scale}; + auto input = makeDeviceInput(res, values, 2); + auto owner = train( + res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); + auto codes = raft::make_device_matrix(res, 3, 2); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); + auto decoded = copyToHost(res, input.data_handle(), values.size()); + auto maximum = + expectReconstructionErrorBound(values.data(), decoded.data(), 3, 2, owner.max_error); + EXPECT_GT(maximum, 0); + EXPECT_TRUE(std::isfinite(owner.max_error)); + EXPECT_NEAR(owner.max_error, maximum, maximum * 2e-6); } +// Squaring these residuals would underflow or overflow, although their L2 norm is finite. +TEST(PerDimensionRangeTest, ReconstructionErrorUnderflow) { testDoubleReconstructionError(1e-300); } +TEST(PerDimensionRangeTest, ReconstructionErrorOverflow) { testDoubleReconstructionError(1e200); } + TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) { - using T = TypeParam; - using bounds_view = typename quantizer_view::per_dim_type; + using T = TypeParam; raft::resources res; auto stream = raft::resource::get_cuda_stream(res); std::vector corpus{0.f, 10.f, 9.f, 254.f, 137.f, 9.f}; - auto input = raft::make_device_matrix(res, 2, 3); - raft::update_device(input.data_handle(), corpus.data(), corpus.size(), stream); + auto input = makeDeviceInput(res, corpus, 3); auto owner = train( res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); auto quantizer = std::as_const(owner).view(); EXPECT_EQ(owner.max_error, 0); EXPECT_EQ(quantizer.max_error, 0); - expectDeviceValues( - res, std::get(quantizer.value).mins.data_handle(), std::vector{0.f, 10.f, 9.f}); - expectDeviceValues(res, - std::get(quantizer.value).maxs.data_handle(), - std::vector{254.f, 137.f, 9.f}); + expectPerDimensionBounds(res, owner, {0.f, 10.f, 9.f}, {254.f, 137.f, 9.f}); // The corpus manifest, rather than query extrema, controls ties and saturation. std::vector queries{127.5f, 73.75f, 1000.f, 128.5f, 72.75f, -1000.f}; raft::update_device(input.data_handle(), queries.data(), queries.size(), stream); @@ -563,10 +553,8 @@ TYPED_TEST(PerDimensionQuantizationTest, CorpusAndQueriesShareQuantizer) TEST(PerDimensionRangeTest, AdjacentHalfBounds) { raft::resources res; - auto stream = raft::resource::get_cuda_stream(res); std::vector values{half{1.f}, half{1.f + 1.f / 1024.f}}; - auto input = raft::make_device_matrix(res, 2, 1); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto input = makeDeviceInput(res, values, 1); auto quantizer = train( res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); auto codes = raft::make_device_matrix(res, 2, 1); @@ -585,32 +573,17 @@ TEST(PerDimensionRangeTest, PerDimensionQuantiles) corpus[i * dim + 1] = 10000.f + 2.f * static_cast(rows - i - 1); corpus[i * dim + 2] = 9.f; } - auto input = raft::make_device_matrix(res, rows, dim); - raft::update_device( - input.data_handle(), corpus.data(), corpus.size(), raft::resource::get_cuda_stream(res)); - auto owner = train( - res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); - auto quantizer = std::as_const(owner).view(); - expectDeviceValues( - res, std::get(quantizer.value).mins.data_handle(), {4.f, 10008.f, 9.f}); - expectDeviceValues( - res, std::get(quantizer.value).maxs.data_handle(), {995.f, 11990.f, 9.f}); - auto full = train( - res, params{1.f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); - expectDeviceValues(res, - std::get(std::as_const(full).view().value).maxs.data_handle(), - {999.f, 11998.f, 9.f}); - expectDeviceValues(res, - std::get(std::as_const(full).view().value).mins.data_handle(), - {0.f, 10000.f, 9.f}); - auto trimmed = train( - res, params{0.5f, quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); - expectDeviceValues(res, - std::get(std::as_const(trimmed).view().value).maxs.data_handle(), - {749.f, 11498.f, 9.f}); - expectDeviceValues(res, - std::get(std::as_const(trimmed).view().value).mins.data_handle(), - {250.f, 10500.f, 9.f}); + auto input = makeDeviceInput(res, corpus, dim); + auto check_bounds = [&](params p, std::vector mins, std::vector maxs) { + auto owner = train(res, p, raft::make_const_mdspan(input.view())); + expectPerDimensionBounds(res, owner, mins, maxs); + }; + check_bounds( + params{.range = quantization_mode::PER_DIMENSION}, {4.f, 10008.f, 9.f}, {995.f, 11990.f, 9.f}); + check_bounds( + params{1.f, quantization_mode::PER_DIMENSION}, {0.f, 10000.f, 9.f}, {999.f, 11998.f, 9.f}); + check_bounds( + params{.5f, quantization_mode::PER_DIMENSION}, {250.f, 10500.f, 9.f}, {749.f, 11498.f, 9.f}); for (float q : {0.f, -1.f, 1.1f, std::numeric_limits::quiet_NaN()}) { EXPECT_THROW( train( @@ -621,8 +594,7 @@ TEST(PerDimensionRangeTest, PerDimensionQuantiles) TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) { - using T = TypeParam; - using bounds_view = typename quantizer_view::per_dim_type; + using T = TypeParam; raft::resources res; auto mins = raft::make_device_vector(res, 2); auto maxs = raft::make_device_vector(res, 2); @@ -636,12 +608,8 @@ TYPED_TEST(PerDimensionQuantizationTest, CustomParametersAndValidation) EXPECT_TRUE(std::isinf(owner.max_error)); EXPECT_TRUE(std::isinf(quantizer.max_error)); EXPECT_TRUE(std::isinf((scalar::quantizer{T{-1}, T{1}}.view().max_error))); - expectDeviceValues(res, std::get(quantizer.value).mins.data_handle(), lower); - expectDeviceValues(res, std::get(quantizer.value).maxs.data_handle(), upper); - auto input = raft::make_device_matrix(res, 1, 2); - std::vector values{104.f, -106.f}; - raft::update_device( - input.data_handle(), values.data(), values.size(), raft::resource::get_cuda_stream(res)); + expectPerDimensionBounds(res, owner, lower, upper); + auto input = makeDeviceInput(res, {104.f, -106.f}, 2); auto codes = raft::make_device_matrix(res, 1, 2); transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); expectDeviceValues(res, codes.data_handle(), {2, -127}); @@ -754,10 +722,8 @@ TEST(PerDimensionRangeTest, QuantizedTypeLimits) { using QuantI = int16_t; raft::resources res; - auto stream = raft::resource::get_cuda_stream(res); std::vector values{-100000.f, -32767.f, 0.f, 32767.f, 100000.f}; - auto input = raft::make_device_matrix(res, values.size(), 1); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto input = makeDeviceInput(res, values, 1); auto codes = raft::make_device_matrix(res, values.size(), 1); auto owner = detail::train(res, params{.5f, quantization_mode::PER_DIMENSION}, @@ -769,51 +735,38 @@ TEST(PerDimensionRangeTest, QuantizedTypeLimits) expectDeviceValues(res, codes.data_handle(), expected); detail::apply_quantizer( res, std::as_const(owner).view(), raft::make_const_mdspan(codes.view()), input.view()); - std::vector decoded(values.size()); - raft::update_host(decoded.data(), input.data_handle(), decoded.size(), stream); - raft::resource::sync_stream(res); - double maximum = 0; - for (size_t i = 0; i < values.size(); ++i) { - EXPECT_FLOAT_EQ(decoded[i], static_cast(expected[i])); - maximum = std::max(maximum, std::abs(double(values[i]) - decoded[i])); - } - EXPECT_GE(owner.max_error, maximum); + auto decoded = copyToHost(res, input.data_handle(), values.size()); + EXPECT_EQ(decoded, (std::vector(expected.begin(), expected.end()))); + auto maximum = expectReconstructionErrorBound( + values.data(), decoded.data(), values.size(), 1, owner.max_error); EXPECT_NEAR(owner.max_error, maximum, maximum * 2e-6); } -TYPED_TEST(PerDimensionQuantizationTest, GlobalBounds) +template +void testGlobalBounds(T min, T max) { - using T = TypeParam; raft::resources res; - auto stream = raft::resource::get_cuda_stream(res); - for (auto bounds : {std::pair{T{-3}, T{7}}, std::pair{T{5}, T{5}}}) { - auto owner = quantizer{bounds.first, bounds.second}; - auto parameters = std::get::global_type>(owner.value); - EXPECT_EQ(parameters.min, bounds.first); - EXPECT_EQ(parameters.max, bounds.second); - std::vector values{T{-100}, bounds.first, bounds.second, T{100}}; - auto input = raft::make_device_matrix(res, 1, values.size()); - auto codes = raft::make_device_matrix(res, 1, values.size()); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); - transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); - inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); - std::vector decoded(values.size()); - raft::update_host(decoded.data(), input.data_handle(), decoded.size(), stream); - raft::resource::sync_stream(res); - EXPECT_EQ(decoded.front(), bounds.first); - EXPECT_EQ(decoded.back(), bounds.second); - EXPECT_EQ(decoded[1], bounds.first); - EXPECT_EQ(decoded[2], bounds.second); - } + auto owner = quantizer{min, max}; + auto bounds = std::get::global_type>(owner.value); + EXPECT_EQ(bounds.min, min); + EXPECT_EQ(bounds.max, max); + auto input = makeDeviceInput(res, {T{-100}, min, max, T{100}}, 4); + auto codes = raft::make_device_matrix(res, 1, 4); + transform(res, owner, raft::make_const_mdspan(input.view()), codes.view()); + inverse_transform(res, owner, raft::make_const_mdspan(codes.view()), input.view()); + expectDeviceValues(res, input.data_handle(), {min, min, max, max}); +} + +TYPED_TEST(PerDimensionQuantizationTest, GlobalBounds) { testGlobalBounds(-3, 7); } +TYPED_TEST(PerDimensionQuantizationTest, ConstantGlobalBounds) +{ + testGlobalBounds(5, 5); } TEST(QuantizationDatasetTest, GlobalRangeDataset) { raft::resources res; - auto input = raft::make_device_matrix(res, 1, 2); - std::vector values{-1.f, 1.f}; - auto stream = raft::resource::get_cuda_stream(res); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto input = makeDeviceInput(res, {-1.f, 1.f}, 2); auto encoded = make_sq_dataset(res, params{1.f}, raft::make_const_mdspan(input.view())); typename decltype(encoded)::const_view_type read_only = encoded.view(); EXPECT_EQ(read_only.const_manifest_view().max_error, 0.f); @@ -829,10 +782,7 @@ TEST(QuantizationDatasetTest, GlobalRangeDataset) TEST(QuantizationDatasetTest, PerDimensionDataset) { raft::resources res; - auto input = raft::make_device_matrix(res, 2, 2); - std::vector values{0.f, 10.f, 254.f, 14.f}; - auto stream = raft::resource::get_cuda_stream(res); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto input = makeDeviceInput(res, {0.f, 10.f, 254.f, 14.f}, 2); auto encoded = make_sq_dataset( res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); typename decltype(encoded)::const_view_type read_only = encoded.view(); @@ -844,11 +794,8 @@ TEST(QuantizationDatasetTest, PerDimensionDataset) auto quantizer = train( res, params{.range = quantization_mode::PER_DIMENSION}, raft::make_const_mdspan(input.view())); transform(res, quantizer, raft::make_const_mdspan(input.view()), codes.view()); - EXPECT_TRUE(devArrMatch(encoded.data_const_view().data_handle(), - codes.data_handle(), - 4, - cuvs::Compare(), - stream.get())); + expectDeviceValues( + res, codes.data_handle(), copyToHost(res, encoded.data_const_view().data_handle(), 4)); } template @@ -863,9 +810,7 @@ void testHostDataset(params p, bool prefetch = false, bool use_pinned = false) if (prefetch) { raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); } - auto stream = raft::resource::get_cuda_stream(res); - auto input = raft::make_device_matrix(res, rows, dim); - raft::update_device(input.data_handle(), values.data(), values.size(), stream); + auto input = makeDeviceInput(res, values, dim); auto pinned = raft::make_pinned_matrix(res, rows, dim); if (use_pinned) { std::copy(values.begin(), values.end(), pinned.data_handle()); } // A negative staging limit is ignored for device input. @@ -876,27 +821,21 @@ void testHostDataset(params p, bool prefetch = false, bool use_pinned = false) auto actual = make_sq_dataset(res, p, host_input); EXPECT_EQ(actual.n_rows(), rows); EXPECT_EQ(actual.dim(), dim); - EXPECT_TRUE(devArrMatch(expected.data_const_view().data_handle(), - actual.data_const_view().data_handle(), - values.size(), - cuvs::Compare(), - stream.get())); + expectDeviceValues(res, + actual.data_const_view().data_handle(), + copyToHost(res, expected.data_const_view().data_handle(), values.size())); auto a = actual.const_manifest_view(); auto e = expected.const_manifest_view(); EXPECT_NEAR(double(a.max_error), double(e.max_error), double(e.max_error) * 1e-6); if (p.range == quantization_mode::PER_DIMENSION) { auto actual_bounds = std::get::per_dim_type>(a.value); auto expected_bounds = std::get::per_dim_type>(e.value); - EXPECT_TRUE(devArrMatch(actual_bounds.maxs.data_handle(), - expected_bounds.maxs.data_handle(), - dim, - cuvs::Compare(), - stream.get())); - EXPECT_TRUE(devArrMatch(actual_bounds.mins.data_handle(), - expected_bounds.mins.data_handle(), - dim, - cuvs::Compare(), - stream.get())); + expectDeviceValues(res, + actual_bounds.maxs.data_handle(), + copyToHost(res, expected_bounds.maxs.data_handle(), dim)); + expectDeviceValues(res, + actual_bounds.mins.data_handle(), + copyToHost(res, expected_bounds.mins.data_handle(), dim)); } // Standalone training must agree without allocating corpus codes. auto trained = train(res, p, host_input); @@ -908,99 +847,74 @@ TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPartialBatch) testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}); } -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetSingleRowBatch) -{ - testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 1}); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetFullBatch) -{ - testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 37}); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetUnlimitedBatch) -{ - testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 0}); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPinned) -{ - testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}, false, true); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPrefetch) -{ - testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}, true); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetPinnedPrefetch) -{ - testHostDataset(params{1.f, quantization_mode::PER_DIMENSION, 7}, true, true); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetClippedQuantile) -{ - testHostDataset(params{.5f, quantization_mode::PER_DIMENSION, 7}); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetDefaultQuantile) -{ - testHostDataset(params{.99f, quantization_mode::PER_DIMENSION, 7}); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetGlobalRange) -{ - testHostDataset(params{1.f, quantization_mode::GLOBAL, 7}); -} - -TYPED_TEST(PerDimensionQuantizationTest, HostDatasetGlobalClippedQuantile) -{ - testHostDataset(params{.5f, quantization_mode::GLOBAL, 7}); -} - -TEST(QuantizationDatasetTest, HostBuildBatchWorkspace) +struct HostDatasetInputs { + const char* name; + params quantization; + bool prefetch = false; + bool pinned = false; +}; +class HostDatasetTest : public ::testing::TestWithParam {}; +TEST_P(HostDatasetTest, MatchesDeviceDataset) +{ + auto const& p = GetParam(); + testHostDataset(p.quantization, p.prefetch, p.pinned); +} + +INSTANTIATE_TEST_SUITE_P( + HostDatasetCases, + HostDatasetTest, + ::testing::Values( + HostDatasetInputs{"SingleRowBatch", {1.f, quantization_mode::PER_DIMENSION, 1}}, + HostDatasetInputs{"FullBatch", {1.f, quantization_mode::PER_DIMENSION, 37}}, + HostDatasetInputs{"UnlimitedBatch", {1.f, quantization_mode::PER_DIMENSION, 0}}, + HostDatasetInputs{"Pinned", {1.f, quantization_mode::PER_DIMENSION, 7}, false, true}, + HostDatasetInputs{"Prefetch", {1.f, quantization_mode::PER_DIMENSION, 7}, true}, + HostDatasetInputs{"PinnedPrefetch", {1.f, quantization_mode::PER_DIMENSION, 7}, true, true}, + HostDatasetInputs{"ClippedQuantile", {.5f, quantization_mode::PER_DIMENSION, 7}}, + HostDatasetInputs{"DefaultQuantile", {.99f, quantization_mode::PER_DIMENSION, 7}}, + HostDatasetInputs{"GlobalRange", {1.f, quantization_mode::GLOBAL, 7}}, + HostDatasetInputs{"GlobalClippedQuantile", {.5f, quantization_mode::GLOBAL, 7}}), + [](auto const& info) { return info.param.name; }); + +void testHostBuildWorkspace(float quantile, bool prefetch = false) { constexpr int64_t rows = 4097, dim = 35, batch_rows = 128; std::vector values(rows * dim, 1.f); auto input = raft::make_host_matrix_view(values.data(), rows, dim); - for (bool prefetch : {false, true}) { - raft::resources res; - if (prefetch) { - raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); - } - for (float quantile : {1.f, .5f}) { - params p{quantile, quantization_mode::PER_DIMENSION, batch_rows}; - auto action = [&](raft::resources const& h) { - auto encoded = make_sq_dataset(h, p, input); - if (!raft::resource::get_dry_run_flag(h)) { - EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); - } - }; - execute_with_dry_run_check(res, action, alloc_behavior::ARGUMENT_DRIVEN); - raft::memory_stats_resources stats(res); - action(stats); - auto peak = stats.get_bytes_peak(); - auto device_bytes = peak.device_workspace + peak.device_large_workspace + peak.device_global + - peak.device_managed; - // Includes all int8 codes; excludes enough memory for even one full FP32 corpus copy. - EXPECT_LT(device_bytes, values.size() * sizeof(float)); - } + raft::resources res; + if (prefetch) { + raft::resource::set_cuda_stream_pool(res, std::make_shared(1)); } - EXPECT_THROW( - make_sq_dataset(raft::resources{}, params{1.f, quantization_mode::PER_DIMENSION, -1}, input), - raft::logic_error); + auto action = [&](raft::resources const& h) { + auto encoded = + make_sq_dataset(h, params{quantile, quantization_mode::PER_DIMENSION, batch_rows}, input); + if (!raft::resource::get_dry_run_flag(h)) { + EXPECT_EQ(encoded.const_manifest_view().max_error, 0.f); + } + }; + execute_with_dry_run_check(res, action, alloc_behavior::ARGUMENT_DRIVEN); + raft::memory_stats_resources stats(res); + action(stats); + auto peak = stats.get_bytes_peak(); + // Includes all int8 codes; excludes even one full FP32 corpus copy. + EXPECT_LT( + peak.device_workspace + peak.device_large_workspace + peak.device_global + peak.device_managed, + values.size() * sizeof(float)); } -TEST_P(PerDimensionRangeTest, Result) { this->testPerDimensionQuantization(); } +TEST(QuantizationDatasetTest, HostBuildBatchWorkspace) { testHostBuildWorkspace(1.f); } +TEST(QuantizationDatasetTest, HostBuildClippedWorkspace) { testHostBuildWorkspace(.5f); } +TEST(QuantizationDatasetTest, HostBuildPrefetchWorkspace) { testHostBuildWorkspace(1.f, true); } -typedef QuantizationTest QuantizationTest_float_int8t; -TEST_P(QuantizationTest_float_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } - -typedef QuantizationTest QuantizationTest_double_int8t; -TEST_P(QuantizationTest_double_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } - -typedef QuantizationTest QuantizationTest_half_int8t; -TEST_P(QuantizationTest_half_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } +TEST(QuantizationDatasetTest, HostBuildRejectsNegativeBatchSize) +{ + std::vector values{1.f}; + EXPECT_THROW( + make_sq_dataset(raft::resources{}, + params{1.f, quantization_mode::PER_DIMENSION, -1}, + raft::make_host_matrix_view(values.data(), 1, 1)), + raft::logic_error); +} template const std::vector> inputs = { @@ -1016,34 +930,14 @@ const std::vector> inputs = { {{0.95}, 10, 20, T(5.0), T(5.0)}, }; -const std::vector dataset_inputs = { - // Each nonconstant dimension uses the full code range despite different input ranges. - {3, - 3, - {-127.f, -127.f / 128.f, 9.f, 1.f, 1.f / 128.f, 9.f, 127.f, 127.f / 128.f, 9.f}, - {-127, -127, 0, 1, 1, 0, 127, 127, 0}, - 1e-6f}, - {2, 2, {10.f, -7.f, 10.f, -7.f}, {0, 0, 0, 0}, 0.f}, - // Equal subnormal bounds must retain their value and reconstruct exactly. - {1, - 2, - {std::numeric_limits::denorm_min(), -std::numeric_limits::denorm_min()}, - {0, 0}, - 0.f}, - {2, - 1, - {-std::numeric_limits::max(), std::numeric_limits::max()}, - {-127, 127}, - std::numeric_limits::max() / 254.f}, - {2, - 1, - {-std::ldexp(124.f, 104), std::numeric_limits::max()}, - {-127, 127}, - (std::ldexp(124.f, 103) + std::numeric_limits::max() * 0.5f) / 127.f * 0.5f}}; - -INSTANTIATE_TEST_CASE_P(PerDimensionRangeTests, - PerDimensionRangeTest, - ::testing::ValuesIn(dataset_inputs)); +typedef QuantizationTest QuantizationTest_float_int8t; +TEST_P(QuantizationTest_float_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } + +typedef QuantizationTest QuantizationTest_double_int8t; +TEST_P(QuantizationTest_double_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } + +typedef QuantizationTest QuantizationTest_half_int8t; +TEST_P(QuantizationTest_half_int8t, ScalarQuantizationTest) { this->testScalarQuantization(); } INSTANTIATE_TEST_CASE_P(QuantizationTest, QuantizationTest_float_int8t,