feat(io_uring): add new IO Type: UringIO

Signed-off-by: LHT129 <tianlan.lht@antgroup.com>

Author: LHT129

CMakeLists.txt

@@ -73,6 +73,7 @@ option (DISABLE_NEON_FORCE "Force disable neon instructions" OFF)
 option (DISABLE_SVE_FORCE "Force disable sve instructions" OFF)
 # AIO library configuration
 option (ENABLE_LIBAIO "Whether to enable libaio support" ON)
+option (ENABLE_LIBURING "Whether to enable liburing support" ON)
 
 if (ENABLE_CXX11_ABI)
     add_definitions (-D_GLIBCXX_USE_CXX11_ABI=1)
@@ -133,25 +134,8 @@ else ()
     endif ()
 endif ()
 
-# AIO library detection
-if (ENABLE_LIBAIO)
-    find_library (AIO_LIBRARY aio)
-    if (AIO_LIBRARY)
-        message (STATUS "Found libaio: ${AIO_LIBRARY}")
-        set (HAVE_LIBAIO 1)
-        add_definitions (-DHAVE_LIBAIO=1)
-    else ()
-        message (WARNING "libaio not found, disabling async I/O support")
-        set (HAVE_LIBAIO 0)
-        add_definitions (-DHAVE_LIBAIO=0)
-        add_definitions (-DNO_LIBAIO=1)
-    endif ()
-else ()
-    message (STATUS "libaio support disabled by user")
-    set (HAVE_LIBAIO 0)
-    add_definitions (-DHAVE_LIBAIO=0)
-    add_definitions (-DNO_LIBAIO=1)
-endif ()
+include (cmake/FindLibaio.cmake)
+include (cmake/FindLiburing.cmake)
 
 # ccache
 find_program (CCACHE_FOUND ccache)

cmake/FindLibaio.cmake

@@ -0,0 +1,34 @@
+
+# Copyright 2024-present the vsag project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# AIO library detection
+if (ENABLE_LIBAIO)
+    find_library (AIO_LIBRARY aio)
+    if (AIO_LIBRARY)
+        message (STATUS "Found libaio: ${AIO_LIBRARY}")
+        set (HAVE_LIBAIO 1)
+        add_definitions (-DHAVE_LIBAIO=1)
+    else ()
+        message (WARNING "libaio not found, disabling async I/O support")
+        set (HAVE_LIBAIO 0)
+        add_definitions (-DHAVE_LIBAIO=0)
+        add_definitions (-DNO_LIBAIO=1)
+    endif ()
+else ()
+    message (STATUS "libaio support disabled by user")
+    set (HAVE_LIBAIO 0)
+    add_definitions (-DHAVE_LIBAIO=0)
+    add_definitions (-DNO_LIBAIO=1)
+endif ()

cmake/FindLiburing.cmake

@@ -0,0 +1,43 @@
+
+# Copyright 2024-present the vsag project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if (ENABLE_LIBURING)
+    find_package(PkgConfig QUIET)
+    if (PkgConfig_FOUND)
+        pkg_check_modules(PC_LIBURING QUIET liburing)
+    endif()
+
+    if (PC_LIBURING_FOUND)
+        message(STATUS "Found liburing via pkg-config: ${PC_LIBURING_LIBRARIES}")
+        set(HAVE_LIBURING 1)
+        add_definitions(-DHAVE_LIBURING=1)
+    else()
+        # fallback to find_library
+        find_library(URING_LIBRARY NAMES uring PATHS /usr/lib /usr/lib64 /usr/lib/x86_64-linux-gnu)
+        if (URING_LIBRARY)
+            message(STATUS "Found liburing: ${URING_LIBRARY}")
+            set(HAVE_LIBURING 1)
+            add_definitions(-DHAVE_LIBURING=1)
+        else()
+            message(WARNING "liburing not found, disabling io_uring support")
+            set(HAVE_LIBURING 0)
+            add_definitions(-DHAVE_LIBURING=0 -DNO_LIBURING=1)
+        endif()
+    endif()
+else()
+    message(STATUS "liburing support disabled by user")
+    set(HAVE_LIBURING 0)
+    add_definitions(-DHAVE_LIBURING=0 -DNO_LIBURING=1)
+endif()

docs/hgraph.md

@@ -121,7 +121,7 @@ For examples, refer to [103_index_hgraph.cpp](https://github.com/antgroup/vsag/b
 ### base_io_type
 - **Parameter Type**: string
 - **Parameter Description**: Storage type for base quantization codes
-- **Optional Values**: "memory_io", "block_memory_io", "buffer_io", "async_io", "mmap_io"
+- **Optional Values**: "memory_io", "block_memory_io", "buffer_io", "async_io", "mmap_io", "uring_io"
 - **Default Value**: "block_memory_io"
 
 ### base_file_path
@@ -133,7 +133,7 @@ For examples, refer to [103_index_hgraph.cpp](https://github.com/antgroup/vsag/b
 ### precise_io_type
 - **Parameter Type**: string
 - **Parameter Description**: Storage type for precise quantization codes, same as base_io_type but for reordering codes
-- **Optional Values**: "memory_io", "block_memory_io", "buffer_io", "async_io", "mmap_io"
+- **Optional Values**: "memory_io", "block_memory_io", "buffer_io", "async_io", "mmap_io", "uring_io"
 - **Default Value**: "block_memory_io"
 
 ### precise_file_path

scripts/deps/install_deps_centos.sh

@@ -1,5 +1,5 @@
 # install python3-devel
-yum install -y gfortran python3-devel libaio-devel libcurl-devel
+yum install -y gfortran python3-devel libaio-devel libcurl-devel liburing-devel
 
 # install openmp for alibaba clang 11
 yum install -y current -y libomp11-devel libomp11

src/algorithm/hgraph.cpp

@@ -502,7 +502,6 @@ HGraph::map_hgraph_param(const JsonType& hgraph_json) {
     std::string str = format_map(hgraph_params_template, DEFAULT_MAP);
     auto inner_json = JsonType::Parse(str);
     mapping_external_param_to_inner(hgraph_json, external_mapping, inner_json);
-
     return inner_json;
 }
 

src/datacell/bucket_interface.cpp

@@ -109,6 +109,9 @@ BucketInterface::MakeInstance(const BucketDataCellParamPtr& param,
     if (io_type_name == IO_TYPE_VALUE_BUFFER_IO) {
         return make_instance<NonContinuousIO<BufferIO>>(param, common_param);
     }
+    if (io_type_name == IO_TYPE_VALUE_URING_IO) {
+        return make_instance<NonContinuousIO<UringIO>>(param, common_param);
+    }
     return nullptr;
 }
 }  // namespace vsag

src/datacell/flatten_interface.cpp

@@ -190,6 +190,9 @@ FlattenInterface::MakeInstance(const FlattenInterfaceParamPtr& param,
     if (io_type_name == IO_TYPE_VALUE_READER_IO) {
         return make_instance<ReaderIO>(param, common_param);
     }
+    if (io_type_name == IO_TYPE_VALUE_URING_IO) {
+        return make_instance<UringIO>(param, common_param);
+    }
     return nullptr;
 }
 

src/inner_string_params.h

@@ -55,6 +55,7 @@ const char* const IO_TYPE_VALUE_BUFFER_IO = "buffer_io";
 const char* const IO_TYPE_VALUE_MMAP_IO = "mmap_io";
 const char* const IO_TYPE_VALUE_READER_IO = "reader_io";
 const char* const IO_TYPE_VALUE_ASYNC_IO = "async_io";
+const char* const IO_TYPE_VALUE_URING_IO = "uring_io";
 const char* const IO_TYPE_VALUE_BLOCK_MEMORY_IO = "block_memory_io";
 const char* const BLOCK_IO_BLOCK_SIZE_KEY = "block_size";
 

src/io/CMakeLists.txt

@@ -28,6 +28,8 @@ set (IO_SRC
         memory_block_io.cpp
         reader_io.cpp
         reader_io_parameter.cpp
+        uring_io.cpp
+        uring_io_parameter.cpp
 )
 
 add_library (io OBJECT ${IO_SRC})
@@ -41,6 +43,16 @@ else ()
     message (STATUS "io module: libaio not available, async I/O disabled")
 endif ()
 
+if (HAVE_LIBURING)
+    target_compile_definitions (io PUBLIC -DHAVE_LIBURING=1)
+    target_link_libraries (io PUBLIC fmt::fmt uring coverage_config)
+    message (STATUS "io module: linking with liburing")
+else ()
+    target_compile_definitions (io PUBLIC -DHAVE_LIBURING=0)
+    target_link_libraries (io PUBLIC fmt::fmt coverage_config)
+    message (STATUS "io module: liburing not available, io_uring disabled")
+endif ()
+
 maybe_add_dependencies (io spdlog)
 
 if (ENABLE_TESTS)