Merge pull request #1123 from iceljc/features/refine-vector-store

Features/refine vector store

Author: iceljc

Directory.Packages.props

@@ -46,7 +46,7 @@
     <PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
     <PackageVersion Include="NCrontab" Version="3.3.3" />
     <PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.5" />
-    <PackageVersion Include="OpenAI" Version="2.2.0" />
+    <PackageVersion Include="OpenAI" Version="2.3.0" />
     <PackageVersion Include="MailKit" Version="4.11.0" />
     <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
     <PackageVersion Include="MySql.Data" Version="9.0.0" />
@@ -74,7 +74,7 @@
     <PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
     <PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
     <PackageVersion Include="pythonnet" Version="3.0.4" />
-    <PackageVersion Include="Qdrant.Client" Version="1.13.0" />
+    <PackageVersion Include="Qdrant.Client" Version="1.15.0" />
     <PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
     <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Graph.Models;
+using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Abstraction.Knowledges;
@@ -13,6 +14,7 @@ public interface IKnowledgeService
     Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
+    Task<IEnumerable<VectorCollectionData>> GetVectorCollectionData(string collectionName, IEnumerable<string> ids, VectorQueryOptions? options = null);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
     Task<bool> DeleteVectorCollectionAllData(string collectionName);
     Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
@@ -69,6 +71,11 @@ Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileNa
     Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName);
     #endregion
 
+    #region Index
+    Task<SuccessFailResponse<string>> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable<CreateVectorCollectionIndexOptions> options);
+    Task<SuccessFailResponse<string>> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable<DeleteVectorCollectionIndexOptions> options);
+    #endregion
+
     #region Common
     Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
     #endregion

src/Infrastructure/BotSharp.Abstraction/Models/SuccessFailResponse.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Models;
+
+public class SuccessFailResponse<T>
+{
+    public List<T> Success { get; set; } = [];
+    public List<T> Fail { get; set; } = [];
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs

@@ -8,6 +8,8 @@ public class ConversationStateKeysFilter
     public bool PreLoad { get; set; }
     public List<string>? AgentIds { get; set; }
     public List<string>? UserIds { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public ConversationStateKeysFilter()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs

@@ -8,6 +8,8 @@ public class InstructLogFilter : Pagination
     public List<string>? TemplateNames { get; set; }
     public List<string>? UserIds { get; set; }
     public List<KeyValue>? States { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public static InstructLogFilter Empty()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs

@@ -8,6 +8,8 @@ public class InstructLogKeysFilter
     public bool PreLoad { get; set; }
     public List<string>? AgentIds { get; set; }
     public List<string>? UserIds { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public InstructLogKeysFilter()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -56,7 +56,7 @@ bool UpdateRole(Role role, bool updateRoleAgents = false)
     void UpdateUserPhone(string userId, string Iphone, string regionCode) => throw new NotImplementedException();
     void UpdateUserIsDisable(string userId, bool isDisable) => throw new NotImplementedException();
     void UpdateUsersIsDisable(List<string> userIds, bool isDisable) => throw new NotImplementedException();
-    PagedItems<User> GetUsers(UserFilter filter) => throw new NotImplementedException();
+    ValueTask<PagedItems<User>> GetUsers(UserFilter filter) => throw new NotImplementedException();
     List<User> SearchLoginUsers(User filter, string source = UserSource.Internal) =>throw new NotImplementedException();
     User? GetUserDetails(string userId, bool includeAgent = false) => throw new NotImplementedException();
     bool UpdateUser(User user, bool updateUserAgents = false) => throw new NotImplementedException();
@@ -93,7 +93,7 @@ bool AppendAgentLabels(string agentId, List<string> labels)
     #endregion
 
     #region Agent Task
-    PagedItems<AgentTask> GetAgentTasks(AgentTaskFilter filter)
+    ValueTask<PagedItems<AgentTask>> GetAgentTasks(AgentTaskFilter filter)
         => throw new NotImplementedException();
     AgentTask? GetAgentTask(string agentId, string taskId)
         => throw new NotImplementedException();
@@ -126,7 +126,7 @@ void UpdateConversationStatus(string conversationId, string status)
         => throw new NotImplementedException();
     Conversation GetConversation(string conversationId, bool isLoadStates = false)
         => throw new NotImplementedException();
-    PagedItems<Conversation> GetConversations(ConversationFilter filter)
+    ValueTask<PagedItems<Conversation>> GetConversations(ConversationFilter filter)
         => throw new NotImplementedException();
     void UpdateConversationTitle(string conversationId, string title)
         => throw new NotImplementedException();
@@ -179,7 +179,7 @@ DateTimePagination<ConversationStateLogModel> GetConversationStateLogs(string co
     bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
         => throw new NotImplementedException();
 
-    PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+    ValueTask<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter)
         => throw new NotImplementedException();
 
     List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
@@ -227,7 +227,7 @@ bool SaveKnolwedgeBaseFileMeta(KnowledgeDocMetaData metaData)
     /// <returns></returns>
     bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
          => throw new NotImplementedException();
-    PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
+    ValueTask<PagedItems<KnowledgeDocMetaData>> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
          => throw new NotImplementedException();
     #endregion
 
@@ -236,7 +236,7 @@ bool UpsertCrontabItem(CrontabItem cron)
         => throw new NotImplementedException();
     bool DeleteCrontabItem(string conversationId)
         => throw new NotImplementedException();
-    PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
+    ValueTask<PagedItems<CrontabItem>> GetCrontabItems(CrontabItemFilter filter)
         => throw new NotImplementedException();
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs

@@ -7,6 +7,7 @@ namespace BotSharp.Abstraction.Users;
 public interface IUserService
 {
     Task<User> GetUser(string id);
+    Task<List<User>> GetUsers(List<string> ids);
     Task<PagedItems<User>> GetUsers(UserFilter filter);
     Task<List<User>> SearchLoginUsers(User filter);
     Task<User?> GetUserDetails(string userId, bool includeAgent = false);

src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs

@@ -50,6 +50,6 @@ public string GetCacheKey()
 
 public class PagedItems<T>
 {
-    public int Count { get; set; }
+    public long Count { get; set; }
     public IEnumerable<T> Items { get; set; } = new List<T>();
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -14,17 +14,15 @@ Task<IEnumerable<string>> GetCollections()
         => throw new NotImplementedException();
     Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
         => throw new NotImplementedException();
-    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
+    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, VectorQueryOptions? options = null)
         => throw new NotImplementedException();
     Task<bool> CreateCollection(string collectionName, int dimension)
         => throw new NotImplementedException();
     Task<bool> DeleteCollection(string collectionName)
         => throw new NotImplementedException();
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
         => throw new NotImplementedException();
-    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields,
-        int limit = 5, float confidence = 0.5f, bool withVector = false)
+    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
         => throw new NotImplementedException();
     Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
         => throw new NotImplementedException();
@@ -40,4 +38,9 @@ Task<bool> RecoverCollectionFromShapshot(string collectionName, string snapshotF
         => throw new NotImplementedException();
     Task<bool> DeleteCollectionShapshot(string collectionName, string snapshotName)
         => throw new NotImplementedException();
+
+    Task<bool> CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options)
+        => throw new NotImplementedException();
 }