Features/refine vector store

Directory.Packages.props

@@ -46,7 +46,7 @@
     <PackageVersion Include="Whisper.net.Runtime" Version="1.8.1" />
     <PackageVersion Include="NCrontab" Version="3.3.3" />
     <PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.5" />
-    <PackageVersion Include="OpenAI" Version="2.2.0" />
+    <PackageVersion Include="OpenAI" Version="2.3.0" />
     <PackageVersion Include="MailKit" Version="4.11.0" />
     <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
     <PackageVersion Include="MySql.Data" Version="9.0.0" />
@@ -74,7 +74,7 @@
     <PackageVersion Include="Sdcb.PaddleOCR.Models.LocalV3" Version="2.7.0.1" />
     <PackageVersion Include="System.Drawing.Common" Version="8.0.14" />
     <PackageVersion Include="pythonnet" Version="3.0.4" />
-    <PackageVersion Include="Qdrant.Client" Version="1.13.0" />
+    <PackageVersion Include="Qdrant.Client" Version="1.15.0" />
     <PackageVersion Include="Selenium.WebDriver" Version="4.27.0" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.12.0" />
     <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.16.0" />

src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Graph.Models;
+using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.VectorStorage.Models;
 
 namespace BotSharp.Abstraction.Knowledges;
@@ -13,6 +14,7 @@ public interface IKnowledgeService
     Task<VectorCollectionDetails?> GetVectorCollectionDetails(string collectionName);
     Task<IEnumerable<VectorSearchResult>> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options);
     Task<StringIdPagedItems<VectorSearchResult>> GetPagedVectorCollectionData(string collectionName, VectorFilter filter);
+    Task<IEnumerable<VectorCollectionData>> GetVectorCollectionData(string collectionName, IEnumerable<string> ids, VectorQueryOptions? options = null);
     Task<bool> DeleteVectorCollectionData(string collectionName, string id);
     Task<bool> DeleteVectorCollectionAllData(string collectionName);
     Task<bool> CreateVectorCollectionData(string collectionName, VectorCreateModel create);
@@ -69,6 +71,11 @@ Task<bool> ImportDocumentContentToKnowledge(string collectionName, string fileNa
     Task<bool> DeleteVectorCollectionSnapshot(string collectionName, string snapshotName);
     #endregion
 
+    #region Index
+    Task<SuccessFailResponse<string>> CreateVectorCollectionPayloadIndexes(string collectionName, IEnumerable<CreateVectorCollectionIndexOptions> options);
+    Task<SuccessFailResponse<string>> DeleteVectorCollectionPayloadIndexes(string collectionName, IEnumerable<DeleteVectorCollectionIndexOptions> options);
+    #endregion
+
     #region Common
     Task<bool> RefreshVectorKnowledgeConfigs(VectorCollectionConfigsModel configs);
     #endregion

src/Infrastructure/BotSharp.Abstraction/Models/SuccessFailResponse.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Models;
+
+public class SuccessFailResponse<T>
+{
+    public List<T> Success { get; set; } = [];
+    public List<T> Fail { get; set; } = [];
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs

@@ -8,6 +8,8 @@ public class ConversationStateKeysFilter
     public bool PreLoad { get; set; }
     public List<string>? AgentIds { get; set; }
     public List<string>? UserIds { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public ConversationStateKeysFilter()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs

@@ -8,6 +8,8 @@ public class InstructLogFilter : Pagination
     public List<string>? TemplateNames { get; set; }
     public List<string>? UserIds { get; set; }
     public List<KeyValue>? States { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public static InstructLogFilter Empty()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs

@@ -8,6 +8,8 @@ public class InstructLogKeysFilter
     public bool PreLoad { get; set; }
     public List<string>? AgentIds { get; set; }
     public List<string>? UserIds { get; set; }
+    public DateTime? StartTime { get; set; }
+    public DateTime? EndTime { get; set; }
 
     public InstructLogKeysFilter()
     {

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -43,7 +43,7 @@
     List<User> GetUsersByAffiliateId(string affiliateId) => throw new NotImplementedException();
     User? GetUserByUserName(string userName) => throw new NotImplementedException();
     void UpdateUserName(string userId, string userName) => throw new NotImplementedException();
     Dashboard? GetDashboard(string id = null) => throw new NotImplementedException();
     void CreateUser(User user) => throw new NotImplementedException();
     void UpdateExistUser(string userId, User user) => throw new NotImplementedException();
     void UpdateUserVerified(string userId) => throw new NotImplementedException();
@@ -56,7 +56,7 @@
     void UpdateUserPhone(string userId, string Iphone, string regionCode) => throw new NotImplementedException();
     void UpdateUserIsDisable(string userId, bool isDisable) => throw new NotImplementedException();
     void UpdateUsersIsDisable(List<string> userIds, bool isDisable) => throw new NotImplementedException();
-    PagedItems<User> GetUsers(UserFilter filter) => throw new NotImplementedException();
+    ValueTask<PagedItems<User>> GetUsers(UserFilter filter) => throw new NotImplementedException();
     List<User> SearchLoginUsers(User filter, string source = UserSource.Internal) =>throw new NotImplementedException();
     User? GetUserDetails(string userId, bool includeAgent = false) => throw new NotImplementedException();
     bool UpdateUser(User user, bool updateUserAgents = false) => throw new NotImplementedException();
@@ -93,7 +93,7 @@
     #endregion
 
     #region Agent Task
-    PagedItems<AgentTask> GetAgentTasks(AgentTaskFilter filter)
+    ValueTask<PagedItems<AgentTask>> GetAgentTasks(AgentTaskFilter filter)
         => throw new NotImplementedException();
     AgentTask? GetAgentTask(string agentId, string taskId)
         => throw new NotImplementedException();
@@ -126,7 +126,7 @@
         => throw new NotImplementedException();
     Conversation GetConversation(string conversationId, bool isLoadStates = false)
         => throw new NotImplementedException();
-    PagedItems<Conversation> GetConversations(ConversationFilter filter)
+    ValueTask<PagedItems<Conversation>> GetConversations(ConversationFilter filter)
         => throw new NotImplementedException();
     void UpdateConversationTitle(string conversationId, string title)
         => throw new NotImplementedException();
@@ -179,7 +179,7 @@
     bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
         => throw new NotImplementedException();
 
-    PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
+    ValueTask<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter)
         => throw new NotImplementedException();
 
     List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
@@ -227,7 +227,7 @@
     /// <returns></returns>
     bool DeleteKnolwedgeBaseFileMeta(string collectionName, string vectorStoreProvider, Guid? fileId = null)
          => throw new NotImplementedException();
-    PagedItems<KnowledgeDocMetaData> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
+    ValueTask<PagedItems<KnowledgeDocMetaData>> GetKnowledgeBaseFileMeta(string collectionName, string vectorStoreProvider, KnowledgeFileFilter filter)
          => throw new NotImplementedException();
     #endregion
 
@@ -236,7 +236,7 @@
         => throw new NotImplementedException();
     bool DeleteCrontabItem(string conversationId)
         => throw new NotImplementedException();
-    PagedItems<CrontabItem> GetCrontabItems(CrontabItemFilter filter)
+    ValueTask<PagedItems<CrontabItem>> GetCrontabItems(CrontabItemFilter filter)
         => throw new NotImplementedException();
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs

@@ -7,6 +7,7 @@ namespace BotSharp.Abstraction.Users;
 public interface IUserService
 {
     Task<User> GetUser(string id);
+    Task<List<User>> GetUsers(List<string> ids);
     Task<PagedItems<User>> GetUsers(UserFilter filter);
     Task<List<User>> SearchLoginUsers(User filter);
     Task<User?> GetUserDetails(string userId, bool includeAgent = false);

src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs

@@ -50,6 +50,6 @@ public string GetCacheKey()
 
 public class PagedItems<T>
 {
-    public int Count { get; set; }
+    public long Count { get; set; }
     public IEnumerable<T> Items { get; set; } = new List<T>();
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs

@@ -14,17 +14,15 @@ Task<IEnumerable<string>> GetCollections()
         => throw new NotImplementedException();
     Task<StringIdPagedItems<VectorCollectionData>> GetPagedCollectionData(string collectionName, VectorFilter filter)
         => throw new NotImplementedException();
-    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids,
-        bool withPayload = false, bool withVector = false)
+    Task<IEnumerable<VectorCollectionData>> GetCollectionData(string collectionName, IEnumerable<Guid> ids, VectorQueryOptions? options = null)
         => throw new NotImplementedException();
     Task<bool> CreateCollection(string collectionName, int dimension)
         => throw new NotImplementedException();
     Task<bool> DeleteCollection(string collectionName)
         => throw new NotImplementedException();
     Task<bool> Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary<string, object>? payload = null)
         => throw new NotImplementedException();
-    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, IEnumerable<string>? fields,
-        int limit = 5, float confidence = 0.5f, bool withVector = false)
+    Task<IEnumerable<VectorCollectionData>> Search(string collectionName, float[] vector, VectorSearchOptions? options = null)
         => throw new NotImplementedException();
     Task<bool> DeleteCollectionData(string collectionName, List<Guid> ids)
         => throw new NotImplementedException();
@@ -40,4 +38,9 @@ Task<bool> RecoverCollectionFromShapshot(string collectionName, string snapshotF
         => throw new NotImplementedException();
     Task<bool> DeleteCollectionShapshot(string collectionName, string snapshotName)
         => throw new NotImplementedException();
+
+    Task<bool> CreateCollectionPayloadIndex(string collectionName, CreateVectorCollectionIndexOptions options)
+        => throw new NotImplementedException();
+    Task<bool> DeleteCollectionPayloadIndex(string collectionName, DeleteVectorCollectionIndexOptions options)
+        => throw new NotImplementedException();
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionIndexOptions.cs

@@ -0,0 +1,18 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorCollectionIndexOptions
+{
+    [JsonPropertyName("field_name")]
+    public string FieldName { get; set; } = null!;
+}
+
+public class CreateVectorCollectionIndexOptions : VectorCollectionIndexOptions
+{
+    [JsonPropertyName("field_schema_type")]
+    public string FieldSchemaType { get; set; } = null!;
+}
+
+public class DeleteVectorCollectionIndexOptions : VectorCollectionIndexOptions
+{
+
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs

@@ -6,15 +6,23 @@ public class VectorFilter : StringIdPagination
     public bool WithVector { get; set; }
 
     /// <summary>
-    /// For keyword search
+    /// Filter group: each item contains a logical operator and a list of key-value pairs
     /// </summary>
-    [JsonPropertyName("search_pairs")]
-    public IEnumerable<KeyValue>? SearchPairs { get; set; }
+    [JsonPropertyName("filter_groups")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
 
+    /// <summary>
+    /// Order by a specific field
+    /// </summary>
+    [JsonPropertyName("order_by")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public VectorSort? OrderBy { get; set; }
 
     /// <summary>
-    /// Included payload keys
+    /// Included payload fields
     /// </summary>
-    [JsonPropertyName("included_payloads")]
-    public IEnumerable<string>? IncludedPayloads { get; set; }
+    [JsonPropertyName("fields")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IEnumerable<string>? Fields { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilterGroup.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorFilterGroup
+{
+    [JsonPropertyName("filters")]
+    public IEnumerable<KeyValue>? Filters { get; set; }
+
+    [JsonPropertyName("filter_operator")]
+    public string FilterOperator { get; set; } = "or";
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorQueryOptions.cs

@@ -0,0 +1,12 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorQueryOptions
+{
+    public bool WithPayload { get; set; }
+    public bool WithVector { get; set; }
+
+    public static VectorQueryOptions Default()
+    {
+        return new();
+    }
+}

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchOptions.cs

@@ -4,8 +4,21 @@ namespace BotSharp.Abstraction.VectorStorage.Models;
 
 public class VectorSearchOptions
 {
-    public IEnumerable<string>? Fields { get; set; } = new List<string> { KnowledgePayloadName.Text, KnowledgePayloadName.Answer };
+    public IEnumerable<string>? Fields { get; set; } = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer];
+    public IEnumerable<VectorFilterGroup>? FilterGroups { get; set; }
     public int? Limit { get; set; } = 5;
     public float? Confidence { get; set; } = 0.5f;
     public bool WithVector { get; set; }
+
+    public static VectorSearchOptions Default()
+    {
+        return new()
+        {
+            Fields = [KnowledgePayloadName.Text, KnowledgePayloadName.Answer],
+            FilterGroups = null,
+            Limit = 5,
+            Confidence = 0.5f,
+            WithVector = false
+        };
+    }
 }

src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSort.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.VectorStorage.Models;
+
+public class VectorSort
+{
+    [JsonPropertyName("field")]
+    public string? Field { get; set; }
+
+    [JsonPropertyName("order")]
+    public string? Order { get; set; } = "desc";
+}

src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs

@@ -44,7 +44,7 @@ public CrontabService(IServiceProvider services, ILogger<CrontabService> logger)
     public async Task<List<CrontabItem>> GetCrontable()
     {
         var repo = _services.GetRequiredService<IBotSharpRepository>();
-        var crontable = repo.GetCrontabItems(CrontabItemFilter.Empty());
+        var crontable = await repo.GetCrontabItems(CrontabItemFilter.Empty());
 
         // Add fixed crontab items from cronsources
         var fixedCrantabItems = crontable.Items.ToList();

src/Infrastructure/BotSharp.Core.SideCar/Services/BotSharpConversationSideCar.cs

@@ -112,7 +112,7 @@ public async Task<RoleDialogModel> SendMessage(
         var response = await InnerExecute(agentId, text, postback, states);
         AfterExecute();
 
-        _logger.LogInformation($"Existing side car conversation...");
+        _logger.LogInformation($"Exiting side car conversation...");
         return response;
     }
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -87,7 +87,7 @@ public async Task<PagedItems<Conversation>> GetConversations(ConversationFilter
         }
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var conversations = db.GetConversations(filter);
+        var conversations = await db.GetConversations(filter);
         return conversations;
     }
 

src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs

@@ -18,22 +18,18 @@ public async Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLo
 
         filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null;
 
-        var agents = new List<Agent>();
         var users = new List<User>();
 
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var logs = db.GetInstructionLogs(filter);
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var logs = await db.GetInstructionLogs(filter);
         var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList();
         var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList();
-        agents = db.GetAgents(new AgentFilter
-        {
-            AgentIds = agentIds,
-            Pager = new Pagination { Size = filter.Size }
-        });
+        var agents = await agentService.GetAgentOptions(agentIds);
 
         if (isAdmin)
         {
-            users = db.GetUserByIds(userIds);
+            users = await userService.GetUsers(userIds);
         }
 
         var items = logs.Items.Select(x =>