diff --git a/App/backend-api/Microsoft.GS.DPS.Host/API/ChatHost/Chat.cs b/App/backend-api/Microsoft.GS.DPS.Host/API/ChatHost/Chat.cs index 8660106f..74ba4937 100644 --- a/App/backend-api/Microsoft.GS.DPS.Host/API/ChatHost/Chat.cs +++ b/App/backend-api/Microsoft.GS.DPS.Host/API/ChatHost/Chat.cs @@ -17,7 +17,7 @@ public static void AddAPIs(WebApplication app) ChatRequestValidator validator, ChatHost chatHost) => { - if(validator.Validate(request).IsValid == false) + if (!validator.Validate(request).IsValid) { return Results.BadRequest(); } @@ -37,7 +37,7 @@ public static void AddAPIs(WebApplication app) ChatRequestValidator validator, ChatHost chatHost) => { - if (validator.Validate(request).IsValid == false) + if (!validator.Validate(request).IsValid) { return Results.BadRequest(); } diff --git a/App/backend-api/Microsoft.GS.DPS.Host/API/KernelMemory/KernelMemory.cs b/App/backend-api/Microsoft.GS.DPS.Host/API/KernelMemory/KernelMemory.cs index 6951d43a..5337075c 100644 --- a/App/backend-api/Microsoft.GS.DPS.Host/API/KernelMemory/KernelMemory.cs +++ b/App/backend-api/Microsoft.GS.DPS.Host/API/KernelMemory/KernelMemory.cs @@ -93,10 +93,13 @@ DPS.API.KernelMemory kernelMemory await kernelMemory.DeleteDocument(documentId); return Results.Ok(new DocumentDeletedResult() { IsDeleted = true }); } + #pragma warning disable CA1031 // Must catch all to log and keep the process alive catch (Exception ex) { + app.Logger.LogError(ex, "An error occurred while deleting a document."); return Results.BadRequest(new DocumentDeletedResult() { IsDeleted = false }); } + #pragma warning restore CA1031 }) .DisableAntiforgery(); diff --git a/App/backend-api/Microsoft.GS.DPS.Host/API/UserInterface/UserInterface.cs b/App/backend-api/Microsoft.GS.DPS.Host/API/UserInterface/UserInterface.cs index 3493950a..d2311fff 100644 --- a/App/backend-api/Microsoft.GS.DPS.Host/API/UserInterface/UserInterface.cs +++ b/App/backend-api/Microsoft.GS.DPS.Host/API/UserInterface/UserInterface.cs @@ -30,9 +30,9 @@ public static void AddAPIs(WebApplication app) var document = await documentRepository.FindByDocumentIdAsync(DocumentId); //Check if the thumbnail is already in the cache - if (thumbnails.ContainsKey(document.MimeType)) + if (thumbnails.TryGetValue(document.MimeType, out var thumbnail)) { - return Results.File(thumbnails[document.MimeType], "image/png"); + return Results.File(thumbnail, "image/png"); } else { @@ -108,14 +108,7 @@ public static void AddAPIs(WebApplication app) { DPS.Storage.Document.Entities.Document result = await documents.GetDocument(DocumentId); - if (result == null) - { - return Results.NotFound(); - } - else - { - return Results.Ok(result); - } + return result == null ? Results.NotFound() : Results.Ok(result); } ) .DisableAntiforgery(); diff --git a/App/backend-api/Microsoft.GS.DPS.Host/DependencyConfiguration/ServiceDependencies.cs b/App/backend-api/Microsoft.GS.DPS.Host/DependencyConfiguration/ServiceDependencies.cs index 359841e4..0d263f7a 100644 --- a/App/backend-api/Microsoft.GS.DPS.Host/DependencyConfiguration/ServiceDependencies.cs +++ b/App/backend-api/Microsoft.GS.DPS.Host/DependencyConfiguration/ServiceDependencies.cs @@ -28,7 +28,6 @@ public static void Inject(IHostApplicationBuilder builder) .AddSingleton() .AddSingleton(x => { - var aiService = x.GetRequiredService>().Value; return Kernel.CreateBuilder() .AddAzureOpenAIChatCompletion(deploymentName: builder.Configuration.GetSection("Application:AIServices:GPT-4o-mini")["ModelName"] ?? "", endpoint: builder.Configuration.GetSection("Application:AIServices:GPT-4o-mini")["Endpoint"] ?? "", diff --git a/App/backend-api/Microsoft.GS.DPS.Host/Helpers/AzureCredentialHelper.cs b/App/backend-api/Microsoft.GS.DPS.Host/Helpers/AzureCredentialHelper.cs index 49fc01f5..bbfc01dc 100644 --- a/App/backend-api/Microsoft.GS.DPS.Host/Helpers/AzureCredentialHelper.cs +++ b/App/backend-api/Microsoft.GS.DPS.Host/Helpers/AzureCredentialHelper.cs @@ -1,34 +1,28 @@ -using System; -using System.Threading.Tasks; -using Azure.Core; -using Azure.Identity; - -namespace Microsoft.GS.DPSHost.Helpers -{ - /// - /// The Azure Credential Helper class - /// - public static class AzureCredentialHelper - { - /// - /// Get the Azure Credentials based on the environment type - /// - /// The client Id in case of User assigned Managed identity - /// The Credential Object - public static TokenCredential GetAzureCredential(string? clientId = null) - { - var env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Production"; - - if (string.Equals(env, "Development", StringComparison.OrdinalIgnoreCase)) - { - return new DefaultAzureCredential(); // CodeQL [SM05139] Okay use of DefaultAzureCredential as it is only used in development - } - else - { - return clientId != null - ? new ManagedIdentityCredential(clientId) - : new ManagedIdentityCredential(); - } - } - } +using System; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Identity; + +namespace Microsoft.GS.DPSHost.Helpers +{ + /// + /// The Azure Credential Helper class + /// + public static class AzureCredentialHelper + { + /// + /// Get the Azure Credentials based on the environment type + /// + /// The client Id in case of User assigned Managed identity + /// The Credential Object + public static TokenCredential GetAzureCredential(string? clientId = null) + { + var env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Production"; + + // CodeQL [SM05139] Okay use of DefaultAzureCredential as it is only used in development + return string.Equals(env, "Development", StringComparison.OrdinalIgnoreCase) + ? new DefaultAzureCredential() // CodeQL [SM05139] Okay use of DefaultAzureCredential as it is only used in development + : (clientId != null ? new ManagedIdentityCredential(clientId) : new ManagedIdentityCredential()); + } + } } \ No newline at end of file diff --git a/App/backend-api/Microsoft.GS.DPS/API/ChatHost/ChatHost.cs b/App/backend-api/Microsoft.GS.DPS/API/ChatHost/ChatHost.cs index e238330a..ee90e8b9 100644 --- a/App/backend-api/Microsoft.GS.DPS/API/ChatHost/ChatHost.cs +++ b/App/backend-api/Microsoft.GS.DPS/API/ChatHost/ChatHost.cs @@ -49,7 +49,7 @@ static ChatHost() var assemblyLocation = Assembly.GetExecutingAssembly().Location; var assemblyDirectory = System.IO.Path.GetDirectoryName(assemblyLocation); // binding assembly directory with file path (Prompts/Chat_SystemPrompt.txt) - var systemPromptFilePath = System.IO.Path.Combine(assemblyDirectory, "Prompts/Chat_SystemPrompt.txt"); + var systemPromptFilePath = System.IO.Path.Combine(assemblyDirectory, "Prompts", "Chat_SystemPrompt.txt"); ChatHost.s_systemPrompt = System.IO.File.ReadAllText(systemPromptFilePath); ChatHost.s_assistancePrompt = @" @@ -74,15 +74,7 @@ Please feel free to ask me any questions related to those documents and contents private async Task makeNewSession(string? chatSessionId) { - var sessionId = string.Empty; - if(string.IsNullOrEmpty(chatSessionId)) - { - sessionId = Guid.NewGuid().ToString(); - } - else - { - sessionId = chatSessionId; - } + var sessionId = string.IsNullOrEmpty(chatSessionId) ? Guid.NewGuid().ToString() : chatSessionId; //Create New Chat History this.chatHistory = new ChatHistory(); @@ -92,7 +84,7 @@ private async Task makeNewSession(string? chatSessionId) //Create a new ChatSession Entity for Saving into Azure Cosmos return new ChatSession() { - SessionId = sessionId, // New Session ID + SessionId = this.sessionId, // New Session ID StartTime = DateTime.UtcNow // Session Created Time }; diff --git a/App/backend-api/Microsoft.GS.DPS/API/KernelMemory/KernelMemory.cs b/App/backend-api/Microsoft.GS.DPS/API/KernelMemory/KernelMemory.cs index 1c167612..854d9f81 100644 --- a/App/backend-api/Microsoft.GS.DPS/API/KernelMemory/KernelMemory.cs +++ b/App/backend-api/Microsoft.GS.DPS/API/KernelMemory/KernelMemory.cs @@ -36,7 +36,7 @@ static KernelMemory() var assemblyLocation = Assembly.GetExecutingAssembly().Location; var assemblyDirectory = System.IO.Path.GetDirectoryName(assemblyLocation); // binding assembly directory with file path (Prompts/KeywordExtract_SystemPrompt.txt) - var systemPromptFilePath = System.IO.Path.Combine(assemblyDirectory, "Prompts/KeywordExtract_SystemPrompt.txt"); + var systemPromptFilePath = System.IO.Path.Combine(assemblyDirectory, "Prompts", "KeywordExtract_SystemPrompt.txt"); KernelMemory.keywordExtractorPrompt = System.IO.File.ReadAllText(systemPromptFilePath); } @@ -197,7 +197,7 @@ private async Task getSummary(string documentId, string fileName) return keywordDict; } - catch (Exception ex) + catch (Exception) { return new Dictionary(); } diff --git a/App/backend-api/Microsoft.GS.DPS/API/UserInterface/DataCacheManager.cs b/App/backend-api/Microsoft.GS.DPS/API/UserInterface/DataCacheManager.cs index e3d6dcc8..379298a3 100644 --- a/App/backend-api/Microsoft.GS.DPS/API/UserInterface/DataCacheManager.cs +++ b/App/backend-api/Microsoft.GS.DPS/API/UserInterface/DataCacheManager.cs @@ -1,86 +1,83 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using System.Timers; -using Microsoft.GS.DPS.Storage.Document; -using Timers =System.Timers; - -namespace Microsoft.GS.DPS.API.UserInterface -{ - public class DataCacheManager - { - private readonly DocumentRepository _documentRepository; - private Dictionary> _keywordCache; - private readonly Timers.Timer _cacheTimer; - private readonly object _cacheLock = new object(); - - public DataCacheManager(DocumentRepository documentRepository) - { - _documentRepository = documentRepository; - _keywordCache = new Dictionary>(); - _cacheTimer = new Timers.Timer(5 * 60 * 1000); // 5 minutes - _cacheTimer.Elapsed += async (sender, e) => await RefreshCacheAsync(); - _cacheTimer.Start(); - } - - public async Task>> GetConsolidatedKeywordsAsync() - { - if (_keywordCache.Count == 0) - { - await RefreshCacheAsync(); - } - - lock (_cacheLock) - { - return new Dictionary>(_keywordCache); - } - } - - public async Task RefreshCacheAsync() - { - var consolidatedKeywords = new Dictionary>(); - var documents = await _documentRepository.GetAllDocuments(); - - foreach (var document in documents) - { - if (document.Keywords != null) - { - foreach (var keywordDict in document.Keywords) - { - if (!consolidatedKeywords.ContainsKey(keywordDict.Key)) - { - consolidatedKeywords[keywordDict.Key] = new List(); - } - - var values = keywordDict.Value.Split(',').Select(v => v.Trim()).ToArray(); - - foreach (var value in values) - { - if (!consolidatedKeywords[keywordDict.Key].Contains(value)) - { - consolidatedKeywords[keywordDict.Key].Add(value); - } - } - - consolidatedKeywords[keywordDict.Key] = consolidatedKeywords[keywordDict.Key].OrderBy(v => v).ToList(); - } - } - } - - consolidatedKeywords = consolidatedKeywords.OrderBy(k => k.Key).ToDictionary(k => k.Key, v => v.Value); - - lock (_cacheLock) - { - _keywordCache = consolidatedKeywords; - } - } - - public void ManualRefresh() - { - _cacheTimer.Stop(); - _cacheTimer.Start(); - Task.Run(async () => await RefreshCacheAsync()); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System.Timers; +using Microsoft.GS.DPS.Storage.Document; +using Timers =System.Timers; + +namespace Microsoft.GS.DPS.API.UserInterface +{ + public class DataCacheManager + { + private readonly DocumentRepository _documentRepository; + private Dictionary> _keywordCache; + private readonly Timers.Timer _cacheTimer; + private readonly object _cacheLock = new object(); + + public DataCacheManager(DocumentRepository documentRepository) + { + _documentRepository = documentRepository; + _keywordCache = new Dictionary>(); + _cacheTimer = new Timers.Timer(5 * 60 * 1000); // 5 minutes + _cacheTimer.Elapsed += async (sender, e) => await RefreshCacheAsync(); + _cacheTimer.Start(); + } + + public async Task>> GetConsolidatedKeywordsAsync() + { + if (_keywordCache.Count == 0) + { + await RefreshCacheAsync(); + } + + lock (_cacheLock) + { + return new Dictionary>(_keywordCache); + } + } + + public async Task RefreshCacheAsync() + { + var consolidatedKeywords = new Dictionary>(); + var documents = await _documentRepository.GetAllDocuments(); + + foreach (var document in documents.Where(d => d.Keywords != null)) + { + foreach (var keywordDict in document.Keywords) + { + if (!consolidatedKeywords.ContainsKey(keywordDict.Key)) + { + consolidatedKeywords[keywordDict.Key] = new List(); + } + + var values = keywordDict.Value.Split(',').Select(v => v.Trim()).ToArray(); + + foreach (var value in values) + { + if (!consolidatedKeywords[keywordDict.Key].Contains(value)) + { + consolidatedKeywords[keywordDict.Key].Add(value); + } + } + + consolidatedKeywords[keywordDict.Key] = consolidatedKeywords[keywordDict.Key].OrderBy(v => v).ToList(); + } + } + + consolidatedKeywords = consolidatedKeywords.OrderBy(k => k.Key).ToDictionary(k => k.Key, v => v.Value); + + lock (_cacheLock) + { + _keywordCache = consolidatedKeywords; + } + } + + public void ManualRefresh() + { + _cacheTimer.Stop(); + _cacheTimer.Start(); + Task.Run(async () => await RefreshCacheAsync()); + } + } +} diff --git a/App/backend-api/Microsoft.GS.DPS/API/UserInterface/Documents.cs b/App/backend-api/Microsoft.GS.DPS/API/UserInterface/Documents.cs index 981e0744..51c1170e 100644 --- a/App/backend-api/Microsoft.GS.DPS/API/UserInterface/Documents.cs +++ b/App/backend-api/Microsoft.GS.DPS/API/UserInterface/Documents.cs @@ -1,253 +1,247 @@ -using Microsoft.GS.DPS.Storage.Document; -using Entities = Microsoft.GS.DPS.Storage.Document.Entities; -using Microsoft.KernelMemory; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.GS.DPS.Storage.Document.Entities; -using System.Reflection.Metadata; -using System.Text.Json; -using static Microsoft.Extensions.Logging.EventSource.LoggingEventSource; -namespace Microsoft.GS.DPS.API.UserInterface -{ - public class Documents - { - private readonly DocumentRepository _documentRepository; - private readonly MemoryWebClient _memoryWebClient; - private readonly DataCacheManager _dataCache; - - public Documents(DocumentRepository documentRepository, MemoryWebClient memoryWebClient, DataCacheManager dataCache) - { - _documentRepository = documentRepository; - _memoryWebClient = memoryWebClient; - _dataCache = dataCache; - } - - private async Task GetAllDocumentsByPageAsync(int pageNumber, - int pageSize, - DateTime? startDate, - DateTime? endDate) - { - return await _documentRepository.GetAllDocumentsByPageAsync(pageNumber, pageSize, startDate, endDate); - } - - - public async Task GetDocuments(int pageNumber, - int pageSize, - DateTime? startDate, - DateTime? endDate) - { - var resultSet = await this.GetAllDocumentsByPageAsync(pageNumber, pageSize,startDate, endDate); - //var keywordFilterInfo = GetConsolidatedKeywords(resultSet.Results); - //var keywordFilterInfo = await GetConsolidatedKeywords(); - var keywordFilterInfo = await _dataCache.GetConsolidatedKeywordsAsync(); - - - return new Model.UserInterface.DocumentQuerySet - { - documents = resultSet.Results, - keywordFilterInfo = keywordFilterInfo, - TotalPages = resultSet.TotalPages, - CurrentPage = resultSet.CurrentPage, - TotalRecords = resultSet.TotalRecords - }; - } - - public async Task GetDocument(string documentId) - { - return await _documentRepository.FindByDocumentIdAsync(documentId); - } - - //public async Task GetDocumentsByDocumentIds(string[] documentIds) - //{ - // var documents = await _documentRepository.FindByDocumentIdsAsync(documentIds); - // return new Model.UserInterface.DocumentQuerySet - // { - // documents = documents.Results, - // keywordFilterInfo = GetConsolidatedKeywords(documents.Results), - // TotalPages = documents.TotalPages, - // CurrentPage = documents.CurrentPage, - // TotalRecords = documents.TotalRecords - // }; - //} - - //public async Task GetDocumentsByTagAsync(Dictionary tags, int pageNumber, int pageSize) - //{ - // var documents = await _documentRepository.FindByTagsAsync(tags, pageNumber, pageSize); - - // return new Model.UserInterface.DocumentQuerySet - // { - // documents = documents.Results, - // keywordFilterInfo = GetConsolidatedKeywords(documents.Results), - // TotalPages = documents.TotalPages, - // CurrentPage = documents.CurrentPage, - // TotalRecords = documents.TotalRecords - // }; - //} - - //private async Task DownloadSummaryFromBlob(string documentId, string fileName) - //{ - // StreamableFileContent file = await _memoryWebClient.ExportFileAsync(documentId, $"{fileName}.summarize.0.txt"); - // Stream summarizedFileStream = await file.GetStreamAsync(); - // return await new StreamReader(summarizedFileStream).ReadToEndAsync(); - //} - - /// - /// Search by Keywords and Tags with Paging - /// - /// Page Number - /// Page Size (Item Numbers per Page) - /// Search Keyword - /// Tags - /// - public async Task GetDocumentsWithQuery(int pageNumber, - int pageSize, - string? query, - Dictionary? tags, - DateTime? searchStartDate, - DateTime? searchEndDate) - { - //Search from Memory then get the documents - List filters = new List(); - - if (tags != null && tags.Count > 0) - { - //The payload will be key and string values with comma separated - //every values should be added to the filter with same key - foreach (var kvp in tags) - { - var values = kvp.Value.Split(',').Select(v => v.Trim()).ToArray(); - foreach (var item in values) - { - filters.Add(new MemoryFilter().ByTag(kvp.Key, item)); - } - } - } - - if ((string.IsNullOrEmpty(query) || query.Contains("*")) && filters.Count == 0) - { - return await this.GetDocuments(pageNumber, pageSize, searchStartDate, searchEndDate); - } - else - { - //when query string contains space, it should be add within [string] to avoiding separate search - if(!string.IsNullOrEmpty(query) && query.Contains(" ")) - { - //make a double quote to avoid separate search - query = $"\"{query}\""; - } - - if(!string.IsNullOrEmpty(query) && query.Contains("*")) - { - query = null; - } - - SearchResult result = await this._memoryWebClient.SearchAsync(query ?? String.Empty, filters: filters, minRelevance: 0.0166666676); - - //Get Document Ids from result - var documentIds = result.Results.Select(r => r.DocumentId).ToArray(); - - //Get Documents from Repository - QueryResultSet resultSet = await _documentRepository.FindByDocumentIdsAsync(documentIds, pageNumber, pageSize); - var filteredDocuments = resultSet.Results.Where(document => (!searchStartDate.HasValue || document.ImportedTime >= searchStartDate.Value) && (!searchEndDate.HasValue || document.ImportedTime <= searchEndDate.Value)).ToList(); - - return new Model.UserInterface.DocumentQuerySet - { - documents = filteredDocuments, - //keywordFilterInfo = GetConsolidatedKeywords(resultSet.Results), - //keywordFilterInfo = await GetConsolidatedKeywords(), - keywordFilterInfo = await _dataCache.GetConsolidatedKeywordsAsync(), - TotalPages = resultSet.TotalPages, - CurrentPage = resultSet.CurrentPage, - TotalRecords = resultSet.TotalRecords - }; - } - } - - public Dictionary> GetConsolidatedKeywords(IEnumerable documents) - { - //var documents = await this.EntityCollection.GetAllAsync(); - var consolidatedKeywords = new Dictionary>(); - - foreach (var document in documents) - { - if (document.Keywords != null) - { - foreach (var keywordDict in document.Keywords) - { - if (!consolidatedKeywords.ContainsKey(keywordDict.Key)) - { - consolidatedKeywords[keywordDict.Key] = new List(); - } - - //Before adding Value, check the value is already existing - //Split comma separated values and add to the list - var values = keywordDict.Value.Split(',').Select(v => v.Trim()).ToArray(); - - foreach (var value in values) - { - if (!consolidatedKeywords[keywordDict.Key].Contains(value)) - { - consolidatedKeywords[keywordDict.Key].Add(value); - } - - //set order values under same Key by asc. - consolidatedKeywords[keywordDict.Key] = consolidatedKeywords[keywordDict.Key].OrderBy(v => v).ToList(); - } - } - } - } - - //set order key by asc - consolidatedKeywords = consolidatedKeywords.OrderBy(k => k.Key).ToDictionary(k => k.Key, v => v.Value); - return consolidatedKeywords; - } - - - - public async Task>> GetConsolidatedKeywords() - { - //var documents = await this.EntityCollection.GetAllAsync(); - var consolidatedKeywords = new Dictionary>(); - - //Get All Records only Keywords field. - var documents = await _documentRepository.GetAllDocuments(); - - foreach (var document in documents) - { - if (document.Keywords != null) - { - foreach (var keywordDict in document.Keywords) - { - if (!consolidatedKeywords.ContainsKey(keywordDict.Key)) - { - consolidatedKeywords[keywordDict.Key] = new List(); - } - - //Before adding Value, check the value is already existing - //Split comma separated values and add to the list - var values = keywordDict.Value.Split(',').Select(v => v.Trim()).ToArray(); - - foreach (var value in values) - { - if (!consolidatedKeywords[keywordDict.Key].Contains(value)) - { - consolidatedKeywords[keywordDict.Key].Add(value); - } - - //set order values under same Key by asc. - consolidatedKeywords[keywordDict.Key] = consolidatedKeywords[keywordDict.Key].OrderBy(v => v).ToList(); - - } - } - } - } - - //set order key by asc - consolidatedKeywords = consolidatedKeywords.OrderBy(k => k.Key).ToDictionary(k => k.Key, v => v.Value); - return consolidatedKeywords; - } - } -} +using Microsoft.GS.DPS.Storage.Document; +using Entities = Microsoft.GS.DPS.Storage.Document.Entities; +using Microsoft.KernelMemory; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.GS.DPS.Storage.Document.Entities; +using System.Reflection.Metadata; +using System.Text.Json; +using static Microsoft.Extensions.Logging.EventSource.LoggingEventSource; +namespace Microsoft.GS.DPS.API.UserInterface +{ + public class Documents + { + private readonly DocumentRepository _documentRepository; + private readonly MemoryWebClient _memoryWebClient; + private readonly DataCacheManager _dataCache; + + public Documents(DocumentRepository documentRepository, MemoryWebClient memoryWebClient, DataCacheManager dataCache) + { + _documentRepository = documentRepository; + _memoryWebClient = memoryWebClient; + _dataCache = dataCache; + } + + private async Task GetAllDocumentsByPageAsync(int pageNumber, + int pageSize, + DateTime? startDate, + DateTime? endDate) + { + return await _documentRepository.GetAllDocumentsByPageAsync(pageNumber, pageSize, startDate, endDate); + } + + + public async Task GetDocuments(int pageNumber, + int pageSize, + DateTime? startDate, + DateTime? endDate) + { + var resultSet = await this.GetAllDocumentsByPageAsync(pageNumber, pageSize,startDate, endDate); + //var keywordFilterInfo = GetConsolidatedKeywords(resultSet.Results); + //var keywordFilterInfo = await GetConsolidatedKeywords(); + var keywordFilterInfo = await _dataCache.GetConsolidatedKeywordsAsync(); + + + return new Model.UserInterface.DocumentQuerySet + { + documents = resultSet.Results, + keywordFilterInfo = keywordFilterInfo, + TotalPages = resultSet.TotalPages, + CurrentPage = resultSet.CurrentPage, + TotalRecords = resultSet.TotalRecords + }; + } + + public async Task GetDocument(string documentId) + { + return await _documentRepository.FindByDocumentIdAsync(documentId); + } + + //public async Task GetDocumentsByDocumentIds(string[] documentIds) + //{ + // var documents = await _documentRepository.FindByDocumentIdsAsync(documentIds); + // return new Model.UserInterface.DocumentQuerySet + // { + // documents = documents.Results, + // keywordFilterInfo = GetConsolidatedKeywords(documents.Results), + // TotalPages = documents.TotalPages, + // CurrentPage = documents.CurrentPage, + // TotalRecords = documents.TotalRecords + // }; + //} + + //public async Task GetDocumentsByTagAsync(Dictionary tags, int pageNumber, int pageSize) + //{ + // var documents = await _documentRepository.FindByTagsAsync(tags, pageNumber, pageSize); + + // return new Model.UserInterface.DocumentQuerySet + // { + // documents = documents.Results, + // keywordFilterInfo = GetConsolidatedKeywords(documents.Results), + // TotalPages = documents.TotalPages, + // CurrentPage = documents.CurrentPage, + // TotalRecords = documents.TotalRecords + // }; + //} + + //private async Task DownloadSummaryFromBlob(string documentId, string fileName) + //{ + // StreamableFileContent file = await _memoryWebClient.ExportFileAsync(documentId, $"{fileName}.summarize.0.txt"); + // Stream summarizedFileStream = await file.GetStreamAsync(); + // return await new StreamReader(summarizedFileStream).ReadToEndAsync(); + //} + + /// + /// Search by Keywords and Tags with Paging + /// + /// Page Number + /// Page Size (Item Numbers per Page) + /// Search Keyword + /// Tags + /// + public async Task GetDocumentsWithQuery(int pageNumber, + int pageSize, + string? query, + Dictionary? tags, + DateTime? searchStartDate, + DateTime? searchEndDate) + { + //Search from Memory then get the documents + List filters = new List(); + + if (tags != null && tags.Count > 0) + { + //The payload will be key and string values with comma separated + //every values should be added to the filter with same key + foreach (var kvp in tags) + { + var values = kvp.Value.Split(',').Select(v => v.Trim()).ToArray(); + foreach (var item in values) + { + filters.Add(new MemoryFilter().ByTag(kvp.Key, item)); + } + } + } + + if ((string.IsNullOrEmpty(query) || query.Contains("*")) && filters.Count == 0) + { + return await this.GetDocuments(pageNumber, pageSize, searchStartDate, searchEndDate); + } + else + { + //when query string contains space, it should be add within [string] to avoiding separate search + if(!string.IsNullOrEmpty(query) && query.Contains(" ")) + { + //make a double quote to avoid separate search + query = $"\"{query}\""; + } + + if(!string.IsNullOrEmpty(query) && query.Contains("*")) + { + query = null; + } + + SearchResult result = await this._memoryWebClient.SearchAsync(query ?? String.Empty, filters: filters, minRelevance: 0.0166666676); + + //Get Document Ids from result + var documentIds = result.Results.Select(r => r.DocumentId).ToArray(); + + //Get Documents from Repository + QueryResultSet resultSet = await _documentRepository.FindByDocumentIdsAsync(documentIds, pageNumber, pageSize); + var filteredDocuments = resultSet.Results.Where(document => (!searchStartDate.HasValue || document.ImportedTime >= searchStartDate.Value) && (!searchEndDate.HasValue || document.ImportedTime <= searchEndDate.Value)).ToList(); + + return new Model.UserInterface.DocumentQuerySet + { + documents = filteredDocuments, + //keywordFilterInfo = GetConsolidatedKeywords(resultSet.Results), + //keywordFilterInfo = await GetConsolidatedKeywords(), + keywordFilterInfo = await _dataCache.GetConsolidatedKeywordsAsync(), + TotalPages = resultSet.TotalPages, + CurrentPage = resultSet.CurrentPage, + TotalRecords = resultSet.TotalRecords + }; + } + } + + public Dictionary> GetConsolidatedKeywords(IEnumerable documents) + { + //var documents = await this.EntityCollection.GetAllAsync(); + var consolidatedKeywords = new Dictionary>(); + + foreach (var document in documents.Where(d => d.Keywords != null)) + { + foreach (var keywordDict in document.Keywords) + { + if (!consolidatedKeywords.ContainsKey(keywordDict.Key)) + { + consolidatedKeywords[keywordDict.Key] = new List(); + } + + //Before adding Value, check the value is already existing + //Split comma separated values and add to the list + var values = keywordDict.Value.Split(',').Select(v => v.Trim()).ToArray(); + + foreach (var value in values) + { + if (!consolidatedKeywords[keywordDict.Key].Contains(value)) + { + consolidatedKeywords[keywordDict.Key].Add(value); + } + + //set order values under same Key by asc. + consolidatedKeywords[keywordDict.Key] = consolidatedKeywords[keywordDict.Key].OrderBy(v => v).ToList(); + } + } + } + + //set order key by asc + consolidatedKeywords = consolidatedKeywords.OrderBy(k => k.Key).ToDictionary(k => k.Key, v => v.Value); + return consolidatedKeywords; + } + + + + public async Task>> GetConsolidatedKeywords() + { + //var documents = await this.EntityCollection.GetAllAsync(); + var consolidatedKeywords = new Dictionary>(); + + //Get All Records only Keywords field. + var documents = await _documentRepository.GetAllDocuments(); + + foreach (var document in documents.Where(d => d.Keywords != null)) + { + foreach (var keywordDict in document.Keywords) + { + if (!consolidatedKeywords.ContainsKey(keywordDict.Key)) + { + consolidatedKeywords[keywordDict.Key] = new List(); + } + + //Before adding Value, check the value is already existing + //Split comma separated values and add to the list + var values = keywordDict.Value.Split(',').Select(v => v.Trim()).ToArray(); + + foreach (var value in values) + { + if (!consolidatedKeywords[keywordDict.Key].Contains(value)) + { + consolidatedKeywords[keywordDict.Key].Add(value); + } + + //set order values under same Key by asc. + consolidatedKeywords[keywordDict.Key] = consolidatedKeywords[keywordDict.Key].OrderBy(v => v).ToList(); + + } + } + } + + //set order key by asc + consolidatedKeywords = consolidatedKeywords.OrderBy(k => k.Key).ToDictionary(k => k.Key, v => v.Value); + return consolidatedKeywords; + } + } +} diff --git a/App/backend-api/Microsoft.GS.DPS/Storage/AISearch/TagUpdater.cs b/App/backend-api/Microsoft.GS.DPS/Storage/AISearch/TagUpdater.cs index 3b6d1a41..8c7cb4b4 100644 --- a/App/backend-api/Microsoft.GS.DPS/Storage/AISearch/TagUpdater.cs +++ b/App/backend-api/Microsoft.GS.DPS/Storage/AISearch/TagUpdater.cs @@ -47,7 +47,7 @@ public async Task UpdateTags(string documentId, List updatingTags) try { var response = await _searchClient.MergeOrUploadDocumentsAsync(new[] { updateDocument }); - Console.WriteLine($"Document with ID {document["id"]} updated successfully. - {response.GetRawResponse().ToString()}"); + Console.WriteLine($"Document with ID {document["id"]} updated successfully. - {response.GetRawResponse()}"); } catch (Exception ex) { diff --git a/App/backend-api/Microsoft.GS.DPS/Storage/Components/BusinessTransactionRepository.cs b/App/backend-api/Microsoft.GS.DPS/Storage/Components/BusinessTransactionRepository.cs index a5a20a83..d12b3092 100644 --- a/App/backend-api/Microsoft.GS.DPS/Storage/Components/BusinessTransactionRepository.cs +++ b/App/backend-api/Microsoft.GS.DPS/Storage/Components/BusinessTransactionRepository.cs @@ -80,9 +80,7 @@ public async Task> GetAllAsync() public async Task> GetAllAsync(IEnumerable identifiers) { - List results = new List(); - IMongoCollection collection = _database.GetCollection(typeof(TEntity).Name.ToLowerInvariant()); foreach (var i in identifiers) { results.Add(await this.GetAsync(i)); diff --git a/App/backend-api/Microsoft.GS.DPS/Storage/Documents/DocumentRepository.cs b/App/backend-api/Microsoft.GS.DPS/Storage/Documents/DocumentRepository.cs index 2de31a28..413aa26c 100644 --- a/App/backend-api/Microsoft.GS.DPS/Storage/Documents/DocumentRepository.cs +++ b/App/backend-api/Microsoft.GS.DPS/Storage/Documents/DocumentRepository.cs @@ -146,14 +146,7 @@ private int GetTotalPages(int pageSize, double recordsCount) public async Task UpdateAsync(Entities.Document document) { var result = await _collection.ReplaceOneAsync(Builders.Filter.Eq(x => x.id, document.id), document); - if (result.IsAcknowledged && result.ModifiedCount > 0) - { - return document; - } - else - { - return null; - } + return (result.IsAcknowledged && result.ModifiedCount > 0) ? document : null; } public async Task DeleteAsync(Guid id) @@ -192,14 +185,12 @@ async public Task FindByDocumentIdsAsync(string[] documentIds, //Just in case StartDate and EndDate is not null, define filter between StartDate and EndDate //endDate should be converted from datetime to DateTime of end of day Day:23:59:59 - FilterDefinition filter = Builders.Filter.Empty; - if (endDate.HasValue) { endDate = endDate?.Date.AddHours(23).AddMinutes(59).AddSeconds(59); - filter = Builders.Filter.Gte(x => x.ImportedTime, startDate ?? DateTime.Now) & - Builders.Filter.Lte(x => x.ImportedTime, endDate ?? endDate); - + var timeFilter = Builders.Filter.Gte(x => x.ImportedTime, startDate ?? DateTime.Now) & + Builders.Filter.Lte(x => x.ImportedTime, endDate.Value); + filterDefinition &= timeFilter; } diff --git a/App/frontend-app/src/App.tsx b/App/frontend-app/src/App.tsx index 10d8065c..6473315d 100644 --- a/App/frontend-app/src/App.tsx +++ b/App/frontend-app/src/App.tsx @@ -1,9 +1,7 @@ import React, { Suspense } from "react"; import { BrowserRouter } from "react-router-dom"; import { Layout } from "./components/layout/layout"; -import { Telemetry } from "./utils/telemetry/telemetry"; -import { AppInsightsContext, ReactPlugin } from "@microsoft/applicationinsights-react-js"; -import { FluentProvider, makeStyles, webLightTheme } from "@fluentui/react-components"; +import { FluentProvider, webLightTheme } from "@fluentui/react-components"; import resolveConfig from "tailwindcss/resolveConfig"; import TailwindConfig from "../tailwind.config"; import AppRoutes from "./AppRoutes"; diff --git a/App/frontend-app/src/AppContext.tsx b/App/frontend-app/src/AppContext.tsx index b886f017..cf9526f3 100644 --- a/App/frontend-app/src/AppContext.tsx +++ b/App/frontend-app/src/AppContext.tsx @@ -1,6 +1,5 @@ import { ReactNode, createContext, useState } from 'react'; import { ChatApiResponse } from './api/apiTypes/chatTypes'; -import { SearchFacet } from './types/searchRequest'; export interface IAppContext { conversationAnswers: [prompt: string, response: ChatApiResponse, userTimestamp?: Date, answerTimestamp?: Date][]; diff --git a/App/frontend-app/src/AppRoutes.tsx b/App/frontend-app/src/AppRoutes.tsx index 81f56eaf..0cbce159 100644 --- a/App/frontend-app/src/AppRoutes.tsx +++ b/App/frontend-app/src/AppRoutes.tsx @@ -2,8 +2,6 @@ import { Routes, Route } from "react-router-dom"; import { Home } from "./pages/home/home"; import { ChatPage } from "./pages/chat/chatPage"; // import { PersonalDocumentsPage } from "./pages/personalDocuments/personalDocumentsPage"; -import { useState } from "react"; -import { SearchFacet } from "./types/searchRequest"; function App() { diff --git a/App/frontend-app/src/api/apiTypes/documentResults.ts b/App/frontend-app/src/api/apiTypes/documentResults.ts index c5a965f9..dccc69de 100644 --- a/App/frontend-app/src/api/apiTypes/documentResults.ts +++ b/App/frontend-app/src/api/apiTypes/documentResults.ts @@ -1,4 +1,3 @@ -import { Facets } from "../../types/facets"; export interface DocumentResults { documents: Document[] diff --git a/App/frontend-app/src/api/documentsService.ts b/App/frontend-app/src/api/documentsService.ts index 0d8abada..6b2c3b21 100644 --- a/App/frontend-app/src/api/documentsService.ts +++ b/App/frontend-app/src/api/documentsService.ts @@ -2,7 +2,6 @@ import { SearchRequest } from "../types/searchRequest"; import { httpClient } from "../utils/httpClient/httpClient"; import { DocumentResults } from "./apiTypes/documentResults"; import { Embedded } from "./apiTypes/embedded"; -import { SingleDocument } from "./apiTypes/singleDocument"; @@ -57,25 +56,23 @@ export const importDocuments = async (formData: FormData): Promise => { }; - - -function formatKeywords(keywords: { [key: string]: string }): { [key: string]: string } { - // This function formats keywords into the desired comma-separated string for each category - const formattedKeywords: { [key: string]: string } = {}; - - Object.keys(keywords).forEach((category) => { - const keywordList = keywords[category]; - if (Array.isArray(keywordList)) { - // If the keywords are in an array, join them into a comma-separated string - formattedKeywords[category] = keywordList.join(', '); - } else { - // If already a string (or incorrect format), preserve it - formattedKeywords[category] = keywordList; - } - }); - - return formattedKeywords; -} +// function formatKeywords(keywords: { [key: string]: string }): { [key: string]: string } { +// // This function formats keywords into the desired comma-separated string for each category +// const formattedKeywords: { [key: string]: string } = {}; + +// Object.keys(keywords).forEach((category) => { +// const keywordList = keywords[category]; +// if (Array.isArray(keywordList)) { +// // If the keywords are in an array, join them into a comma-separated string +// formattedKeywords[category] = keywordList.join(', '); +// } else { +// // If already a string (or incorrect format), preserve it +// formattedKeywords[category] = keywordList; +// } +// }); + +// return formattedKeywords; +// } // Update in your documentsService file export const downloadDocument = async (documentId: string, fileName: string): Promise => { diff --git a/App/frontend-app/src/components/chat/FeedbackForm.tsx b/App/frontend-app/src/components/chat/FeedbackForm.tsx index 05346b33..ea9738db 100644 --- a/App/frontend-app/src/components/chat/FeedbackForm.tsx +++ b/App/frontend-app/src/components/chat/FeedbackForm.tsx @@ -15,7 +15,7 @@ import { } from "@fluentui/react-components"; import { AddCircle24Regular, Dismiss24Regular, SubtractCircle24Regular } from "@fluentui/react-icons"; import { useState } from "react"; -import { ChatMessage, History, Reference } from "../../api/apiTypes/chatTypes"; +import { History, Reference } from "../../api/apiTypes/chatTypes"; import { ChatOptions } from "../../api/apiTypes/chatTypes"; import { PostFeedback } from "../../api/chatService"; import { useTranslation } from "react-i18next"; diff --git a/App/frontend-app/src/components/chat/chatRoom.tsx b/App/frontend-app/src/components/chat/chatRoom.tsx index 89952cb9..84bcc84b 100644 --- a/App/frontend-app/src/components/chat/chatRoom.tsx +++ b/App/frontend-app/src/components/chat/chatRoom.tsx @@ -9,18 +9,15 @@ import { DialogSurface, DialogTitle, Tag, - Tooltip, makeStyles, } from "@fluentui/react-components"; import { DocDialog } from "../documentViewer/documentViewer"; import { Textarea } from "@fluentai/textarea"; import type { TextareaSubmitEvents, TextareaValueData } from "@fluentai/textarea"; import { CopilotChat, UserMessage, CopilotMessage } from "@fluentai/react-copilot-chat"; -import { ChatAdd24Regular, DocumentOnePageLink20Regular } from "@fluentui/react-icons"; -import { AttachmentTag } from "@fluentai/attachments"; +import { ChatAdd24Regular } from "@fluentui/react-icons"; import styles from "./chatRoom.module.scss"; -import { CopilotProvider, FeedbackButtons, Suggestion } from "@fluentai/react-copilot"; -import { Result, SingleDocument, Tokens } from "../../api/apiTypes/singleDocument"; +import { CopilotProvider, Suggestion } from "@fluentai/react-copilot"; //import { getDocument } from "../../api/documentsService"; import { Completion, PostFeedback } from "../../api/chatService"; import { FeedbackForm } from "./FeedbackForm"; @@ -48,21 +45,19 @@ interface ChatRoomProps { } export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDocument, disableOptionsPanel, clearChatFlag }: ChatRoomProps) { - const customStyles = useStyles(); + const { t } = useTranslation(); const [chatSessionId, setChatSessionId] = useState(null); const [isLoading, setIsLoading] = useState(false); const [disableSources, setDisableSources] = useState(false); - const [error, setError] = useState(); const [model, setModel] = useState("chat_35"); const [source, setSource] = useState("rag"); const [temperature, setTemperature] = useState(0.8); - const [maxTokens, setMaxTokens] = useState(750); + const [maxTokens] = useState(750); const [selectedDocument, setSelectedDocument] = useState(chatWithDocument); const [button, setButton] = useState(""); - const [dialogMetadata, setDialogMetadata] = useState(null); + const [dialogMetadata] = useState(null); const [isDialogOpen, setIsDialogOpen] = useState(false); - const [tokens, setTokens] = useState(null); const [open, setOpen] = useState(false); const [isFeedbackFormOpen, setIsFeedbackFormOpen] = useState(false); const [options, setOptions] = useState({ @@ -75,7 +70,7 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc const [textAreaValue, setTextAreaValue] = useState(""); const [textareaKey, setTextareaKey] = useState(0); const inputRef = useRef(null); - const [allChunkTexts, setAllChunkTexts] = useState([]); + const [allChunkTexts] = useState([]); const { conversationAnswers, setConversationAnswers } = useContext(AppContext); const [isSticky, setIsSticky] = useState(false); @@ -129,11 +124,6 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc setDisableSources(true); setIsLoading(true); - // A simple function to check if the text contains Markdown - const isMarkdown = (text: string) => { - const markdownPattern = /(^|\s)(#{1,6}|\*\*|__|[-*]|\d+\.\s|\[.*\]\(.*\)|```|`[^`]*`)/; - return markdownPattern.test(text); - }; const userTimestamp = new Date(); // Ensure we have a chatSessionId or create one if null/undefined @@ -171,8 +161,6 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc | Average delinquency of NPLs sold | 2.8 years | FHFA Non-Performing Loan Sales Report | Page 2 | | Average current mark-to-market LTV ratio of NPLs | 83% | FHFA Non-Performing Loan Sales Report | Page 2 |`; - const htmlString = await marked.parse(markdown); - const htmlString2 = markdownToHtmlString(markdown); setConversationAnswers((prevAnswers) => [ ...prevAnswers, @@ -183,26 +171,11 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc }], ]); - - - - let filterByDocumentIds: string[] = []; - - const transformDocuments = (documents: any[]) => { return documents.map(doc => doc.documentId); // Extracting documentId from each document }; const formattedDocuments = transformDocuments(selectedDocuments); - - if (button === "Selected Document" && selectedDocument) { - filterByDocumentIds = [selectedDocument[0].documentId]; - } else if (button === "Search Results") { - filterByDocumentIds = searchResultDocuments.map((document) => document.documentId); - } else if (button === "Selected Documents") { - filterByDocumentIds = selectedDocuments.map((document) => document.documentId); - } - try { @@ -233,8 +206,8 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc try { if (response && response.answer) { - const formattedAnswer = removeNewlines(response.answer) - const chatResp = await marked.parse(formattedAnswer) // Convert to HTML if Markdown detected + const formattedAnswer = removeNewlines(response.answer); + const chatResp = await marked.parse(formattedAnswer); // Convert to HTML if Markdown detected @@ -325,41 +298,41 @@ export function ChatRoom({ searchResultDocuments, selectedDocuments, chatWithDoc setIsFeedbackFormOpen(false); }; - const handlePositiveFeedback = async (sources: Reference[]) => { - setIsLoading(true); - - const request = { - isPositive: true, - reason: "Correct answer", - comment: "", - history: history, - options: options, - sources: sources.map((ref) => ({ ...ref })), - filterByDocumentIds: - button === "Selected Documents" - ? selectedDocuments.map((document) => document.documentId) - : button === "Search Results" - ? searchResultDocuments.map((document) => document.documentId) - : button === "Selected Document" - ? [selectedDocument[0]?.documentId || ""] - : [], - groundTruthAnswer: "", - documentURLs: [], - chunkTexts: [], - }; + // const handlePositiveFeedback = async (sources: Reference[]) => { + // setIsLoading(true); + + // const request = { + // isPositive: true, + // reason: "Correct answer", + // comment: "", + // history: history, + // options: options, + // sources: sources.map((ref) => ({ ...ref })), + // filterByDocumentIds: + // button === "Selected Documents" + // ? selectedDocuments.map((document) => document.documentId) + // : button === "Search Results" + // ? searchResultDocuments.map((document) => document.documentId) + // : button === "Selected Document" + // ? [selectedDocument[0]?.documentId || ""] + // : [], + // groundTruthAnswer: "", + // documentURLs: [], + // chunkTexts: [], + // }; - try { - const response = await PostFeedback(request); + // try { + // const response = await PostFeedback(request); - if (response) { - setIsLoading(false); - setOpen(true); - } - } catch (error) { - setIsLoading(false); - console.error("An error occurred while submitting the feedback:", error); - } - }; + // if (response) { + // setIsLoading(false); + // setOpen(true); + // } + // } catch (error) { + // setIsLoading(false); + // console.error("An error occurred while submitting the feedback:", error); + // } + // }; const handleSubmittedFeedback = (submitted: boolean) => { if (submitted) { diff --git a/App/frontend-app/src/components/chat/modelSwitch.tsx b/App/frontend-app/src/components/chat/modelSwitch.tsx index b3c0d49b..b494b1e7 100644 --- a/App/frontend-app/src/components/chat/modelSwitch.tsx +++ b/App/frontend-app/src/components/chat/modelSwitch.tsx @@ -1,5 +1,4 @@ import { useState } from "react"; -import { Text, Tooltip } from "@fluentui/react-components"; import { useTranslation } from "react-i18next"; interface ModelSwitchProps { @@ -7,18 +6,8 @@ interface ModelSwitchProps { } export function ModelSwitch({ onSwitchChange }: ModelSwitchProps) { - const { t } = useTranslation(); - const GPT35 = "chat_35"; - const GPT4 = "chat_4"; const GPT4O = "chat_4o"; - const [activeSwitch, setActiveSwitch] = useState(GPT4O); - - const handleClick = (model: string) => { - setActiveSwitch(model); - onSwitchChange(model); - }; - return (
{/*
diff --git a/App/frontend-app/src/components/chat/optionsPanel.tsx b/App/frontend-app/src/components/chat/optionsPanel.tsx index 4126771f..bd57246b 100644 --- a/App/frontend-app/src/components/chat/optionsPanel.tsx +++ b/App/frontend-app/src/components/chat/optionsPanel.tsx @@ -1,4 +1,4 @@ -import { Label } from "@fluentui/react-components"; +import { } from "@fluentui/react-components"; import { useState, useEffect } from "react"; import { useTranslation } from "react-i18next"; import './OptionsPanel.css'; // CSS file for slider diff --git a/App/frontend-app/src/components/datePicker/dateFilterDropdownMenu.tsx b/App/frontend-app/src/components/datePicker/dateFilterDropdownMenu.tsx index bf58f254..bda4694f 100644 --- a/App/frontend-app/src/components/datePicker/dateFilterDropdownMenu.tsx +++ b/App/frontend-app/src/components/datePicker/dateFilterDropdownMenu.tsx @@ -1,6 +1,5 @@ import React, { useState, useEffect, useCallback } from "react"; import { Dropdown, Option, makeStyles } from "@fluentui/react-components"; -import { CustomDatePicker } from "./customDatePicker"; const useStyles = makeStyles({ root: { diff --git a/App/frontend-app/src/components/documentViewer/documentViewer.tsx b/App/frontend-app/src/components/documentViewer/documentViewer.tsx index e248c32f..eb0d0ba3 100644 --- a/App/frontend-app/src/components/documentViewer/documentViewer.tsx +++ b/App/frontend-app/src/components/documentViewer/documentViewer.tsx @@ -2,28 +2,21 @@ import { Dialog, DialogBody, DialogSurface, - Divider, SelectTabData, SelectTabEvent, TabListProps, - Text, makeStyles, shorthands, } from "@fluentui/react-components"; import { useEffect, useState } from "react"; import { MetadataTable } from "./metadataTable"; -import TableTab from "./tableTab"; -import { Embedded, Result } from "../../api/apiTypes/embedded"; import { Document } from "../../api/apiTypes/embedded"; -import { downloadDocument, getEmbedded } from "../../api/documentsService"; -import { Tokens } from "../../api/apiTypes/documentResults"; import { IFrameComponent } from "./iFrameComponent"; import { DialogContentComponent } from "./dialogContentComponent"; import { PagesTab } from "./PagesTab"; import { PageNumberTab } from "./pageNumberTab"; import { DialogTitleBar } from "./dialogTitleBar"; import { AIKnowledgeTab } from "./aIKnowledgeTab"; -import { useTranslation } from "react-i18next"; const useStyles = makeStyles({ dialog: { @@ -68,14 +61,12 @@ interface Cell { export function DocDialog( { metadata, isOpen, allChunkTexts, clearChatFlag, onClose, ...props }: DocDialogProps & Partial ) { - const {t} = useTranslation(); const styles = useStyles(); const [selectedTab, setSelectedTab] = useState("Document"); const [urlWithSasToken, setUrlWithSasToken] = useState(undefined); const [selectedPage, setSelectedPage] = useState(null); - const [tablesData, setTablesData] = useState([]); - const [pageMetadata, setPageMetadata] = useState(null); + const [pageMetadata] = useState(null); const [iframeKey, setIframeKey] = useState(0); const [isExpanded, setIsExpanded] = useState(false); const [clearedChatFlag, setClearChatFlag] = useState(clearChatFlag); @@ -211,7 +202,6 @@ export function DocDialog( }, [iframeKey]); const AI_KNOWLEDGE_FIELDS = window.ENV.AI_KNOWLEDGE_FIELDS; - const METADATA_EXCLUSION_LIST = window.ENV.METADATA_EXCLUSION_LIST; let aiKnowledgeMetadata = {}; diff --git a/App/frontend-app/src/components/documentViewer/imageCarousel.tsx b/App/frontend-app/src/components/documentViewer/imageCarousel.tsx index 25c1870d..0e703192 100644 --- a/App/frontend-app/src/components/documentViewer/imageCarousel.tsx +++ b/App/frontend-app/src/components/documentViewer/imageCarousel.tsx @@ -1,13 +1,7 @@ import { useEffect, useState } from "react"; import { Document } from "../../api/apiTypes/embedded"; import { GetImage } from "../../api/storageService"; -import { Button, Image, Spinner } from "@fluentui/react-components"; -import { - ChevronCircleLeftRegular, - ChevronCircleRightRegular, - ChevronLeftFilled, - ChevronRightFilled, -} from "@fluentui/react-icons"; +import { Image, Spinner } from "@fluentui/react-components"; import { useTranslation } from "react-i18next"; interface ImageCarouselProps { diff --git a/App/frontend-app/src/components/documentViewer/metadataTable.tsx b/App/frontend-app/src/components/documentViewer/metadataTable.tsx index 805ab68c..f6bbf785 100644 --- a/App/frontend-app/src/components/documentViewer/metadataTable.tsx +++ b/App/frontend-app/src/components/documentViewer/metadataTable.tsx @@ -1,6 +1,5 @@ import { Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow } from "@fluentui/react-components"; import { useState } from "react"; -import { Document } from "../../api/apiTypes/embedded"; import { ChevronCircleDown24Regular, ChevronCircleRight24Regular } from "@fluentui/react-icons"; import { mapMetadataKeys } from "../../utils/mapper/metadataMapper"; import { useTranslation } from "react-i18next"; @@ -47,7 +46,7 @@ export function MetadataTable({ metadata }: MetadataTableProps) { ]; } - if (typeof value === "object" && value !== null) { + if (typeof value === "object") { return [ { key: { label: key }, diff --git a/App/frontend-app/src/components/filter/filter.tsx b/App/frontend-app/src/components/filter/filter.tsx index 7251eb38..dab0652f 100644 --- a/App/frontend-app/src/components/filter/filter.tsx +++ b/App/frontend-app/src/components/filter/filter.tsx @@ -1,7 +1,6 @@ import React, { useEffect, useState, memo, useRef } from "react"; import { Checkbox } from "@fluentui/react-checkbox"; -import { useTranslation } from "react-i18next"; -import { Button, Accordion, AccordionHeader, AccordionItem, AccordionPanel, makeStyles } from "@fluentui/react-components"; +import { Accordion, AccordionHeader, AccordionItem, AccordionPanel, makeStyles } from "@fluentui/react-components"; const useStyles = makeStyles({ clearAll: { @@ -49,7 +48,6 @@ export function Filter({ clearFilters, onFilterCleared }: FilterProps) { - const { t } = useTranslation(); const classes = useStyles(); const localStorageKey = "selectedKeywords"; diff --git a/App/frontend-app/src/components/headerBar/headerBar.tsx b/App/frontend-app/src/components/headerBar/headerBar.tsx index fe6e9099..c941a813 100644 --- a/App/frontend-app/src/components/headerBar/headerBar.tsx +++ b/App/frontend-app/src/components/headerBar/headerBar.tsx @@ -2,8 +2,6 @@ import React, { MouseEventHandler, useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; import { Link, useNavigate } from "react-router-dom"; import { useMsal } from "@azure/msal-react"; -import { Auth } from "../../utils/auth/auth"; -import { RedirectRequest } from "@azure/msal-browser"; import { Avatar, makeStyles, @@ -16,7 +14,6 @@ import { } from "@fluentui/react-components"; import resolveConfig from "tailwindcss/resolveConfig"; import TailwindConfig from "../../../tailwind.config"; -import { isPlatformAdmin } from "../../utils/auth/roles"; const fullConfig = resolveConfig(TailwindConfig); const useStylesAvatar = makeStyles({ @@ -58,7 +55,6 @@ export function HeaderBar({ location }: { location?: NavLocation }) { const linkClasses = "cursor-pointer hover:no-underline hover:border-b-[3px] h-9 min-h-0 block text-white"; const linkCurrent = "pointer-events-none border-b-[3px]"; const isAuthenticated = accounts.length > 0; - const isAdmin = isPlatformAdmin(accounts); const navItems: (NavItem | null)[] = useMemo( () => [ @@ -102,10 +98,6 @@ export function HeaderBar({ location }: { location?: NavLocation }) { setOpenDrawer((openDrawer) => !openDrawer); } - function signIn() { - instance.loginRedirect(Auth.getAuthenticationRequest() as RedirectRequest); - } - function signOut() { instance.logoutRedirect(); } diff --git a/App/frontend-app/src/components/headerMenu/HeaderMenuTabs.tsx b/App/frontend-app/src/components/headerMenu/HeaderMenuTabs.tsx index 92575b9b..15c73acc 100644 --- a/App/frontend-app/src/components/headerMenu/HeaderMenuTabs.tsx +++ b/App/frontend-app/src/components/headerMenu/HeaderMenuTabs.tsx @@ -1,146 +1,146 @@ -import { - Button, - Menu, - MenuItem, - MenuList, - MenuPopover, - MenuTrigger, - Tab, - TabList, -} from "@fluentui/react-components"; -import { Search20Regular, Dismiss24Regular, ChatSparkle20Filled } from "@fluentui/react-icons"; -import { useLocation, useNavigate } from "react-router-dom"; -import { Document, Tokens } from "../../api/apiTypes/documentResults"; -import { useState } from "react"; -import { DocDialog } from "../documentViewer/documentViewer"; -import { useTranslation } from "react-i18next"; - -interface HeaderMenuTabsTabsProps { - className?: string; - searchResultDocuments?: Document[]; - selectedDocuments: Document[]; - tokens: Tokens | undefined; - updateSelectedDocuments: (document: Document) => void; -} - -export function HeaderMenuTabs({ - className, - searchResultDocuments, - selectedDocuments, - tokens, - updateSelectedDocuments, -}: HeaderMenuTabsTabsProps) { - const { t } = useTranslation(); - const navigate = useNavigate(); - const location = useLocation(); - const currentPath = location.pathname.substring(1); - - const [isDocViewerOpen, setIsDocViewerOpen] = useState(false); - const [isDocToBeOpened, setIsDocToBeOpened] = useState(null); - - const tabs = [ - { value: "search", label: "Search", icon: }, - { value: "chat", label: "Chat", icon: }, - ]; - - const commonTabClassName = "mr-2"; - - const handleClick = (path: string) => { - if (path === "/chat") { - navigate(path, { - state: { - searchResultDocuments: searchResultDocuments, - selectedDocuments: selectedDocuments, - tokens: tokens, - }, - }); - return; - } - navigate(path, { - state: { - selectedDocuments: selectedDocuments, - }, - }); - }; - - const openDocumentViewer = (document: Document) => { - setIsDocToBeOpened(document); - setIsDocViewerOpen(true); - }; - - const handleDocumentViewerClose = () => { - setIsDocViewerOpen(false); - }; - - return ( - <> -
- - {tabs.map((tab) => ( - handleClick("/" + tab.value)} - > - {tab.label} - - ))} - - - {selectedDocuments && selectedDocuments.length > 0 && ( -
- - ))} - - - - )} - - )} - - {selectedDocuments && selectedDocuments.length > 0 && isDocViewerOpen && tokens && ( - - )} -
- - ); -} +import { + Button, + Menu, + MenuItem, + MenuList, + MenuPopover, + MenuTrigger, + Tab, + TabList, +} from "@fluentui/react-components"; +import { Search20Regular, Dismiss24Regular, ChatSparkle20Filled } from "@fluentui/react-icons"; +import { useLocation, useNavigate } from "react-router-dom"; +import { Document, Tokens } from "../../api/apiTypes/documentResults"; +import { useState } from "react"; +import { DocDialog } from "../documentViewer/documentViewer"; +import { useTranslation } from "react-i18next"; + +interface HeaderMenuTabsTabsProps { + className?: string; + searchResultDocuments?: Document[]; + selectedDocuments: Document[]; + tokens: Tokens | undefined; + updateSelectedDocuments: (document: Document) => void; +} + +export function HeaderMenuTabs({ + className, + searchResultDocuments, + selectedDocuments, + tokens, + updateSelectedDocuments, +}: HeaderMenuTabsTabsProps) { + const { t } = useTranslation(); + const navigate = useNavigate(); + const location = useLocation(); + const currentPath = location.pathname.substring(1); + + const [isDocViewerOpen, setIsDocViewerOpen] = useState(false); + const [isDocToBeOpened, setIsDocToBeOpened] = useState(null); + + const tabs = [ + { value: "search", label: "Search", icon: }, + { value: "chat", label: "Chat", icon: }, + ]; + + const commonTabClassName = "mr-2"; + + const handleClick = (path: string) => { + if (path === "/chat") { + navigate(path, { + state: { + searchResultDocuments: searchResultDocuments, + selectedDocuments: selectedDocuments, + tokens: tokens, + }, + }); + return; + } + navigate(path, { + state: { + selectedDocuments: selectedDocuments, + }, + }); + }; + + const openDocumentViewer = (document: Document) => { + setIsDocToBeOpened(document); + setIsDocViewerOpen(true); + }; + + const handleDocumentViewerClose = () => { + setIsDocViewerOpen(false); + }; + + return ( + <> +
+ + {tabs.map((tab) => ( + handleClick("/" + tab.value)} + > + {tab.label} + + ))} + + + {selectedDocuments && selectedDocuments.length > 0 && ( +
+ + ))} + + + + )} + + )} + + {selectedDocuments && selectedDocuments.length > 0 && isDocViewerOpen && tokens && ( + + )} +
+ + ); +} diff --git a/App/frontend-app/src/components/layout/layout.tsx b/App/frontend-app/src/components/layout/layout.tsx index 1c197e4f..6da11d53 100644 --- a/App/frontend-app/src/components/layout/layout.tsx +++ b/App/frontend-app/src/components/layout/layout.tsx @@ -1,7 +1,6 @@ import React from "react"; import { InteractionStatus } from "@azure/msal-browser"; import { useMsal } from "@azure/msal-react"; -import { Footer } from "../footer/footer"; export function Layout({ children }: { children?: React.ReactNode }) { const { inProgress } = useMsal(); diff --git a/App/frontend-app/src/components/pagination/pagination.tsx b/App/frontend-app/src/components/pagination/pagination.tsx index 126a228b..dc3a13fd 100644 --- a/App/frontend-app/src/components/pagination/pagination.tsx +++ b/App/frontend-app/src/components/pagination/pagination.tsx @@ -1,6 +1,5 @@ import { Button } from "@fluentui/react-components"; import { DOTS, usePagination } from "../../utils/customHooks/usePagination"; -import { useState } from "react"; import { ChevronLeft24Regular, ChevronRight24Regular } from "@fluentui/react-icons"; interface PaginagtionProps { diff --git a/App/frontend-app/src/components/searchBox/searchBox.tsx b/App/frontend-app/src/components/searchBox/searchBox.tsx index b356c0d8..391344b3 100644 --- a/App/frontend-app/src/components/searchBox/searchBox.tsx +++ b/App/frontend-app/src/components/searchBox/searchBox.tsx @@ -5,10 +5,8 @@ import { Button, InputOnChangeData, Tooltip, useId } from "@fluentui/react-compo import { useDebouncedCallback } from "use-debounce"; import { Keyboard24Regular, - Mic24Regular, SearchVisual24Regular, - Search24Regular, - ArrowUpload24Filled, + Search24Regular } from "@fluentui/react-icons"; import "./searchInput.scss"; import { UploadMultipleFiles } from "../../api/storageService"; @@ -28,18 +26,6 @@ interface SearchBoxProps { onKeyDown?: (event: KeyboardEvent) => void; // Include onKeyDown as a prop } -const MicButton = () => { - return