Skip to content

Commit cee74fd

Browse files
maumarmaumar
committed
Cosmos Full Text Search support
- Adding model building API to configure property as full-text search enabled, as well as setup the index for it, - Adding model validation (e.g. FTS index not matching FTS property), - Adding EF.Functions stubs and translations for FullTextContains, FullTextContainsAll, FullTextContainsAny, FullTextScore and RRF (for hybrid), - Adding logic in SelectExpression to produce ORDER BY RANK when necessary, - Adding validation when attempting to mix with ORDER BY RANK with regular ORDER BY, - Rewrite OFFSET/LIMIT from parameter to constant when ORDER BY RANK is present. Also fixed / added support for vector search on owned types (since it shares logic with FTS) and added some tests. outstanding work: - support for FTS Container building using Azure.ResourceManager.CosmosDb (currently blocked on updated package being released) - add model building support for default language (superfluous for now, since only one language is supported), - add more model validation tests for invalid scenarios (index on multiple columns, FTS on non-string etc) - clean up exception messages and put them in a resource file, Fixes #35476 Fixes #35853 Fixes #35867 Fixes #35852
1 parent ce01986 commit cee74fd

29 files changed

Lines changed: 1884 additions & 61 deletions

‎Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
<PackageVersion Include="Microsoft.DotNet.Build.Tasks.Templating" Version="$(MicrosoftDotNetBuildTasksTemplatingVersion)" />
3838

3939
<!-- Azure SDK for .NET dependencies -->
40-
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.46.0" />
40+
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.49.0-preview.0" />
4141

4242
<!-- SQL Server dependencies -->
4343
<PackageVersion Include="Microsoft.Data.SqlClient" Version="6.0.1" />

‎src/EFCore.Cosmos/EFCore.Cosmos.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<NoWarn>$(NoWarn);EF9101</NoWarn> <!-- Metrics is experimental -->
1313
<NoWarn>$(NoWarn);EF9102</NoWarn> <!-- Paging is experimental -->
1414
<NoWarn>$(NoWarn);EF9103</NoWarn> <!-- Vector search is experimental -->
15+
<NoWarn>$(NoWarn);EF9104</NoWarn> <!-- Full-text search is experimental -->
1516
</PropertyGroup>
1617

1718
<ItemGroup>

‎src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,60 @@ public static T CoalesceUndefined<T>(
5252
T expression2)
5353
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(CoalesceUndefined)));
5454

55+
/// <summary>
56+
/// Checks if the specified property contains the given keyword using full-text search.
57+
/// </summary>
58+
/// <param name="_">The <see cref="DbFunctions" /> instance.</param>
59+
/// <param name="property">The property to search.</param>
60+
/// <param name="keyword">The keyword to search for.</param>
61+
/// <returns><see langword="true" /> if the property contains the keyword; otherwise, <see langword="false" />.</returns>
62+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
63+
public static bool FullTextContains(this DbFunctions _, string property, string keyword)
64+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContains)));
65+
66+
/// <summary>
67+
/// Checks if the specified property contains all the given keywords using full-text search.
68+
/// </summary>
69+
/// <param name="_">The <see cref="DbFunctions" /> instance.</param>
70+
/// <param name="property">The property to search.</param>
71+
/// <param name="keywords">The keywords to search for.</param>
72+
/// <returns><see langword="true" /> if the property contains all the keywords; otherwise, <see langword="false" />.</returns>
73+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
74+
public static bool FullTextContainsAll(this DbFunctions _, string property, params string[] keywords)
75+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAll)));
76+
77+
/// <summary>
78+
/// Checks if the specified property contains any of the given keywords using full-text search.
79+
/// </summary>
80+
/// <param name="_">The <see cref="DbFunctions" /> instance.</param>
81+
/// <param name="property">The property to search.</param>
82+
/// <param name="keywords">The keywords to search for.</param>
83+
/// <returns><see langword="true" /> if the property contains any of the keywords; otherwise, <see langword="false" />.</returns>
84+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
85+
public static bool FullTextContainsAny(this DbFunctions _, string property, params string[] keywords)
86+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAny)));
87+
88+
/// <summary>
89+
/// Returns the full-text search score for the specified property and keywords.
90+
/// </summary>
91+
/// <param name="_">The <see cref="DbFunctions" /> instance.</param>
92+
/// <param name="property">The property to search.</param>
93+
/// <param name="keywords">The keywords to search for.</param>
94+
/// <returns>The full-text search score.</returns>
95+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
96+
public static double FullTextScore(this DbFunctions _, string property, string[] keywords)
97+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextScore)));
98+
99+
/// <summary>
100+
/// Combines scores provided by two or more specified functions.
101+
/// </summary>
102+
/// <param name="_">The <see cref="DbFunctions" /> instance.</param>
103+
/// <param name="functions">The functions to compute the score for.</param>
104+
/// <returns>The combined score.</returns>
105+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
106+
public static double Rrf(this DbFunctions _, params double[] functions)
107+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(Rrf)));
108+
55109
/// <summary>
56110
/// Returns the distance between two vectors, using the distance function and data type defined using
57111
/// <see

‎src/EFCore.Cosmos/Extensions/CosmosIndexBuilderExtensions.cs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,4 +96,84 @@ public static bool CanSetVectorIndexType(
9696
VectorIndexType? indexType,
9797
bool fromDataAnnotation = false)
9898
=> indexBuilder.CanSetAnnotation(CosmosAnnotationNames.VectorIndexType, indexType, fromDataAnnotation);
99+
100+
/// <summary>
101+
/// Configures the index as a full-text index.
102+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
103+
/// </summary>
104+
/// <remarks>
105+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
106+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
107+
/// </remarks>
108+
/// <param name="indexBuilder">The builder for the index being configured.</param>
109+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
110+
/// <returns>A builder to further configure the index.</returns>
111+
public static IndexBuilder ForFullText(this IndexBuilder indexBuilder, bool? value = true)
112+
{
113+
indexBuilder.Metadata.SetFullTextIndex(value);
114+
115+
return indexBuilder;
116+
}
117+
118+
/// <summary>
119+
/// Configures the index as a full-text index.
120+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
121+
/// </summary>
122+
/// <remarks>
123+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
124+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
125+
/// </remarks>
126+
/// <param name="indexBuilder">The builder for the index being configured.</param>
127+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
128+
/// <returns>A builder to further configure the index.</returns>
129+
public static IndexBuilder<TEntity> ForFullText<TEntity>(
130+
this IndexBuilder<TEntity> indexBuilder,
131+
bool? value = true)
132+
=> (IndexBuilder<TEntity>)ForFullText((IndexBuilder)indexBuilder, value);
133+
134+
/// <summary>
135+
/// Configures the index as a full-text index.
136+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
137+
/// </summary>
138+
/// <remarks>
139+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
140+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
141+
/// </remarks>
142+
/// <param name="indexBuilder">The builder for the index being configured.</param>
143+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
144+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
145+
/// <returns>
146+
/// The same builder instance if the configuration was applied,
147+
/// <see langword="null" /> otherwise.
148+
/// </returns>
149+
public static IConventionIndexBuilder? ForFullText(
150+
this IConventionIndexBuilder indexBuilder,
151+
bool? value,
152+
bool fromDataAnnotation = false)
153+
{
154+
if (indexBuilder.CanSetFullTextIndex(fromDataAnnotation))
155+
{
156+
indexBuilder.Metadata.SetFullTextIndex(value, fromDataAnnotation);
157+
return indexBuilder;
158+
}
159+
160+
return null;
161+
}
162+
163+
/// <summary>
164+
/// Returns a value indicating whether the index can be configured as a Full-text index.
165+
/// </summary>
166+
/// <remarks>
167+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
168+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
169+
/// </remarks>
170+
/// <param name="indexBuilder">The builder for the index being configured.</param>
171+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
172+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
173+
/// <returns><see langword="true" /> if the index can be configured as a Full-text index.</returns>
174+
public static bool CanSetFullTextIndex(
175+
this IConventionIndexBuilder indexBuilder,
176+
bool? value,
177+
bool fromDataAnnotation = false)
178+
=> indexBuilder.CanSetAnnotation(CosmosAnnotationNames.FullTextIndex, value, fromDataAnnotation);
99179
}

‎src/EFCore.Cosmos/Extensions/CosmosIndexExtensions.cs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,49 @@ public static void SetVectorIndexType(this IMutableIndex index, VectorIndexType?
6161
/// <returns>The <see cref="ConfigurationSource" /> for whether the index is clustered.</returns>
6262
public static ConfigurationSource? GetVectorIndexTypeConfigurationSource(this IConventionIndex property)
6363
=> property.FindAnnotation(CosmosAnnotationNames.VectorIndexType)?.GetConfigurationSource();
64+
65+
/// <summary>
66+
/// Returns the value indicating whether the index is configured for Full-text search.
67+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
68+
/// </summary>
69+
/// <param name="index">The index.</param>
70+
/// <returns>The index type to use, or <see langword="null" /> if none is set.</returns>
71+
public static bool? IsFullTextIndex(this IReadOnlyIndex index)
72+
=> (index is RuntimeIndex)
73+
? throw new InvalidOperationException(CoreStrings.RuntimeModelMissingData)
74+
: (bool?)index[CosmosAnnotationNames.FullTextIndex];
75+
76+
/// <summary>
77+
/// Configures the index for Full-text search.
78+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
79+
/// </summary>
80+
/// <param name="index">The index.</param>
81+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
82+
public static void SetFullTextIndex(this IMutableIndex index, bool? value)
83+
=> index.SetAnnotation(CosmosAnnotationNames.FullTextIndex, value);
84+
85+
/// <summary>
86+
/// Configures the index for Full-text search.
87+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
88+
/// </summary>
89+
/// <param name="index">The index.</param>
90+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
91+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
92+
/// <returns>The configured value.</returns>
93+
public static string? SetFullTextIndex(
94+
this IConventionIndex index,
95+
bool? value,
96+
bool fromDataAnnotation = false)
97+
=> (string?)index.SetAnnotation(
98+
CosmosAnnotationNames.FullTextIndex,
99+
value,
100+
fromDataAnnotation)?.Value;
101+
102+
/// <summary>
103+
/// Returns the <see cref="ConfigurationSource" /> for whether the <see cref="IsFullTextIndex" />.
104+
/// </summary>
105+
/// <param name="property">The property.</param>
106+
/// <returns>The <see cref="ConfigurationSource" /> for whether the index is clustered.</returns>
107+
public static ConfigurationSource? IsFullTextIndexConfigurationSource(this IConventionIndex property)
108+
=> property.FindAnnotation(CosmosAnnotationNames.FullTextIndex)?.GetConfigurationSource();
64109
}

‎src/EFCore.Cosmos/Extensions/CosmosPropertyBuilderExtensions.cs‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,4 +243,95 @@ private static CosmosVectorType CreateVectorType(DistanceFunction distanceFuncti
243243
? new CosmosVectorType(distanceFunction, dimensions)
244244
: throw new ArgumentException(
245245
CoreStrings.InvalidEnumValue(distanceFunction, nameof(distanceFunction), typeof(DistanceFunction)));
246+
247+
/// <summary>
248+
/// Configures the property to be used with a full-text search.
249+
/// </summary>
250+
/// <remarks>
251+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
252+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
253+
/// </remarks>
254+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
255+
/// <param name="language">The language for the full-text search.</param>
256+
/// <returns>The same builder instance so that multiple calls can be chained.</returns>
257+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
258+
public static PropertyBuilder IsFullText(
259+
this PropertyBuilder propertyBuilder,
260+
string language = "en-US")
261+
{
262+
propertyBuilder.Metadata.SetFullTextSearchLanguage(language);
263+
return propertyBuilder;
264+
}
265+
266+
/// <summary>
267+
/// Configures the property to enable Full-text search for Azure Cosmos DB using a specified language.
268+
/// </summary>
269+
/// <remarks>
270+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
271+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
272+
/// </remarks>
273+
/// <remarks>
274+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
275+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
276+
/// </remarks>
277+
/// <typeparam name="TProperty">The type of the property being configured.</typeparam>
278+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
279+
/// <param name="language">The language for the full-text search.</param>
280+
/// <returns>The same builder instance so that multiple calls can be chained.</returns>
281+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
282+
public static PropertyBuilder<TProperty> IsFullText<TProperty>(
283+
this PropertyBuilder<TProperty> propertyBuilder,
284+
string language = "en-US")
285+
=> (PropertyBuilder<TProperty>)IsFullText((PropertyBuilder)propertyBuilder, language);
286+
287+
/// <summary>
288+
/// Configures the property to enable Full-text search for Azure Cosmos DB using a specified language.
289+
/// </summary>
290+
/// <remarks>
291+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
292+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
293+
/// </remarks>
294+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
295+
/// <param name="language">The language for the full-text search.</param>
296+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
297+
/// <returns>
298+
/// The same builder instance if the configuration was applied,
299+
/// <see langword="null" /> otherwise.
300+
/// </returns>
301+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
302+
public static IConventionPropertyBuilder? IsFullText(
303+
this IConventionPropertyBuilder propertyBuilder,
304+
string language,
305+
bool fromDataAnnotation = false)
306+
{
307+
if (!propertyBuilder.CanSetIsFullText(language, fromDataAnnotation))
308+
{
309+
return null;
310+
}
311+
312+
propertyBuilder.Metadata.SetFullTextSearchLanguage(language, fromDataAnnotation);
313+
314+
return propertyBuilder;
315+
}
316+
317+
/// <summary>
318+
/// Returns a value indicating whether the Full-text search language can be set.
319+
/// </summary>
320+
/// <remarks>
321+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
322+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
323+
/// </remarks>
324+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
325+
/// <param name="language">The language for the full-text search.</param>
326+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
327+
/// <returns><see langword="true" /> if the vector type can be set.</returns>
328+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
329+
public static bool CanSetIsFullText(
330+
this IConventionPropertyBuilder propertyBuilder,
331+
string language,
332+
bool fromDataAnnotation = false)
333+
=> propertyBuilder.CanSetAnnotation(
334+
CosmosAnnotationNames.FullTextSearchLanguage,
335+
language,
336+
fromDataAnnotation);
246337
}

0 commit comments

Comments
 (0)