Skip to content

Commit f38b9c0

Browse files
MackinnonBuckjeffhandleyadamsitnik
authored
Use Microsoft.Extensions.DataIngestion in AI Chat Web template (#7023)
* Add Markdown support * Remove PDF support * Revert "Remove PDF support" This reverts commit e1d066034962c9686bf8150984b6adf0e25846c8. * Add 'Example_GPS_Watch.md' * Add MEDI dependencies * Revert "[MEDI] Remove collection key type workaround (#7010)" This reverts commit a369be9. * MEDI integration into chat template * Remove PdfPig dependency * Fix citation + normalize identifier path * Undo changes to `M.E.DI.csproj` * Update snapshots * Update DataIngestion unit tests to handle keys as either strings or guids * Update SK and fix MEDI version * Remove SK workaround * Fix sandbox paths to allow running tests multiple times * Reliable data ingestion * Enable MEDI tracing * Simplify log message * Add `PdfPigReader` for non-Aspire template * Invert PdfPigReader exclusion condition * Use Markitdown MCP * Update snapshots * Undo changes to `IngestionPipelineTests.cs` * Update src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/Ingestion/DocumentReader.cs Co-authored-by: Jeff Handley <[email protected]> * Update snapshots * Improve template execution test failure output * Support .NET 10 in aichatweb, using it by default * Show a message when loading documents by loading docs as a separate tool * disable the incremental ingestion * map every PDF page to a single section * drop SK dependency * Add system prompt instructions for calling the LoadDocuments tool. Fix code formatting. --------- Co-authored-by: Jeff Handley <[email protected]> Co-authored-by: Adam Sitnik <[email protected]>
1 parent e742371 commit f38b9c0

File tree

131 files changed

+2566
-1546
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

131 files changed

+2566
-1546
lines changed

eng/build.proj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.Build.Traversal">
22
<ItemGroup>
33
<_SnapshotsToExclude Include="$(MSBuildThisFileDirectory)..\test\**\Snapshots\**\*.*proj" />
4-
<_GeneratedContentToExclude Include="$(MSBuildThisFileDirectory)..\test\**\ExecutionTemplateSandbox\**\*.*proj" />
4+
<_GeneratedContentToExclude Include="$(MSBuildThisFileDirectory)..\test\**\ExecutionTestSandbox\**\*.*proj" />
55

66
<!-- We recursively add all of the projects inside the src directory, except for the exclusions above -->
77
<_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\**\*.csproj" />

src/ProjectTemplates/GeneratedContent.targets

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626
<PackageVersionVariableReference Include="$(RepoRoot)src\Libraries\Microsoft.Extensions.AI\Microsoft.Extensions.AI.csproj"></PackageVersionVariableReference>
2727
<PackageVersionVariableReference Include="$(RepoRoot)src\Libraries\Microsoft.Extensions.AI.OpenAI\Microsoft.Extensions.AI.OpenAI.csproj"></PackageVersionVariableReference>
2828
<PackageVersionVariableReference Include="$(RepoRoot)src\Libraries\Microsoft.Extensions.Http.Resilience\Microsoft.Extensions.Http.Resilience.csproj"></PackageVersionVariableReference>
29+
<PackageVersionVariableReference Include="$(RepoRoot)src\Libraries\Microsoft.Extensions.DataIngestion\Microsoft.Extensions.DataIngestion.csproj"></PackageVersionVariableReference>
30+
<PackageVersionVariableReference Include="$(RepoRoot)src\Libraries\Microsoft.Extensions.DataIngestion.Markdig\Microsoft.Extensions.DataIngestion.Markdig.csproj"></PackageVersionVariableReference>
31+
<PackageVersionVariableReference Include="$(RepoRoot)src\Libraries\Microsoft.Extensions.DataIngestion.MarkItDown\Microsoft.Extensions.DataIngestion.MarkItDown.csproj"></PackageVersionVariableReference>
2932
</ItemGroup>
3033
</Target>
3134

@@ -44,8 +47,9 @@
4447
<TemplatePackageVersion_CommunityToolkitAspire>9.8.1-beta.413</TemplatePackageVersion_CommunityToolkitAspire>
4548
<TemplatePackageVersion_MicrosoftExtensionsHosting>10.0.0-rc.2.25502.107</TemplatePackageVersion_MicrosoftExtensionsHosting>
4649
<TemplatePackageVersion_MicrosoftExtensionsServiceDiscovery>9.5.1</TemplatePackageVersion_MicrosoftExtensionsServiceDiscovery>
47-
<TemplatePackageVersion_MicrosoftSemanticKernel>1.66.0</TemplatePackageVersion_MicrosoftSemanticKernel>
48-
<TemplatePackageVersion_MicrosoftSemanticKernel_Preview>1.66.0-preview</TemplatePackageVersion_MicrosoftSemanticKernel_Preview>
50+
<TemplatePackageVersion_MicrosoftMLTokenizers>2.0.0-preview.25503.2</TemplatePackageVersion_MicrosoftMLTokenizers>
51+
<TemplatePackageVersion_MicrosoftSemanticKernel>1.67.1</TemplatePackageVersion_MicrosoftSemanticKernel>
52+
<TemplatePackageVersion_MicrosoftSemanticKernel_Preview>1.67.1-preview</TemplatePackageVersion_MicrosoftSemanticKernel_Preview>
4953
<TemplatePackageVersion_ModelContextProtocol>0.4.0-preview.1</TemplatePackageVersion_ModelContextProtocol>
5054
<TemplatePackageVersion_OllamaSharp>5.4.8</TemplatePackageVersion_OllamaSharp>
5155
<TemplatePackageVersion_OpenTelemetry>1.13.0</TemplatePackageVersion_OpenTelemetry>
@@ -72,6 +76,7 @@
7276
TemplatePackageVersion_CommunityToolkitAspire=$(TemplatePackageVersion_CommunityToolkitAspire);
7377
TemplatePackageVersion_MicrosoftExtensionsHosting=$(TemplatePackageVersion_MicrosoftExtensionsHosting);
7478
TemplatePackageVersion_MicrosoftExtensionsServiceDiscovery=$(TemplatePackageVersion_MicrosoftExtensionsServiceDiscovery);
79+
TemplatePackageVersion_MicrosoftMLTokenizers=$(TemplatePackageVersion_MicrosoftMLTokenizers);
7580
TemplatePackageVersion_MicrosoftSemanticKernel=$(TemplatePackageVersion_MicrosoftSemanticKernel);
7681
TemplatePackageVersion_MicrosoftSemanticKernel_Preview=$(TemplatePackageVersion_MicrosoftSemanticKernel_Preview);
7782
TemplatePackageVersion_ModelContextProtocol=$(TemplatePackageVersion_ModelContextProtocol);

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/THIRD-PARTY-NOTICES.TXT

Lines changed: 0 additions & 261 deletions
Original file line numberDiff line numberDiff line change
@@ -817,267 +817,6 @@ https://github.com/mozilla/pdf.js
817817

818818
END OF TERMS AND CONDITIONS
819819

820-
License notice for PdfPig
821-
-------------------------
822-
823-
https://github.com/UglyToad/PdfPig
824-
825-
Apache License
826-
Version 2.0, January 2004
827-
http://www.apache.org/licenses/
828-
829-
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
830-
831-
1. Definitions.
832-
833-
"License" shall mean the terms and conditions for use, reproduction,
834-
and distribution as defined by Sections 1 through 9 of this document.
835-
836-
"Licensor" shall mean the copyright owner or entity authorized by
837-
the copyright owner that is granting the License.
838-
839-
"Legal Entity" shall mean the union of the acting entity and all
840-
other entities that control, are controlled by, or are under common
841-
control with that entity. For the purposes of this definition,
842-
"control" means (i) the power, direct or indirect, to cause the
843-
direction or management of such entity, whether by contract or
844-
otherwise, or (ii) ownership of fifty percent (50%) or more of the
845-
outstanding shares, or (iii) beneficial ownership of such entity.
846-
847-
"You" (or "Your") shall mean an individual or Legal Entity
848-
exercising permissions granted by this License.
849-
850-
"Source" form shall mean the preferred form for making modifications,
851-
including but not limited to software source code, documentation
852-
source, and configuration files.
853-
854-
"Object" form shall mean any form resulting from mechanical
855-
transformation or translation of a Source form, including but
856-
not limited to compiled object code, generated documentation,
857-
and conversions to other media types.
858-
859-
"Work" shall mean the work of authorship, whether in Source or
860-
Object form, made available under the License, as indicated by a
861-
copyright notice that is included in or attached to the work
862-
(an example is provided in the Appendix below).
863-
864-
"Derivative Works" shall mean any work, whether in Source or Object
865-
form, that is based on (or derived from) the Work and for which the
866-
editorial revisions, annotations, elaborations, or other modifications
867-
represent, as a whole, an original work of authorship. For the purposes
868-
of this License, Derivative Works shall not include works that remain
869-
separable from, or merely link (or bind by name) to the interfaces of,
870-
the Work and Derivative Works thereof.
871-
872-
"Contribution" shall mean any work of authorship, including
873-
the original version of the Work and any modifications or additions
874-
to that Work or Derivative Works thereof, that is intentionally
875-
submitted to Licensor for inclusion in the Work by the copyright owner
876-
or by an individual or Legal Entity authorized to submit on behalf of
877-
the copyright owner. For the purposes of this definition, "submitted"
878-
means any form of electronic, verbal, or written communication sent
879-
to the Licensor or its representatives, including but not limited to
880-
communication on electronic mailing lists, source code control systems,
881-
and issue tracking systems that are managed by, or on behalf of, the
882-
Licensor for the purpose of discussing and improving the Work, but
883-
excluding communication that is conspicuously marked or otherwise
884-
designated in writing by the copyright owner as "Not a Contribution."
885-
886-
"Contributor" shall mean Licensor and any individual or Legal Entity
887-
on behalf of whom a Contribution has been received by Licensor and
888-
subsequently incorporated within the Work.
889-
890-
2. Grant of Copyright License. Subject to the terms and conditions of
891-
this License, each Contributor hereby grants to You a perpetual,
892-
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
893-
copyright license to reproduce, prepare Derivative Works of,
894-
publicly display, publicly perform, sublicense, and distribute the
895-
Work and such Derivative Works in Source or Object form.
896-
897-
3. Grant of Patent License. Subject to the terms and conditions of
898-
this License, each Contributor hereby grants to You a perpetual,
899-
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
900-
(except as stated in this section) patent license to make, have made,
901-
use, offer to sell, sell, import, and otherwise transfer the Work,
902-
where such license applies only to those patent claims licensable
903-
by such Contributor that are necessarily infringed by their
904-
Contribution(s) alone or by combination of their Contribution(s)
905-
with the Work to which such Contribution(s) was submitted. If You
906-
institute patent litigation against any entity (including a
907-
cross-claim or counterclaim in a lawsuit) alleging that the Work
908-
or a Contribution incorporated within the Work constitutes direct
909-
or contributory patent infringement, then any patent licenses
910-
granted to You under this License for that Work shall terminate
911-
as of the date such litigation is filed.
912-
913-
4. Redistribution. You may reproduce and distribute copies of the
914-
Work or Derivative Works thereof in any medium, with or without
915-
modifications, and in Source or Object form, provided that You
916-
meet the following conditions:
917-
918-
(a) You must give any other recipients of the Work or
919-
Derivative Works a copy of this License; and
920-
921-
(b) You must cause any modified files to carry prominent notices
922-
stating that You changed the files; and
923-
924-
(c) You must retain, in the Source form of any Derivative Works
925-
that You distribute, all copyright, patent, trademark, and
926-
attribution notices from the Source form of the Work,
927-
excluding those notices that do not pertain to any part of
928-
the Derivative Works; and
929-
930-
(d) If the Work includes a "NOTICE" text file as part of its
931-
distribution, then any Derivative Works that You distribute must
932-
include a readable copy of the attribution notices contained
933-
within such NOTICE file, excluding those notices that do not
934-
pertain to any part of the Derivative Works, in at least one
935-
of the following places: within a NOTICE text file distributed
936-
as part of the Derivative Works; within the Source form or
937-
documentation, if provided along with the Derivative Works; or,
938-
within a display generated by the Derivative Works, if and
939-
wherever such third-party notices normally appear. The contents
940-
of the NOTICE file are for informational purposes only and
941-
do not modify the License. You may add Your own attribution
942-
notices within Derivative Works that You distribute, alongside
943-
or as an addendum to the NOTICE text from the Work, provided
944-
that such additional attribution notices cannot be construed
945-
as modifying the License.
946-
947-
You may add Your own copyright statement to Your modifications and
948-
may provide additional or different license terms and conditions
949-
for use, reproduction, or distribution of Your modifications, or
950-
for any such Derivative Works as a whole, provided Your use,
951-
reproduction, and distribution of the Work otherwise complies with
952-
the conditions stated in this License.
953-
954-
5. Submission of Contributions. Unless You explicitly state otherwise,
955-
any Contribution intentionally submitted for inclusion in the Work
956-
by You to the Licensor shall be under the terms and conditions of
957-
this License, without any additional terms or conditions.
958-
Notwithstanding the above, nothing herein shall supersede or modify
959-
the terms of any separate license agreement you may have executed
960-
with Licensor regarding such Contributions.
961-
962-
6. Trademarks. This License does not grant permission to use the trade
963-
names, trademarks, service marks, or product names of the Licensor,
964-
except as required for reasonable and customary use in describing the
965-
origin of the Work and reproducing the content of the NOTICE file.
966-
967-
7. Disclaimer of Warranty. Unless required by applicable law or
968-
agreed to in writing, Licensor provides the Work (and each
969-
Contributor provides its Contributions) on an "AS IS" BASIS,
970-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
971-
implied, including, without limitation, any warranties or conditions
972-
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
973-
PARTICULAR PURPOSE. You are solely responsible for determining the
974-
appropriateness of using or redistributing the Work and assume any
975-
risks associated with Your exercise of permissions under this License.
976-
977-
8. Limitation of Liability. In no event and under no legal theory,
978-
whether in tort (including negligence), contract, or otherwise,
979-
unless required by applicable law (such as deliberate and grossly
980-
negligent acts) or agreed to in writing, shall any Contributor be
981-
liable to You for damages, including any direct, indirect, special,
982-
incidental, or consequential damages of any character arising as a
983-
result of this License or out of the use or inability to use the
984-
Work (including but not limited to damages for loss of goodwill,
985-
work stoppage, computer failure or malfunction, or any and all
986-
other commercial damages or losses), even if such Contributor
987-
has been advised of the possibility of such damages.
988-
989-
9. Accepting Warranty or Additional Liability. While redistributing
990-
the Work or Derivative Works thereof, You may choose to offer,
991-
and charge a fee for, acceptance of support, warranty, indemnity,
992-
or other liability obligations and/or rights consistent with this
993-
License. However, in accepting such obligations, You may act only
994-
on Your own behalf and on Your sole responsibility, not on behalf
995-
of any other Contributor, and only if You agree to indemnify,
996-
defend, and hold each Contributor harmless for any liability
997-
incurred by, or claims asserted against, such Contributor by reason
998-
of your accepting any such warranty or additional liability.
999-
1000-
END OF TERMS AND CONDITIONS
1001-
1002-
EXTERNAL COMPONENTS
1003-
1004-
PdfPig includes a number of components with separate copyright notices
1005-
and license terms. Your use of these components is subject to the terms and
1006-
conditions of the following licenses.
1007-
1008-
Contributions made to the original PDFBox and FontBox projects:
1009-
1010-
Copyright (c) 2002-2007, www.pdfbox.org
1011-
All rights reserved.
1012-
1013-
Redistribution and use in source and binary forms, with or without
1014-
modification, are permitted provided that the following conditions are met:
1015-
1016-
1. Redistributions of source code must retain the above copyright notice,
1017-
this list of conditions and the following disclaimer.
1018-
1019-
2. Redistributions in binary form must reproduce the above copyright
1020-
notice, this list of conditions and the following disclaimer in the
1021-
documentation and/or other materials provided with the distribution.
1022-
1023-
3. Neither the name of pdfbox; nor the names of its contributors may be
1024-
used to endorse or promote products derived from this software without
1025-
specific prior written permission.
1026-
1027-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1028-
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1029-
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1030-
ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
1031-
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
1032-
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
1033-
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
1034-
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
1035-
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
1036-
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
1037-
SUCH DAMAGE.
1038-
1039-
Adobe Font Metrics (AFM) for PDF Core 14 Fonts
1040-
1041-
This file and the 14 PostScript(R) AFM files it accompanies may be used,
1042-
copied, and distributed for any purpose and without charge, with or without
1043-
modification, provided that all copyright notices are retained; that the
1044-
AFM files are not distributed without this file; that all modifications
1045-
to this file or any of the AFM files are prominently noted in the modified
1046-
file(s); and that this paragraph is not modified. Adobe Systems has no
1047-
responsibility or obligation to support the use of the AFM files.
1048-
1049-
CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads)
1050-
1051-
Copyright 1990-2009 Adobe Systems Incorporated.
1052-
All rights reserved.
1053-
1054-
Redistribution and use in source and binary forms, with or without
1055-
modification, are permitted provided that the following conditions
1056-
are met:
1057-
1058-
Redistributions of source code must retain the above copyright notice,
1059-
this list of conditions and the following disclaimer.
1060-
1061-
Redistributions in binary form must reproduce the above copyright notice,
1062-
this list of conditions and the following disclaimer in the documentation
1063-
and/or other materials provided with the distribution.
1064-
1065-
Neither the name of Adobe Systems Incorporated nor the names of its
1066-
contributors may be used to endorse or promote products derived from this
1067-
software without specific prior written permission.
1068-
1069-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1070-
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1071-
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1072-
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
1073-
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
1074-
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
1075-
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
1076-
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
1077-
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
1078-
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
1079-
THE POSSIBILITY OF SUCH DAMAGE.
1080-
1081820
License notice for OllamaSharp
1082821
------------------------------
1083822

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/.template.config/template.json

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,12 @@
7070
"ChatWithCustomData-CSharp.Web/OllamaResilienceHandlerExtensions.cs"
7171
]
7272
},
73+
{
74+
"condition": "(IsAspire)",
75+
"exclude": [
76+
"ChatWithCustomData-CSharp.Web/Services/Ingestion/PdfPigReader.cs"
77+
]
78+
},
7379
{
7480
"condition": "(IsAspire)",
7581
"exclude": [
@@ -101,13 +107,17 @@
101107
"description": "The target framework for the project.",
102108
"datatype": "choice",
103109
"choices": [
110+
{
111+
"choice": "net10.0",
112+
"description": ".NET 10"
113+
},
104114
{
105115
"choice": "net9.0",
106-
"description": "Target net9.0"
116+
"description": ".NET 9"
107117
}
108118
],
109-
"replaces": "net9.0",
110-
"defaultValue": "net9.0",
119+
"replaces": "net10.0",
120+
"defaultValue": "net10.0",
111121
"displayName": "Framework"
112122
},
113123
"hostIdentifier": {
@@ -185,6 +195,10 @@
185195
"defaultValue": "false",
186196
"description": "Create the project as a distributed application using Aspire."
187197
},
198+
"IsNET9": {
199+
"type": "computed",
200+
"value": "(Framework == \"net9.0\")"
201+
},
188202
"IsManagedIdentity": {
189203
"type": "computed",
190204
"value": "(UseManagedIdentity)"

0 commit comments

Comments
 (0)