diff --git a/.fantomasignore b/.fantomasignore new file mode 100644 index 000000000..cfdc4db2d --- /dev/null +++ b/.fantomasignore @@ -0,0 +1,2 @@ +paket-files/ +tests/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 62a9917a7..37251f12f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,14 +32,22 @@ Type providers consist of two components: CSV provider, this component does the type inference and generates types (that are mapped to runtime components by the compiler). -We need a _runtime component_ for .NET Standard 2.0 (netstandard2.0). We also need a _design time_ -component for each, to be able to host the type provider in .NET Core-based tooling. +We need _runtime components_ for .NET Standard 2.0 (netstandard2.0). We also need a _design time_ +component, to be able to host the type providers in .NET Core-based tooling. -The _runtime_ components are in the following project: +The _core_ runtime components are the following projects. No type providers are activated if you reference these: + + * **FSharp.Data.Http** + * **FSharp.Data.Csv.Core** + * **FSharp.Data.Html.Core** + * **FSharp.Data.Json.Core** + * **FSharp.Data.Xml.Core** + +The _enhanced_ runtime component that mentions the associated the design-time component is in the following project: * **FSharp.Data** -The _design time_ components are in the following project: +The design-time component is the following project: * **FSharp.Data.DesignTime** diff --git a/Directory.Build.props b/Directory.Build.props index 3459447c8..03c303658 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,5 +11,13 @@ https://fsprojects.github.io/FSharp.Data https://raw.githubusercontent.com/fsprojects/FSharp.Data/master/docs/img/logo.png git + true + $(MSBuildThisFileDirectory)src\keyfile.snk + false + $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb + true + true + true + Embedded diff --git a/FSharp.Data.sln b/FSharp.Data.sln index 3f7d10b36..019afa28d 100755 --- a/FSharp.Data.sln +++ b/FSharp.Data.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26730.16 +# Visual Studio Version 17 +VisualStudioVersion = 17.4.32814.64 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{7E48C05A-8C42-4871-A618-180BEEF696AA}" ProjectSection(SolutionItems) = preProject @@ -53,17 +53,17 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ja", "ja", "{CA5C64CE-E68C-4320-A00A-E67CA883B9D2}" ProjectSection(SolutionItems) = preProject docs\ja\contributing.md = docs\ja\contributing.md - docs\ja\fsharpdata.md = docs\ja\fsharpdata.md - docs\ja\index.md = docs\ja\index.md docs\ja\library\CsvFile.fsx = docs\ja\library\CsvFile.fsx docs\ja\library\CsvProvider.fsx = docs\ja\library\CsvProvider.fsx + docs\ja\fsharpdata.md = docs\ja\fsharpdata.md docs\ja\library\Http.fsx = docs\ja\library\Http.fsx + docs\ja\index.md = docs\ja\index.md docs\ja\library\JsonProvider.fsx = docs\ja\library\JsonProvider.fsx + docs\ja\tutorials\JsonToXml.fsx = docs\ja\tutorials\JsonToXml.fsx docs\ja\library\JsonValue.fsx = docs\ja\library\JsonValue.fsx + docs\tools\templates\ja\template.cshtml = docs\tools\templates\ja\template.cshtml docs\ja\library\WorldBank.fsx = docs\ja\library\WorldBank.fsx docs\ja\library\XmlProvider.fsx = docs\ja\library\XmlProvider.fsx - docs\ja\tutorials\JsonToXml.fsx = docs\ja\tutorials\JsonToXml.fsx - docs\tools\templates\ja\template.cshtml = docs\tools\templates\ja\template.cshtml EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{36E72EB1-5847-4B38-8993-B951648CB0D9}" @@ -77,21 +77,32 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProjectSection EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data", "src\FSharp.Data\FSharp.Data.fsproj", "{6EBFDE55-9687-40A9-8C1A-6E204ECB117F}" - ProjectSection(ProjectDependencies) = postProject - {B85F245B-3FB9-4253-8251-16F98F05B6EC} = {B85F245B-3FB9-4253-8251-16F98F05B6EC} - EndProjectSection -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.DesignTime", "src\FSharp.Data.DesignTime\FSharp.Data.DesignTime.fsproj", "{B85F245B-3FB9-4253-8251-16F98F05B6EC}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{1F33D53A-C007-408C-AF6C-B7D62288F941}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.Reference.Tests", "tests\FSharp.Data.Reference.Tests\FSharp.Data.Reference.Tests.fsproj", "{DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Reference.Tests", "tests\FSharp.Data.Reference.Tests\FSharp.Data.Reference.Tests.fsproj", "{DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Core.Tests", "tests\FSharp.Data.Core.Tests\FSharp.Data.Core.Tests.fsproj", "{1746A3E0-32A2-49A7-9C8A-A0BCB52683B0}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.DesignTime.Tests", "tests\FSharp.Data.DesignTime.Tests\FSharp.Data.DesignTime.Tests.fsproj", "{A5B31ACC-56FB-4EC2-917F-BEB3754EF9AC}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.Tests", "tests\FSharp.Data.Tests\FSharp.Data.Tests.fsproj", "{1746A3E0-32A2-49A7-9C8A-A0BCB52683B0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FSharp.Data.Core.Tests.CSharp", "tests\FSharp.Data.Core.Tests.CSharp\FSharp.Data.Core.Tests.CSharp.csproj", "{290FED0C-D7C8-486F-AACF-3D7A1304C863}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.DesignTime.Tests", "tests\FSharp.Data.DesignTime.Tests\FSharp.Data.DesignTime.Tests.fsproj", "{A5B31ACC-56FB-4EC2-917F-BEB3754EF9AC}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Json.Core", "src\FSharp.Data.Json.Core\FSharp.Data.Json.Core.fsproj", "{DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FSharp.Data.Tests.CSharp", "tests\FSharp.Data.Tests.CSharp\FSharp.Data.Tests.CSharp.csproj", "{290FED0C-D7C8-486F-AACF-3D7A1304C863}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Tests", "tests\FSharp.Data.Tests\FSharp.Data.Tests.fsproj", "{750148EC-6A05-421D-96A4-E5AC9D18AF58}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.DesignTime", "src\FSharp.Data.DesignTime\FSharp.Data.DesignTime.fsproj", "{44E0DF97-D8FD-4805-8A84-B888D9589C8A}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Html.Core", "src\FSharp.Data.Html.Core\FSharp.Data.Html.Core.fsproj", "{E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Http", "src\FSharp.Data.Http\FSharp.Data.Http.fsproj", "{29EDED03-D2D6-415C-A17D-00806C52035A}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Xml.Core", "src\FSharp.Data.Xml.Core\FSharp.Data.Xml.Core.fsproj", "{1ECEFFEE-1040-40ED-9EB5-CE720A33058D}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.Csv.Core", "src\FSharp.Data.Csv.Core\FSharp.Data.Csv.Core.fsproj", "{0A1B8B61-268D-4061-B567-A47141C608E4}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Data.WorldBank.Core", "src\FSharp.Data.WorldBank.Core\FSharp.Data.WorldBank.Core.fsproj", "{A69D007B-EAF0-4866-A8B4-A2EDF2614E56}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -103,10 +114,6 @@ Global {6EBFDE55-9687-40A9-8C1A-6E204ECB117F}.Debug|Any CPU.Build.0 = Debug|Any CPU {6EBFDE55-9687-40A9-8C1A-6E204ECB117F}.Release|Any CPU.ActiveCfg = Release|Any CPU {6EBFDE55-9687-40A9-8C1A-6E204ECB117F}.Release|Any CPU.Build.0 = Release|Any CPU - {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B85F245B-3FB9-4253-8251-16F98F05B6EC}.Release|Any CPU.Build.0 = Release|Any CPU {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}.Debug|Any CPU.Build.0 = Debug|Any CPU {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -123,17 +130,50 @@ Global {290FED0C-D7C8-486F-AACF-3D7A1304C863}.Debug|Any CPU.Build.0 = Debug|Any CPU {290FED0C-D7C8-486F-AACF-3D7A1304C863}.Release|Any CPU.ActiveCfg = Release|Any CPU {290FED0C-D7C8-486F-AACF-3D7A1304C863}.Release|Any CPU.Build.0 = Release|Any CPU + {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DAEBFBCF-84CD-40BB-B8F6-99B1A9C4641F}.Release|Any CPU.Build.0 = Release|Any CPU + {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Debug|Any CPU.Build.0 = Debug|Any CPU + {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Release|Any CPU.ActiveCfg = Release|Any CPU + {750148EC-6A05-421D-96A4-E5AC9D18AF58}.Release|Any CPU.Build.0 = Release|Any CPU + {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {44E0DF97-D8FD-4805-8A84-B888D9589C8A}.Release|Any CPU.Build.0 = Release|Any CPU + {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E91BF68E-257C-43E6-BDE9-672D4E3BFAFA}.Release|Any CPU.Build.0 = Release|Any CPU + {29EDED03-D2D6-415C-A17D-00806C52035A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {29EDED03-D2D6-415C-A17D-00806C52035A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {29EDED03-D2D6-415C-A17D-00806C52035A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {29EDED03-D2D6-415C-A17D-00806C52035A}.Release|Any CPU.Build.0 = Release|Any CPU + {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1ECEFFEE-1040-40ED-9EB5-CE720A33058D}.Release|Any CPU.Build.0 = Release|Any CPU + {0A1B8B61-268D-4061-B567-A47141C608E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0A1B8B61-268D-4061-B567-A47141C608E4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0A1B8B61-268D-4061-B567-A47141C608E4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0A1B8B61-268D-4061-B567-A47141C608E4}.Release|Any CPU.Build.0 = Release|Any CPU + {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A69D007B-EAF0-4866-A8B4-A2EDF2614E56}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {34E4FA16-B344-439D-A789-1E8355E5659F} - EndGlobalSection GlobalSection(NestedProjects) = preSolution {DE36F8D0-7895-4ABD-9755-921F1BEAD3C9} = {1F33D53A-C007-408C-AF6C-B7D62288F941} {1746A3E0-32A2-49A7-9C8A-A0BCB52683B0} = {1F33D53A-C007-408C-AF6C-B7D62288F941} {A5B31ACC-56FB-4EC2-917F-BEB3754EF9AC} = {1F33D53A-C007-408C-AF6C-B7D62288F941} {290FED0C-D7C8-486F-AACF-3D7A1304C863} = {1F33D53A-C007-408C-AF6C-B7D62288F941} + {750148EC-6A05-421D-96A4-E5AC9D18AF58} = {1F33D53A-C007-408C-AF6C-B7D62288F941} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {34E4FA16-B344-439D-A789-1E8355E5659F} EndGlobalSection EndGlobal diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d09668a20..21da41fe3 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,13 @@ +### 5.0.1 - Aug 15 2022 + +* There are now multiple packages + * FSharp.Data -- includes everything + * FSharp.Data.Http -- http types/helpers + * FSharp.Data.Csv.Core -- csv types/helpers + * FSharp.Data.Json.Core -- json types/helpers + * FSharp.Data.Html.Core -- html types/helpers + * FSharp.Data.Xml.Core -- xml types/helpers + ### 4.2.10 - Aug 12 2022 * Implement "inline schemas": ability to add type hints into the type providers' source documents by @mlaily in https://github.com/fsprojects/FSharp.Data/pull/1447 diff --git a/build.fsx b/build.fsx index 0eead60f0..46d0a3f5f 100644 --- a/build.fsx +++ b/build.fsx @@ -36,9 +36,15 @@ let summary = "Library of F# type providers and data access tools" let description = """ - The FSharp.Data package contains type providers and utilities to access - common data formats (CSV, HTML, JSON and XML in your F# applications and scripts. It also - contains helpers for parsing CSV, HTML and JSON files and for sending HTTP requests.""" + The FSharp.Data packages contain type providers and utilities to access + common data formats (CSV, HTML, JSON and XML in your F# applications and scripts. + + * FSharp.Data -- includes everything + * FSharp.Data.Http -- http types/helpers + * FSharp.Data.Csv.Core -- csv types/helpers + * FSharp.Data.Json.Core -- json types/helpers + * FSharp.Data.Html.Core -- html types/helpers + * FSharp.Data.Xml.Core -- xml types/helpers""" let tags = "F# fsharp data typeprovider WorldBank CSV HTML CSS JSON XML HTTP linqpad-samples" @@ -120,9 +126,9 @@ Target.create "RunTests" (fun _ -> "FSharp.Data.sln" |> DotNet.test setParams) // -------------------------------------------------------------------------------------- -// Build a NuGet package +// Build packages -Target.create "NuGet" (fun _ -> +Target.create "Pack" (fun _ -> // Format the release notes let releaseNotes = release.Notes |> String.concat "\n" @@ -136,12 +142,7 @@ Target.create "NuGet" (fun _ -> ("PackageLicenseExpression", license) ("PackageReleaseNotes", releaseNotes) ("Summary", summary) - ("PackageDescription", description) - ("EnableSourceLink", "true") - ("PublishRepositoryUrl", "true") - ("EmbedUntrackedSources", "true") - ("IncludeSymbols", "true") - ("SymbolPackageFormat", "snupkg") ] + ("PackageDescription", description) ] DotNet.pack (fun p -> @@ -149,7 +150,7 @@ Target.create "NuGet" (fun _ -> Configuration = DotNet.BuildConfiguration.Release OutputPath = Some "bin" MSBuildParams = { p.MSBuildParams with Properties = properties } }) - "src/FSharp.Data/FSharp.Data.fsproj") + "FSharp.Data.sln") // -------------------------------------------------------------------------------------- // Generate the documentation @@ -178,7 +179,7 @@ Target.create "Help" (fun _ -> printfn " * Build" printfn " * RunTests" printfn " * GenerateDocs" - printfn " * NuGet (creates package only, doesn't publish)" + printfn " * Pack (creates package only, doesn't publish)" printfn " * All (calls previous 5)" printfn "" printfn " Other targets:" @@ -228,7 +229,7 @@ Target.create "All" ignore ==> "GenerateDocs" ==> "All" -"Build" ==> "NuGet" ==> "All" +"Build" ==> "Pack" ==> "All" "Build" ==> "All" "Build" ==> "RunTests" ==> "All" diff --git a/docs/library/CsvFile.fsx b/docs/library/CsvFile.fsx index ef3316694..70bc22bf1 100644 --- a/docs/library/CsvFile.fsx +++ b/docs/library/CsvFile.fsx @@ -6,18 +6,20 @@ index: 2 --- *) (*** condition: prepare ***) +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -48,12 +50,14 @@ points to a live CSV file on the Yahoo finance web site: *) // Download the stock prices -let msft = CsvFile.Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv").Cache() +let msft = + CsvFile + .Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv") + .Cache() // Print the prices in the HLOC format for row in msft.Rows |> Seq.truncate 10 do - printfn "HLOC: (%s, %s, %s)" - (row.GetColumn "High") (row.GetColumn "Low") (row.GetColumn "Date") + printfn "HLOC: (%s, %s, %s)" (row.GetColumn "High") (row.GetColumn "Low") (row.GetColumn "Date") (*** include-fsi-merged-output ***) (** @@ -92,8 +96,7 @@ The following example shows how to process the sample previous CSV sample using open FSharp.Data.CsvExtensions for row in msft.Rows |> Seq.truncate 10 do - printfn "HLOC: (%f, %M, %O)" - (row.["High"].AsFloat()) (row?Low.AsDecimal()) (row?Date.AsDateTime()) + printfn "HLOC: (%f, %M, %O)" (row.["High"].AsFloat()) (row?Low.AsDecimal()) (row?Date.AsDateTime()) (*** include-fsi-merged-output ***) @@ -108,7 +111,8 @@ separator and quote characters when saving. *) // Saving the first 10 stock prices where the closing price is higher than the opening price in TSV format: -msft.Filter(fun row -> row?Close.AsFloat() > row?Open.AsFloat()) +msft + .Filter(fun row -> row?Close.AsFloat() > row?Open.AsFloat()) .Truncate(10) .SaveToString('\t') diff --git a/docs/library/CsvProvider.fsx b/docs/library/CsvProvider.fsx index 059fff799..d62078e90 100644 --- a/docs/library/CsvProvider.fsx +++ b/docs/library/CsvProvider.fsx @@ -6,18 +6,20 @@ index: 1 --- *) (*** condition: prepare ***) +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  @@ -84,7 +86,10 @@ The following sample calls the `Load` method with an URL that points to a live C *) // Download the stock prices -let msft = Stocks.Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv").Cache() +let msft = + Stocks + .Load(__SOURCE_DIRECTORY__ + "/../data/MSFT.csv") + .Cache() // Look at the most recent row. Note the 'Date' property // is of type 'DateTime' and 'Open' has a type 'decimal' @@ -94,7 +99,7 @@ let lastOpen = firstRow.Open // Print the first 10 prices in the HLOC format for row in msft.Rows |> Seq.truncate 10 do - printfn "HLOC: (%A, %A, %A, %A)" row.High row.Low row.Open row.Close + printfn "HLOC: (%A, %A, %A, %A)" row.High row.Low row.Open row.Close (*** include-fsi-merged-output ***) @@ -127,7 +132,8 @@ a static argument. Also note that in this case we're using the same data at runt so we use the `GetSample` method instead of calling `Load` and passing the same parameter again. *) -let small = CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>.GetSample() +let small = + CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>.GetSample () (*** include-fsi-merged-output ***) @@ -135,7 +141,8 @@ let small = CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFold We can also use the default constructor instead of the `GetSample` static method: *) -let small2 = new CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>() +let small2 = + new CsvProvider<"../data/SmallTest.csv", ResolutionFolder=ResolutionFolder>() (*** include-fsi-merged-output ***) @@ -151,9 +158,10 @@ following simple calculation: open FSharp.Data.UnitSystems.SI.UnitNames for row in small.Rows do - let speed = row.Distance / row.Time - if speed > 15.0M then - printfn "%s (%A m/s)" row.Name speed + let speed = row.Distance / row.Time + + if speed > 15.0M then + printfn "%s (%A m/s)" row.Name speed (*** include-fsi-merged-output ***) @@ -178,8 +186,8 @@ type AirQuality = CsvProvider<"../data/AirQuality.csv", ";", ResolutionFolder=Re let airQuality = new AirQuality() for row in airQuality.Rows |> Seq.truncate 10 do - if row.Month > 6 then - printfn "Temp: %i Ozone: %f " row.Temp row.Ozone + if row.Month > 6 then + printfn "Temp: %i Ozone: %f " row.Temp row.Ozone (*** include-fsi-merged-output ***) @@ -195,18 +203,21 @@ we also set `IgnoreErrors` static parameter to `true` so that lines with incorre are automatically skipped (the sample file ([`data/MortalityNY.csv`](../data/MortalityNY.tsv)) contains additional unstructured data at the end): *) -let mortalityNy = CsvProvider<"../data/MortalityNY.tsv", IgnoreErrors=true, ResolutionFolder=ResolutionFolder>.GetSample() +let mortalityNy = + CsvProvider<"../data/MortalityNY.tsv", IgnoreErrors=true, ResolutionFolder=ResolutionFolder>.GetSample () // Find the name of a cause based on code // (Pedal cyclist injured in an accident) -let cause = mortalityNy.Rows |> Seq.find (fun r -> - r.``Cause of death Code`` = "V13.4") +let cause = + mortalityNy.Rows + |> Seq.find (fun r -> r.``Cause of death Code`` = "V13.4") // Print the number of injured cyclists printfn "CAUSE: %s" cause.``Cause of death`` + for r in mortalityNy.Rows do - if r.``Cause of death Code`` = "V13.4" then - printfn "%s (%d cases)" r.County r.Count + if r.``Cause of death Code`` = "V13.4" then + printfn "%s (%d cases)" r.County r.Count (*** include-fsi-merged-output ***) @@ -229,8 +240,9 @@ the `MissingValues` static parameter of `CsvProvider` as a comma-separated strin For example, to ignore `this` and `that` we could do: *) -CsvProvider<"X,Y,Z\nthis,that,1.0", - MissingValues="this,that">.GetSample().Rows +CsvProvider<"X,Y,Z\nthis,that,1.0", MissingValues="this,that"> + .GetSample() + .Rows (*** include-fsi-merged-output ***) (** @@ -241,11 +253,11 @@ each row, then remove missing values and then use the standard `Seq.average` fun open System let mean = - airQuality.Rows - |> Seq.toArray - |> Array.map (fun row -> row.Ozone) - |> Array.filter (fun elem -> not (Double.IsNaN elem)) - |> Array.average + airQuality.Rows + |> Seq.toArray + |> Array.map (fun row -> row.Ozone) + |> Array.filter (fun elem -> not (Double.IsNaN elem)) + |> Array.average (*** include-fsi-merged-output ***) @@ -320,15 +332,12 @@ consider that row as a data row. In that case, the columns will be named `Column names are overridden using the `Schema` parameter. Note that you can override only the name in the `Schema` parameter and still have the provider infer the type for you. Example: *) -type OneTwoThree = - CsvProvider<"1,2,3", HasHeaders = false, Schema = "Duration (float),foo,float option"> +type OneTwoThree = CsvProvider<"1,2,3", HasHeaders=false, Schema="Duration (float),foo,float option"> let csv = OneTwoThree.GetSample() + for row in csv.Rows do - printfn "%f %d %f" - (row.Duration/1.0) - row.Foo - (defaultArg row.Column3 1.0) + printfn "%f %d %f" (row.Duration / 1.0) row.Foo (defaultArg row.Column3 1.0) (*** include-fsi-merged-output ***) @@ -342,14 +351,12 @@ the other columns blank in the schema (you also don't need to add all the traili *) type Titanic1 = - CsvProvider<"../data/Titanic.csv", - Schema=",,Passenger Class,,,float", - ResolutionFolder=ResolutionFolder> + CsvProvider<"../data/Titanic.csv", Schema=",,Passenger Class,,,float", ResolutionFolder=ResolutionFolder> let titanic1 = Titanic1.GetSample() + for row in titanic1.Rows |> Seq.truncate 10 do - printfn "%s Class = %d Fare = %g" - row.Name row.``Passenger Class`` row.Fare + printfn "%s Class = %d Fare = %g" row.Name row.``Passenger Class`` row.Fare (*** include-fsi-merged-output ***) @@ -359,14 +366,12 @@ Alternatively, you can rename and override the type of any column by name instea *) type Titanic2 = - CsvProvider<"../data/Titanic.csv", - Schema="Fare=float,PClass->Passenger Class", - ResolutionFolder=ResolutionFolder> + CsvProvider<"../data/Titanic.csv", Schema="Fare=float,PClass->Passenger Class", ResolutionFolder=ResolutionFolder> let titanic2 = Titanic2.GetSample() + for row in titanic2.Rows |> Seq.truncate 10 do - printfn "%s Class = %d Fare = %g" - row.Name row.``Passenger Class`` row.Fare + printfn "%s Class = %d Fare = %g" row.Name row.``Passenger Class`` row.Fare (*** include-fsi-merged-output ***) @@ -385,11 +390,11 @@ the `Save` method. You can also use the `SaveToString()` to get the output direc // Saving the first 10 rows that don't have missing values to a new csv file airQuality - .Filter(fun row -> - not (Double.IsNaN row.Ozone) && - not (Double.IsNaN row.``Solar.R``)) - .Truncate(10) - .SaveToString() + .Filter(fun row -> + not (Double.IsNaN row.Ozone) + && not (Double.IsNaN row.``Solar.R``)) + .Truncate(10) + .SaveToString() (*** include-fsi-merged-output ***) @@ -400,10 +405,7 @@ It's also possible to transform the columns themselves by using `Map` and the co *) let doubleOzone = - airQuality.Map(fun row -> - AirQuality.Row - ( row.Ozone * 2.0, row.``Solar.R``, - row.Wind, row.Temp, row.Month, row.Day)) + airQuality.Map(fun row -> AirQuality.Row(row.Ozone * 2.0, row.``Solar.R``, row.Wind, row.Temp, row.Month, row.Day)) (*** include-fsi-merged-output ***) @@ -414,12 +416,12 @@ You can also append new rows, either by creating them directly as in the previou *) let newRows = - AirQuality.ParseRows - ("""41;190;7.4;67;5;1 - 36;118;8;72;5;2""") + AirQuality.ParseRows( + """41;190;7.4;67;5;1 + 36;118;8;72;5;2""" + ) -let airQualityWithExtraRows = - airQuality.Append newRows +let airQualityWithExtraRows = airQuality.Append newRows (*** include-fsi-merged-output ***) @@ -429,13 +431,11 @@ It's even possible to create csv files without parsing at all: *) -type MyCsvType = - CsvProvider +type MyCsvType = CsvProvider let myRows = - [ MyCsvType.Row(1, "a", None) - MyCsvType.Row(2, "B", Some System.DateTime.Now) ] + [ MyCsvType.Row(1, "a", None) + MyCsvType.Row(2, "B", Some System.DateTime.Now) ] let myCsv = new MyCsvType(myRows) myCsv.SaveToString() diff --git a/docs/library/HtmlCssSelectors.fsx b/docs/library/HtmlCssSelectors.fsx index 06c68177e..a93bb1450 100644 --- a/docs/library/HtmlCssSelectors.fsx +++ b/docs/library/HtmlCssSelectors.fsx @@ -6,18 +6,20 @@ index: 4 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Html.Core.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -52,11 +54,15 @@ Then we can , for example, use the direct descendants selector to select another id `ires`. The CSS selector to do so is `div#search > div#ires`: *) let links = - doc.CssSelect("div#search > div#ires div.g > div.s div.kv cite") - |> List.map (fun n -> - match n.InnerText() with - | t when (t.StartsWith("https://") || t.StartsWith("http://"))-> t - | t -> "http://" + t ) + doc.CssSelect("div#search > div#ires div.g > div.s div.kv cite") + |> List.map (fun n -> + match n.InnerText() with + | t when + (t.StartsWith("https://") + || t.StartsWith("http://")) + -> + t + | t -> "http://" + t) (*** include-fsi-merged-output ***) @@ -87,8 +93,8 @@ let doc2 = HtmlDocument.Load(fsys) let books = doc2.CssSelect("div.g h3.r a") - |> List.map(fun a -> a.InnerText().Trim(), a.AttributeValue("href")) - |> List.filter(fun (title, href) -> title.Contains("F#")) + |> List.map (fun a -> a.InnerText().Trim(), a.AttributeValue("href")) + |> List.filter (fun (title, href) -> title.Contains("F#")) (*** include-fsi-merged-output ***) @@ -103,7 +109,9 @@ You can also refer to the table below for a complete list of supported selectors Finds all links with an english hreflang attribute. *) -let englishDoc = HtmlDocument.Parse(""" +let englishDoc = + HtmlDocument.Parse( + """ @@ -111,10 +119,10 @@ let englishDoc = HtmlDocument.Parse(""" Some other text will not be outlined - """) + """ + ) -let englishLinks = - englishDoc.CssSelect("a[hreflang|=en]") +let englishLinks = englishDoc.CssSelect("a[hreflang|=en]") (*** include-fsi-merged-output ***) (** @@ -122,7 +130,9 @@ let englishLinks = Finds all inputs with a name containing "man". This includes results where "man" is a substring: *) -let manDoc = HtmlDocument.Parse(""" +let manDoc = + HtmlDocument.Parse( + """ @@ -134,10 +144,10 @@ let manDoc = HtmlDocument.Parse(""" - """) + """ + ) -let manElems = - manDoc.CssSelect("input[name*='man']") +let manElems = manDoc.CssSelect("input[name*='man']") (*** include-fsi-merged-output ***) (** @@ -145,8 +155,7 @@ let manElems = Finds all inputs with a name containing the word "man". This requires a whitespace around the word: *) -let manWordElems = - manDoc.CssSelect("input[name~='man']") +let manWordElems = manDoc.CssSelect("input[name~='man']") (*** include-fsi-merged-output ***) @@ -155,8 +164,7 @@ let manWordElems = Finds all inputs with a name ending with "man". *) -let manEndElemes = - manDoc.CssSelect("input[name$='man']") +let manEndElemes = manDoc.CssSelect("input[name$='man']") (*** include-fsi-merged-output ***) @@ -166,8 +174,7 @@ let manEndElemes = Finds all inputs with a name equal to "man". *) -let manEqElemes = - manDoc.CssSelect("input[name='man']") +let manEqElemes = manDoc.CssSelect("input[name='man']") (*** include-fsi-merged-output ***) @@ -176,8 +183,7 @@ let manEqElemes = Finds all inputs with a name different to "man". *) -let notManElems = - manDoc.CssSelect("input[name!='man']") +let notManElems = manDoc.CssSelect("input[name!='man']") (*** include-fsi-merged-output ***) @@ -187,8 +193,7 @@ let notManElems = Finds all inputs with a name starting with "man". *) -let manStartElems = - manDoc.CssSelect("input[name^='man']") +let manStartElems = manDoc.CssSelect("input[name^='man']") (*** include-fsi-merged-output ***) @@ -198,7 +203,9 @@ let manStartElems = There are some syntax shortcuts to find forms controls. *) -let htmlForm = HtmlDocument.Parse(""" +let htmlForm = + HtmlDocument.Parse( + """ @@ -221,7 +228,8 @@ let htmlForm = HtmlDocument.Parse(""" - """) + """ + ) (** You can use `:prop` to find CSS elements with the specified value of the `type` attribute @@ -331,4 +339,3 @@ Selector name|Status|specification [1] :root Selector seems to be useless in our case because with the HTML parser the root is always the html node. *) - diff --git a/docs/library/HtmlParser.fsx b/docs/library/HtmlParser.fsx index 28f2872f5..78668932a 100644 --- a/docs/library/HtmlParser.fsx +++ b/docs/library/HtmlParser.fsx @@ -6,18 +6,20 @@ index: 3 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Html.Core.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -57,11 +59,10 @@ which in this case is the url that the search result is pointing to, and additio we are looking at. *) let links = - results.Descendants ["a"] + results.Descendants [ "a" ] |> Seq.choose (fun x -> - x.TryGetAttribute("href") - |> Option.map (fun a -> x.InnerText(), a.Value()) - ) + x.TryGetAttribute("href") + |> Option.map (fun a -> x.InnerText(), a.Value())) |> Seq.truncate 10 |> Seq.toList @@ -78,8 +79,13 @@ and `Seq.map`. let searchResults = links |> List.filter (fun (name, url) -> - name <> "Cached" && name <> "Similar" && url.StartsWith("/url?")) - |> List.map (fun (name, url) -> name, url.Substring(0, url.IndexOf("&sa=")).Replace("/url?q=", "")) + name <> "Cached" + && name <> "Similar" + && url.StartsWith("/url?")) + |> List.map (fun (name, url) -> + name, + url + .Substring(0, url.IndexOf("&sa=")) + .Replace("/url?q=", "")) (*** include-fsi-merged-output ***) - diff --git a/docs/library/HtmlProvider.fsx b/docs/library/HtmlProvider.fsx index 488d23baa..532baee1c 100644 --- a/docs/library/HtmlProvider.fsx +++ b/docs/library/HtmlProvider.fsx @@ -6,18 +6,21 @@ index: 2 --- *) (*** condition: prepare ***) +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Html.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -58,9 +61,10 @@ first row is headers. (This behaviour is likely to get smarter in later releases *) [] -let ResolutionFolder = __SOURCE_DIRECTORY__ +let F1_2017_URL = + "https://en.wikipedia.org/wiki/2017_FIA_Formula_One_World_Championship" -type F1_2017 = HtmlProvider<"../data/2017_F1.htm", ResolutionFolder=ResolutionFolder> +type F1_2017 = HtmlProvider (** The generated type provides a type space of tables that it has managed to parse out of the given HTML Document. @@ -69,13 +73,8 @@ entities exist then the table will simply be named `Tablexx` where xx is the pos The `Load` method allows reading the data from a file or web resource. We could also have used a web URL instead of a local file in the sample parameter of the type provider. The following sample calls the `Load` method with an URL that points to a live version of the same page on wikipedia. *) -let url = - "https://en.wikipedia.org/wiki/" + - "2017_FIA_Formula_One_World_Championship" - // Download the latest market depth information -let f1Calendar = - F1_2017.Load(url).Tables.``Season calendar`` +let f1Calendar = F1_2017.Load(F1_2017_URL).Tables.``Season calendarEdit`` // Look at the most recent row. Note the 'Date' property // is of type 'DateTime' and 'Open' has a type 'decimal' @@ -86,8 +85,7 @@ let date = firstRow.Date // Print the bid / offer volumes for each row for row in f1Calendar.Rows do - printfn "Race, round %A is hosted at %A on %A" - row.Round row.``Grand Prix`` row.Date + printfn "Race, round %A is hosted at %A on %A" row.Round row.``Grand Prix`` row.Date (*** include-fsi-merged-output ***) @@ -113,24 +111,30 @@ Note that we're using the live URL as the sample, so we can just use the default // Configure the type provider -type NugetStats = - HtmlProvider<"https://www.nuget.org/packages/FSharp.Data"> +type NugetStats = HtmlProvider<"https://www.nuget.org/packages/FSharp.Data"> // load the live package stats for FSharp.Data let rawStats = NugetStats().Tables.Table4 // helper function to analyze version numbers from nuget -let getMinorVersion (v:string) = - System.Text.RegularExpressions.Regex(@"\d.\d").Match(v).Value +let getMinorVersion (v: string) = + System + .Text + .RegularExpressions + .Regex( + @"\d.\d" + ) + .Match( + v + ) + .Value // group by minor version and calculate download count let stats = - rawStats.Rows - |> Seq.groupBy (fun r -> - getMinorVersion r.Version) - |> Seq.map (fun (k, xs) -> - k, xs |> Seq.sumBy (fun x -> x.Downloads)) - |> Seq.toArray + rawStats.Rows + |> Seq.groupBy (fun r -> getMinorVersion r.Version) + |> Seq.map (fun (k, xs) -> k, xs |> Seq.sumBy (fun x -> x.Downloads)) + |> Seq.toArray (*** include-fsi-merged-output ***) @@ -143,22 +147,23 @@ This sample shows some more screen scraping from Wikipedia: *) (*** define-output:doctorWhoChart ***) -let [] DrWho = - "https://en.wikipedia.org/wiki/List_of_Doctor_Who_episodes_(1963%E2%80%931989)" +[] +let DrWho = + "https://en.wikipedia.org/wiki/List_of_Doctor_Who_episodes_(1963%E2%80%931989)" let doctorWho = new HtmlProvider() // Get the average number of viewers for each doctor's series run let viewersByDoctor = - doctorWho.Tables.``Season 1 (1963-1964) Edit``.Rows - |> Seq.groupBy (fun season -> season.``Directed by``) - |> Seq.map (fun (doctor, seasons) -> - let averaged = - seasons - |> Seq.averageBy (fun season -> - season.``UK viewers (millions)``) - doctor, averaged) - |> Seq.toArray + doctorWho.Tables.``Season 1 (1963-1964) Edit``.Rows + |> Seq.groupBy (fun season -> season.``Directed by``) + |> Seq.map (fun (doctor, seasons) -> + let averaged = + seasons + |> Seq.averageBy (fun season -> season.``UK viewers (millions)``) + + doctor, averaged) + |> Seq.toArray (*** include-fsi-merged-output ***) diff --git a/docs/library/Http.fsx b/docs/library/Http.fsx index 31f92da29..a69fecd1a 100644 --- a/docs/library/Http.fsx +++ b/docs/library/Http.fsx @@ -6,18 +6,18 @@ index: 1 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -56,8 +56,10 @@ can use `cref:M:FSharp.Data.Http.RequestString` and `cref:M:FSharp.Data.Http.Asy Http.RequestString("http://tomasp.net") // Download web site asynchronously -async { let! html = Http.AsyncRequestString("http://tomasp.net") - printfn "%d" html.Length } +async { + let! html = Http.AsyncRequestString("http://tomasp.net") + printfn "%d" html.Length +} |> Async.Start (*** include-fsi-merged-output ***) @@ -74,9 +76,7 @@ can pass them using the optional parameter `query`. The following example also e specifies the GET method, but it will be set automatically for you if you omit it: *) -Http.RequestString - ( "http://httpbin.org/get", - query=["test", "foo"], httpMethod="GET" ) +Http.RequestString("http://httpbin.org/get", query = [ "test", "foo" ], httpMethod = "GET") (*** include-fsi-merged-output ***) @@ -96,10 +96,12 @@ provide your API key: let apiKey = "" // Run the HTTP web request -Http.RequestString - ( "http://api.themoviedb.org/3/search/movie", httpMethod = "GET", - query = [ "api_key", apiKey; "query", "batman" ], - headers = [ "Accept", "application/json" ]) +Http.RequestString( + "http://api.themoviedb.org/3/search/movie", + httpMethod = "GET", + query = [ "api_key", apiKey; "query", "batman" ], + headers = [ "Accept", "application/json" ] +) (** The library supports a simple and unchecked string based API (used in the previous example), @@ -115,10 +117,11 @@ open FSharp.Data.HttpRequestHeaders (*** do-not-eval ***) // Run the HTTP web request -Http.RequestString - ( "http://api.themoviedb.org/3/search/movie", - query = [ "api_key", apiKey; "query", "batman" ], - headers = [ Accept HttpContentTypes.Json ]) +Http.RequestString( + "http://api.themoviedb.org/3/search/movie", + query = [ "api_key", apiKey; "query", "batman" ], + headers = [ Accept HttpContentTypes.Json ] +) (** ## Getting extra information @@ -143,7 +146,8 @@ instead of the `cref:M:FSharp.Data.Http.RequestString` method: *) -let response = Http.Request("http://api.themoviedb.org/3/search/movie", silentHttpErrors = true) +let response = + Http.Request("http://api.themoviedb.org/3/search/movie", silentHttpErrors = true) // Examine information about the response response.Headers @@ -168,7 +172,7 @@ The following example uses the [httpbin.org](http://httpbin.org) service which returns the request details: *) -Http.RequestString("http://httpbin.org/post", body = FormValues ["test", "foo"]) +Http.RequestString("http://httpbin.org/post", body = FormValues [ "test", "foo" ]) (*** include-fsi-merged-output ***) @@ -178,10 +182,11 @@ or `application/octet-stream`, depending on which kind of `HttpRequestBody` you this behaviour by adding `content-type` to the list of headers using the optional argument `headers`: *) -Http.RequestString - ( "http://httpbin.org/post", +Http.RequestString( + "http://httpbin.org/post", headers = [ ContentType HttpContentTypes.Json ], - body = TextRequest """ {"test": 42} """) + body = TextRequest """ {"test": 42} """ +) (*** include-fsi-merged-output ***) @@ -198,8 +203,8 @@ The following is an old sample showing how this is set. // Build URL with documentation for a given class let msdnUrl className = - let root = "http://msdn.microsoft.com" - sprintf "%s/en-gb/library/%s.aspx" root className + let root = "http://msdn.microsoft.com" + sprintf "%s/en-gb/library/%s.aspx" root className // Get the page and search for F# code let docInCSharp = Http.RequestString(msdnUrl "system.web.httprequest") @@ -209,16 +214,19 @@ open System.Net let cc = CookieContainer() // Send a request to switch the language -Http.RequestString - ( msdnUrl "system.datetime", - query = ["cs-save-lang", "1"; "cs-lang","fsharp"], - cookieContainer = cc) |> ignore +Http.RequestString( + msdnUrl "system.datetime", + query = + [ "cs-save-lang", "1" + "cs-lang", "fsharp" ], + cookieContainer = cc +) +|> ignore // Request the documentation again & search for F# let docInFSharp = - Http.RequestString - ( msdnUrl "system.web.httprequest", - cookieContainer = cc ) + Http.RequestString(msdnUrl "system.web.httprequest", cookieContainer = cc) + docInFSharp.Contains "F#" (** @@ -230,11 +238,10 @@ The `cref:M:FSharp.Data.Http.RequestString` method will always return the respon *) let logoUrl = "https://raw.github.com/fsharp/FSharp.Data/master/misc/logo.png" + match Http.Request(logoUrl).Body with -| Text text -> - printfn "Got text content: %s" text -| Binary bytes -> - printfn "Got %d bytes of binary content" bytes.Length +| Text text -> printfn "Got text content: %s" text +| Binary bytes -> printfn "Got %d bytes of binary content" bytes.Length (** ## Customizing the HTTP request @@ -254,14 +261,16 @@ Assuming the certificate is stored in `myCertificate.pfx`: open System.Security.Cryptography.X509Certificates // Load the certificate from local file -let clientCert = - new X509Certificate2(".\myCertificate.pfx", "password") +let clientCert = new X509Certificate2(".\myCertificate.pfx", "password") // Send the request with certificate -Http.Request - ( "http://yourprotectedresouce.com/data", - customizeHttpRequest = fun req -> - req.ClientCertificates.Add(clientCert) |> ignore; req) +Http.Request( + "http://yourprotectedresouce.com/data", + customizeHttpRequest = + fun req -> + req.ClientCertificates.Add(clientCert) |> ignore + req +) (** ## Handling multipart form data @@ -277,14 +286,14 @@ uploads of arbitrary size. let largeFilePath = "//path/to/large/file.mp4" let data = System.IO.File.OpenRead(largeFilePath) :> System.IO.Stream -Http.Request - ( "http://endpoint/for/multipart/data", - body = Multipart( - boundary = "define a custom boundary here", // this is used to separate the items you're streaming - parts = [ - MultipartItem("formFieldName", System.IO.Path.GetFileName(largeFilePath), data) - ] - )) +Http.Request( + "http://endpoint/for/multipart/data", + body = + Multipart( + boundary = "define a custom boundary here", // this is used to separate the items you're streaming + parts = [ MultipartItem("formFieldName", System.IO.Path.GetFileName(largeFilePath), data) ] + ) +) (** ## Related articles diff --git a/docs/library/JsonProvider.fsx b/docs/library/JsonProvider.fsx index 6f1a261d7..a49a3aa9e 100644 --- a/docs/library/JsonProvider.fsx +++ b/docs/library/JsonProvider.fsx @@ -6,18 +6,20 @@ index: 3 --- *) (*** condition: prepare ***) +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -124,14 +126,15 @@ following example uses two records - one with `name` and `age` and the second wi If we want to just use the same text used for the schema at runtime, we can use the `GetSamples` method: *) -type People = JsonProvider<""" +type People = + JsonProvider<""" [ { "name":"John", "age":94 }, { "name":"Tomas" } ] """> for item in People.GetSamples() do - printf "%s " item.Name - item.Age |> Option.iter (printf "(%d)") - printfn "" + printf "%s " item.Name + item.Age |> Option.iter (printf "(%d)") + printfn "" (*** include-fsi-merged-output ***) @@ -150,10 +153,10 @@ as follows: type Values = JsonProvider<""" [{"value":94 }, {"value":"Tomas" }] """> for item in Values.GetSamples() do - match item.Value.Number, item.Value.String with - | Some num, _ -> printfn "Numeric: %d" num - | _, Some str -> printfn "Text: %s" str - | _ -> printfn "Some other value!" + match item.Value.Number, item.Value.String with + | Some num, _ -> printfn "Numeric: %d" num + | _, Some str -> printfn "Text: %s" str + | _ -> printfn "Some other value!" (*** include-fsi-merged-output ***) @@ -174,7 +177,8 @@ Applied to the previous example this would be: *) -type People2 = JsonProvider<""" +type People2 = + JsonProvider<""" [ { "name":"John", "age":94 }, { "name":"Tomas" } ] """, SampleIsList=true> @@ -204,12 +208,12 @@ Let's consider an example where this can be useful: *) type AmbiguousEntity = - JsonProvider + """, SampleIsList=true> + let code = (AmbiguousEntity.GetSamples()[1]).Code let length = (AmbiguousEntity.GetSamples()[1]).Length @@ -226,13 +230,12 @@ Now let's enable inline schemas: open FSharp.Data.Runtime.StructuralInference type AmbiguousEntity2 = - JsonProvider", "length":"typeof>" } { "code":"123", "length":"42" } { "code":"4E5", "length":"1.83" } - """, - SampleIsList = true, - InferenceMode = InferenceMode.ValuesAndInlineSchemasOverrides> + """, SampleIsList=true, InferenceMode=InferenceMode.ValuesAndInlineSchemasOverrides> + let code2 = (AmbiguousEntity2.GetSamples()[1]).Code let length2 = (AmbiguousEntity2.GetSamples()[1]).Length @@ -260,7 +263,7 @@ Inline schemas also enable support for units of measure. In the previous example, the `Length` property is now inferred as a `float` with the `metre` unit of measure (from the default SI units). -Warning: units of measures are discarded when merged with types without a unit or with a different unit. +Warning: units of measures are discarded when merged with types without a unit or with a different unit. As mentioned previously, with the `ValuesAndInlineSchemasHints` inference mode, inline schemas types are merged with other inferred types with the same precedence. Since values-inferred types never have units, inline-schemas-inferred types will lose their @@ -304,11 +307,10 @@ let doc = WorldBank.GetSample() (** Note that we can also load the data directly from the web both in the `Load` method and in the type provider sample parameter, and there's an asynchronous `AsyncLoad` method available too: *) let wbReq = - "http://api.worldbank.org/country/cz/indicator/" + - "GC.DOD.TOTL.GD.ZS?format=json" + "http://api.worldbank.org/country/cz/indicator/" + + "GC.DOD.TOTL.GD.ZS?format=json" -let docAsync = - WorldBank.AsyncLoad(wbReq) +let docAsync = WorldBank.AsyncLoad(wbReq) (*** include-fsi-merged-output ***) @@ -320,13 +322,12 @@ provider infers that there is only one record and one array. We can print the da // Print general information let info = doc.Record -printfn "Showing page %d of %d. Total records %d" - info.Page info.Pages info.Total +printfn "Showing page %d of %d. Total records %d" info.Page info.Pages info.Total // Print all data points for record in doc.Array do - record.Value |> Option.iter (fun value -> - printfn "%d: %f" record.Date value) + record.Value + |> Option.iter (fun value -> printfn "%d: %f" record.Date value) (*** include-fsi-merged-output ***) @@ -348,11 +349,13 @@ provider that the sample is actually a _list of samples_: *) type Tweet = JsonProvider<"../data/TwitterStream.json", SampleIsList=true, ResolutionFolder=ResolutionFolder> -let text = (*[omit:(omitted)]*)""" {"in_reply_to_status_id_str":null,"text":"\u5927\u91d1\u6255\u3063\u3066\u904a\u3070\u3057\u3066\u3082\u3089\u3046\u3002\u3082\u3046\u3053\u306e\u4e0a\u306a\u3044\u8d05\u6ca2\u3002\u3067\u3082\uff0c\u5b9f\u969b\u306b\u306f\u305d\u306e\u8d05\u6ca2\u306e\u672c\u8cea\u3092\u6e80\u55ab\u3067\u304d\u308b\u4eba\u306f\u9650\u3089\u308c\u3066\u308b\u3002\u305d\u3053\u306b\u76ee\u306b\u898b\u3048\u306a\u3044\u968e\u5c64\u304c\u3042\u308b\u3068\u304a\u3082\u3046\u3002","in_reply_to_user_id_str":null,"retweet_count":0,"geo":null,"source":"web","retweeted":false,"truncated":false,"id_str":"263290764686155776","entities":{"user_mentions":[],"hashtags":[],"urls":[]},"in_reply_to_user_id":null,"in_reply_to_status_id":null,"place":null,"coordinates":null,"in_reply_to_screen_name":null,"created_at":"Tue Oct 30 14:46:24 +0000 2012","user":{"notifications":null,"contributors_enabled":false,"time_zone":"Tokyo","profile_background_color":"FFFFFF","location":"Kodaira Tokyo Japan","profile_background_tile":false,"profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","default_profile_image":false,"follow_request_sent":null,"profile_sidebar_fill_color":"17451B","description":"KS(Green62)\/WasedaUniv.(Schl Adv Sci\/Eng)\/SynBio\/ChronoBio\/iGEM2010-2012\/Travel\/Airplane\/ \u5bfa\u30fb\u5ead\u3081\u3050\u308a","favourites_count":17,"screen_name":"Merlin_wand","profile_sidebar_border_color":"000000","id_str":"94788486","verified":false,"lang":"ja","statuses_count":8641,"profile_use_background_image":true,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","listed_count":31,"geo_enabled":true,"created_at":"Sat Dec 05 13:07:32 +0000 2009","profile_text_color":"000000","name":"Marin","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","friends_count":629,"url":null,"id":94788486,"is_translator":false,"default_profile":false,"following":null,"profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","utc_offset":32400,"profile_link_color":"ADADAD","followers_count":426},"id":263290764686155776,"contributors":null,"favorited":false} """(*[/omit]*) + +let text = (*[omit:(omitted)]*) + """ {"in_reply_to_status_id_str":null,"text":"\u5927\u91d1\u6255\u3063\u3066\u904a\u3070\u3057\u3066\u3082\u3089\u3046\u3002\u3082\u3046\u3053\u306e\u4e0a\u306a\u3044\u8d05\u6ca2\u3002\u3067\u3082\uff0c\u5b9f\u969b\u306b\u306f\u305d\u306e\u8d05\u6ca2\u306e\u672c\u8cea\u3092\u6e80\u55ab\u3067\u304d\u308b\u4eba\u306f\u9650\u3089\u308c\u3066\u308b\u3002\u305d\u3053\u306b\u76ee\u306b\u898b\u3048\u306a\u3044\u968e\u5c64\u304c\u3042\u308b\u3068\u304a\u3082\u3046\u3002","in_reply_to_user_id_str":null,"retweet_count":0,"geo":null,"source":"web","retweeted":false,"truncated":false,"id_str":"263290764686155776","entities":{"user_mentions":[],"hashtags":[],"urls":[]},"in_reply_to_user_id":null,"in_reply_to_status_id":null,"place":null,"coordinates":null,"in_reply_to_screen_name":null,"created_at":"Tue Oct 30 14:46:24 +0000 2012","user":{"notifications":null,"contributors_enabled":false,"time_zone":"Tokyo","profile_background_color":"FFFFFF","location":"Kodaira Tokyo Japan","profile_background_tile":false,"profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","default_profile_image":false,"follow_request_sent":null,"profile_sidebar_fill_color":"17451B","description":"KS(Green62)\/WasedaUniv.(Schl Adv Sci\/Eng)\/SynBio\/ChronoBio\/iGEM2010-2012\/Travel\/Airplane\/ \u5bfa\u30fb\u5ead\u3081\u3050\u308a","favourites_count":17,"screen_name":"Merlin_wand","profile_sidebar_border_color":"000000","id_str":"94788486","verified":false,"lang":"ja","statuses_count":8641,"profile_use_background_image":true,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1172376796\/70768_100000537851636_3599485_q_normal.jpg","listed_count":31,"geo_enabled":true,"created_at":"Sat Dec 05 13:07:32 +0000 2009","profile_text_color":"000000","name":"Marin","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","friends_count":629,"url":null,"id":94788486,"is_translator":false,"default_profile":false,"following":null,"profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/612807391\/twitter_free1.br.jpg","utc_offset":32400,"profile_link_color":"ADADAD","followers_count":426},"id":263290764686155776,"contributors":null,"favorited":false} """ (*[/omit]*) + let tweet = Tweet.Parse(text) -printfn "%s (retweeted %d times)\n:%s" - tweet.User.Value.Name tweet.RetweetCount.Value tweet.Text.Value +printfn "%s (retweeted %d times)\n:%s" tweet.User.Value.Name tweet.RetweetCount.Value tweet.Text.Value (*** include-fsi-merged-output ***) @@ -393,7 +396,8 @@ we need to post a JSON value similar to this: *) [] -let issueSample = """ +let issueSample = + """ { "title": "Found a bug", "body": "I'm having a problem with this.", @@ -418,12 +422,14 @@ create an instance, and send a POST request: type GitHubIssue = JsonProvider let newIssue = - GitHubIssue.Issue - ( "Test issue", - "This is a test issue created in FSharp.Data documentation", - assignee = "", - labels = [| |], - milestone = 0) + GitHubIssue.Issue( + "Test issue", + "This is a test issue created in FSharp.Data documentation", + assignee = "", + labels = [||], + milestone = 0 + ) + newIssue.JsonValue.Request "https://api.github.com/repos/fsharp/FSharp.Data/issues" (** @@ -442,9 +448,8 @@ static parameter `EmbeddedResource` (don't forget then to [include the file](htt project file). If you are building a library `MyLib.dll`, you can write: *) -type WB = JsonProvider<"../data/WorldBank.json", - EmbeddedResource="MyLib, MyLib.data.worldbank.json", - ResolutionFolder=ResolutionFolder> +type WB = + JsonProvider<"../data/WorldBank.json", EmbeddedResource="MyLib, MyLib.data.worldbank.json", ResolutionFolder=ResolutionFolder> (** You still need to specify the local path, but this is only used when compiling `MyLib.dll`. diff --git a/docs/library/JsonValue.fsx b/docs/library/JsonValue.fsx index 7a0451a29..7d6092286 100644 --- a/docs/library/JsonValue.fsx +++ b/docs/library/JsonValue.fsx @@ -6,18 +6,20 @@ index: 5 --- *) (*** condition: prepare ***) +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -45,9 +47,11 @@ to parse strings formatted using JSON as follows: *) let info = - JsonValue.Parse(""" + JsonValue.Parse( + """ { "name": "Tomas", "born": 1985, - "siblings": [ "Jan", "Alexander" ] } """) + "siblings": [ "Jan", "Alexander" ] } """ + ) (*** include-fsi-merged-output ***) @@ -103,7 +107,7 @@ printfn "%s (%d)" (info?name.AsString()) (info?born.AsInteger()) // Print names of all siblings for sib in info?siblings do - printfn "%s" (sib.AsString()) + printfn "%s" (sib.AsString()) (*** include-fsi-merged-output ***) @@ -142,11 +146,10 @@ let value = JsonValue.Load(__SOURCE_DIRECTORY__ + "../../data/WorldBank.json") asynchronous version available too: *) let wbReq = - "http://api.worldbank.org/country/cz/indicator/" + - "GC.DOD.TOTL.GD.ZS?format=json" + "http://api.worldbank.org/country/cz/indicator/" + + "GC.DOD.TOTL.GD.ZS?format=json" -let valueAsync = - JsonValue.AsyncLoad(wbReq) +let valueAsync = JsonValue.AsyncLoad(wbReq) (*** include-fsi-merged-output ***) @@ -159,15 +162,12 @@ match value with | JsonValue.Array [| info; data |] -> // Print overall information let page, pages, total = info?page, info?pages, info?total - printfn - "Showing page %d of %d. Total records %d" - (page.AsInteger()) (pages.AsInteger()) (total.AsInteger()) + printfn "Showing page %d of %d. Total records %d" (page.AsInteger()) (pages.AsInteger()) (total.AsInteger()) // Print every non-null data point for record in data do - if record?value <> JsonValue.Null then - printfn "%d: %f" (record?date.AsInteger()) - (record?value.AsFloat()) + if record?value <> JsonValue.Null then + printfn "%d: %f" (record?date.AsInteger()) (record?value.AsFloat()) | _ -> printfn "failed" (*** include-fsi-merged-output ***) diff --git a/docs/library/WorldBank.fsx b/docs/library/WorldBank.fsx index fa2afe250..d6ce9936f 100644 --- a/docs/library/WorldBank.fsx +++ b/docs/library/WorldBank.fsx @@ -6,18 +6,20 @@ index: 5 --- *) (*** condition: prepare ***) +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.WorldBank.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects/FSharp.Data/gh-pages?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -54,9 +56,7 @@ open FSharp.Data let data = WorldBankData.GetDataContext() -data - .Countries.``United Kingdom`` - .Indicators.``Gross capital formation (% of GDP)`` +data.Countries.``United Kingdom``.Indicators.``Gross capital formation (% of GDP)`` |> Seq.maxBy fst (*** include-fsi-merged-output ***) @@ -103,20 +103,20 @@ let wb = WorldBank.GetDataContext() // Create a list of countries to process let countries = - [| wb.Countries.``Arab World`` - wb.Countries.``European Union`` - wb.Countries.Australia - wb.Countries.Brazil - wb.Countries.Canada - wb.Countries.Chile - wb.Countries.``Czech Republic`` - wb.Countries.Denmark - wb.Countries.France - wb.Countries.Greece - wb.Countries.``Low income`` - wb.Countries.``High income`` - wb.Countries.``United Kingdom`` - wb.Countries.``United States`` |] + [| wb.Countries.``Arab World`` + wb.Countries.``European Union`` + wb.Countries.Australia + wb.Countries.Brazil + wb.Countries.Canada + wb.Countries.Chile + wb.Countries.``Czech Republic`` + wb.Countries.Denmark + wb.Countries.France + wb.Countries.Greece + wb.Countries.``Low income`` + wb.Countries.``High income`` + wb.Countries.``United Kingdom`` + wb.Countries.``United States`` |] (** To download the information in parallel, we can create a list of asynchronous @@ -124,8 +124,7 @@ computations, compose them using `Async.Parallel` and then run the (single) obta computation to perform all the downloads: *) -[ for c in countries -> - c.Indicators.``Gross capital formation (% of GDP)`` ] +[ for c in countries -> c.Indicators.``Gross capital formation (% of GDP)`` ] |> Async.Parallel |> Async.RunSynchronously diff --git a/docs/library/XmlProvider.fsx b/docs/library/XmlProvider.fsx index bd71471d2..a0f24d825 100644 --- a/docs/library/XmlProvider.fsx +++ b/docs/library/XmlProvider.fsx @@ -6,18 +6,21 @@ index: 4 --- *) (*** condition: prepare ***) +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Xml.Core.dll" #r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** [![Binder](../img/badge-binder.svg)](https://mybinder.org/v2/gh/fsprojects.github.io/FSharp.Data/main?filepath={{fsdocs-source-basename}}.ipynb)  [![Script](../img/badge-script.svg)]({{root}}/{{fsdocs-source-basename}}.fsx)  @@ -48,6 +51,7 @@ is located in the `../../bin` directory, we can load it in F# Interactive as fol `XDocument` type internally): *) #r "System.Xml.Linq.dll" + open FSharp.Data (** @@ -99,7 +103,9 @@ all, what if a node contains some value, but also has some attributes? *) type Detailed = XmlProvider<"""Karl Popper"""> -let info = Detailed.Parse("""Thomas Kuhn""") + +let info = + Detailed.Parse("""Thomas Kuhn""") printfn "%s (full=%b)" info.Name.Value info.Name.Full @@ -122,7 +128,7 @@ contains multiple `` nodes (note that if we leave out the parameter to th type Test = XmlProvider<"13"> for v in Test.GetSample().Values do - printfn "%d" v + printfn "%d" v (** The type provider generates a property `Values` that returns an array with the @@ -145,12 +151,12 @@ Let's consider an example where this can be useful: *) type AmbiguousEntity = - XmlProvider - """, - SampleIsList = true> + """, SampleIsList=true> + let code = (AmbiguousEntity.GetSamples()[1]).Code let length = (AmbiguousEntity.GetSamples()[1]).Length @@ -167,13 +173,12 @@ Now let's enable inline schemas: open FSharp.Data.Runtime.StructuralInference type AmbiguousEntity2 = - XmlProvider - """, - SampleIsList = true, - InferenceMode = InferenceMode.ValuesAndInlineSchemasOverrides> + """, SampleIsList=true, InferenceMode=InferenceMode.ValuesAndInlineSchemasOverrides> + let code2 = (AmbiguousEntity2.GetSamples()[1]).Code let length2 = (AmbiguousEntity2.GetSamples()[1]).Length @@ -199,7 +204,7 @@ Inline schemas also enable support for units of measure. In the previous example, the `Length` property is now inferred as a `float` with the `metre` unit of measure (from the default SI units). -Warning: units of measures are discarded when merged with types without a unit or with a different unit. +Warning: units of measures are discarded when merged with types without a unit or with a different unit. As mentioned previously, with the `ValuesAndInlineSchemasHints` inference mode, inline schemas types are merged with other inferred types with the same precedence. Since values-inferred types never have units, inline-schemas-inferred types will lose their @@ -225,7 +230,8 @@ At runtime, we use the generated type provider to parse the following string one of the `author` nodes also contains a `died` attribute): *) -let authors = """ +let authors = + """ @@ -245,6 +251,7 @@ type Authors = XmlProvider<"../data/Writers.xml", ResolutionFolder=ResolutionFol let topic = Authors.Parse(authors) printfn "%s" topic.Topic + for author in topic.Authors do printf " - %s" author.Name author.Born |> Option.iter (printf " (%d)") @@ -297,11 +304,12 @@ that takes `Html.Div` and acts as follows: *) /// Prints the content of a
element -let rec printDiv (div:Html.Div) = - div.Spans |> Seq.iter (printfn "%s") - div.Divs |> Seq.iter printDiv - if div.Spans.Length = 0 && div.Divs.Length = 0 then - div.Value |> Option.iter (printfn "%s") +let rec printDiv (div: Html.Div) = + div.Spans |> Seq.iter (printfn "%s") + div.Divs |> Seq.iter printDiv + + if div.Spans.Length = 0 && div.Divs.Length = 0 then + div.Value |> Option.iter (printfn "%s") // Print the root
element with all children printDiv html @@ -355,7 +363,7 @@ let data = Census.Load("https://api.census.gov/data.xml") let apiLinks = data.Datasets - |> Array.map (fun ds -> ds.Title,ds.Distribution.AccessUrl) + |> Array.map (fun ds -> ds.Title, ds.Distribution.AccessUrl) |> Array.truncate 10 (*** include-fsi-merged-output ***) @@ -376,16 +384,19 @@ and then post each link over (for example) a message queue. This is where `AsyncLoad` comes into play: *) -let enqueue (title,apiUrl) = - // do the real message enqueueing here instead of - printfn "%s -> %s" title apiUrl +let enqueue (title, apiUrl) = + // do the real message enqueueing here instead of + printfn "%s -> %s" title apiUrl // helper task which gets scheduled on some background thread somewhere... -let cacheJanitor() = async { - let! reloadData = Census.AsyncLoad("https://api.census.gov/data.xml") - reloadData.Datasets |> Array.map (fun ds -> ds.Title,ds.Distribution.AccessUrl) - |> Array.iter enqueue -} +let cacheJanitor () = + async { + let! reloadData = Census.AsyncLoad("https://api.census.gov/data.xml") + + reloadData.Datasets + |> Array.map (fun ds -> ds.Title, ds.Distribution.AccessUrl) + |> Array.iter enqueue + } (*** include-fsi-merged-output ***) @@ -419,7 +430,7 @@ printfn "%s" blog.Channel.Title // Get all item nodes and print title with link for item in blog.Channel.Items do - printfn " - %s (%s)" item.Title item.Link + printfn " - %s (%s)" item.Title item.Link (*** include-fsi-merged-output ***) @@ -432,7 +443,8 @@ Consider the problem of flattening a data set. Let's say you have xml data that *) [] -let customersXmlSample = """ +let customersXmlSample = + """ @@ -455,7 +467,8 @@ and you want to transform it into something like this: *) [] -let orderLinesXmlSample = """ +let orderLinesXmlSample = + """ @@ -471,15 +484,11 @@ type InputXml = XmlProvider type OutputXml = XmlProvider let orderLines = - OutputXml.OrderLines [| - for customer in InputXml.GetSample().Customers do - for order in customer.Orders do - for line in order.OrderLines do - yield OutputXml.OrderLine - ( customer.Name, - order.Number, - line.Item, - line.Quantity ) |] + OutputXml.OrderLines + [| for customer in InputXml.GetSample().Customers do + for order in customer.Orders do + for line in order.OrderLines do + yield OutputXml.OrderLine(customer.Name, order.Number, line.Item, line.Quantity) |] (*** include-fsi-merged-output ***) @@ -492,7 +501,8 @@ The value of the parameter can be either the name of a schema file or plain text like in the following example: *) -type Person = XmlProvider @@ -505,7 +515,9 @@ type Person = XmlProvider """> -let turing = Person.Parse """ +let turing = + Person.Parse + """ Turing 1912-06-23 @@ -529,7 +541,7 @@ When the file includes other schema files, the `ResolutionFolder` parameter can The uri may also refer to online resources: *) -type RssXsd = XmlProvider +type RssXsd = XmlProvider (** @@ -537,7 +549,8 @@ The schema is expected to define a root element (a global element with complex t In case of multiple root elements: *) -type TwoRoots = XmlProvider @@ -560,15 +573,15 @@ the provided type has an optional property for each alternative: *) let e1 = TwoRoots.Parse "" + match e1.Root1, e1.Root2 with -| Some x, None -> - printfn "Foo = %s and Fow = %A" x.Foo x.Fow +| Some x, None -> printfn "Foo = %s and Fow = %A" x.Foo x.Fow | _ -> failwith "Unexpected" let e2 = TwoRoots.Parse "" + match e2.Root1, e2.Root2 with -| None, Some x -> - printfn "Bar = %s and Baz = %O" x.Bar x.Baz +| None, Some x -> printfn "Bar = %s and Baz = %O" x.Bar x.Baz | _ -> failwith "Unexpected" (*** include-fsi-merged-output ***) @@ -583,7 +596,8 @@ The following xsd defines `foo` as a sequence made of an arbitrary number of `bar` elements followed by a single `baz` element. *) -type FooSequence = XmlProvider @@ -600,7 +614,9 @@ type FooSequence = XmlProvider 42 43 @@ -614,7 +630,8 @@ printfn "%d" fooSequence.Baz.Year // 1957 (** Instead of a sequence we may have a `choice`: *) -type FooChoice = XmlProvider @@ -636,12 +653,15 @@ in type providers) but also preferred because it improves discoverability: intellisense can show both alternatives. There is a lack of precision but this is not the main goal. *) -let fooChoice = FooChoice.Parse """ +let fooChoice = + FooChoice.Parse + """ 1957-08-13 """ printfn "%d items" fooChoice.Bars.Length // 0 items + match fooChoice.Baz with | Some date -> printfn "%d" date.Year // 1957 | None -> () @@ -659,7 +679,8 @@ XML Schema provides various extensibility mechanisms. The following example is a terse summary mixing substitution groups with abstract recursive definitions. *) -type Prop = XmlProvider @@ -673,7 +694,9 @@ type Prop = XmlProvider """> -let formula = Prop.Parse """ +let formula = + Prop.Parse + """ p1 diff --git a/docs/tutorials/JsonAnonymizer.fsx b/docs/tutorials/JsonAnonymizer.fsx index 0f76f40c1..f7e1096b2 100644 --- a/docs/tutorials/JsonAnonymizer.fsx +++ b/docs/tutorials/JsonAnonymizer.fsx @@ -6,18 +6,19 @@ index: 1 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** # Anonymizing JSON @@ -48,86 +49,112 @@ open FSharp.Data type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) = - let propertiesToSkip = Set.ofList (defaultArg propertiesToSkip []) - let valuesToSkip = Set.ofList (defaultArg valuesToSkip []) - - let rng = Random() - - let digits = [| '0' .. '9' |] - let lowerLetters = [| 'a' .. 'z' |] - let upperLetters = [| 'A' .. 'Z' |] - - let getRandomChar (c:char) = - if Char.IsDigit c then digits.[rng.Next(10)] - elif Char.IsLetter c then - if Char.IsLower c - then lowerLetters.[rng.Next(26)] - else upperLetters.[rng.Next(26)] - else c - - let randomize (str:string) = - String(str.ToCharArray() |> Array.map getRandomChar) - - let isType testType typ = - match typ with - | Runtime.StructuralTypes.InferedType.Primitive (typ, _, _, _) -> typ = testType - | _ -> false - - let rec anonymize json = - match json with - | JsonValue.String s when valuesToSkip.Contains s -> json - | JsonValue.String s -> - let typ = - Runtime.StructuralInference.inferPrimitiveType - Runtime.StructuralInference.defaultUnitsOfMeasureProvider - Runtime.StructuralInference.InferenceMode'.ValuesOnly - CultureInfo.InvariantCulture s None - - ( if typ |> isType typeof then Guid.NewGuid().ToString() - elif typ |> isType typeof || - typ |> isType typeof then s - elif typ |> isType typeof then s + let propertiesToSkip = Set.ofList (defaultArg propertiesToSkip []) + let valuesToSkip = Set.ofList (defaultArg valuesToSkip []) + + let rng = Random() + + let digits = [| '0' .. '9' |] + let lowerLetters = [| 'a' .. 'z' |] + let upperLetters = [| 'A' .. 'Z' |] + + let getRandomChar (c: char) = + if Char.IsDigit c then + digits.[rng.Next(10)] + elif Char.IsLetter c then + if Char.IsLower c then + lowerLetters.[rng.Next(26)] else - let prefix, s = - if s.StartsWith "http://" then - "http://", s.Substring("http://".Length) - elif s.StartsWith "https://" then - "https://", s.Substring("https://".Length) - else "", s - prefix + randomize s ) - |> JsonValue.String - | JsonValue.Number d -> - let typ = - Runtime.StructuralInference.inferPrimitiveType - Runtime.StructuralInference.defaultUnitsOfMeasureProvider - Runtime.StructuralInference.InferenceMode'.ValuesOnly - CultureInfo.InvariantCulture (d.ToString()) None - if typ |> isType typeof || - typ |> isType typeof then json - else d.ToString() |> randomize |> Decimal.Parse |> JsonValue.Number - | JsonValue.Float f -> - f.ToString() - |> randomize - |> Double.Parse - |> JsonValue.Float - | JsonValue.Boolean _ | JsonValue.Null -> json - | JsonValue.Record props -> - props - |> Array.map (fun (key, value) -> - let newValue = if propertiesToSkip.Contains key then value else anonymize value - key, newValue) - |> JsonValue.Record - | JsonValue.Array array -> - array - |> Array.map anonymize - |> JsonValue.Array - - member __.Anonymize json = anonymize json - -let json = JsonValue.Load (__SOURCE_DIRECTORY__ + "../../data/TwitterStream.json") + upperLetters.[rng.Next(26)] + else + c + + let randomize (str: string) = + String(str.ToCharArray() |> Array.map getRandomChar) + + let isType testType typ = + match typ with + | Runtime.StructuralTypes.InferedType.Primitive (typ, _, _, _) -> typ = testType + | _ -> false + + let rec anonymize json = + match json with + | JsonValue.String s when valuesToSkip.Contains s -> json + | JsonValue.String s -> + let typ = + Runtime.StructuralInference.inferPrimitiveType + Runtime.StructuralInference.defaultUnitsOfMeasureProvider + Runtime.StructuralInference.InferenceMode'.ValuesOnly + CultureInfo.InvariantCulture + s + None + + (if typ |> isType typeof then + Guid.NewGuid().ToString() + elif typ |> isType typeof + || typ |> isType typeof then + s + elif typ |> isType typeof then + s + else + let prefix, s = + if s.StartsWith "http://" then + "http://", s.Substring("http://".Length) + elif s.StartsWith "https://" then + "https://", s.Substring("https://".Length) + else + "", s + + prefix + randomize s) + |> JsonValue.String + | JsonValue.Number d -> + let typ = + Runtime.StructuralInference.inferPrimitiveType + Runtime.StructuralInference.defaultUnitsOfMeasureProvider + Runtime.StructuralInference.InferenceMode'.ValuesOnly + CultureInfo.InvariantCulture + (d.ToString()) + None + + if typ |> isType typeof + || typ |> isType typeof then + json + else + d.ToString() + |> randomize + |> Decimal.Parse + |> JsonValue.Number + | JsonValue.Float f -> + f.ToString() + |> randomize + |> Double.Parse + |> JsonValue.Float + | JsonValue.Boolean _ + | JsonValue.Null -> json + | JsonValue.Record props -> + props + |> Array.map (fun (key, value) -> + let newValue = + if propertiesToSkip.Contains key then + value + else + anonymize value + + key, newValue) + |> JsonValue.Record + | JsonValue.Array array -> array |> Array.map anonymize |> JsonValue.Array + + member _.Anonymize json = anonymize json + +let json = + JsonValue.Load( + __SOURCE_DIRECTORY__ + + "../../data/TwitterStream.json" + ) + printfn "%O" json -let anonymizedJson = (JsonAnonymizer ["lang"]).Anonymize json +let anonymizedJson = (JsonAnonymizer [ "lang" ]).Anonymize json printfn "%O" anonymizedJson (** diff --git a/docs/tutorials/JsonToXml.fsx b/docs/tutorials/JsonToXml.fsx index 7862b442c..88e55b78d 100644 --- a/docs/tutorials/JsonToXml.fsx +++ b/docs/tutorials/JsonToXml.fsx @@ -6,18 +6,21 @@ index: 2 --- *) (*** condition: prepare ***) -#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Http.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Csv.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Json.Core.dll" +#r "../../src/FSharp.Data/bin/Release/netstandard2.0/FSharp.Data.Xml.Core.dll" (*** condition: fsx ***) #if FSX #r "nuget: FSharp.Data,{{fsdocs-package-version}}" -#endif // FSX +#endif (*** condition: ipynb ***) #if IPYNB #r "nuget: FSharp.Data,{{fsdocs-package-version}}" Formatter.SetPreferredMimeTypesFor(typeof, "text/plain") -Formatter.Register(fun (x:obj) (writer: TextWriter) -> fprintfn writer "%120A" x ) -#endif // IPYNB +Formatter.Register(fun (x: obj) (writer: TextWriter) -> fprintfn writer "%120A" x) +#endif (** # Converting between JSON and XML @@ -47,6 +50,7 @@ We will be using the LINQ to XML API (available in `System.Xml.Linq.dll`) and th *) #r "System.Xml.Linq.dll" + open System.Xml.Linq open FSharp.Data @@ -92,33 +96,32 @@ attributes to a corresponding JSON type: *) /// Creates a JSON representation of a XML element -let rec fromXml (xml:XElement) = - - // Create a collection of key/value pairs for all attributes - let attrs = - [ for attr in xml.Attributes() -> - (attr.Name.LocalName, JsonValue.String attr.Value) ] - - // Function that turns a collection of XElement values - // into an array of JsonValue (using fromXml recursively) - let createArray xelems = - [| for xelem in xelems -> fromXml xelem |] - |> JsonValue.Array - - // Group child elements by their name and then turn all single- - // element groups into a record (recursively) and all multi- - // element groups into a JSON array using createArray - let children = - xml.Elements() - |> Seq.groupBy (fun x -> x.Name.LocalName) - |> Seq.map (fun (key, childs) -> - match Seq.toList childs with - | [child] -> key, fromXml child - | children -> key + "s", createArray children ) - - // Concatenate elements produced for child elements & attributes - Array.append (Array.ofList attrs) (Array.ofSeq children) - |> JsonValue.Record +let rec fromXml (xml: XElement) = + + // Create a collection of key/value pairs for all attributes + let attrs = + [ for attr in xml.Attributes() -> (attr.Name.LocalName, JsonValue.String attr.Value) ] + + // Function that turns a collection of XElement values + // into an array of JsonValue (using fromXml recursively) + let createArray xelems = + [| for xelem in xelems -> fromXml xelem |] + |> JsonValue.Array + + // Group child elements by their name and then turn all single- + // element groups into a record (recursively) and all multi- + // element groups into a JSON array using createArray + let children = + xml.Elements() + |> Seq.groupBy (fun x -> x.Name.LocalName) + |> Seq.map (fun (key, childs) -> + match Seq.toList childs with + | [ child ] -> key, fromXml child + | children -> key + "s", createArray children) + + // Concatenate elements produced for child elements & attributes + Array.append (Array.ofList attrs) (Array.ofSeq children) + |> JsonValue.Record (** @@ -155,44 +158,48 @@ and record construct nested element(s) or attribute: /// Creates an XML representation of a JSON value (works /// only when the top-level value is an object or an array) -let toXml(x:JsonValue) = - // Helper functions for constructing XML - // attributes and XML elements - let attr name value = - XAttribute(XName.Get name, value) :> XObject - let elem name (value:obj) = - XElement(XName.Get name, value) :> XObject - - // Inner recursive function that implements the conversion - let rec toXml = function - // Primitive values are returned as objects - | JsonValue.Null -> null - | JsonValue.Boolean b -> b :> obj - | JsonValue.Number number -> number :> obj - | JsonValue.Float number -> number :> obj - | JsonValue.String s -> s :> obj - - // JSON object becomes a collection of XML - // attributes (for primitives) or child elements - | JsonValue.Record properties -> - properties - |> Array.map (fun (key, value) -> - match value with - | JsonValue.String s -> attr key s - | JsonValue.Boolean b -> attr key b - | JsonValue.Number n -> attr key n - | JsonValue.Float n -> attr key n - | _ -> elem key (toXml value)) :> obj - - // JSON array is turned into a - // sequence of elements - | JsonValue.Array elements -> - elements |> Array.map (fun item -> - elem "item" (toXml item)) :> obj - - // Perform the conversion and cast the result to sequence - // of objects (may fail for unexpected inputs!) - (toXml x) :?> XObject seq +let toXml (x: JsonValue) = + // Helper functions for constructing XML + // attributes and XML elements + let attr name value = + XAttribute(XName.Get name, value) :> XObject + + let elem name (value: obj) = + XElement(XName.Get name, value) :> XObject + + // Inner recursive function that implements the conversion + let rec toXml = + function + // Primitive values are returned as objects + | JsonValue.Null -> null + | JsonValue.Boolean b -> b :> obj + | JsonValue.Number number -> number :> obj + | JsonValue.Float number -> number :> obj + | JsonValue.String s -> s :> obj + + // JSON object becomes a collection of XML + // attributes (for primitives) or child elements + | JsonValue.Record properties -> + properties + |> Array.map (fun (key, value) -> + match value with + | JsonValue.String s -> attr key s + | JsonValue.Boolean b -> attr key b + | JsonValue.Number n -> attr key n + | JsonValue.Float n -> attr key n + | _ -> elem key (toXml value)) + :> obj + + // JSON array is turned into a + // sequence of elements + | JsonValue.Array elements -> + elements + |> Array.map (fun item -> elem "item" (toXml item)) + :> obj + + // Perform the conversion and cast the result to sequence + // of objects (may fail for unexpected inputs!) + (toXml x) :?> XObject seq (** diff --git a/nuget/FSharp.Data.nuspec b/nuget/FSharp.Data.nuspec deleted file mode 100755 index f8c36e670..000000000 --- a/nuget/FSharp.Data.nuspec +++ /dev/null @@ -1,39 +0,0 @@ - - - - @project@ - FSharp.Data - @build.number@ - @authors@ - @authors@ - Apache-2.0 - https://fsprojects.github.io/FSharp.Data - https://raw.github.com/fsharp/FSharp.Data/master/misc/logo.png - false - @summary@ - @description@ - @releaseNotes@ - Copyright 2021 - @tags@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/nuget/linqpad-samples/GitHub.linq b/nuget/linqpad-samples/GitHub.linq deleted file mode 100644 index 2bd722d42..000000000 --- a/nuget/linqpad-samples/GitHub.linq +++ /dev/null @@ -1,19 +0,0 @@ - - FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a - <ProgramFilesX86>\Reference Assemblies\Microsoft\FSharp\.NETFramework\v4.0\4.3.0.0\FSharp.Core.dll - FSharp.Data - - -open FSharp.Data - -// Beware of rate limiting while running this sample: https://developer.github.com/v3/#rate-limiting -type GitHub = JsonProvider<"https://api.github.com/repos/fsharp/FSharp.Data/issues"> - -let topRecentlyUpdatedIssues = - GitHub.GetSamples() - |> Seq.filter (fun issue -> issue.State = "open") - |> Seq.sortBy (fun issue -> System.DateTime.Now - issue.UpdatedAt) - |> Seq.truncate 5 - -for issue in topRecentlyUpdatedIssues do - printfn "#%d %s" issue.Number issue.Title \ No newline at end of file diff --git a/nuget/linqpad-samples/Jira.linq b/nuget/linqpad-samples/Jira.linq deleted file mode 100644 index b6569d997..000000000 --- a/nuget/linqpad-samples/Jira.linq +++ /dev/null @@ -1,15 +0,0 @@ - - FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a - <ProgramFilesX86>\Reference Assemblies\Microsoft\FSharp\.NETFramework\v4.0\4.3.0.0\FSharp.Core.dll - FSharp.Data - - -open FSharp.Data - -[] -let apiUrl = "https://jira.atlassian.com/rest/api/2/search?filter=-4" // all issues -type Jira = JsonProvider -let jira = Jira.Load(apiUrl) - -let tickets = jira.Issues |> Array.map (fun ticket -> (ticket.Id, ticket.Fields.Summary)) -tickets |> Dump \ No newline at end of file diff --git a/nuget/linqpad-samples/TypeInference.linq b/nuget/linqpad-samples/TypeInference.linq deleted file mode 100644 index 193198cd4..000000000 --- a/nuget/linqpad-samples/TypeInference.linq +++ /dev/null @@ -1,27 +0,0 @@ - - FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a - <ProgramFilesX86>\Reference Assemblies\Microsoft\FSharp\.NETFramework\v4.0\4.3.0.0\FSharp.Core.dll - FSharp.Data - - -open FSharp.Data - -// The JsonProvider<...> takes one static parameter of type string. The parameter can be either a sample string or a sample file -// (relative to the current folder or online accessible via http or https). It is not likely that this could lead to ambiguities. -type Simple = JsonProvider<""" { "name":"John", "age":94 } """> -let simple = Simple.Parse(""" { "name":"Tomas", "age":4 } """) -simple.Age |> Dump -simple.Name |> Dump - -// A list may mix integers and floats. When the sample is a collection, the type provider generates a type -// that can be used to store all values in the sample. In this case, the resulting type is decimal, because one of the values is not an integer -type Numbers = JsonProvider<""" [1, 2, 3, 3.14] """> -Numbers.Parse(""" [1.2, 45.1, 98.2, 5] """) |> Dump - -// Other primitive types cannot be combined into a single type. For example, if the list contains numbers and strings. -// In this case, the provider generates two methods that can be used to get values that match one of the types: -type Mixed = JsonProvider<""" [1, 2, "hello", "world"] """> -let mixed = Mixed.Parse(""" [4, 5, "hello", "world" ] """) - -mixed.Numbers |> Seq.sum |> Dump -mixed.Strings |> String.concat ", " |> Dump \ No newline at end of file diff --git a/nuget/publish.cmd b/nuget/publish.cmd deleted file mode 100644 index 6a30dc533..000000000 --- a/nuget/publish.cmd +++ /dev/null @@ -1 +0,0 @@ -@for %%f in (..\bin\*.nupkg) do @..\packages\NuGet.CommandLine\tools\NuGet.exe push %%f \ No newline at end of file diff --git a/src/AssemblyInfo.Csv.Core.fs b/src/AssemblyInfo.Csv.Core.fs new file mode 100644 index 000000000..c0ef17e0b --- /dev/null +++ b/src/AssemblyInfo.Csv.Core.fs @@ -0,0 +1,27 @@ +// Auto-Generated by FAKE; do not edit +namespace System + +open System.Reflection + +[] +[] +[] +[] +[] +do () + +module internal AssemblyVersionInformation = + [] + let AssemblyTitle = "FSharp.Data.Csv.Core" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.DesignTime.fs b/src/AssemblyInfo.DesignTime.fs index 8e2005f92..f870a91a5 100644 --- a/src/AssemblyInfo.DesignTime.fs +++ b/src/AssemblyInfo.DesignTime.fs @@ -1,17 +1,27 @@ // Auto-Generated by FAKE; do not edit namespace System + open System.Reflection [] [] [] -[] -[] +[] +[] do () module internal AssemblyVersionInformation = - let [] AssemblyTitle = "FSharp.Data.DesignTime" - let [] AssemblyProduct = "FSharp.Data" - let [] AssemblyDescription = "Library of F# type providers and data access tools" - let [] AssemblyVersion = "4.2.8.0" - let [] AssemblyFileVersion = "4.2.8.0" + [] + let AssemblyTitle = "FSharp.Data.DesignTime" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.Html.Core.fs b/src/AssemblyInfo.Html.Core.fs new file mode 100644 index 000000000..bd3974889 --- /dev/null +++ b/src/AssemblyInfo.Html.Core.fs @@ -0,0 +1,27 @@ +// Auto-Generated by FAKE; do not edit +namespace System + +open System.Reflection + +[] +[] +[] +[] +[] +do () + +module internal AssemblyVersionInformation = + [] + let AssemblyTitle = "FSharp.Data.Html.Core" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.Http.fs b/src/AssemblyInfo.Http.fs new file mode 100644 index 000000000..e502ea06a --- /dev/null +++ b/src/AssemblyInfo.Http.fs @@ -0,0 +1,27 @@ +// Auto-Generated by FAKE; do not edit +namespace System + +open System.Reflection + +[] +[] +[] +[] +[] +do () + +module internal AssemblyVersionInformation = + [] + let AssemblyTitle = "FSharp.Data.Http" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.Json.Core.fs b/src/AssemblyInfo.Json.Core.fs new file mode 100644 index 000000000..2b5815538 --- /dev/null +++ b/src/AssemblyInfo.Json.Core.fs @@ -0,0 +1,27 @@ +// Auto-Generated by FAKE; do not edit +namespace System + +open System.Reflection + +[] +[] +[] +[] +[] +do () + +module internal AssemblyVersionInformation = + [] + let AssemblyTitle = "FSharp.Data.Json.Core" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.WorldBank.Core.fs b/src/AssemblyInfo.WorldBank.Core.fs new file mode 100644 index 000000000..0c7839464 --- /dev/null +++ b/src/AssemblyInfo.WorldBank.Core.fs @@ -0,0 +1,27 @@ +// Auto-Generated by FAKE; do not edit +namespace System + +open System.Reflection + +[] +[] +[] +[] +[] +do () + +module internal AssemblyVersionInformation = + [] + let AssemblyTitle = "FSharp.Data.WorldBank.Core" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.Xml.Core.fs b/src/AssemblyInfo.Xml.Core.fs new file mode 100644 index 000000000..a77ba79eb --- /dev/null +++ b/src/AssemblyInfo.Xml.Core.fs @@ -0,0 +1,27 @@ +// Auto-Generated by FAKE; do not edit +namespace System + +open System.Reflection + +[] +[] +[] +[] +[] +do () + +module internal AssemblyVersionInformation = + [] + let AssemblyTitle = "FSharp.Data.Xml.Core" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/AssemblyInfo.fs b/src/AssemblyInfo.fs index baa16c53a..1490e0647 100644 --- a/src/AssemblyInfo.fs +++ b/src/AssemblyInfo.fs @@ -1,17 +1,27 @@ // Auto-Generated by FAKE; do not edit namespace System + open System.Reflection [] [] [] -[] -[] +[] +[] do () module internal AssemblyVersionInformation = - let [] AssemblyTitle = "FSharp.Data" - let [] AssemblyProduct = "FSharp.Data" - let [] AssemblyDescription = "Library of F# type providers and data access tools" - let [] AssemblyVersion = "4.2.8.0" - let [] AssemblyFileVersion = "4.2.8.0" + [] + let AssemblyTitle = "FSharp.Data" + + [] + let AssemblyProduct = "FSharp.Data" + + [] + let AssemblyDescription = "Library of F# type providers and data access tools" + + [] + let AssemblyVersion = "5.0.1.0" + + [] + let AssemblyFileVersion = "5.0.1.0" diff --git a/src/CommonProviderImplementation/ConversionsGenerator.fs b/src/CommonProviderImplementation/ConversionsGenerator.fs index 3a5d56fd4..3a9af5722 100644 --- a/src/CommonProviderImplementation/ConversionsGenerator.fs +++ b/src/CommonProviderImplementation/ConversionsGenerator.fs @@ -67,7 +67,7 @@ let getBackConversionQuotation missingValuesStr cultureStr typ value : Expr and converts it to /// an expression of other type - the type is specified by `field` -let convertStringValue missingValuesStr cultureStr (field: PrimitiveInferedProperty) = +let internal convertStringValue missingValuesStr cultureStr (field: PrimitiveInferedProperty) = let fieldName = field.Name let field = field.Value diff --git a/src/CommonProviderImplementation/Helpers.fs b/src/CommonProviderImplementation/Helpers.fs index a4f1d7163..1ad38bd5e 100644 --- a/src/CommonProviderImplementation/Helpers.fs +++ b/src/CommonProviderImplementation/Helpers.fs @@ -103,21 +103,21 @@ type DisposableTypeProviderForNamespaces(config, ?assemblyReplacementMap) as x = use _holder = logTime "DisposingEvent" (sprintf "%O [%d]" x id) dispose None) - member __.Id = id + member _.Id = id - member __.SetFileToWatch(fullTypeName, path) = + member _.SetFileToWatch(fullTypeName, path) = lock filesToWatch (fun () -> filesToWatch.[fullTypeName] <- path) - member __.GetFileToWath(fullTypeName) = + member _.GetFileToWath(fullTypeName) = lock filesToWatch (fun () -> match filesToWatch.TryGetValue(fullTypeName) with | true, path -> Some path | _ -> None) - member __.AddDisposeAction action = + member _.AddDisposeAction action = lock disposeActions (fun () -> disposeActions.Add action) - member __.InvalidateOneType typeName = + member _.InvalidateOneType typeName = (use _holder = logTime "InvalidateOneType" (sprintf "%s in %O [%d]" typeName x id) dispose (Some typeName) log (sprintf "Calling invalidate for %O [%d]" x id)) @@ -743,5 +743,6 @@ module internal ProviderHelpers = spec.GeneratedType) -[] +[] +[] do () diff --git a/src/CommonRuntime/Caching.fs b/src/CommonRuntime/Caching.fs index 9bc9c85a3..37afa2a48 100644 --- a/src/CommonRuntime/Caching.fs +++ b/src/CommonRuntime/Caching.fs @@ -1,5 +1,5 @@ -/// Implements caching using in-memory and local file system -module FSharp.Data.Runtime.Caching +/// Implements caching using in-memory and local file system +module internal FSharp.Data.Runtime.Caching open System open System.Collections.Concurrent @@ -34,7 +34,7 @@ let createInMemoryCache (expiration: TimeSpan) = } { new ICache<_, _> with - member __.Set(key, value) = + member _.Set(key, value) = dict.[key] <- (value, DateTime.UtcNow) invalidationFunction key |> Async.Start @@ -47,7 +47,7 @@ let createInMemoryCache (expiration: TimeSpan) = Some value | _ -> None - member __.Remove(key) = + member _.Remove(key) = match dict.TryRemove(key) with | true, _ -> log (sprintf "Explicitly removed from cache: %O" key) | _ -> () } @@ -89,7 +89,7 @@ let createInternetFileCache prefix expiration = let cache = { new ICache with - member __.Set(key, value) = + member _.Set(key, value) = let cacheFile = cacheFile key try @@ -101,7 +101,7 @@ let createInternetFileCache prefix expiration = e.Message ) - member __.TryRetrieve(key, ?extendCacheExpiration) = + member _.TryRetrieve(key, ?extendCacheExpiration) = if extendCacheExpiration = Some true then failwith "Not implemented" @@ -119,7 +119,7 @@ let createInternetFileCache prefix expiration = Debug.WriteLine("Caching: Failed to read file {0} with an exception: {1}", cacheFile, e.Message) None - member __.Remove(key) = + member _.Remove(key) = let cacheFile = cacheFile key try diff --git a/src/CommonRuntime/IO.fs b/src/CommonRuntime/IO.fs index e4c08cbc5..b91d07420 100644 --- a/src/CommonRuntime/IO.fs +++ b/src/CommonRuntime/IO.fs @@ -1,4 +1,4 @@ -/// Helper functions called from the generated code for working with files +/// Helper functions called from the generated code for working with files module FSharp.Data.Runtime.IO open System @@ -113,7 +113,7 @@ let internal logTime category (instance: string) = s.Start() { new IDisposable with - member __.Dispose() = + member _.Dispose() = s.Stop() Interlocked.Decrement &indentation |> ignore log (sprintf "Finished %s [%dms]" category s.ElapsedMilliseconds) @@ -126,11 +126,11 @@ let internal logTime category (instance: string) = let internal dummyDisposable = { new IDisposable with - member __.Dispose() = () } + member _.Dispose() = () } -let inline internal log (_: string) = () -let inline internal logWithStackTrace (_: string) = () -let inline internal logTime (_: string) (_: string) = dummyDisposable +let internal log (_: string) = () +let internal logWithStackTrace (_: string) = () +let internal logTime (_: string) (_: string) = dummyDisposable #endif @@ -165,9 +165,9 @@ type private FileWatcher(path) = watcher.Renamed.Add(checkForChanges "renamed") watcher.Deleted.Add(checkForChanges "deleted") - member __.Subscribe(name, action) = subscriptions.Add(name, action) + member _.Subscribe(name, action) = subscriptions.Add(name, action) - member __.Unsubscribe(name) = + member _.Unsubscribe(name) = if subscriptions.Remove(name) then log (sprintf "Unsubscribed %s from %s watcher" name path) @@ -205,7 +205,7 @@ let watchForChanges path (owner, onChange) = watcher) { new IDisposable with - member __.Dispose() = + member _.Dispose() = lock watchers (fun () -> if watcher.Unsubscribe(owner) then watchers.Remove(path) |> ignore) } diff --git a/src/CommonRuntime/NameUtils.fs b/src/CommonRuntime/NameUtils.fs index 5462007b0..1d494b771 100644 --- a/src/CommonRuntime/NameUtils.fs +++ b/src/CommonRuntime/NameUtils.fs @@ -2,7 +2,6 @@ module FSharp.Data.Runtime.NameUtils open System -open System.Globalization open System.Collections.Generic open FSharp.Data.Runtime diff --git a/src/CommonRuntime/StructuralInference.fs b/src/CommonRuntime/StructuralInference.fs index cbd3941e8..e2f2c3904 100644 --- a/src/CommonRuntime/StructuralInference.fs +++ b/src/CommonRuntime/StructuralInference.fs @@ -1,6 +1,8 @@ /// Implements type inference for unstructured documents like XML or JSON module FSharp.Data.Runtime.StructuralInference +#nowarn "44" + open System open System.Diagnostics open System.Collections.Generic @@ -26,6 +28,7 @@ type InferenceMode = | ValuesAndInlineSchemasOverrides = 4 /// This is the internal DU representing all the valid cases we support, mapped from the public InferenceMode. +[] type InferenceMode' = | NoInference /// Backward compatible mode. @@ -50,8 +53,9 @@ type InferenceMode' = | InferenceMode.ValuesAndInlineSchemasOverrides -> InferenceMode'.ValuesAndInlineSchemasOverrides | _ -> failwithf "Unexpected inference mode value %A" inferenceMode -let asOption = - function +[] +let asOption inp = + match inp with | true, x -> Some x | false, _ -> None @@ -93,12 +97,14 @@ let private primitiveTypes = @ numericTypes /// Checks whether a type supports unit of measure +[] let supportsUnitsOfMeasure typ = List.exists ((=) typ) numericTypes /// Returns a tag of a type - a tag represents a 'kind' of type /// (essentially it describes the different bottom types we have) -let typeTag = - function +[] +let typeTag inferredType = + match inferredType with | InferedType.Record (name = n) -> InferedTypeTag.Record n | InferedType.Collection _ -> InferedTypeTag.Collection | InferedType.Null @@ -214,7 +220,7 @@ let private (|SubtypePrimitives|_|) allowEmptyValues = /// The contract that should hold about the function is that given two types with the /// same `InferedTypeTag`, the result also has the same `InferedTypeTag`. /// -let rec subtypeInfered allowEmptyValues ot1 ot2 = +let rec internal subtypeInfered allowEmptyValues ot1 ot2 = match ot1, ot2 with // Subtype of matching types or one of equal types | SubtypePrimitives allowEmptyValues t -> InferedType.Primitive t @@ -355,7 +361,7 @@ and private unionCollectionTypes allowEmptyValues cases1 cases2 = tag, (m, t) | _ -> failwith "unionCollectionTypes: pairBy returned None, None") -and unionCollectionOrder order1 order2 = +and internal unionCollectionOrder order1 order2 = order1 @ (order2 |> List.filter (fun x -> not (List.exists ((=) x) order1))) @@ -363,7 +369,7 @@ and unionCollectionOrder order1 order2 = /// Get the union of record types (merge their properties) /// This matches the corresponding members and marks them as `Optional` /// if one may be missing. It also returns subtype of their types. -and unionRecordTypes allowEmptyValues t1 t2 = +and internal unionRecordTypes allowEmptyValues t1 t2 = List.pairBy (fun (p: InferedProperty) -> p.Name) t1 t2 |> List.map (fun (name, fst, snd) -> match fst, snd with @@ -381,7 +387,7 @@ and unionRecordTypes allowEmptyValues t1 t2 = /// Infer the type of the collection based on multiple sample types /// (group the types by tag, count their multiplicity) -let inferCollectionType allowEmptyValues types = +let internal inferCollectionType allowEmptyValues types = let groupedTypes = types |> Seq.groupBy typeTag @@ -392,11 +398,13 @@ let inferCollectionType allowEmptyValues types = InferedType.Collection(List.map fst groupedTypes, Map.ofList groupedTypes) +[] type IUnitsOfMeasureProvider = abstract SI: str: string -> System.Type abstract Product: measure1: System.Type * measure2: System.Type -> System.Type abstract Inverse: denominator: System.Type -> System.Type +[] let defaultUnitsOfMeasureProvider = { new IUnitsOfMeasureProvider with member x.SI(_) : Type = null @@ -408,6 +416,7 @@ let private uomTransformations = [ "³"; "^3" ], (fun (provider: IUnitsOfMeasureProvider) t -> provider.Product(provider.Product(t, t), t)) [ "^-1" ], (fun (provider: IUnitsOfMeasureProvider) t -> provider.Inverse(t)) ] +[] let parseUnitOfMeasure (provider: IUnitsOfMeasureProvider) (str: string) = let unit = uomTransformations @@ -432,6 +441,7 @@ let parseUnitOfMeasure (provider: IUnitsOfMeasureProvider) (str: string) = /// The inferred types may be set explicitly via inline schemas. /// This table specifies the mapping from (the names that users can use) to (the types used). +[] let nameToType = [ "int", (typeof, TypeWrapper.None) "int64", (typeof, TypeWrapper.None) @@ -465,6 +475,7 @@ let private validInlineSchema = /// This can be of the form: type|measure|type<measure> /// type{measure} is also supported to ease definition in xml values. /// +[] let parseTypeAndUnit unitsOfMeasureProvider (nameToType: IDictionary) str = let m = typeAndUnitRegex.Value.Match(str) @@ -516,6 +527,7 @@ module private Helpers = /// with the desiredUnit applied, /// or a value parsed from an inline schema. /// (For inline schemas, the unit parsed from the schema takes precedence over desiredUnit when present) +[] let inferPrimitiveType (unitsOfMeasureProvider: IUnitsOfMeasureProvider) (inferenceMode: InferenceMode') @@ -615,5 +627,6 @@ let inferPrimitiveType |> Option.defaultValue fallbackType /// Infers the type of a simple string value +[] let getInferedTypeFromString unitsOfMeasureProvider inferenceMode cultureInfo value unit = inferPrimitiveType unitsOfMeasureProvider inferenceMode cultureInfo value unit diff --git a/src/CommonRuntime/StructuralTypes.fs b/src/CommonRuntime/StructuralTypes.fs index 11483deb1..2da962f37 100644 --- a/src/CommonRuntime/StructuralTypes.fs +++ b/src/CommonRuntime/StructuralTypes.fs @@ -11,6 +11,7 @@ open FSharp.Data.Runtime /// /// Types that represent the result of static type inference. /// +[] type InferedProperty = { Name: string mutable Type: InferedType } @@ -19,6 +20,7 @@ type InferedProperty = /// For heterogeneous types (types that have multiple possible forms /// such as differently named XML nodes or records and arrays mixed together) /// this type represents the number of occurrences of individual forms +[] type InferedMultiplicity = | Single | OptionalSingle @@ -27,6 +29,7 @@ type InferedMultiplicity = /// For heterogeneous types, this represents the tag that defines the form /// (that is either primitive type, collection, named record etc.) [] +[] type InferedTypeTag = // Unknown type | Null @@ -61,6 +64,7 @@ type InferedTypeTag = /// we would lose information about multiplicity and so we would not be able /// to generate nicer types! [] +[] type InferedType = | Primitive of typ: Type * unit: option * optional: bool * shouldOverrideOnMerge: bool | Record of name: string option * fields: InferedProperty list * optional: bool @@ -141,7 +145,7 @@ type InferedType = // ------------------------------------------------------------------------------------------------ // Additional operations for working with the inferred representation -type InferedTypeTag with +type internal InferedTypeTag with member x.NiceName = match x with | Null -> failwith "Null nodes should be skipped" @@ -184,20 +188,24 @@ type InferedTypeTag with /// Dummy type to represent that only "0" was found. /// Will be generated as 'int', unless it's converted to Bit. +[] type Bit0 = Bit0 /// Dummy type to represent that only "1" was found /// Will be generated as 'int', unless it's converted to Bit +[] type Bit1 = Bit1 /// Dummy type to represent that only one of "0" and "1" were found /// Will be generated as a 'bool', unless it's converted to another numerical type +[] type Bit = Bit // ------------------------------------------------------------------------------------------------ /// Represents a transformation of a type [] +[] type TypeWrapper = /// No transformation will be made to the type | None @@ -211,7 +219,7 @@ type TypeWrapper = /// Represents type information about a primitive value (used mainly in the CSV provider) /// This type captures the type, unit of measure and handling of missing values (if we /// infer that the value may be missing, we can generate option or nullable) -type PrimitiveInferedValue = +type internal PrimitiveInferedValue = { InferedType: Type RuntimeType: Type UnitOfMeasure: Type option @@ -236,7 +244,7 @@ type PrimitiveInferedValue = /// Represents type information about a primitive property (used mainly in the CSV provider) /// This type captures the type, unit of measure and handling of missing values (if we /// infer that the value may be missing, we can generate option or nullable) -type PrimitiveInferedProperty = +type internal PrimitiveInferedProperty = { Name: string Value: PrimitiveInferedValue } static member Create(name, typ, (typWrapper: TypeWrapper), unit) = diff --git a/src/CommonRuntime/TextRuntime.fs b/src/CommonRuntime/TextRuntime.fs index 3d06bc8ff..178cac246 100644 --- a/src/CommonRuntime/TextRuntime.fs +++ b/src/CommonRuntime/TextRuntime.fs @@ -1,16 +1,16 @@ -namespace FSharp.Data.Runtime +namespace FSharp.Data.Runtime open System +open System.Collections.Generic open System.Globalization open FSharp.Data -open FSharp.Data.Runtime /// Static helper methods called from the generated code for working with text type TextRuntime = [] [] - static val mutable private cultureInfoCache: Collections.Generic.Dictionary + static val mutable private cultureInfoCache: Dictionary /// Returns CultureInfo matching the specified culture string /// (or InvariantCulture if the argument is null or empty) @@ -21,7 +21,7 @@ type TextRuntime = let mutable cache = TextRuntime.cultureInfoCache if cache = null then - cache <- Collections.Generic.Dictionary() + cache <- Dictionary() TextRuntime.cultureInfoCache <- cache match cache.TryGetValue cultureStr with diff --git a/src/Csv/CsvFile.fs b/src/Csv/CsvFile.fs index e8767fad1..3161d19e2 100644 --- a/src/Csv/CsvFile.fs +++ b/src/Csv/CsvFile.fs @@ -17,21 +17,21 @@ open System.Text type CsvRow(parent: CsvFile, columns: string[]) = /// The columns of the row - member __.Columns = columns + member _.Columns = columns /// Gets a column by index - member __.GetColumn index = columns.[index] + member _.GetColumn index = columns.[index] /// Gets a column by name - member __.GetColumn columnName = + member _.GetColumn columnName = columns.[parent.GetColumnIndex columnName] /// Gets a column by index - member __.Item + member _.Item with get index = columns.[index] /// Gets a column by name - member __.Item + member _.Item with get columnName = columns.[parent.GetColumnIndex columnName] /// @@ -74,10 +74,10 @@ and CsvFile | None -> [] |> dict /// Returns the index of the column with the given name - member __.GetColumnIndex columnName = headerDic.[columnName] + member _.GetColumnIndex columnName = headerDic.[columnName] /// Returns the index of the column with the given name, or returns None if no column is found - member __.TryGetColumnIndex columnName = + member _.TryGetColumnIndex columnName = match headerDic.TryGetValue columnName with | true, index -> Some index | false, _ -> None diff --git a/src/Csv/CsvInference.fs b/src/Csv/CsvInference.fs index afee7a315..d228e03a0 100644 --- a/src/Csv/CsvInference.fs +++ b/src/Csv/CsvInference.fs @@ -3,7 +3,6 @@ module FSharp.Data.Runtime.CsvInference open System open System.IO -open System.Runtime.InteropServices open System.Text.RegularExpressions open FSharp.Data open FSharp.Data.Runtime @@ -415,7 +414,7 @@ type CsvFile with /// - Assumes all columns can have missing values /// - when set to true, inference will prefer to use the option type instead of nullable types, double.NaN or "" for missing values /// - optional function to resolve Units of Measure - member x.InferColumnTypes + member internal x.InferColumnTypes ( inferRows, missingValues, diff --git a/src/Csv/CsvRuntime.fs b/src/Csv/CsvRuntime.fs index 89a6899aa..e62249ee0 100644 --- a/src/Csv/CsvRuntime.fs +++ b/src/Csv/CsvRuntime.fs @@ -256,22 +256,22 @@ type CsvFile<'RowType> ) = /// The rows with data - member __.Rows = rows + member _.Rows = rows /// The names of the columns - member __.Headers = headers + member _.Headers = headers /// The number of columns - member __.NumberOfColumns = numberOfColumns + member _.NumberOfColumns = numberOfColumns /// The character(s) used as column separator(s) - member __.Separators = separators + member _.Separators = separators /// The quotation mark use for surrounding values containing separator chars - member __.Quote = quote + member _.Quote = quote interface IDisposable with - member __.Dispose() = disposer.Dispose() + member _.Dispose() = disposer.Dispose() /// [] diff --git a/src/FSharp.Data.Csv.Core/FSharp.Data.Csv.Core.fsproj b/src/FSharp.Data.Csv.Core/FSharp.Data.Csv.Core.fsproj new file mode 100644 index 000000000..5e3bbbabf --- /dev/null +++ b/src/FSharp.Data.Csv.Core/FSharp.Data.Csv.Core.fsproj @@ -0,0 +1,29 @@ + + + + Library + netstandard2.0 + $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 + true + false + logo.png + + true + + + + + + + + + + + + + + + + + + diff --git a/src/FSharp.Data.Csv.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Csv.Core/InternalsVisibleTo.fs new file mode 100644 index 000000000..04aff9db8 --- /dev/null +++ b/src/FSharp.Data.Csv.Core/InternalsVisibleTo.fs @@ -0,0 +1,10 @@ +namespace FSharp.Data + +open System.Runtime.CompilerServices + +[] +[] +[] +[] +[] +do () diff --git a/src/FSharp.Data.Csv.Core/paket.references b/src/FSharp.Data.Csv.Core/paket.references new file mode 100644 index 000000000..c89b441a0 --- /dev/null +++ b/src/FSharp.Data.Csv.Core/paket.references @@ -0,0 +1,2 @@ +Microsoft.SourceLink.GitHub +FSharp.Core diff --git a/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj b/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj index 26ca346ef..aa6e5fd5a 100755 --- a/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj +++ b/src/FSharp.Data.DesignTime/FSharp.Data.DesignTime.fsproj @@ -1,27 +1,16 @@ + false netstandard2.0 IS_DESIGNTIME;NO_GENERATIVE;$(DefineConstants) - --warnon:1182 + $(OtherFlags) --warnon:1182 --nowarn:44 false false - true - ..\keyfile.snk - false true - - - - - - - - - ProvidedTypes.fsi @@ -32,41 +21,29 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + diff --git a/src/FSharp.Data.Html.Core/FSharp.Data.Html.Core.fsproj b/src/FSharp.Data.Html.Core/FSharp.Data.Html.Core.fsproj new file mode 100644 index 000000000..2814db6e4 --- /dev/null +++ b/src/FSharp.Data.Html.Core/FSharp.Data.Html.Core.fsproj @@ -0,0 +1,35 @@ + + + + Library + netstandard2.0 + $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 + true + false + logo.png + + true + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/FSharp.Data.Html.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Html.Core/InternalsVisibleTo.fs new file mode 100644 index 000000000..73b5e6d63 --- /dev/null +++ b/src/FSharp.Data.Html.Core/InternalsVisibleTo.fs @@ -0,0 +1,9 @@ +namespace FSharp.Data + +open System.Runtime.CompilerServices + +[] +[] +[] +[] +do () diff --git a/src/FSharp.Data.Html.Core/paket.references b/src/FSharp.Data.Html.Core/paket.references new file mode 100644 index 000000000..c89b441a0 --- /dev/null +++ b/src/FSharp.Data.Html.Core/paket.references @@ -0,0 +1,2 @@ +Microsoft.SourceLink.GitHub +FSharp.Core diff --git a/src/FSharp.Data.Http/FSharp.Data.Http.fsproj b/src/FSharp.Data.Http/FSharp.Data.Http.fsproj new file mode 100644 index 000000000..d03aa1b2b --- /dev/null +++ b/src/FSharp.Data.Http/FSharp.Data.Http.fsproj @@ -0,0 +1,31 @@ + + + + Library + netstandard2.0 + $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 + true + false + logo.png + + true + + + + + + + + + + + + + + + + + + + + diff --git a/src/FSharp.Data.Http/InternalsVisibleTo.fs b/src/FSharp.Data.Http/InternalsVisibleTo.fs new file mode 100644 index 000000000..b514a2b6b --- /dev/null +++ b/src/FSharp.Data.Http/InternalsVisibleTo.fs @@ -0,0 +1,14 @@ +namespace FSharp.Data + +open System.Runtime.CompilerServices + +[] +[] +[] +[] +[] +[] +[] +[] +[] +do () diff --git a/src/FSharp.Data.Http/paket.references b/src/FSharp.Data.Http/paket.references new file mode 100644 index 000000000..c89b441a0 --- /dev/null +++ b/src/FSharp.Data.Http/paket.references @@ -0,0 +1,2 @@ +Microsoft.SourceLink.GitHub +FSharp.Core diff --git a/src/FSharp.Data.Json.Core/FSharp.Data.Json.Core.fsproj b/src/FSharp.Data.Json.Core/FSharp.Data.Json.Core.fsproj new file mode 100644 index 000000000..0b1911d8d --- /dev/null +++ b/src/FSharp.Data.Json.Core/FSharp.Data.Json.Core.fsproj @@ -0,0 +1,31 @@ + + + + Library + netstandard2.0 + $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 + true + false + logo.png + + true + + + + + + + + + + + + + + + + + + + + diff --git a/src/FSharp.Data.Json.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Json.Core/InternalsVisibleTo.fs new file mode 100644 index 000000000..a59db0192 --- /dev/null +++ b/src/FSharp.Data.Json.Core/InternalsVisibleTo.fs @@ -0,0 +1,11 @@ +namespace FSharp.Data + +open System.Runtime.CompilerServices + +[] +[] +[] +[] +[] +[] +do () diff --git a/src/FSharp.Data.Json.Core/paket.references b/src/FSharp.Data.Json.Core/paket.references new file mode 100644 index 000000000..c89b441a0 --- /dev/null +++ b/src/FSharp.Data.Json.Core/paket.references @@ -0,0 +1,2 @@ +Microsoft.SourceLink.GitHub +FSharp.Core diff --git a/src/FSharp.Data.WorldBank.Core/FSharp.Data.WorldBank.Core.fsproj b/src/FSharp.Data.WorldBank.Core/FSharp.Data.WorldBank.Core.fsproj new file mode 100644 index 000000000..ebd2c67dc --- /dev/null +++ b/src/FSharp.Data.WorldBank.Core/FSharp.Data.WorldBank.Core.fsproj @@ -0,0 +1,27 @@ + + + + Library + netstandard2.0 + $(OtherFlags) --warnon:1182 --nowarn:10001 + true + false + logo.png + + true + + + + + + + + + + + + + + + + diff --git a/src/FSharp.Data.WorldBank.Core/InternalsVisibleTo.fs b/src/FSharp.Data.WorldBank.Core/InternalsVisibleTo.fs new file mode 100644 index 000000000..73b5e6d63 --- /dev/null +++ b/src/FSharp.Data.WorldBank.Core/InternalsVisibleTo.fs @@ -0,0 +1,9 @@ +namespace FSharp.Data + +open System.Runtime.CompilerServices + +[] +[] +[] +[] +do () diff --git a/src/FSharp.Data.WorldBank.Core/paket.references b/src/FSharp.Data.WorldBank.Core/paket.references new file mode 100644 index 000000000..c89b441a0 --- /dev/null +++ b/src/FSharp.Data.WorldBank.Core/paket.references @@ -0,0 +1,2 @@ +Microsoft.SourceLink.GitHub +FSharp.Core diff --git a/src/FSharp.Data.Xml.Core/FSharp.Data.Xml.Core.fsproj b/src/FSharp.Data.Xml.Core/FSharp.Data.Xml.Core.fsproj new file mode 100644 index 000000000..68cacad38 --- /dev/null +++ b/src/FSharp.Data.Xml.Core/FSharp.Data.Xml.Core.fsproj @@ -0,0 +1,30 @@ + + + + Library + netstandard2.0 + $(OtherFlags) --warnon:1182 --nowarn:10001 --nowarn:44 + true + false + logo.png + + true + + + + + + + + + + + + + + + + + + + diff --git a/src/FSharp.Data.Xml.Core/InternalsVisibleTo.fs b/src/FSharp.Data.Xml.Core/InternalsVisibleTo.fs new file mode 100644 index 000000000..73b5e6d63 --- /dev/null +++ b/src/FSharp.Data.Xml.Core/InternalsVisibleTo.fs @@ -0,0 +1,9 @@ +namespace FSharp.Data + +open System.Runtime.CompilerServices + +[] +[] +[] +[] +do () diff --git a/src/FSharp.Data.Xml.Core/paket.references b/src/FSharp.Data.Xml.Core/paket.references new file mode 100644 index 000000000..c89b441a0 --- /dev/null +++ b/src/FSharp.Data.Xml.Core/paket.references @@ -0,0 +1,2 @@ +Microsoft.SourceLink.GitHub +FSharp.Core diff --git a/src/FSharp.Data/FSharp.Data.fsproj b/src/FSharp.Data/FSharp.Data.fsproj index 174700c5c..797a49377 100755 --- a/src/FSharp.Data/FSharp.Data.fsproj +++ b/src/FSharp.Data/FSharp.Data.fsproj @@ -3,58 +3,34 @@ Library netstandard2.0 - --warnon:1182 + $(OtherFlags) --warnon:1182 --nowarn:10001 true false logo.png typeproviders typeproviders - false - true - ..\keyfile.snk true - - - - - - - - - - - - - - - - - - - - - - - - - - - - + true all + + + + + + - + diff --git a/src/Html/HtmlActivePatterns.fs b/src/Html/HtmlActivePatterns.fs index c2b3cd45a..f370f66c3 100644 --- a/src/Html/HtmlActivePatterns.fs +++ b/src/Html/HtmlActivePatterns.fs @@ -1,5 +1,6 @@ -namespace FSharp.Data +namespace FSharp.Data +[] module HtmlActivePatterns = let (|HtmlElement|HtmlText|HtmlComment|HtmlCData|) (node: HtmlNode) = match node with diff --git a/src/Html/HtmlCssSelectors.fs b/src/Html/HtmlCssSelectors.fs index 93ec31aaf..dc40698ff 100644 --- a/src/Html/HtmlCssSelectors.fs +++ b/src/Html/HtmlCssSelectors.fs @@ -1,7 +1,9 @@ -namespace FSharp.Data +namespace FSharp.Data open System +#nowarn "26" + module internal HtmlCssSelectors = type SelectorToken = diff --git a/src/Html/HtmlInference.fs b/src/Html/HtmlInference.fs index 8882a4736..03db0ffcb 100644 --- a/src/Html/HtmlInference.fs +++ b/src/Html/HtmlInference.fs @@ -7,14 +7,14 @@ open FSharp.Data.Runtime open FSharp.Data.Runtime.StructuralInference open FSharp.Data.Runtime.StructuralTypes -type Parameters = +type internal Parameters = { MissingValues: string[] CultureInfo: CultureInfo UnitsOfMeasureProvider: IUnitsOfMeasureProvider PreferOptionals: bool InferenceMode: InferenceMode' } -let inferColumns parameters (headerNamesAndUnits: _[]) rows = +let internal inferColumns parameters (headerNamesAndUnits: _[]) rows = let inferRows = 0 let schema = Array.init headerNamesAndUnits.Length (fun _ -> None) @@ -32,7 +32,7 @@ let inferColumns parameters (headerNamesAndUnits: _[]) rows = parameters.PreferOptionals parameters.UnitsOfMeasureProvider -let inferHeaders parameters (rows: string[][]) = +let internal inferHeaders parameters (rows: string[][]) = if rows.Length <= 2 then false, None, None, None //Not enough info to infer anything, assume first row data else @@ -51,7 +51,7 @@ let inferHeaders parameters (rows: string[][]) = let headerNames, units = Array.unzip headerNamesAndUnits true, Some headerNames, Some units, Some dataRowsType -let inferListType parameters (values: string[]) = +let internal inferListType parameters (values: string[]) = if values.Length > 0 then let inferedtype value = diff --git a/src/Html/HtmlNode.fs b/src/Html/HtmlNode.fs new file mode 100644 index 000000000..115b63ee6 --- /dev/null +++ b/src/Html/HtmlNode.fs @@ -0,0 +1,233 @@ +#nowarn "10001" +namespace FSharp.Data + +open System +open System.ComponentModel +open System.Text + +// -------------------------------------------------------------------------------------- + +/// Represents an HTML attribute. The name is always normalized to lowercase +/// +/// Contains the primary types for the FSharp.Data package. +/// +/// +type HtmlAttribute = + + internal + | HtmlAttribute of name: string * value: string + + /// + /// Creates an html attribute + /// + /// The name of the attribute + /// The value of the attribute + static member New(name: string, value: string) = + HtmlAttribute(name.ToLowerInvariant(), value) + +[] +[] +/// Represents an HTML node. The names of elements are always normalized to lowercase +type HtmlNode = + + internal + | HtmlElement of name: string * attributes: HtmlAttribute list * elements: HtmlNode list + | HtmlText of content: string + | HtmlComment of content: string + | HtmlCData of content: string + + /// + /// Creates an html element + /// + /// The name of the element + static member NewElement(name: string) = + HtmlElement(name.ToLowerInvariant(), [], []) + + /// + /// Creates an html element + /// + /// The name of the element + /// The HtmlAttribute(s) of the element + static member NewElement(name: string, attrs: seq<_>) = + let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList + HtmlElement(name.ToLowerInvariant(), attrs, []) + + /// + /// Creates an html element + /// + /// The name of the element + /// The children elements of this element + static member NewElement(name: string, children: seq<_>) = + HtmlElement(name.ToLowerInvariant(), [], List.ofSeq children) + + + /// + /// Creates an html element + /// + /// The name of the element + /// The HtmlAttribute(s) of the element + /// The children elements of this element + static member NewElement(name: string, attrs: seq<_>, children: seq<_>) = + let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList + HtmlElement(name.ToLowerInvariant(), attrs, List.ofSeq children) + + /// + /// Creates a text content element + /// + /// The actual content + static member NewText content = HtmlText(content) + + /// + /// Creates a comment element + /// + /// The actual content + static member NewComment content = HtmlComment(content) + + /// + /// Creates a CData element + /// + /// The actual content + static member NewCData content = HtmlCData(content) + + override x.ToString() = + let isVoidElement = + let set = + [| "area" + "base" + "br" + "col" + "command" + "embed" + "hr" + "img" + "input" + "keygen" + "link" + "meta" + "param" + "source" + "track" + "wbr" |] + |> Set.ofArray + + fun name -> Set.contains name set + + let rec serialize (sb: StringBuilder) indentation canAddNewLine html = + let append (str: string) = sb.Append str |> ignore + + let appendEndTag name = + append "" + + let newLine plus = + sb.AppendLine() |> ignore + String(' ', indentation + plus) |> append + + match html with + | HtmlElement (name, attributes, elements) -> + let onlyText = + elements + |> List.forall (function + | HtmlText _ -> true + | _ -> false) + + if canAddNewLine && not onlyText then newLine 0 + append "<" + append name + + for HtmlAttribute (name, value) in attributes do + append " " + append name + append "=\"" + append value + append "\"" + + if isVoidElement name then + append " />" + elif elements.IsEmpty then + append ">" + appendEndTag name + else + append ">" + if not onlyText then newLine 2 + let mutable canAddNewLine = false + + for element in elements do + serialize sb (indentation + 2) canAddNewLine element + canAddNewLine <- true + + if not onlyText then newLine 0 + appendEndTag name + | HtmlText str -> append str + | HtmlComment str -> + append "" + | HtmlCData str -> + append "" + + let sb = StringBuilder() + serialize sb 0 false x |> ignore + sb.ToString() + + /// + [] + [] + member x._Print = + let str = x.ToString() + + if str.Length > 512 then + str.Substring(0, 509) + "..." + else + str + +[] +/// Represents an HTML document +type HtmlDocument = + internal + | HtmlDocument of docType: string * elements: HtmlNode list + + /// + /// Creates an html document + /// + /// The document type specifier string + /// The child elements of this document + static member New(docType, children: seq<_>) = + HtmlDocument(docType, List.ofSeq children) + + /// + /// Creates an html document + /// + /// The child elements of this document + static member New(children: seq<_>) = HtmlDocument("", List.ofSeq children) + + override x.ToString() = + match x with + | HtmlDocument (docType, elements) -> + (if String.IsNullOrEmpty docType then + "" + else + "" + Environment.NewLine) + + (elements + |> List.map (fun x -> x.ToString()) + |> String.Concat) + + /// + [] + [] + member x._Print = + let str = x.ToString() + + if str.Length > 512 then + str.Substring(0, 509) + "..." + else + str diff --git a/src/Html/HtmlOperations.fs b/src/Html/HtmlOperations.fs index 75b9e5a09..df79bd85b 100644 --- a/src/Html/HtmlOperations.fs +++ b/src/Html/HtmlOperations.fs @@ -49,13 +49,13 @@ module HtmlNode = /// Gets the given nodes name let name n = match n with - | HtmlElement (name = name) -> name + | HtmlNode.HtmlElement (name = name) -> name | _ -> "" /// Gets all of the nodes immediately under this node let elements n = match n with - | HtmlElement (elements = elements) -> elements + | HtmlNode.HtmlElement (elements = elements) -> elements | _ -> [] /// @@ -193,7 +193,7 @@ module HtmlNode = /// Gets all of the attributes of this node let attributes n = match n with - | HtmlElement (attributes = attributes) -> attributes + | HtmlNode.HtmlElement (attributes = attributes) -> attributes | _ -> [] /// @@ -259,16 +259,16 @@ module HtmlNode = let private innerTextExcluding' recurse exclusions n = let rec innerText' n = match n with - | HtmlElement (name, _, content) when List.forall ((<>) name) exclusions -> + | HtmlNode.HtmlElement (name, _, content) when List.forall ((<>) name) exclusions -> seq { for e in content do match e with - | HtmlText (text) -> yield text - | HtmlComment (_) -> yield "" + | HtmlNode.HtmlText (text) -> yield text + | HtmlNode.HtmlComment (_) -> yield "" | elem -> if recurse then yield innerText' elem else yield "" } |> String.Concat - | HtmlText (text) -> text + | HtmlNode.HtmlText (text) -> text | _ -> "" innerText' n diff --git a/src/Html/HtmlParser.fs b/src/Html/HtmlParser.fs index 61990ba30..1c9f4519d 100644 --- a/src/Html/HtmlParser.fs +++ b/src/Html/HtmlParser.fs @@ -2,7 +2,6 @@ namespace FSharp.Data open System -open System.ComponentModel open System.IO open System.Text open System.Text.RegularExpressions @@ -11,233 +10,6 @@ open FSharp.Data.Runtime open System.Runtime.InteropServices open System.Collections.Generic -// -------------------------------------------------------------------------------------- - -/// Represents an HTML attribute. The name is always normalized to lowercase -/// -/// Contains the primary types for the FSharp.Data package. -/// -/// -type HtmlAttribute = - - internal - | HtmlAttribute of name: string * value: string - - /// - /// Creates an html attribute - /// - /// The name of the attribute - /// The value of the attribute - static member New(name: string, value: string) = - HtmlAttribute(name.ToLowerInvariant(), value) - -[] -/// Represents an HTML node. The names of elements are always normalized to lowercase -type HtmlNode = - - internal - | HtmlElement of name: string * attributes: HtmlAttribute list * elements: HtmlNode list - | HtmlText of content: string - | HtmlComment of content: string - | HtmlCData of content: string - - /// - /// Creates an html element - /// - /// The name of the element - static member NewElement(name: string) = - HtmlElement(name.ToLowerInvariant(), [], []) - - /// - /// Creates an html element - /// - /// The name of the element - /// The HtmlAttribute(s) of the element - static member NewElement(name: string, attrs: seq<_>) = - let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList - HtmlElement(name.ToLowerInvariant(), attrs, []) - - /// - /// Creates an html element - /// - /// The name of the element - /// The children elements of this element - static member NewElement(name: string, children: seq<_>) = - HtmlElement(name.ToLowerInvariant(), [], List.ofSeq children) - - - /// - /// Creates an html element - /// - /// The name of the element - /// The HtmlAttribute(s) of the element - /// The children elements of this element - static member NewElement(name: string, attrs: seq<_>, children: seq<_>) = - let attrs = attrs |> Seq.map HtmlAttribute.New |> Seq.toList - HtmlElement(name.ToLowerInvariant(), attrs, List.ofSeq children) - - /// - /// Creates a text content element - /// - /// The actual content - static member NewText content = HtmlText(content) - - /// - /// Creates a comment element - /// - /// The actual content - static member NewComment content = HtmlComment(content) - - /// - /// Creates a CData element - /// - /// The actual content - static member NewCData content = HtmlCData(content) - - override x.ToString() = - let isVoidElement = - let set = - [| "area" - "base" - "br" - "col" - "command" - "embed" - "hr" - "img" - "input" - "keygen" - "link" - "meta" - "param" - "source" - "track" - "wbr" |] - |> Set.ofArray - - fun name -> Set.contains name set - - let rec serialize (sb: StringBuilder) indentation canAddNewLine html = - let append (str: string) = sb.Append str |> ignore - - let appendEndTag name = - append "" - - let newLine plus = - sb.AppendLine() |> ignore - String(' ', indentation + plus) |> append - - match html with - | HtmlElement (name, attributes, elements) -> - let onlyText = - elements - |> List.forall (function - | HtmlText _ -> true - | _ -> false) - - if canAddNewLine && not onlyText then newLine 0 - append "<" - append name - - for HtmlAttribute (name, value) in attributes do - append " " - append name - append "=\"" - append value - append "\"" - - if isVoidElement name then - append " />" - elif elements.IsEmpty then - append ">" - appendEndTag name - else - append ">" - if not onlyText then newLine 2 - let mutable canAddNewLine = false - - for element in elements do - serialize sb (indentation + 2) canAddNewLine element - canAddNewLine <- true - - if not onlyText then newLine 0 - appendEndTag name - | HtmlText str -> append str - | HtmlComment str -> - append "" - | HtmlCData str -> - append "" - - let sb = StringBuilder() - serialize sb 0 false x |> ignore - sb.ToString() - - /// - [] - [] - member x._Print = - let str = x.ToString() - - if str.Length > 512 then - str.Substring(0, 509) + "..." - else - str - -[] -/// Represents an HTML document -type HtmlDocument = - internal - | HtmlDocument of docType: string * elements: HtmlNode list - - /// - /// Creates an html document - /// - /// The document type specifier string - /// The child elements of this document - static member New(docType, children: seq<_>) = - HtmlDocument(docType, List.ofSeq children) - - /// - /// Creates an html document - /// - /// The child elements of this document - static member New(children: seq<_>) = HtmlDocument("", List.ofSeq children) - - override x.ToString() = - match x with - | HtmlDocument (docType, elements) -> - (if String.IsNullOrEmpty docType then - "" - else - "" + Environment.NewLine) - + (elements - |> List.map (fun x -> x.ToString()) - |> String.Concat) - - /// - [] - [] - member x._Print = - let str = x.ToString() - - if str.Length > 512 then - str.Substring(0, 509) + "..." - else - str - - // -------------------------------------------------------------------------------------- module private TextParser = @@ -321,20 +93,20 @@ module internal HtmlParser = | CDATAMode -> "cdata" type HtmlState = - { Attributes: (CharList * CharList) list ref - CurrentTag: CharList ref - Content: CharList ref - HasFormattedParent: bool ref - InsertionMode: InsertionMode ref - Tokens: HtmlToken list ref + { mutable Attributes: (CharList * CharList) list + mutable CurrentTag: CharList + mutable Content: CharList + mutable HasFormattedParent: bool + mutable InsertionMode: InsertionMode + mutable Tokens: HtmlToken list Reader: TextReader } static member Create(reader: TextReader) = - { Attributes = ref [] - CurrentTag = ref CharList.Empty - Content = ref CharList.Empty - HasFormattedParent = ref false - InsertionMode = ref DefaultMode - Tokens = ref [] + { Attributes = [] + CurrentTag = CharList.Empty + Content = CharList.Empty + HasFormattedParent = false + InsertionMode = DefaultMode + Tokens = [] Reader = reader } member x.Pop() = x.Reader.Read() |> ignore @@ -344,30 +116,28 @@ module internal HtmlParser = [| 0 .. (count - 1) |] |> Array.map (fun _ -> x.Reader.ReadChar()) - member x.Contents = (!x.Content).ToString() - member x.ContentLength = (!x.Content).Length + member x.Contents = x.Content.ToString() + member x.ContentLength = x.Content.Length member x.NewAttribute() = - x.Attributes - := (CharList.Empty, CharList.Empty) - :: (!x.Attributes) + x.Attributes <- (CharList.Empty, CharList.Empty) :: x.Attributes member x.ConsAttrName() = - match !x.Attributes with + match x.Attributes with | [] -> x.NewAttribute() x.ConsAttrName() | (h, _) :: _ -> h.Cons(Char.ToLowerInvariant(x.Reader.ReadChar())) - member x.CurrentTagName() = (!x.CurrentTag).ToString().Trim() + member x.CurrentTagName() = x.CurrentTag.ToString().Trim() member x.CurrentAttrName() = - match !x.Attributes with + match x.Attributes with | [] -> String.Empty | (h, _) :: _ -> h.ToString() member x.ConsAttrValue(c) = - match !x.Attributes with + match x.Attributes with | [] -> x.NewAttribute() x.ConsAttrValue(c) @@ -376,7 +146,7 @@ module internal HtmlParser = member x.ConsAttrValue() = x.ConsAttrValue(x.Reader.ReadChar()) member x.GetAttributes() = - !x.Attributes + x.Attributes |> List.choose (fun (key, value) -> if key.Length > 0 then Some @@ -386,12 +156,12 @@ module internal HtmlParser = |> List.rev member x.EmitSelfClosingTag() = - let name = (!x.CurrentTag).ToString().Trim() + let name = x.CurrentTag.ToString().Trim() let result = Tag(true, name, x.GetAttributes()) - x.CurrentTag := CharList.Empty - x.InsertionMode := DefaultMode - x.Attributes := [] - x.Tokens := result :: !x.Tokens + x.CurrentTag <- CharList.Empty + x.InsertionMode <- DefaultMode + x.Attributes <- [] + x.Tokens <- result :: x.Tokens member x.IsFormattedTag = match x.CurrentTagName().ToLower() with @@ -405,7 +175,7 @@ module internal HtmlParser = | _ -> false member x.EmitTag(isEnd) = - let name = (!x.CurrentTag).ToString().Trim() + let name = x.CurrentTag.ToString().Trim() let result = if isEnd then @@ -420,38 +190,37 @@ module internal HtmlParser = // pre is the only default formatted tag, nested pres are not // allowed in the spec. if x.IsFormattedTag then - x.HasFormattedParent := not isEnd + x.HasFormattedParent <- not isEnd else - x.HasFormattedParent - := !x.HasFormattedParent || x.IsFormattedTag + x.HasFormattedParent <- x.HasFormattedParent || x.IsFormattedTag - x.InsertionMode - := if x.IsScriptTag && (not isEnd) then - ScriptMode - else - DefaultMode + x.InsertionMode <- + if x.IsScriptTag && (not isEnd) then + ScriptMode + else + DefaultMode - x.CurrentTag := CharList.Empty - x.Attributes := [] - x.Tokens := result :: !x.Tokens + x.CurrentTag <- CharList.Empty + x.Attributes <- [] + x.Tokens <- result :: x.Tokens member x.EmitToAttributeValue() = - assert (!x.InsertionMode = InsertionMode.CharRefMode) - let content = (!x.Content).ToString() |> HtmlCharRefs.substitute + assert (x.InsertionMode = InsertionMode.CharRefMode) + let content = x.Content.ToString() |> HtmlCharRefs.substitute for c in content.ToCharArray() do x.ConsAttrValue c - x.Content := CharList.Empty - x.InsertionMode := DefaultMode + x.Content <- CharList.Empty + x.InsertionMode <- DefaultMode member x.Emit() : unit = let result = - let content = (!x.Content).ToString() + let content = x.Content.ToString() - match !x.InsertionMode with + match x.InsertionMode with | DefaultMode -> - if !x.HasFormattedParent then + if x.HasFormattedParent then Text content else let normalizedContent = wsRegex.Value.Replace(content, " ") @@ -466,24 +235,24 @@ module internal HtmlParser = | DocTypeMode -> DocType content | CDATAMode -> CData(content.Replace("", "")) - x.Content := CharList.Empty - x.InsertionMode := DefaultMode + x.Content <- CharList.Empty + x.InsertionMode <- DefaultMode match result with | Text t when String.IsNullOrEmpty(t) -> () - | _ -> x.Tokens := result :: !x.Tokens + | _ -> x.Tokens <- result :: x.Tokens - member x.Cons() = (!x.Content).Cons(x.Reader.ReadChar()) - member x.Cons(char) = (!x.Content).Cons(char) - member x.Cons(char) = Array.iter ((!x.Content).Cons) char + member x.Cons() = x.Content.Cons(x.Reader.ReadChar()) + member x.Cons(char) = x.Content.Cons(char) + member x.Cons(char) = Array.iter (x.Content.Cons) char member x.Cons(char: string) = x.Cons(char.ToCharArray()) member x.ConsTag() = match x.Reader.ReadChar() with | TextParser.Whitespace _ -> () - | a -> (!x.CurrentTag).Cons(Char.ToLowerInvariant a) + | a -> x.CurrentTag.Cons(Char.ToLowerInvariant a) - member x.ClearContent() = (!x.Content).Clear() + member x.ClearContent() = x.Content.Clear() // Tokenises a stream into a sequence of HTML tokens. let private tokenise reader = @@ -497,15 +266,15 @@ module internal HtmlParser = else state.Pop() tagOpen state - | TextParser.EndOfFile _ -> state.Tokens := EOF :: !state.Tokens + | TextParser.EndOfFile _ -> state.Tokens <- EOF :: state.Tokens | '&' -> if state.ContentLength > 0 then state.Emit() else - state.InsertionMode := CharRefMode + state.InsertionMode <- CharRefMode charRef state | _ -> - match !state.InsertionMode with + match state.InsertionMode with | DefaultMode -> state.Cons() data state @@ -769,7 +538,7 @@ module internal HtmlParser = | '>' -> state.Cons([| '<'; '/' |]) state.Cons(state.CurrentTagName()) - (!state.CurrentTag).Clear() + state.CurrentTag.Clear() script state | TextParser.Letter _ -> state.ConsTag() @@ -826,7 +595,7 @@ module internal HtmlParser = | _ -> state.Cons([| '<'; '/' |]) state.Cons(state.CurrentTagName()) - (!state.CurrentTag).Clear() + state.CurrentTag.Clear() script state and charRef state = @@ -863,7 +632,7 @@ module internal HtmlParser = and bogusComment state = let rec bogusComment' (state: HtmlState) = let exitBogusComment state = - state.InsertionMode := CommentMode + state.InsertionMode <- CommentMode state.Emit() match state.Peek() with @@ -895,10 +664,10 @@ module internal HtmlParser = cData (i + 1) state | '>' when i = 2 -> state.Cons() - state.InsertionMode := CDATAMode + state.InsertionMode <- CDATAMode state.Emit() | TextParser.EndOfFile _ -> - state.InsertionMode := CDATAMode + state.InsertionMode <- CDATAMode state.Emit() | _ -> state.Cons() @@ -908,7 +677,7 @@ module internal HtmlParser = match state.Peek() with | '>' -> state.Pop() - state.InsertionMode := DocTypeMode + state.InsertionMode <- DocTypeMode state.Emit() | _ -> state.Cons() @@ -920,7 +689,7 @@ module internal HtmlParser = state.Pop() commentEndDash state | TextParser.EndOfFile _ -> - state.InsertionMode := CommentMode + state.InsertionMode <- CommentMode state.Emit() | _ -> state.Cons() @@ -932,7 +701,7 @@ module internal HtmlParser = state.Pop() commentEndState state | TextParser.EndOfFile _ -> - state.InsertionMode := CommentMode + state.InsertionMode <- CommentMode state.Emit() | _ -> state.Cons() @@ -942,10 +711,10 @@ module internal HtmlParser = match state.Peek() with | '>' -> state.Pop() - state.InsertionMode := CommentMode + state.InsertionMode <- CommentMode state.Emit() | TextParser.EndOfFile _ -> - state.InsertionMode := CommentMode + state.InsertionMode <- CommentMode state.Emit() | _ -> state.Cons() @@ -1071,7 +840,7 @@ module internal HtmlParser = state.EmitTag(false) | '&' -> assert (state.ContentLength = 0) - state.InsertionMode := InsertionMode.CharRefMode + state.InsertionMode <- InsertionMode.CharRefMode attributeValueUnquotedCharRef [ '/'; '>' ] state | _ -> state.ConsAttrValue() @@ -1093,7 +862,7 @@ module internal HtmlParser = afterAttributeValueQuoted state | '&' -> assert (state.ContentLength = 0) - state.InsertionMode := InsertionMode.CharRefMode + state.InsertionMode <- InsertionMode.CharRefMode attributeValueQuotedCharRef quote state | _ -> state.ConsAttrValue() @@ -1147,13 +916,13 @@ module internal HtmlParser = state.NewAttribute() attributeName state - let next = ref (state.Reader.Peek()) + let mutable next = state.Reader.Peek() - while !next <> -1 do + while next <> -1 do data state - next := state.Reader.Peek() + next <- state.Reader.Peek() - !state.Tokens |> List.rev + state.Tokens |> List.rev let private parse reader = let canNotHaveChildren (name: string) = @@ -1244,19 +1013,19 @@ module internal HtmlParser = (dt, tokens, content) else let _, elements, expectedTagEnd, parentTagName, name, attributes = callstack.Pop() - let e = HtmlElement(name, attributes, content) + let e = HtmlNode.HtmlElement(name, attributes, content) parse' dt (e :: elements) expectedTagEnd parentTagName tokens match tokens with | DocType dt :: rest -> parse' (dt.Trim()) elements expectedTagEnd parentTagName rest | Tag (_, "br", []) :: rest -> - let t = HtmlText Environment.NewLine + let t = HtmlNode.HtmlText Environment.NewLine parse' docType (t :: elements) expectedTagEnd parentTagName rest | Tag (true, name, attributes) :: rest -> - let e = HtmlElement(name, attributes, []) + let e = HtmlNode.HtmlElement(name, attributes, []) parse' docType (e :: elements) expectedTagEnd parentTagName rest | Tag (false, name, attributes) :: rest when canNotHaveChildren name -> - let e = HtmlElement(name, attributes, []) + let e = HtmlNode.HtmlElement(name, attributes, []) parse' docType (e :: elements) expectedTagEnd parentTagName rest | Tag (_, name, _) :: _ when isImplicitlyClosedByStartTag expectedTagEnd name -> // insert missing or when starting new row/cell/header @@ -1291,20 +1060,20 @@ module internal HtmlParser = // ignore this token parse' docType elements expectedTagEnd parentTagName rest else - let t = HtmlText(a + b) + let t = HtmlNode.HtmlText(a + b) parse' docType (t :: elements) expectedTagEnd parentTagName rest | Text cont :: rest -> if cont = "" then // ignore this token parse' docType elements expectedTagEnd parentTagName rest else - let t = HtmlText cont + let t = HtmlNode.HtmlText cont parse' docType (t :: elements) expectedTagEnd parentTagName rest | Comment cont :: rest -> - let c = HtmlComment cont + let c = HtmlNode.HtmlComment cont parse' docType (c :: elements) expectedTagEnd parentTagName rest | CData cont :: rest -> - let c = HtmlCData cont + let c = HtmlNode.HtmlCData cont parse' docType (c :: elements) expectedTagEnd parentTagName rest | EOF :: _ -> recursiveReturn (docType, [], List.rev elements) | [] -> recursiveReturn (docType, [], List.rev elements) @@ -1326,44 +1095,46 @@ module internal HtmlParser = /// All br tags will be replaced by newlines let parseFragment reader = parse reader |> snd -// -------------------------------------------------------------------------------------- - -type HtmlDocument with - - /// Parses the specified HTML string - static member Parse(text) = - use reader = new StringReader(text) - HtmlParser.parseDocument reader - - /// Loads HTML from the specified stream - static member Load(stream: Stream) = - use reader = new StreamReader(stream) - HtmlParser.parseDocument reader - - /// Loads HTML from the specified reader - static member Load(reader: TextReader) = HtmlParser.parseDocument reader - - /// Loads HTML from the specified uri asynchronously - static member AsyncLoad(uri: string, [] ?encoding) = - async { - let encoding = defaultArg encoding Encoding.UTF8 - let! reader = IO.asyncReadTextAtRuntime false "" "" "HTML" encoding.WebName uri - return HtmlParser.parseDocument reader - } - - /// Loads HTML from the specified uri - static member Load(uri: string, [] ?encoding) = - HtmlDocument.AsyncLoad(uri, ?encoding = encoding) - |> Async.RunSynchronously - -type HtmlNode with - - /// Parses the specified HTML string to a list of HTML nodes - static member Parse(text) = - use reader = new StringReader(text) - HtmlParser.parseFragment reader - - /// Parses the specified HTML string to a list of HTML nodes - static member ParseRooted(rootName, text) = - use reader = new StringReader(text) - HtmlElement(rootName, [], HtmlParser.parseFragment reader) +[] +module HtmlAutoOpens = + // -------------------------------------------------------------------------------------- + + type HtmlDocument with + + /// Parses the specified HTML string + static member Parse(text) = + use reader = new StringReader(text) + HtmlParser.parseDocument reader + + /// Loads HTML from the specified stream + static member Load(stream: Stream) = + use reader = new StreamReader(stream) + HtmlParser.parseDocument reader + + /// Loads HTML from the specified reader + static member Load(reader: TextReader) = HtmlParser.parseDocument reader + + /// Loads HTML from the specified uri asynchronously + static member AsyncLoad(uri: string, [] ?encoding) = + async { + let encoding = defaultArg encoding Encoding.UTF8 + let! reader = IO.asyncReadTextAtRuntime false "" "" "HTML" encoding.WebName uri + return HtmlParser.parseDocument reader + } + + /// Loads HTML from the specified uri + static member Load(uri: string, [] ?encoding) = + HtmlDocument.AsyncLoad(uri, ?encoding = encoding) + |> Async.RunSynchronously + + type HtmlNode with + + /// Parses the specified HTML string to a list of HTML nodes + static member Parse(text) = + use reader = new StringReader(text) + HtmlParser.parseFragment reader + + /// Parses the specified HTML string to a list of HTML nodes + static member ParseRooted(rootName, text) = + use reader = new StringReader(text) + HtmlNode.HtmlElement(rootName, [], HtmlParser.parseFragment reader) diff --git a/src/Html/HtmlRuntime.fs b/src/Html/HtmlRuntime.fs index 8ddaa8290..e8394059c 100644 --- a/src/Html/HtmlRuntime.fs +++ b/src/Html/HtmlRuntime.fs @@ -1,10 +1,8 @@ -namespace FSharp.Data.Runtime +namespace FSharp.Data.Runtime open System open System.Globalization -open System.IO open System.Text -open System.Text.RegularExpressions open FSharp.Data open FSharp.Data.HtmlExtensions open FSharp.Data.Runtime @@ -32,17 +30,34 @@ type HtmlTableCell = | Cell (_, d) -> d /// Representation of an HTML table cell -type HtmlTable = - { Name: string - HeaderNamesAndUnits: (string * Type option)[] option // always set at designtime, never at runtime - InferedProperties: PrimitiveInferedProperty list option // sometimes set at designtime, never at runtime - HasHeaders: bool option // always set at designtime, never at runtime - Rows: string[][] - Html: HtmlNode } - override x.ToString() = +type HtmlTable + internal + ( + name: string, + headerNamesAndUnits: (string * Type option)[] option, // always set at designtime, never at runtime + inferedProperties: PrimitiveInferedProperty list option, // sometimes set at designtime, never at runtime + hasHeaders: bool option, // always set at designtime, never at runtime + rows: string[][], + html: HtmlNode + ) = + member _.Name = name + + // always set at designtime, never at runtime + member internal _.HeaderNamesAndUnits = headerNamesAndUnits + + // sometimes set at designtime, never at runtime + member internal _.InferedProperties = inferedProperties + + member _.HasHeaders = hasHeaders // always set at designtime, never at runtime + + member _.Rows = rows + + member _.Html = html + + override _.ToString() = let sb = StringBuilder() - sb.AppendLine x.Name |> ignore - let data = array2D x.Rows + sb.AppendLine name |> ignore + let data = array2D rows let rows = data.GetLength(0) let columns = data.GetLength(1) let widths = Array.zeroCreate columns @@ -91,10 +106,11 @@ type HtmlDefinitionList = sb.ToString() /// Representation of an HTML table, list, or definition list -type HtmlObject = +type HtmlObjectDescription = | Table of HtmlTable | List of HtmlList | DefinitionList of HtmlDefinitionList + member x.Name = match x with | Table t -> t.Name @@ -281,13 +297,13 @@ module HtmlRuntime = | HtmlElement ("th", _, contents) -> Cell(true, getContents contents) | _ -> Empty - let col_i = ref colindex + let mutable col_i = colindex - while !col_i < res.[rowindex].Length - && res.[rowindex].[!col_i] <> Empty do - incr (col_i) + while col_i < res.[rowindex].Length + && res.[rowindex].[col_i] <> Empty do + col_i <- col_i + 1 - for j in [ !col_i .. (!col_i + colSpan cell - 1) ] do + for j in [ col_i .. (col_i + colSpan cell - 1) ] do for i in [ rowindex .. (rowindex + rowSpan cell - 1) ] do if i < rows.Length && j < numberOfColumns then res.[i].[j] <- data @@ -341,12 +357,7 @@ module HtmlRuntime = let rows = res |> Array.map (Array.map (fun x -> x.Data)) - { Name = name - HeaderNamesAndUnits = headerNamesAndUnits - InferedProperties = inferedProperties - HasHeaders = hasHeaders - Rows = rows - Html = table } + HtmlTable(name, headerNamesAndUnits, inferedProperties, hasHeaders, rows, table) |> Some let private parseList makeUnique index (list: HtmlNode, parents: HtmlNode list) = @@ -407,7 +418,7 @@ module HtmlRuntime = Html = definitionList } |> Some - let getTables inferenceParameters includeLayoutTables (doc: HtmlDocument) = + let internal getTables inferenceParameters includeLayoutTables (doc: HtmlDocument) = let tableElements = doc.DescendantsWithPath "table" |> List.ofSeq let tableElements = @@ -439,7 +450,7 @@ module HtmlRuntime = |> List.mapi (parseDefinitionList (NameUtils.uniqueGenerator id)) |> List.choose id - let getHtmlObjects inferenceParameters includeLayoutTables (doc: HtmlDocument) = + let internal getHtmlObjects inferenceParameters includeLayoutTables (doc: HtmlDocument) = Seq.concat [ doc |> getTables inferenceParameters includeLayoutTables @@ -462,7 +473,7 @@ open FSharp.Data.Runtime /// Underlying representation of the root types generated by HtmlProvider type HtmlDocument internal (doc, tables, lists, definitionLists) = - member __.Html = doc + member _.Html = doc /// [] @@ -499,7 +510,7 @@ type HtmlDocument internal (doc, tables, lists, definitionLists) = 10001, IsHidden = true, IsError = false)>] - member __.GetTable(id: string) = tables |> Map.find id + member _.GetTable(id: string) = tables |> Map.find id /// [] @@ -507,7 +518,7 @@ type HtmlDocument internal (doc, tables, lists, definitionLists) = 10001, IsHidden = true, IsError = false)>] - member __.GetList(id: string) = lists |> Map.find id + member _.GetList(id: string) = lists |> Map.find id /// [] @@ -515,15 +526,15 @@ type HtmlDocument internal (doc, tables, lists, definitionLists) = 10001, IsHidden = true, IsError = false)>] - member __.GetDefinitionList(id: string) = definitionLists |> Map.find id + member _.GetDefinitionList(id: string) = definitionLists |> Map.find id /// Underlying representation of table types generated by HtmlProvider type HtmlTable<'RowType> internal (name: string, headers: string[] option, values: 'RowType[], html: HtmlNode) = - member __.Name = name - member __.Headers = headers - member __.Rows = values - member __.Html = html + member _.Name = name + member _.Headers = headers + member _.Rows = values + member _.Html = html /// [] @@ -545,9 +556,9 @@ type HtmlTable<'RowType> internal (name: string, headers: string[] option, value /// Underlying representation of list types generated by HtmlProvider type HtmlList<'ItemType> internal (name: string, values: 'ItemType[], html) = - member __.Name = name - member __.Values = values - member __.Html = html + member _.Name = name + member _.Values = values + member _.Html = html [] [ and converts it to /// an expression of other type - the type is specified by `field` -let convertJsonValue missingValuesStr cultureStr canPassAllConversionCallingTypes (field: PrimitiveInferedValue) = +let internal convertJsonValue + missingValuesStr + cultureStr + canPassAllConversionCallingTypes + (field: PrimitiveInferedValue) + = let returnType = diff --git a/src/Json/JsonDocument.fs b/src/Json/JsonDocument.fs new file mode 100644 index 000000000..61447c204 --- /dev/null +++ b/src/Json/JsonDocument.fs @@ -0,0 +1,97 @@ +// -------------------------------------------------------------------------------------- +// JSON type provider - methods that are called from the generated erased code +// -------------------------------------------------------------------------------------- +namespace FSharp.Data.Runtime.BaseTypes + +open System.ComponentModel +open System.IO +open FSharp.Data + +#nowarn "10001" + +/// +type IJsonDocument = + abstract JsonValue: JsonValue + + [] + [] + abstract Path: unit -> string + + [] + [] + abstract CreateNew: value: JsonValue * pathIncrement: string -> IJsonDocument + +/// Underlying representation of types generated by JsonProvider +/// +/// Contains the runtime base types used by generated row types for FSharp.Data type providers. +/// +[] +type JsonDocument = + + private + { + /// + Json: JsonValue + /// + Path: string + } + + interface IJsonDocument with + member x.JsonValue = x.Json + member x.Path() = x.Path + + member x.CreateNew(value, pathIncrement) = + JsonDocument.Create(value, x.Path + pathIncrement) + + /// The underlying JsonValue + member x.JsonValue = x.Json + + /// + [] + [] + override x.ToString() = x.JsonValue.ToString() + + /// + [] + [] + static member Create(value, path) = + { Json = value; Path = path } :> IJsonDocument + + /// + [] + [] + static member Create(reader: TextReader) = + use reader = reader + let text = reader.ReadToEnd() + let value = JsonValue.Parse(text) + JsonDocument.Create(value, "") + + /// + [] + [] + static member CreateList(reader: TextReader) = + use reader = reader + let text = reader.ReadToEnd() + + match JsonValue.ParseMultiple(text) |> Seq.toArray with + | [| JsonValue.Array array |] -> array + | array -> array + |> Array.mapi (fun i value -> JsonDocument.Create(value, "[" + (string i) + "]")) diff --git a/src/Json/JsonInference.fs b/src/Json/JsonInference.fs index f42e41da9..1699e66b9 100644 --- a/src/Json/JsonInference.fs +++ b/src/Json/JsonInference.fs @@ -14,7 +14,7 @@ open FSharp.Data.Runtime.StructuralInference /// functionality is handled in `StructureInference` (most notably, by /// `inferCollectionType` and various functions to find common subtype), so /// here we just need to infer types of primitive JSON values. -let rec inferType unitsOfMeasureProvider inferenceMode cultureInfo parentName json = +let rec internal inferType unitsOfMeasureProvider inferenceMode cultureInfo parentName json = let inline inRangeDecimal lo hi (v: decimal) : bool = (v >= decimal lo) && (v <= decimal hi) let inline inRangeFloat lo hi (v: float) : bool = (v >= float lo) && (v <= float hi) let inline isIntegerDecimal (v: decimal) : bool = Math.Round v = v diff --git a/src/Json/JsonRuntime.fs b/src/Json/JsonRuntime.fs index 0c121bd9a..385dca19a 100644 --- a/src/Json/JsonRuntime.fs +++ b/src/Json/JsonRuntime.fs @@ -1,110 +1,12 @@ // -------------------------------------------------------------------------------------- // JSON type provider - methods that are called from the generated erased code // -------------------------------------------------------------------------------------- -namespace FSharp.Data.Runtime.BaseTypes - -open System.ComponentModel -open System.IO -open FSharp.Data -open FSharp.Data.Runtime - -#nowarn "10001" - -/// -type IJsonDocument = - abstract JsonValue: JsonValue - - [] - [] - abstract Path: unit -> string - - [] - [] - abstract CreateNew: value: JsonValue * pathIncrement: string -> IJsonDocument - -/// Underlying representation of types generated by JsonProvider -/// -/// Contains the runtime base types used by generated row types for FSharp.Data type providers. -/// -[] -type JsonDocument = - - private - { - /// - Json: JsonValue - /// - Path: string - } - - interface IJsonDocument with - member x.JsonValue = x.Json - member x.Path() = x.Path - - member x.CreateNew(value, pathIncrement) = - JsonDocument.Create(value, x.Path + pathIncrement) - - /// The underlying JsonValue - member x.JsonValue = x.Json - - /// - [] - [] - override x.ToString() = x.JsonValue.ToString() - - /// - [] - [] - static member Create(value, path) = - { Json = value; Path = path } :> IJsonDocument - - /// - [] - [] - static member Create(reader: TextReader) = - use reader = reader - let text = reader.ReadToEnd() - let value = JsonValue.Parse(text) - JsonDocument.Create(value, "") - - /// - [] - [] - static member CreateList(reader: TextReader) = - use reader = reader - let text = reader.ReadToEnd() - - match JsonValue.ParseMultiple(text) |> Seq.toArray with - | [| JsonValue.Array array |] -> array - | array -> array - |> Array.mapi (fun i value -> JsonDocument.Create(value, "[" + (string i) + "]")) - -// -------------------------------------------------------------------------------------- namespace FSharp.Data.Runtime open System open System.Globalization open FSharp.Data -open FSharp.Data.JsonExtensions open FSharp.Data.Runtime open FSharp.Data.Runtime.BaseTypes open FSharp.Data.Runtime.StructuralTypes diff --git a/src/Runtime.fs b/src/Runtime.fs index 739899daa..5ab8e0003 100644 --- a/src/Runtime.fs +++ b/src/Runtime.fs @@ -1,8 +1,8 @@ -namespace global +namespace global open System.Runtime.CompilerServices open FSharp.Core.CompilerServices [] -[] +[] do () diff --git a/src/SetupTesting.fsx b/src/SetupTesting.fsx index 9ce599560..7118966d2 100644 --- a/src/SetupTesting.fsx +++ b/src/SetupTesting.fsx @@ -4,39 +4,55 @@ open System open System.IO open System.Xml.Linq -let generateSetupScript dir proj = +let generateSetupScript dir proj = let getElemName name = XName.Get name - let getElemValue name (parent:XElement) = + let getElemValue name (parent: XElement) = let elem = parent.Element(getElemName name) - if elem = null || String.IsNullOrEmpty elem.Value then None else Some(elem.Value) - - let getAttrValue name (elem:XElement) = + + if elem = null || String.IsNullOrEmpty elem.Value then + None + else + Some(elem.Value) + + let getAttrValue name (elem: XElement) = let attr = elem.Attribute(XName.Get name) - if attr = null || String.IsNullOrEmpty attr.Value then None else Some(attr.Value) + + if attr = null || String.IsNullOrEmpty attr.Value then + None + else + Some(attr.Value) let (|??) (option1: 'a Option) option2 = - if option1.IsSome then option1 else option2 + if option1.IsSome then + option1 + else + option2 let fsProjFile = Path.Combine(dir, proj + ".fsproj") let fsProjXml = XDocument.Load fsProjFile - let refs = + let refs = fsProjXml.Document.Descendants(getElemName "Reference") - |> Seq.choose (fun elem -> getElemValue "HintPath" elem |?? getAttrValue "Include" elem) + |> Seq.choose (fun elem -> + getElemValue "HintPath" elem + |?? getAttrValue "Include" elem) |> Seq.map (fun ref -> ref.Replace(@"\", @"\\").Split(',').[0]) - |> Seq.filter (fun ref -> ref <> "mscorlib" && ref <> "FSharp.Core" && not (ref.EndsWith "FSharp.Core.dll")) + |> Seq.filter (fun ref -> + ref <> "mscorlib" + && ref <> "FSharp.Core" + && not (ref.EndsWith "FSharp.Core.dll")) |> Seq.map (fun ref -> "#r \"" + ref + "\"") |> Seq.toList - let fsFiles = + let fsFiles = fsProjXml.Document.Descendants(getElemName "Compile") |> Seq.choose (fun elem -> getAttrValue "Include" elem) |> Seq.filter (Path.GetExtension >> (<>) ".fsi") |> Seq.filter (Path.GetFileName >> (<>) "Test.fs") |> Seq.map (fun path -> "#load \"" + path.Replace(@"\", @"\\") + "\"") |> Seq.toList - + let tempFile = Path.Combine(dir, "__setup__" + proj + "__.fsx") - File.WriteAllLines(tempFile, refs @ fsFiles) + File.WriteAllLines(tempFile, refs @ fsFiles) diff --git a/src/Test.fsx b/src/Test.fsx index bd7ef9f05..d1314bb87 100644 --- a/src/Test.fsx +++ b/src/Test.fsx @@ -2,6 +2,7 @@ let dir = __SOURCE_DIRECTORY__ + "/FSharp.Data.DesignTime" let proj = "FSharp.Data.DesignTime" SetupTesting.generateSetupScript (__SOURCE_DIRECTORY__ + "/FSharp.Data.DesignTime") "FSharp.Data.DesignTime" + #load "FSharp.Data.DesignTime/__setup__FSharp.Data.DesignTime__.fsx" #load "../paket-files/fsprojects/FSharp.TypeProviders.SDK/src/ProvidedTypesTesting.fs" #load "../tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs" @@ -14,25 +15,44 @@ open FSharp.Data open FSharp.Data.Runtime let (++) a b = Path.Combine(a, b) -let resolutionFolder = __SOURCE_DIRECTORY__ ++ ".." ++ "tests" ++ "FSharp.Data.Tests" ++ "Data" -let outputFolder = __SOURCE_DIRECTORY__ ++ ".." ++ "tests" ++ "FSharp.Data.DesignTime.Tests" ++ "expected" + +let resolutionFolder = + __SOURCE_DIRECTORY__ + ++ ".." + ++ "tests" + ++ "FSharp.Data.Tests" + ++ "Data" + +let outputFolder = + __SOURCE_DIRECTORY__ + ++ ".." + ++ "tests" + ++ "FSharp.Data.DesignTime.Tests" + ++ "expected" + let assemblyName = "FSharp.Data.dll" -let dump signatureOnly ignoreOutput platform saveToFileSystem (inst:TypeProviderInstantiation) = +let dump signatureOnly ignoreOutput saveToFileSystem (inst: TypeProviderInstantiation) = let root = __SOURCE_DIRECTORY__ ++ ".." ++ "bin" - let runtimeAssembly = - match platform with - | NetStandard20 -> root ++ "netstandard2.0" ++ assemblyName - let runtimeAssemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs platform - inst.Dump(resolutionFolder, (if saveToFileSystem then outputFolder else ""), runtimeAssembly, runtimeAssemblyRefs, signatureOnly, ignoreOutput) + let runtimeAssembly = root ++ "netstandard2.0" ++ assemblyName + let runtimeAssemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs() + + inst.Dump( + resolutionFolder, + (if saveToFileSystem then + outputFolder + else + ""), + runtimeAssembly, + runtimeAssemblyRefs, + signatureOnly, + ignoreOutput + ) |> Console.WriteLine -let dumpAll inst = - dump false false NetStandard20 false inst -// dump false false NetStandard16 false inst -// dump false false NetStandard20 false inst +let dumpAll inst = dump false false false inst -let parameters : HtmlInference.Parameters = +let parameters: HtmlInference.Parameters = { MissingValues = TextConversions.DefaultMissingValues CultureInfo = CultureInfo.InvariantCulture UnitsOfMeasureProvider = StructuralInference.defaultUnitsOfMeasureProvider @@ -40,7 +60,7 @@ let parameters : HtmlInference.Parameters = let includeLayout = false -let printTable tableName (url:string) = +let printTable tableName (url: string) = url |> HtmlDocument.Load |> HtmlRuntime.getTables (Some parameters) includeLayout @@ -49,28 +69,31 @@ let printTable tableName (url:string) = printTable "Overview" "https://en.wikipedia.org/wiki/List_of_Doctor_Who_serials" -Html { Sample = "doctor_who3.html" - PreferOptionals = false - IncludeLayoutTables = false - MissingValues = "NaN,NA,N/A,#N/A,:,-,TBA,TBD" - Culture = "" - Encoding = "" - ResolutionFolder = "" - EmbeddedResource = "" } +Html + { Sample = "doctor_who3.html" + PreferOptionals = false + IncludeLayoutTables = false + MissingValues = "NaN,NA,N/A,#N/A,:,-,TBA,TBD" + Culture = "" + Encoding = "" + ResolutionFolder = "" + EmbeddedResource = "" } |> dumpAll -Json { Sample = "optionals.json" - SampleIsList = false - RootName = "" - Culture = "" - Encoding = "" - ResolutionFolder = "" - EmbeddedResource = "" - InferTypesFromValues = true - PreferDictionaries = false } +Json + { Sample = "optionals.json" + SampleIsList = false + RootName = "" + Culture = "" + Encoding = "" + ResolutionFolder = "" + EmbeddedResource = "" + InferTypesFromValues = true + PreferDictionaries = false } |> dumpAll -Xml { Sample = "JsonInXml.xml" +Xml + { Sample = "JsonInXml.xml" SampleIsList = true Global = false Culture = "" @@ -81,7 +104,8 @@ Xml { Sample = "JsonInXml.xml" Schema = "" } |> dumpAll -Csv { Sample = "AirQuality.csv" +Csv + { Sample = "AirQuality.csv" Separators = ";" InferRows = Int32.MaxValue Schema = "" @@ -100,9 +124,13 @@ Csv { Sample = "AirQuality.csv" |> dumpAll let testCases = - __SOURCE_DIRECTORY__ ++ ".." ++ "tests" ++ "FSharp.Data.DesignTime.Tests" ++ "SignatureTestCases.config" + __SOURCE_DIRECTORY__ + ++ ".." + ++ "tests" + ++ "FSharp.Data.DesignTime.Tests" + ++ "SignatureTestCases.config" |> File.ReadAllLines |> Array.map (TypeProviderInstantiation.Parse >> snd) for testCase in testCases do - dump false false NetStandard20 true testCase + dump false false true testCase diff --git a/src/WorldBank/WorldBankRuntime.fs b/src/WorldBank/WorldBankRuntime.fs index 91e7b5379..766e1fc1b 100644 --- a/src/WorldBank/WorldBankRuntime.fs +++ b/src/WorldBank/WorldBankRuntime.fs @@ -292,59 +292,63 @@ type Indicator internal (connection: ServiceConnection, countryOrRegionCode: str let dataDict = lazy (dict data) /// Get the code for the country or region of the indicator - member x.Code = countryOrRegionCode + member _.Code = countryOrRegionCode /// Get the code for the indicator - member x.IndicatorCode = indicatorCode + member _.IndicatorCode = indicatorCode /// Get the name of the indicator - member x.Name = connection.IndicatorsIndexed.[indicatorCode].Name + member _.Name = connection.IndicatorsIndexed.[indicatorCode].Name /// Get the source of the indicator - member x.Source = connection.IndicatorsIndexed.[indicatorCode].Source + member _.Source = connection.IndicatorsIndexed.[indicatorCode].Source /// Get the description of the indicator - member x.Description = connection.IndicatorsIndexed.[indicatorCode].Description + member _.Description = connection.IndicatorsIndexed.[indicatorCode].Description /// Get the indicator value for the given year. If there's no data for that year, NaN is returned - member x.Item + member _.Item with get year = match dataDict.Force().TryGetValue year with | true, value -> value | _ -> Double.NaN /// Get the indicator value for the given year, if present - member x.TryGetValueAt year = + member _.TryGetValueAt year = match dataDict.Force().TryGetValue year with | true, value -> Some value | _ -> None /// Get the years for which the indicator has values - member x.Years = dataDict.Force().Keys + member _.Years = dataDict.Force().Keys /// Get the values for the indicator (without years) - member x.Values = dataDict.Force().Values + member _.Values = dataDict.Force().Values interface seq with - member x.GetEnumerator() = data.GetEnumerator() + member _.GetEnumerator() = data.GetEnumerator() interface IEnumerable with - member x.GetEnumerator() = (data.GetEnumerator() :> _) + member _.GetEnumerator() = (data.GetEnumerator() :> _) /// Metadata for an Indicator [] [] type IndicatorDescription internal (connection: ServiceConnection, topicCode: string, indicatorCode: string) = /// Get the code for the topic of the indicator - member x.Code = topicCode + member _.Code = topicCode + /// Get the code for the indicator - member x.IndicatorCode = indicatorCode + member _.IndicatorCode = indicatorCode + /// Get the name of the indicator - member x.Name = connection.IndicatorsIndexed.[indicatorCode].Name + member _.Name = connection.IndicatorsIndexed.[indicatorCode].Name + /// Get the source of the indicator - member x.Source = connection.IndicatorsIndexed.[indicatorCode].Source + member _.Source = connection.IndicatorsIndexed.[indicatorCode].Source + /// Get the description of the indicator - member x.Description = connection.IndicatorsIndexed.[indicatorCode].Description + member _.Description = connection.IndicatorsIndexed.[indicatorCode].Description /// type IIndicators = @@ -357,17 +361,17 @@ type Indicators internal (connection: ServiceConnection, countryOrRegionCode) = seq { for indicator in connection.Indicators -> Indicator(connection, countryOrRegionCode, indicator.Id) } interface IIndicators with - member x.GetIndicator(indicatorCode) = + member _.GetIndicator(indicatorCode) = Indicator(connection, countryOrRegionCode, indicatorCode) - member x.AsyncGetIndicator(indicatorCode) = + member _.AsyncGetIndicator(indicatorCode) = async { return Indicator(connection, countryOrRegionCode, indicatorCode) } interface seq with - member x.GetEnumerator() = indicators.GetEnumerator() + member _.GetEnumerator() = indicators.GetEnumerator() interface IEnumerable with - member x.GetEnumerator() = indicators.GetEnumerator() :> _ + member _.GetEnumerator() = indicators.GetEnumerator() :> _ /// type IIndicatorsDescriptions = @@ -382,14 +386,14 @@ type IndicatorsDescriptions internal (connection: ServiceConnection, topicCode) } interface IIndicatorsDescriptions with - member x.GetIndicator(indicatorCode) = + member _.GetIndicator(indicatorCode) = IndicatorDescription(connection, topicCode, indicatorCode) interface seq with - member x.GetEnumerator() = indicatorsDescriptions.GetEnumerator() + member _.GetEnumerator() = indicatorsDescriptions.GetEnumerator() interface IEnumerable with - member x.GetEnumerator() = + member _.GetEnumerator() = indicatorsDescriptions.GetEnumerator() :> _ /// @@ -401,17 +405,21 @@ type ICountry = [] type Country internal (connection: ServiceConnection, countryCode: string) = let indicators = new Indicators(connection, countryCode) + /// Get the WorldBank code of the country - member x.Code = countryCode + member _.Code = countryCode + /// Get the name of the country - member x.Name = connection.CountriesIndexed.[countryCode].Name + member _.Name = connection.CountriesIndexed.[countryCode].Name + /// Get the capital city of the country - member x.CapitalCity = connection.CountriesIndexed.[countryCode].CapitalCity + member _.CapitalCity = connection.CountriesIndexed.[countryCode].CapitalCity + /// Get the region of the country - member x.Region = connection.CountriesIndexed.[countryCode].Region + member _.Region = connection.CountriesIndexed.[countryCode].Region interface ICountry with - member x.GetIndicators() = indicators + member _.GetIndicators() = indicators /// type ICountryCollection = @@ -432,13 +440,13 @@ type CountryCollection<'T when 'T :> Country> internal (connection: ServiceConne } interface seq<'T> with - member x.GetEnumerator() = items.GetEnumerator() + member _.GetEnumerator() = items.GetEnumerator() interface IEnumerable with - member x.GetEnumerator() = (items :> IEnumerable).GetEnumerator() + member _.GetEnumerator() = (items :> IEnumerable).GetEnumerator() interface ICountryCollection with - member x.GetCountry(countryCode (*this parameter is only here to help FunScript*) , _countryName) = + member _.GetCountry(countryCode (*this parameter is only here to help FunScript*) , _countryName) = Country(connection, countryCode) /// @@ -452,15 +460,15 @@ type IRegion = type Region internal (connection: ServiceConnection, regionCode: string) = let indicators = new Indicators(connection, regionCode) /// Get the WorldBank code for the region - member x.RegionCode = regionCode + member _.RegionCode = regionCode /// Get the name of the region - member x.Name = connection.RegionsIndexed.[regionCode] + member _.Name = connection.RegionsIndexed.[regionCode] interface IRegion with - member x.GetCountries() = + member _.GetCountries() = CountryCollection(connection, Some regionCode) - member x.GetIndicators() = indicators + member _.GetIndicators() = indicators /// type IRegionCollection = @@ -472,13 +480,13 @@ type RegionCollection<'T when 'T :> Region> internal (connection: ServiceConnect seq { for (code, _) in connection.Regions -> Region(connection, code) :?> 'T } interface seq<'T> with - member x.GetEnumerator() = items.GetEnumerator() + member _.GetEnumerator() = items.GetEnumerator() interface IEnumerable with - member x.GetEnumerator() = (items :> IEnumerable).GetEnumerator() + member _.GetEnumerator() = (items :> IEnumerable).GetEnumerator() interface IRegionCollection with - member x.GetRegion(regionCode) = Region(connection, regionCode) + member _.GetRegion(regionCode) = Region(connection, regionCode) /// type ITopic = @@ -489,15 +497,18 @@ type ITopic = [] type Topic internal (connection: ServiceConnection, topicCode: string) = let indicatorsDescriptions = new IndicatorsDescriptions(connection, topicCode) + /// Get the WorldBank code of the topic - member x.Code = topicCode + member _.Code = topicCode + /// Get the name of the topic - member x.Name = connection.TopicsIndexed.[topicCode].Name + member _.Name = connection.TopicsIndexed.[topicCode].Name + /// Get the description of the topic - member x.Description = connection.TopicsIndexed.[topicCode].Description + member _.Description = connection.TopicsIndexed.[topicCode].Description interface ITopic with - member x.GetIndicators() = indicatorsDescriptions + member _.GetIndicators() = indicatorsDescriptions /// type ITopicCollection = @@ -509,13 +520,13 @@ type TopicCollection<'T when 'T :> Topic> internal (connection: ServiceConnectio seq { for topic in connection.Topics -> Topic(connection, topic.Id) :?> 'T } interface seq<'T> with - member x.GetEnumerator() = items.GetEnumerator() + member _.GetEnumerator() = items.GetEnumerator() interface IEnumerable with - member x.GetEnumerator() = (items :> IEnumerable).GetEnumerator() + member _.GetEnumerator() = (items :> IEnumerable).GetEnumerator() interface ITopicCollection with - member x.GetTopic(topicCode) = Topic(connection, topicCode) + member _.GetTopic(topicCode) = Topic(connection, topicCode) /// type IWorldBankData = @@ -533,8 +544,8 @@ type WorldBankData(serviceUrl: string, sources: string) = let connection = new ServiceConnection(restCache, serviceUrl, sources) interface IWorldBankData with - member x.GetCountries() = + member _.GetCountries() = CountryCollection(connection, None) :> seq<_> - member x.GetRegions() = RegionCollection(connection) :> seq<_> - member x.GetTopics() = TopicCollection(connection) :> seq<_> + member _.GetRegions() = RegionCollection(connection) :> seq<_> + member _.GetTopics() = TopicCollection(connection) :> seq<_> diff --git a/src/Xml/XmlExtensions.fs b/src/Xml/XmlExtensions.fs new file mode 100644 index 000000000..4ea19829d --- /dev/null +++ b/src/Xml/XmlExtensions.fs @@ -0,0 +1,68 @@ +// -------------------------------------------------------------------------------------- +// XML type provider - methods & types used by the generated erased code +// -------------------------------------------------------------------------------------- +namespace FSharp.Data + +open System.Xml.Linq +open System.Runtime.InteropServices + +// XElementExtensions is not a static class with C#-style extension methods because that would +// force to reference System.Xml.Linq.dll everytime you reference FSharp.Data, even when not using +// any of the XML parts +[] +/// Extension methods for XElement +module XElementExtensions = + + type XElement with + + /// Sends the XML to the specified uri. Defaults to a POST request. + member x.Request(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = + let httpMethod = defaultArg httpMethod HttpMethod.Post + let headers = defaultArg (Option.map List.ofSeq headers) [] + + let headers = + if + headers + |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) + then + headers + else + HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" + :: headers + + let headers = + HttpRequestHeaders.ContentType HttpContentTypes.Xml + :: headers + + Http.Request( + uri, + body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), + headers = headers, + httpMethod = httpMethod + ) + + /// Sends the XML to the specified uri. Defaults to a POST request. + member x.RequestAsync(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = + let httpMethod = defaultArg httpMethod HttpMethod.Post + let headers = defaultArg (Option.map List.ofSeq headers) [] + + let headers = + if + headers + |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) + then + headers + else + HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" + :: headers + + let headers = + HttpRequestHeaders.ContentType HttpContentTypes.Xml + :: headers + + Http.AsyncRequest( + uri, + body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), + headers = headers, + httpMethod = httpMethod + ) diff --git a/src/Xml/XmlInference.fs b/src/Xml/XmlInference.fs index 6fb83c285..fc63fd9a0 100644 --- a/src/Xml/XmlInference.fs +++ b/src/Xml/XmlInference.fs @@ -2,7 +2,7 @@ // Implements type inference for XML // -------------------------------------------------------------------------------------- -module ProviderImplementation.XmlInference +module internal ProviderImplementation.XmlInference open System open System.Xml.Linq @@ -12,6 +12,10 @@ open FSharp.Data.Runtime open FSharp.Data.Runtime.StructuralInference open FSharp.Data.Runtime.StructuralTypes +/// Takes a map and succeeds if it is empty +let (|EmptyMap|_|) result (map: Map<_, _>) = + if map.IsEmpty then Some result else None + // The type of XML element is always a non-optional record with a field // for every attribute. If it has some content, then it also // contains a special field named "" which is either a collection diff --git a/src/Xml/XmlRuntime.fs b/src/Xml/XmlRuntime.fs index 3e26cb746..fef40a02e 100644 --- a/src/Xml/XmlRuntime.fs +++ b/src/Xml/XmlRuntime.fs @@ -1,77 +1,9 @@ // -------------------------------------------------------------------------------------- // XML type provider - methods & types used by the generated erased code // -------------------------------------------------------------------------------------- -namespace FSharp.Data - -open System.Xml.Linq -open System.Runtime.InteropServices - -// XElementExtensions is not a static class with C#-style extension methods because that would -// force to reference System.Xml.Linq.dll everytime you reference FSharp.Data, even when not using -// any of the XML parts -[] -/// Extension methods for XElement -module XElementExtensions = - - type XElement with - - /// Sends the XML to the specified uri. Defaults to a POST request. - member x.Request(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = - let httpMethod = defaultArg httpMethod HttpMethod.Post - let headers = defaultArg (Option.map List.ofSeq headers) [] - - let headers = - if - headers - |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) - then - headers - else - HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" - :: headers - - let headers = - HttpRequestHeaders.ContentType HttpContentTypes.Xml - :: headers - - Http.Request( - uri, - body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), - headers = headers, - httpMethod = httpMethod - ) - - /// Sends the XML to the specified uri. Defaults to a POST request. - member x.RequestAsync(uri: string, [] ?httpMethod, [] ?headers: seq<_>) = - let httpMethod = defaultArg httpMethod HttpMethod.Post - let headers = defaultArg (Option.map List.ofSeq headers) [] - - let headers = - if - headers - |> List.exists (fst >> (=) (fst (HttpRequestHeaders.UserAgent ""))) - then - headers - else - HttpRequestHeaders.UserAgent "FSharp.Data XML Type Provider" - :: headers - - let headers = - HttpRequestHeaders.ContentType HttpContentTypes.Xml - :: headers - - Http.AsyncRequest( - uri, - body = TextRequest(x.ToString(SaveOptions.DisableFormatting)), - headers = headers, - httpMethod = httpMethod - ) - -// -------------------------------------------------------------------------------------- namespace FSharp.Data.Runtime.BaseTypes -open System open System.ComponentModel open System.IO open System.Xml.Linq diff --git a/src/Xml/XsdInference.fs b/src/Xml/XsdInference.fs index 5f073db5e..81fc8c739 100644 --- a/src/Xml/XsdInference.fs +++ b/src/Xml/XsdInference.fs @@ -208,7 +208,7 @@ module XsdParsing = /// Element definitions in a schema are mapped to InferedType instances -module XsdInference = +module internal XsdInference = open XsdModel open FSharp.Data.Runtime.StructuralTypes diff --git a/tests/FSharp.Data.Tests.CSharp/CsvExtensionsTests.cs b/tests/FSharp.Data.Core.Tests.CSharp/CsvExtensionsTests.cs similarity index 100% rename from tests/FSharp.Data.Tests.CSharp/CsvExtensionsTests.cs rename to tests/FSharp.Data.Core.Tests.CSharp/CsvExtensionsTests.cs diff --git a/tests/FSharp.Data.Tests.CSharp/FSharp.Data.Tests.CSharp.csproj b/tests/FSharp.Data.Core.Tests.CSharp/FSharp.Data.Core.Tests.CSharp.csproj similarity index 61% rename from tests/FSharp.Data.Tests.CSharp/FSharp.Data.Tests.CSharp.csproj rename to tests/FSharp.Data.Core.Tests.CSharp/FSharp.Data.Core.Tests.CSharp.csproj index 5159e9b6e..6a2b87bfd 100644 --- a/tests/FSharp.Data.Tests.CSharp/FSharp.Data.Tests.CSharp.csproj +++ b/tests/FSharp.Data.Core.Tests.CSharp/FSharp.Data.Core.Tests.CSharp.csproj @@ -7,6 +7,7 @@ true true false + $(OtherFlags) --warnon:1182 --nowarn:44 @@ -19,7 +20,12 @@ - + + + + + + diff --git a/tests/FSharp.Data.Tests.CSharp/HtmlExtensionsTests.cs b/tests/FSharp.Data.Core.Tests.CSharp/HtmlExtensionsTests.cs similarity index 100% rename from tests/FSharp.Data.Tests.CSharp/HtmlExtensionsTests.cs rename to tests/FSharp.Data.Core.Tests.CSharp/HtmlExtensionsTests.cs diff --git a/tests/FSharp.Data.Tests.CSharp/JsonExtensionsTests.cs b/tests/FSharp.Data.Core.Tests.CSharp/JsonExtensionsTests.cs similarity index 100% rename from tests/FSharp.Data.Tests.CSharp/JsonExtensionsTests.cs rename to tests/FSharp.Data.Core.Tests.CSharp/JsonExtensionsTests.cs diff --git a/tests/FSharp.Data.Tests.CSharp/Properties/AssemblyInfo.cs b/tests/FSharp.Data.Core.Tests.CSharp/Properties/AssemblyInfo.cs similarity index 100% rename from tests/FSharp.Data.Tests.CSharp/Properties/AssemblyInfo.cs rename to tests/FSharp.Data.Core.Tests.CSharp/Properties/AssemblyInfo.cs diff --git a/tests/FSharp.Data.Tests.CSharp/paket.references b/tests/FSharp.Data.Core.Tests.CSharp/paket.references similarity index 100% rename from tests/FSharp.Data.Tests.CSharp/paket.references rename to tests/FSharp.Data.Core.Tests.CSharp/paket.references diff --git a/tests/FSharp.Data.Tests/CsvReader.fs b/tests/FSharp.Data.Core.Tests/CsvReader.fs similarity index 100% rename from tests/FSharp.Data.Tests/CsvReader.fs rename to tests/FSharp.Data.Core.Tests/CsvReader.fs diff --git a/tests/FSharp.Data.Core.Tests/FSharp.Data.Core.Tests.fsproj b/tests/FSharp.Data.Core.Tests/FSharp.Data.Core.Tests.fsproj new file mode 100644 index 000000000..48a896c14 --- /dev/null +++ b/tests/FSharp.Data.Core.Tests/FSharp.Data.Core.Tests.fsproj @@ -0,0 +1,43 @@ + + + + net6.0 + false + + true + true + false + $(OtherFlags) --warnon:1182 --nowarn:44 + + true + + + + + + + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/FSharp.Data.Tests/HtmlCharRefs.fs b/tests/FSharp.Data.Core.Tests/HtmlCharRefs.fs similarity index 92% rename from tests/FSharp.Data.Tests/HtmlCharRefs.fs rename to tests/FSharp.Data.Core.Tests/HtmlCharRefs.fs index de6d09a12..6be3f42c7 100644 --- a/tests/FSharp.Data.Tests/HtmlCharRefs.fs +++ b/tests/FSharp.Data.Core.Tests/HtmlCharRefs.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.HtmlCharRefs +module FSharp.Data.Tests.HtmlCharRefs open NUnit.Framework open FsUnit @@ -8,7 +8,7 @@ open FSharp.Data.HtmlNode open FSharp.Data.JsonExtensions let charRefsTestCases = - JsonValue.Load(__SOURCE_DIRECTORY__ + "/Data/charrefs.json")?items.AsArray() + JsonValue.Load(__SOURCE_DIRECTORY__ + "/../FSharp.Data.Tests/Data/charrefs.json")?items.AsArray() |> Array.map (fun x -> [| x?key.AsString(); x?characters.AsString() |]) [] diff --git a/tests/FSharp.Data.Tests/HtmlCssSelectors.fs b/tests/FSharp.Data.Core.Tests/HtmlCssSelectors.fs similarity index 100% rename from tests/FSharp.Data.Tests/HtmlCssSelectors.fs rename to tests/FSharp.Data.Core.Tests/HtmlCssSelectors.fs diff --git a/tests/FSharp.Data.Tests/HtmlOperations.fs b/tests/FSharp.Data.Core.Tests/HtmlOperations.fs similarity index 99% rename from tests/FSharp.Data.Tests/HtmlOperations.fs rename to tests/FSharp.Data.Core.Tests/HtmlOperations.fs index 925e6b4e2..b8cb4223d 100644 --- a/tests/FSharp.Data.Tests/HtmlOperations.fs +++ b/tests/FSharp.Data.Core.Tests/HtmlOperations.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.HtmlOperations +module FSharp.Data.Tests.HtmlOperations open NUnit.Framework open FsUnit diff --git a/tests/FSharp.Data.Tests/HtmlParser.fs b/tests/FSharp.Data.Core.Tests/HtmlParser.fs similarity index 98% rename from tests/FSharp.Data.Tests/HtmlParser.fs rename to tests/FSharp.Data.Core.Tests/HtmlParser.fs index b22feb528..fe9a26fc2 100644 --- a/tests/FSharp.Data.Tests/HtmlParser.fs +++ b/tests/FSharp.Data.Core.Tests/HtmlParser.fs @@ -589,7 +589,7 @@ let ``Can handle html without html tag``() = [] let ``Can find header when nested in a div``() = let tables = - HtmlDocument.Load "Data/wimbledon_wikipedia.html" + HtmlDocument.Load "wimbledon_wikipedia.html" |> getTables false |> List.map (fun t -> t.Name, t) |> Map.ofList @@ -599,7 +599,7 @@ let ``Can find header when nested in a div``() = [] let ``Can parse tables imdb chart``() = - let imdb = HtmlDocument.Load "Data/imdb_chart.htm" + let imdb = HtmlDocument.Load "imdb_chart.htm" let tables = imdb |> getTables false tables.Length |> should equal 2 tables.[0].Name |> should equal "Top 250" @@ -608,12 +608,12 @@ let ``Can parse tables imdb chart``() = [] let ``Can parse tables ebay cars``() = - let ebay = HtmlDocument.Load "Data/ebay_cars.htm" + let _ebay = HtmlDocument.Load "ebay_cars.htm" true |> should equal true [] let ``Does not crash when parsing us presidents``() = - let table = HtmlDocument.Load "Data/us_presidents_wikipedia.html" |> getTables false + let _table = HtmlDocument.Load "us_presidents_wikipedia.html" |> getTables false true |> should equal true [] @@ -859,33 +859,33 @@ let ``Drops whitespace outside pre``() = [] let ``Can parse national rail mobile site correctly``() = - HtmlDocument.Load "Data/UKDepartures.html" + HtmlDocument.Load "UKDepartures.html" |> HtmlDocument.descendantsNamed false [ "li" ] |> Seq.length |> should equal 68 - HtmlDocument.Load "Data/UKLiveProgress.html" + HtmlDocument.Load "UKLiveProgress.html" |> HtmlDocument.descendantsNamed false [ "li" ] |> Seq.length |> should equal 15 - HtmlDocument.Load "Data/UKDepartures.html" + HtmlDocument.Load "UKDepartures.html" |> HtmlDocument.descendantsNamed false [ "li"; "hr" ] |> Seq.length |> should equal 69 - HtmlDocument.Load "Data/UKLiveProgress.html" + HtmlDocument.Load "UKLiveProgress.html" |> HtmlDocument.descendantsNamed false [ "li"; "hr" ] |> Seq.length |> should equal 17 [] let ``Can parse old zoopla site correctly``() = - HtmlDocument.Load "Data/zoopla.html" + HtmlDocument.Load "zoopla.html" |> HtmlDocument.descendants false (fun x -> HtmlNode.hasName "li" x && HtmlNode.hasAttribute "itemtype" "http://schema.org/Place" x) |> Seq.length |> should equal 100 [] let ``Can parse new zoopla site correctly``() = - HtmlDocument.Load "Data/zoopla2.html" + HtmlDocument.Load "zoopla2.html" |> HtmlDocument.descendants false (fun x -> HtmlNode.hasName "li" x && HtmlNode.hasAttribute "itemtype" "http://schema.org/Residence" x) |> Seq.length |> should equal 10 diff --git a/tests/FSharp.Data.Tests/Http.fs b/tests/FSharp.Data.Core.Tests/Http.fs similarity index 99% rename from tests/FSharp.Data.Tests/Http.fs rename to tests/FSharp.Data.Core.Tests/Http.fs index 30b240e7a..f2223a5ca 100644 --- a/tests/FSharp.Data.Tests/Http.fs +++ b/tests/FSharp.Data.Core.Tests/Http.fs @@ -224,14 +224,14 @@ let testFormDataSizesInBytes = [ [] let testFormDataBodySize (size: int) = use localServer = startHttpLocalServer() - let bodyString = seq {for i in 0..size -> "x\n"} |> String.concat "" + let bodyString = seq {for _i in 0..size -> "x\n"} |> String.concat "" let body = FormValues([("input", bodyString)]) Assert.DoesNotThrowAsync(fun () -> Http.AsyncRequest (url= localServer.BaseAddress + "/200", httpMethod="POST", body=body, timeout = 10000) |> Async.Ignore |> Async.StartAsTask :> _) [] let testMultipartFormDataBodySize (size: int) = use localServer = startHttpLocalServer() - let bodyString = seq {for i in 0..size -> "x\n"} |> String.concat "" + let bodyString = seq {for _i in 0..size -> "x\n"} |> String.concat "" let multipartItem = [ MultipartItem("input", "input.txt", new MemoryStream(Encoding.UTF8.GetBytes(bodyString)) :> Stream) ] let body = Multipart(Guid.NewGuid().ToString(), multipartItem) diff --git a/tests/FSharp.Data.Tests/JsonConversions.fs b/tests/FSharp.Data.Core.Tests/JsonConversions.fs similarity index 95% rename from tests/FSharp.Data.Tests/JsonConversions.fs rename to tests/FSharp.Data.Core.Tests/JsonConversions.fs index 43c87bbdf..ef8a8917f 100644 --- a/tests/FSharp.Data.Tests/JsonConversions.fs +++ b/tests/FSharp.Data.Core.Tests/JsonConversions.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.JsonConversions +module FSharp.Data.Tests.JsonConversions open NUnit.Framework open FsUnit diff --git a/tests/FSharp.Data.Tests/JsonParserProperties.fs b/tests/FSharp.Data.Core.Tests/JsonParserProperties.fs similarity index 97% rename from tests/FSharp.Data.Tests/JsonParserProperties.fs rename to tests/FSharp.Data.Core.Tests/JsonParserProperties.fs index c50b003b6..7fcda85be 100644 --- a/tests/FSharp.Data.Tests/JsonParserProperties.fs +++ b/tests/FSharp.Data.Core.Tests/JsonParserProperties.fs @@ -11,16 +11,16 @@ let rec isValidJsonString s = match s with | [] -> true | ['\\'] -> false - | '"' :: t -> false - | h :: t when Globalization.CharUnicodeInfo.GetUnicodeCategory h + | '"' :: _t -> false + | h :: _t when Globalization.CharUnicodeInfo.GetUnicodeCategory h = Globalization.UnicodeCategory.Control -> false | '\\' :: 'u' :: d1 :: d2 :: d3 :: d4 :: t when [d1;d2;d3;d4] |> Seq.forall (fun c -> (Char.IsDigit c || Text.RegularExpressions.Regex("[a-fA-F]").IsMatch(c.ToString()))) -> isValidJsonString t - | '\\' :: i :: t when escaped |> (not << Set.contains i) -> false + | '\\' :: i :: _t when escaped |> (not << Set.contains i) -> false | '\\' :: i :: t when escaped |> Set.contains i -> isValidJsonString t - | h :: t -> isValidJsonString t + | _h :: t -> isValidJsonString t let validJsonStringGen = Arb.generate diff --git a/tests/FSharp.Data.Tests/JsonValue.fs b/tests/FSharp.Data.Core.Tests/JsonValue.fs similarity index 99% rename from tests/FSharp.Data.Tests/JsonValue.fs rename to tests/FSharp.Data.Core.Tests/JsonValue.fs index 2c240fe65..1e0ee911a 100644 --- a/tests/FSharp.Data.Tests/JsonValue.fs +++ b/tests/FSharp.Data.Core.Tests/JsonValue.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.JsonValue +module FSharp.Data.Tests.JsonValue open NUnit.Framework open System diff --git a/tests/FSharp.Data.Tests/NameUtils.fs b/tests/FSharp.Data.Core.Tests/NameUtils.fs similarity index 96% rename from tests/FSharp.Data.Tests/NameUtils.fs rename to tests/FSharp.Data.Core.Tests/NameUtils.fs index 5fc45ceab..9bb3b717c 100644 --- a/tests/FSharp.Data.Tests/NameUtils.fs +++ b/tests/FSharp.Data.Core.Tests/NameUtils.fs @@ -1,4 +1,4 @@ -// -------------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------- // Tests for a utility that generates nice PascalCase and camelCase names for members // -------------------------------------------------------------------------------------- @@ -87,7 +87,7 @@ let ``Handles long and ugly names`` () = [] let ``Unique generator generates unique names`` () = let gen = uniqueGenerator nicePascalName - let names = [ for i in 0 .. 100 -> gen "test" ] + let names = [ for _i in 0 .. 100 -> gen "test" ] Seq.length names |> should equal (Seq.length (set names)) [] diff --git a/tests/FSharp.Data.Core.Tests/Program.fs b/tests/FSharp.Data.Core.Tests/Program.fs new file mode 100644 index 000000000..96a10a38d --- /dev/null +++ b/tests/FSharp.Data.Core.Tests/Program.fs @@ -0,0 +1,6 @@ +open System + +[] +let main _argv = + printfn "Dotnet Core NUnit Tests..." + 0 diff --git a/tests/FSharp.Data.Tests/TextConversions.fs b/tests/FSharp.Data.Core.Tests/TextConversions.fs similarity index 98% rename from tests/FSharp.Data.Tests/TextConversions.fs rename to tests/FSharp.Data.Core.Tests/TextConversions.fs index 5547ed822..16d064145 100644 --- a/tests/FSharp.Data.Tests/TextConversions.fs +++ b/tests/FSharp.Data.Core.Tests/TextConversions.fs @@ -1,4 +1,4 @@ -module FSharp.Data.Tests.Conversions +module FSharp.Data.Tests.Conversions open NUnit.Framework open FsUnit diff --git a/tests/FSharp.Data.Core.Tests/paket.references b/tests/FSharp.Data.Core.Tests/paket.references new file mode 100644 index 000000000..aaa9d970b --- /dev/null +++ b/tests/FSharp.Data.Core.Tests/paket.references @@ -0,0 +1,8 @@ +group Test + + Microsoft.NET.Test.Sdk + NUnit + NUnit3TestAdapter + FsUnit + FsCheck + GitHubActionsTestLogger diff --git a/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj b/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj index cbecb419b..b12489268 100644 --- a/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj +++ b/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj @@ -7,6 +7,7 @@ true false + $(OtherFlags) --nowarn:44 diff --git a/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs b/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs index 72a589f9a..43d9b1ccd 100644 --- a/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs +++ b/tests/FSharp.Data.DesignTime.Tests/InferenceTests.fs @@ -3,6 +3,9 @@ module FSharp.Data.DesignTime.Tests.InferenceTests open FsUnit open System open System.Globalization +open System.Xml +open System.Xml.Linq +open System.Xml.Schema open NUnit.Framework open FSharp.Data open FSharp.Data.Runtime @@ -12,19 +15,18 @@ open FSharp.Data.Runtime.StructuralInference open ProviderImplementation /// A collection containing just one type -let SimpleCollection typ = +let internal SimpleCollection typ = InferedType.Collection([ typeTag typ], Map.ofSeq [typeTag typ, (InferedMultiplicity.Multiple, typ)]) let culture = TextRuntime.GetCulture "" -let inferenceMode = InferenceMode'.ValuesOnly -let unitsOfMeasureProvider = ProviderHelpers.unitsOfMeasureProvider +let internal inferenceMode = InferenceMode'.ValuesOnly +let internal unitsOfMeasureProvider = ProviderHelpers.unitsOfMeasureProvider -let inferType (csv:CsvFile) inferRows missingValues cultureInfo schema assumeMissingValues preferOptionals = +let internal inferType (csv:CsvFile) inferRows missingValues cultureInfo schema assumeMissingValues preferOptionals = let headerNamesAndUnits, schema = parseHeaders csv.Headers csv.NumberOfColumns schema unitsOfMeasureProvider inferType headerNamesAndUnits schema (csv.Rows |> Seq.map (fun x -> x.Columns)) inferRows missingValues inferenceMode cultureInfo assumeMissingValues preferOptionals unitsOfMeasureProvider -let toRecord fields = InferedType.Record(None, fields, false) - +let internal toRecord fields = InferedType.Record(None, fields, false) [] let ``List.pairBy helper function works``() = @@ -383,7 +385,6 @@ let ``Inference with % suffix``() = let expected = toRecord [ propFloat ; propInteger ] actual |> should equal expected - [] let ``Inference with $ prefix``() = let source = CsvFile.Parse("float,integer\n$2.0,$2\n$4.0,$3\n") @@ -393,27 +394,21 @@ let ``Inference with $ prefix``() = let expected = toRecord [ propFloat ; propInteger ] actual |> should equal expected - -open System.Xml -open System.Xml.Linq -open System.Xml.Schema - -let getInferedTypeFromSamples samples = +let internal getInferedTypeFromSamples samples = let culture = System.Globalization.CultureInfo.InvariantCulture samples |> Array.map XElement.Parse |> XmlInference.inferType unitsOfMeasureProvider inferenceMode culture false false |> Seq.fold (subtypeInfered false) InferedType.Top - -let getInferedTypeFromSchema xsd = +let internal getInferedTypeFromSchema xsd = xsd |> XmlSchema.parseSchema "" |> XsdParsing.getElements |> List.ofSeq |> XsdInference.inferElements -let isValid xsd = +let internal isValid xsd = let xmlSchemaSet = XmlSchema.parseSchema "" xsd fun xml -> let settings = XmlReaderSettings(ValidationType = ValidationType.Schema) @@ -426,8 +421,7 @@ let isValid xsd = printfn "%s/n%s" e.Message xml false - -let getInferedTypes xsd xmlSamples = +let internal getInferedTypes xsd xmlSamples = //printfn "%s/n---------------------------------------------" xsd let isValid = isValid xsd for xml in xmlSamples do @@ -440,14 +434,11 @@ let getInferedTypes xsd xmlSamples = //printfn "%A" inferedTypeFromSamples inferedTypeFromSchema, inferedTypeFromSamples - - -let check xsd xmlSamples = +let internal check xsd xmlSamples = //printfn "checking schema and samples" let inferedTypeFromSchema, inferedTypeFromSamples = getInferedTypes xsd xmlSamples inferedTypeFromSchema |> should equal inferedTypeFromSamples - [] let ``at least one global complex element is needed``() = let xsd = """ @@ -506,6 +497,7 @@ let ``recursive schemas don't cause loops``() = """ let inferedTypeFromSchema = getInferedTypeFromSchema xsd + inferedTypeFromSchema |> ignore //printfn "%A" inferedTypeFromSchema let xsd = """] -let main argv = +let main _argv = printfn "Dotnet Core NUnit Tests..." - 0 \ No newline at end of file + 0 diff --git a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config index 63a6c5092..6f6688a33 100644 --- a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config +++ b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config @@ -207,5 +207,4 @@ Html,doctor_who3.html,false,false, Html,SimpleHtmlLists.html,false,false, Html,EmptyDefinitionLists.html,false,false, Html,zoopla.html,false,false, -Html,zoopla2.html,false,false, -WorldBank,World Development Indicators;Global Financial Development,true +Html,zoopla2.html,false,false, \ No newline at end of file diff --git a/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs b/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs index 9347bd109..5636dcbc5 100644 --- a/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs +++ b/tests/FSharp.Data.DesignTime.Tests/SignatureTests.fs @@ -10,52 +10,44 @@ let (++) a b = Path.Combine(a, b) let sourceDirectory = __SOURCE_DIRECTORY__ -let testCasesTuple = +let testCases = sourceDirectory ++ "SignatureTestCases.config" |> File.ReadAllLines // "No data is available for encoding 932. For information on defining a custom encoding, see the documentation for the Encoding.RegisterProvider method." |> Array.filter (fun line -> not (line.Contains ("cp932.csv"))) - |> Array.map TypeProviderInstantiation.Parse - -let testCases = - testCasesTuple - // These WorldBank tests nearly always need updating. COmment out this line if you want to go through the process of - // updating them. - |> Array.filter (snd >> function | WorldBank _ -> false | _ -> true) - |> Array.map snd let expectedDirectory = sourceDirectory ++ "expected" let resolutionFolder = sourceDirectory ++ ".." ++ "FSharp.Data.Tests" ++ "Data" -let assemblyName = "FSharp.Data.dll" -let netstandard2RuntimeAssembly = sourceDirectory ++ ".." ++ ".." ++ "src" ++ "FSharp.Data" ++ "bin" ++ - #if DEBUG - "Debug" - #else - "Release" - #endif - ++ "netstandard2.0" ++ assemblyName -let getRuntimeRefs platform = TypeProviderInstantiation.GetRuntimeAssemblyRefs platform +#if DEBUG +let build = "Debug" +#else +let build = "Release" +#endif + +let netstandard2RuntimeAssembly = sourceDirectory ++ ".." ++ ".." ++ "src" ++ "FSharp.Data" ++ "bin" ++ build ++ "netstandard2.0" ++ "FSharp.Data.dll" let normalize (str:string) = str.Replace("\r\n", "\n").Replace("\r", "\n").Replace("@\"\"", "\"\"") [] [] -let ``Validate signature didn't change `` (testCase:TypeProviderInstantiation) = - let path = testCase.ExpectedPath expectedDirectory - let expected = path |> File.ReadAllText |> normalize - let assemblyRefs = getRuntimeRefs NetStandard20 - printfn "assemblyRefs = %A" assemblyRefs - let outputRaw = testCase.Dump (resolutionFolder, "", netstandard2RuntimeAssembly, assemblyRefs, signatureOnly=false, ignoreOutput=false) - let output = outputRaw |> normalize - if output <> expected then - printfn "Obtained Signature:\n%s" outputRaw - File.WriteAllText(testCase.ExpectedPath expectedDirectory + ".obtained", outputRaw) - output |> should equal expected +let ``Validate signature didn't change `` (testCaseSpec: string) = + let _, testCase = TypeProviderInstantiation.Parse testCaseSpec + let path = testCase.ExpectedPath expectedDirectory + let expected = path |> File.ReadAllText |> normalize + let assemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs() + let outputRaw = testCase.Dump (resolutionFolder, "", netstandard2RuntimeAssembly, assemblyRefs, signatureOnly=false, ignoreOutput=false) + let output = outputRaw |> normalize + if output <> expected then + printfn "Obtained Signature:\n%s" outputRaw + File.WriteAllText(testCase.ExpectedPath expectedDirectory + ".obtained", outputRaw) + output |> should equal expected [] [] -let ``Generating expressions works in netstandard2.0 `` (testCase:TypeProviderInstantiation) = - testCase.Dump(resolutionFolder, "", netstandard2RuntimeAssembly, (getRuntimeRefs NetStandard20), signatureOnly=false, ignoreOutput=true) |> ignore +let ``Generating expressions works in netstandard2.0 `` (testCaseSpec: string) = + let _, testCase = TypeProviderInstantiation.Parse testCaseSpec + let assemblyRefs = TypeProviderInstantiation.GetRuntimeAssemblyRefs() + testCase.Dump(resolutionFolder, "", netstandard2RuntimeAssembly, assemblyRefs, signatureOnly=false, ignoreOutput=true) |> ignore diff --git a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs index 422e0fdde..b632fe664 100644 --- a/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs +++ b/tests/FSharp.Data.DesignTime.Tests/TypeProviderInstantiation.fs @@ -8,7 +8,7 @@ open ProviderImplementation.ProvidedTypesTesting open FSharp.Data.Runtime open FSharp.Data.Runtime.StructuralInference -type CsvProviderArgs = +type internal CsvProviderArgs = { Sample : string Separators : string InferRows : int @@ -26,7 +26,7 @@ type CsvProviderArgs = ResolutionFolder : string EmbeddedResource : string } -type XmlProviderArgs = +type internal XmlProviderArgs = { Sample : string SampleIsList : bool Global : bool @@ -38,7 +38,7 @@ type XmlProviderArgs = Schema : string InferenceMode: InferenceMode } -type JsonProviderArgs = +type internal JsonProviderArgs = { Sample : string SampleIsList : bool RootName : string @@ -50,7 +50,7 @@ type JsonProviderArgs = PreferDictionaries : bool InferenceMode: InferenceMode } -type HtmlProviderArgs = +type internal HtmlProviderArgs = { Sample : string PreferOptionals : bool IncludeLayoutTables : bool @@ -60,13 +60,11 @@ type HtmlProviderArgs = ResolutionFolder : string EmbeddedResource : string } -type WorldBankProviderArgs = +type internal WorldBankProviderArgs = { Sources : string Asynchronous : bool } -type Platform = NetStandard20 - -type TypeProviderInstantiation = +type internal TypeProviderInstantiation = | Csv of CsvProviderArgs | Xml of XmlProviderArgs | Json of JsonProviderArgs @@ -251,7 +249,23 @@ type TypeProviderInstantiation = Asynchronous = args.[2] |> bool.Parse } | _ -> failwithf "Unknown: %s" args.[0] - static member GetRuntimeAssemblyRefs platform = - match platform with - | NetStandard20 -> Targets.DotNetStandard20FSharpRefs() + static member GetRuntimeAssemblyRefs () = + let (++) a b = Path.Combine(a, b) + #if DEBUG + let build = "Debug" + #else + let build = "Release" + #endif + + let extraDlls = + [ "FSharp.Data.Http" + "FSharp.Data.Csv.Core" + "FSharp.Data.Html.Core" + "FSharp.Data.Xml.Core" + "FSharp.Data.Json.Core" + "FSharp.Data.WorldBank.Core" ] + let extraRefs = + [ for j in extraDlls do + __SOURCE_DIRECTORY__ ++ ".." ++ ".." ++ "src" ++ "FSharp.Data" ++ "bin" ++ build ++ "netstandard2.0" ++ (j + ".dll") ] + extraRefs @ Targets.DotNetStandard20FSharpRefs() diff --git a/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj b/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj index ad64764f6..532d035a1 100644 --- a/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj +++ b/tests/FSharp.Data.Reference.Tests/FSharp.Data.Reference.Tests.fsproj @@ -7,6 +7,7 @@ true false + $(OtherFlags) --warnon:1182 --nowarn:44 diff --git a/tests/FSharp.Data.Reference.Tests/Program.fs b/tests/FSharp.Data.Reference.Tests/Program.fs index c94e2a499..96a10a38d 100644 --- a/tests/FSharp.Data.Reference.Tests/Program.fs +++ b/tests/FSharp.Data.Reference.Tests/Program.fs @@ -1,6 +1,6 @@ -open System +open System [] -let main argv = +let main _argv = printfn "Dotnet Core NUnit Tests..." - 0 \ No newline at end of file + 0 diff --git a/tests/FSharp.Data.Tests/CsvProvider.fs b/tests/FSharp.Data.Tests/CsvProvider.fs index 36a69d037..673d456aa 100644 --- a/tests/FSharp.Data.Tests/CsvProvider.fs +++ b/tests/FSharp.Data.Tests/CsvProvider.fs @@ -6,8 +6,8 @@ open System open System.IO open FSharp.Data.UnitSystems.SI.UnitNames open FSharp.Data -open FSharp.Data.Runtime.CsvInference open FSharp.Data.Runtime +open FSharp.Data.Runtime.CsvInference open System.Globalization let [] simpleCsv = """ @@ -72,14 +72,14 @@ let ``Inference of numbers with empty values`` () = let row = rows.[0] - let f1:float = row.Float1 - let f2:float = row.Float2 - let f3:float = row.Float3 - let f4:float = row.Float4 - let i:Nullable = row.Int - let f5:float = row.Float5 - let f6:float = row.Float6 - let d:option = row.Date + let _f1:float = row.Float1 + let _f2:float = row.Float2 + let _f3:float = row.Float3 + let _f4:float = row.Float4 + let _i:Nullable = row.Int + let _f5:float = row.Float5 + let _f6:float = row.Float6 + let _d:option = row.Date let expected = 1.0, 1.0, 1.0, 1.0, Nullable(), Double.NaN, Double.NaN, (None: DateTime option) let actual = row.Float1, row.Float2, row.Float3, row.Float4, row.Int, row.Float5, row.Float6, row.Date @@ -343,9 +343,9 @@ let ``Uses UTF8 for sample file when encoding not specified``() = [] let ``Disposing CsvProvider shouldn't throw``() = - let csv = + let _csv = use csv = CsvProvider<"Data/TabSeparated.csv", HasHeaders=false>.GetSample() - csv.Rows |> Seq.iter (fun x -> ()) + csv.Rows |> Seq.iter (fun _ -> ()) () [] diff --git a/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj b/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj index 0b65efeeb..69617490b 100755 --- a/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj +++ b/tests/FSharp.Data.Tests/FSharp.Data.Tests.fsproj @@ -9,6 +9,7 @@ false true + $(OtherFlags) --warnon:1182 --nowarn:44 @@ -17,21 +18,10 @@ PreserveNewest - - - - - - - - - - - diff --git a/tests/FSharp.Data.Tests/Program.fs b/tests/FSharp.Data.Tests/Program.fs index 6329c59f8..96a10a38d 100644 --- a/tests/FSharp.Data.Tests/Program.fs +++ b/tests/FSharp.Data.Tests/Program.fs @@ -1,6 +1,6 @@ open System [] -let main argv = +let main _argv = printfn "Dotnet Core NUnit Tests..." 0 diff --git a/tests/FSharp.Data.Tests/XmlProvider.fs b/tests/FSharp.Data.Tests/XmlProvider.fs index a13028d6e..199f66410 100644 --- a/tests/FSharp.Data.Tests/XmlProvider.fs +++ b/tests/FSharp.Data.Tests/XmlProvider.fs @@ -151,10 +151,10 @@ type Html = XmlProvider<""" [] let ``Nested xml types compile when only used in type annotations``() = - let divWorks (div:Html.Div) = () - let spanWorks (span:Html.Span) = () - let ulWorks (ul:Html.Ul) = () - let liWorks (li:Html.Li) = () + let _divWorks (_div:Html.Div) = () + let _spanWorks (_span:Html.Span) = () + let _ulWorks (_ul:Html.Ul) = () + let _liWorks (_li:Html.Li) = () () [] @@ -168,8 +168,8 @@ let sameNameDifferentNamespace = """ [] let ``XML elements with same name in different namespaces``() = let xml = XmlProvider.GetSample() - let b1 = xml.B - let b2 = xml.B2 + let _b1 = xml.B + let _b2 = xml.B2 () []