New Bonsai.ML.Lds.Torch package to extract latents from high-dimensional data using TorchSharp

Bonsai.ML.sln

@@ -1,4 +1,4 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.31903.59
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -40,6 +40,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{DEE5DD87
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Tests.Utilities", "tests\Bonsai.ML.Tests.Utilities\Bonsai.ML.Tests.Utilities.csproj", "{DB1090D3-38DD-404B-96DF-66BF3C39E508}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Lds.Torch", "src\Bonsai.ML.Lds.Torch\Bonsai.ML.Lds.Torch.csproj", "{41D4BEC7-AB1F-41E4-95FE-4DB23970FF4B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Lds.Torch.Tests", "tests\Bonsai.ML.Lds.Torch.Tests\Bonsai.ML.Lds.Torch.Tests.csproj", "{0B258929-0B07-4CE7-BE8D-A86BBC46AAD4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Lds.Torch.Design", "src\Bonsai.ML.Lds.Torch.Design\Bonsai.ML.Lds.Torch.Design.csproj", "{1F52DECD-1B2C-4F6C-996C-14C715283B80}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -102,6 +108,18 @@ Global
 		{DB1090D3-38DD-404B-96DF-66BF3C39E508}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DB1090D3-38DD-404B-96DF-66BF3C39E508}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DB1090D3-38DD-404B-96DF-66BF3C39E508}.Release|Any CPU.Build.0 = Release|Any CPU
+		{41D4BEC7-AB1F-41E4-95FE-4DB23970FF4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{41D4BEC7-AB1F-41E4-95FE-4DB23970FF4B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{41D4BEC7-AB1F-41E4-95FE-4DB23970FF4B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{41D4BEC7-AB1F-41E4-95FE-4DB23970FF4B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0B258929-0B07-4CE7-BE8D-A86BBC46AAD4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0B258929-0B07-4CE7-BE8D-A86BBC46AAD4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0B258929-0B07-4CE7-BE8D-A86BBC46AAD4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0B258929-0B07-4CE7-BE8D-A86BBC46AAD4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1F52DECD-1B2C-4F6C-996C-14C715283B80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1F52DECD-1B2C-4F6C-996C-14C715283B80}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1F52DECD-1B2C-4F6C-996C-14C715283B80}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1F52DECD-1B2C-4F6C-996C-14C715283B80}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

docs/README.md

@@ -51,6 +51,9 @@ Interfaces with the [TorchSharp](https://github.com/dotnet/TorchSharp) package,
 > [!NOTE]
 > Bonsai.ML packages can be installed through Bonsai's integrated package manager and are generally ready for immediate use. However, some packages may require additional installation steps. Refer to the specific package section for detailed installation guides and documentation.
 
+### Bonsai.ML.Lds.Torch
+Linear dynamical systems implemented using the `Bonsai.ML.Torch` package for online filtering and smoothing of latent states and parameter estimation.
+
 ## Development Roadmap
 The ultimate goal of the `Bonsai.ML` project is to bring powerful machine learning tools into Bonsai to enable intelligent experimental control. To achieve this, our plan is to incorporate several different packages, models, frameworks, and language integrations. You can follow our progress by going to the [Bonsai ML development roadmap](https://github.com/orgs/bonsai-rx/projects/7).
 

docs/articles/Lds.Torch/lds-torch-overview.md

@@ -0,0 +1,7 @@
+# Bonsai.ML.Lds.Torch - Overview
+
+This package provides an implementation of the Kalman filter, Rauch-Tung-Striebel (RTS) smoother, expectation maximization (EM) algorithm, and stochastic subspace identification, developed for online filtering, smoothing, and parameter estimation from data streams in Bonsai using the TorchSharp package.
+
+## Installation Guide
+
+Install the `Bonsai.ML.Lds.Torch` package from the Bonsai package manager. You will also need to follow the [instructions for setting up the Bonsai.ML.Torch package](../Torch/torch-overview.md) for running on the CPU or GPU.

docs/articles/toc.yml

@@ -19,4 +19,6 @@
   href: PointProcessDecoder/ppd-overview.md
 - name: Torch
 - name: Overview
-  href: Torch/torch-overview.md
+  href: Torch/torch-overview.md
+- name: Lds.Torch
+  href: Lds.Torch/lds-torch-overview.md

src/Bonsai.ML.PointProcessDecoder.Design/OxyColorPresetCycle.cs → src/Bonsai.ML.Design/OxyColorPresetCycle.cs

@@ -1,7 +1,7 @@
 
 using OxyPlot;
 
-namespace Bonsai.ML.PointProcessDecoder.Design
+namespace Bonsai.ML.Design
 {
     /// <summary>
     /// Enumerates the colors and provides a preset collection of colors to cycle through.

src/Bonsai.ML.Lds.Torch.Design/Bonsai.ML.Lds.Torch.Design.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Description>Visualizers for the Bonsai.ML.Lds.Torch library.</Description>
+    <PackageTags>$(PackageTags) Torch LDS Design</PackageTags>
+    <TargetFramework>net472</TargetFramework>
+    <UseWindowsForms>true</UseWindowsForms>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Bonsai.Core" Version="2.9.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Bonsai.ML.Lds.Torch\Bonsai.ML.Lds.Torch.csproj" />
+    <ProjectReference Include="..\Bonsai.ML.Design\Bonsai.ML.Design.csproj" />
+  </ItemGroup>
+  <PropertyGroup>
+    <!-- This property is needed to avoid repacking the native SkiaSharp libraries, which should already be included with the Bonsai.ML.Torch -->
+    <ShouldIncludeNativeSkiaSharp>false</ShouldIncludeNativeSkiaSharp>
+  </PropertyGroup>
+</Project>

src/Bonsai.ML.Lds.Torch.Design/ExpectationMaximizationVisualizer.cs

@@ -0,0 +1,90 @@
+using System;
+using System.Reactive;
+using System.Linq;
+using System.Windows.Forms;
+using System.Collections.Generic;
+
+using Bonsai;
+using Bonsai.Design;
+using Bonsai.ML.Design;
+
+using OxyPlot;
+using OxyPlot.Series;
+
+using static TorchSharp.torch;
+
+[assembly: TypeVisualizer(typeof(Bonsai.ML.Lds.Torch.Design.ExpectationMaximizationVisualizer),
+    Target = typeof(Bonsai.ML.Lds.Torch.ExpectationMaximizationResult))]
+
+namespace Bonsai.ML.Lds.Torch.Design;
+
+/// <summary>
+/// Provides a visualizer for the state means and covariances from a Kalman filter or smoother.
+/// </summary>
+public class ExpectationMaximizationVisualizer : BufferedVisualizer
+{
+    private TimeSeriesOxyPlotBase _plot;
+    private LineSeries _lineSeries;
+
+    /// <summary>
+    /// Gets the underlying plot control.
+    /// </summary>
+    public TimeSeriesOxyPlotBase Plot => _plot;
+
+    /// <inheritdoc/>
+    public override void Load(IServiceProvider provider)
+    {
+        _plot = new TimeSeriesOxyPlotBase()
+        {
+            Dock = DockStyle.Fill,
+            StartTime = DateTime.Now,
+            BufferData = true,
+            ValueLabel = "Log Likelihood"
+        };
+
+        _lineSeries = _plot.AddNewLineSeries("Log Likelihood", OxyColors.Blue);
+
+        var visualizerService = (IDialogTypeVisualizerService)provider.GetService(typeof(IDialogTypeVisualizerService));
+        visualizerService?.AddControl(_plot);
+    }
+
+    /// <inheritdoc/>
+    public override void Show(object value)
+    {
+    }
+
+    /// <inheritdoc/>
+    protected override void Show(DateTime time, object value)
+    {
+        if (value is null) return;
+
+        if (value is not ExpectationMaximizationResult result) return;
+
+        var logLikelihood = result.LogLikelihood;
+        if (logLikelihood is null) return;
+
+        var ll = logLikelihood[-1].to_type(ScalarType.Float64).item<double>();
+
+        _plot.AddToLineSeries(
+            lineSeries: _lineSeries,
+            time: time,
+            value: ll
+        );
+    }
+
+    /// <inheritdoc/>
+    protected override void ShowBuffer(IList<Timestamped<object>> values)
+    {
+        base.ShowBuffer(values);
+        if (values.Count > 0)
+        {
+            _plot.UpdatePlot();
+        }
+    }
+
+    /// <inheritdoc/>
+    public override void Unload()
+    {
+        if (!_plot.IsDisposed) _plot.Dispose();
+    }
+}

src/Bonsai.ML.Lds.Torch.Design/Properties/AssemblyInfo.cs

@@ -0,0 +1,6 @@
+using Bonsai;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: XmlNamespacePrefix("clr-namespace:Bonsai.ML.Lds.Torch.Design", null)]

src/Bonsai.ML.Lds.Torch.Design/Properties/launchSettings.json

@@ -0,0 +1,10 @@
+{
+  "profiles": {
+    "Bonsai": {
+      "commandName": "Executable",
+      "executablePath": "$(BonsaiExecutablePath)",
+      "commandLineArgs": "--lib:\"$(TargetDir).\"",
+      "nativeDebugging": true
+    }
+  }
+}