IDataView to DataFrame

src/Microsoft.Data.Analysis/DataFrame.IDataView.cs

@@ -11,7 +11,7 @@
 namespace Microsoft.Data.Analysis
 {
     public partial class DataFrame : IDataView
-    {        
+    {
         // TODO: support shuffling
         bool IDataView.CanShuffle => false;
 

src/Microsoft.Data.Analysis/DataFrameColumn.cs

@@ -247,6 +247,13 @@ public virtual DataFrameColumn Sort(bool ascending = true)
         /// </param>
         protected internal virtual void AddDataViewColumn(DataViewSchema.Builder builder) => throw new NotImplementedException();
 
+        /// <summary>
+        /// Appends a value to this <see cref="DataFrameColumn"/> using <paramref name="cursor"/>
+        /// </summary>
+        /// <param name="cursor">The row cursor which has the current position</param>
+        /// <param name="schemaColumn">The <see cref="DataViewSchema.Column"/> in <see cref="DataViewSchema"/></param>
+        protected internal virtual void AddValueUsingCursor(DataViewRowCursor cursor, DataViewSchema.Column schemaColumn) => throw new NotImplementedException();
+
         /// <summary>
         /// Clamps values beyond the specified thresholds
         /// </summary>

src/Microsoft.Data.Analysis/IDataView.Extension.cs

@@ -0,0 +1,115 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+
+namespace Microsoft.Data.Analysis
+{
+    public static class IDataViewExtensions
+    {
+        public static DataFrame ToDataFrame(this IDataView dataView, long maxRows = -1)
+        {
+            return ToDataFrame(dataView, maxRows, null);
+        }
+
+        public static DataFrame ToDataFrame(this IDataView dataView, params string[] selectColumns)
+        {
+            return ToDataFrame(dataView, -1, selectColumns);
+        }
+
+        public static DataFrame ToDataFrame(this IDataView dataView, long maxRows, params string[] selectColumns)
+        {
+            DataViewSchema schema = dataView.Schema;
+            List<DataFrameColumn> columns = new List<DataFrameColumn>(schema.Count);
+
+            HashSet<string> selectColumnsSet = null;
+            if (selectColumns != null && selectColumns.Length > 0)
+            {
+                selectColumnsSet = new HashSet<string>(selectColumns);
+            }
+
+            List<DataViewSchema.Column> activeColumns = new List<DataViewSchema.Column>();
+            foreach (DataViewSchema.Column column in schema)
+            {
+                long length = maxRows >= 0 ? maxRows : long.MaxValue;
+                length = Math.Min(length, dataView.GetRowCount() ?? 0);
+                if (column.IsHidden || (selectColumnsSet != null && !selectColumnsSet.Contains(column.Name)))
+                {
+                    continue;
+                }
+
+                activeColumns.Add(column);
+                DataViewType type = column.Type;
+                if (type == BooleanDataViewType.Instance)
+                {
+                    columns.Add(new BooleanDataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.Byte)
+                {
+                    columns.Add(new ByteDataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.Double)
+                {
+                    columns.Add(new DoubleDataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.Single)
+                {
+                    columns.Add(new SingleDataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.Int32)
+                {
+                    columns.Add(new Int32DataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.Int64)
+                {
+                    columns.Add(new Int64DataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.SByte)
+                {
+                    columns.Add(new SByteDataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.Int16)
+                {
+                    columns.Add(new Int16DataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.UInt32)
+                {
+                    columns.Add(new UInt32DataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.UInt64)
+                {
+                    columns.Add(new UInt64DataFrameColumn(column.Name, length));
+                }
+                else if (type == NumberDataViewType.UInt16)
+                {
+                    columns.Add(new UInt16DataFrameColumn(column.Name, length));
+                }
+                else if (type == TextDataViewType.Instance)
+                {
+                    columns.Add(new StringDataFrameColumn(column.Name, length));
+                }
+                else
+                {
+                    throw new NotSupportedException(nameof(type));
+                }
+            }
+
+            DataFrame ret = new DataFrame(columns);
+            DataViewRowCursor cursor = dataView.GetRowCursor(activeColumns);
+            while (cursor.MoveNext())
+            {
+                foreach (DataViewSchema.Column column in activeColumns)
+                {
+                    ret[column.Name].AddValueUsingCursor(cursor, column);
+                }
+            }
+
+            return ret;
+        }
+    }
+
+}

src/Microsoft.Data.Analysis/PrimitiveDataFrameColumn.cs

@@ -775,5 +775,30 @@ private static ValueGetter<ushort> CreateCharValueGetterDelegate(DataViewRowCurs
 
         private static ValueGetter<double> CreateDecimalValueGetterDelegate(DataViewRowCursor cursor, PrimitiveDataFrameColumn<decimal> column) =>
             (ref double value) => value = (double?)column[cursor.Position] ?? double.NaN;
+
+        private ValueGetter<T> getter = null;
+
+        protected internal override void AddValueUsingCursor(DataViewRowCursor cursor, DataViewSchema.Column column)
+        {
+            long row = cursor.Position;
+            T value = default;
+            if (getter == null)
+            {
+                getter = cursor.GetGetter<T>(column);
+            }
+            getter(ref value);
+            if (Length > row)
+            {
+                this[row] = value;
+            }
+            else if (Length == row)
+            {
+                Append(value);
+            }
+            else
+            {
+                throw new IndexOutOfRangeException(nameof(row));
+            }
+        }
     }
 }

src/Microsoft.Data.Analysis/StringDataFrameColumn.cs

@@ -467,5 +467,30 @@ protected internal override Delegate GetDataViewGetter(DataViewRowCursor cursor)
 
         private ValueGetter<ReadOnlyMemory<char>> CreateValueGetterDelegate(DataViewRowCursor cursor) =>
             (ref ReadOnlyMemory<char> value) => value = this[cursor.Position].AsMemory();
+
+        private ValueGetter<ReadOnlyMemory<char>> getter = null;
+
+        protected internal override void AddValueUsingCursor(DataViewRowCursor cursor, DataViewSchema.Column schemaColumn)
+        {
+            long row = cursor.Position;
+            ReadOnlyMemory<char> value = default;
+            if (getter == null)
+            {
+                getter = cursor.GetGetter<ReadOnlyMemory<char>>(schemaColumn);
+            }
+            getter(ref value);
+            if (Length > row)
+            {
+                this[row] = value.ToString();
+            }
+            else if (Length == row)
+            {
+                Append(value.ToString());
+            }
+            else
+            {
+                throw new IndexOutOfRangeException(nameof(row));
+            }
+        }
     }
 }

test/Microsoft.Data.Analysis.Tests/DataFrameTests.IDataView.cs

@@ -10,12 +10,12 @@
 
 namespace Microsoft.Data.Analysis.Tests
 {
-    public partial class DataFrameTests
+    public partial class DataFrameIDataViewTests
     {
         [Fact]
         public void TestIDataView()
         {
-            IDataView dataView = MakeDataFrameWithAllColumnTypes(10, withNulls: false);
+            IDataView dataView = DataFrameTests.MakeDataFrameWithAllColumnTypes(10, withNulls: false);
 
             DataDebuggerPreview preview = dataView.Preview();
             Assert.Equal(10, preview.RowView.Length);
@@ -85,7 +85,7 @@ public void TestIDataView()
         [Fact]
         public void TestIDataViewSchemaInvalidate()
         {
-            DataFrame df = MakeDataFrameWithAllMutableColumnTypes(10, withNulls: false);
+            DataFrame df = DataFrameTests.MakeDataFrameWithAllMutableColumnTypes(10, withNulls: false);
 
             IDataView dataView = df;
 
@@ -113,7 +113,7 @@ public void TestIDataViewSchemaInvalidate()
         public void TestIDataViewWithNulls()
         {
             int length = 10;
-            IDataView dataView = MakeDataFrameWithAllColumnTypes(length, withNulls: true);
+            IDataView dataView = DataFrameTests.MakeDataFrameWithAllColumnTypes(length, withNulls: true);
 
             DataDebuggerPreview preview = dataView.Preview();
             Assert.Equal(length, preview.RowView.Length);
@@ -224,5 +224,59 @@ public void TestIDataViewWithNulls()
             Assert.Equal("", preview.ColumnView[14].Values[5].ToString()); // null row
             Assert.Equal("foo", preview.ColumnView[14].Values[6].ToString());
         }
+
+        [Fact]
+        public void TestDataFrameFromIDataView()
+        {
+            DataFrame df = DataFrameTests.MakeDataFrameWithAllColumnTypes(10, withNulls: false);
+            df.Columns.Remove("Char"); // Because chars are returned as uint16 by IDataView, so end up comparing CharDataFrameColumn to UInt16DataFrameColumn and fail asserts
+            IDataView dfAsIDataView = df;
+            DataFrame newDf = dfAsIDataView.ToDataFrame();
+            Assert.Equal(dfAsIDataView.GetRowCount(), newDf.Rows.Count);
+            Assert.Equal(dfAsIDataView.Schema.Count, newDf.Columns.Count);
+            for (int i = 0; i < df.Columns.Count; i++)
+            {
+                Assert.True(df.Columns[i].ElementwiseEquals(newDf.Columns[i]).All());
+            }
+        }
+
+        [Fact]
+        public void TestDataFrameFromIDataView_SelectColumns()
+        {
+            DataFrame df = DataFrameTests.MakeDataFrameWithAllColumnTypes(10, withNulls: false);
+            IDataView dfAsIDataView = df;
+            DataFrame newDf = dfAsIDataView.ToDataFrame("Int", "Double");
+            Assert.Equal(dfAsIDataView.GetRowCount(), newDf.Rows.Count);
+            Assert.Equal(2, newDf.Columns.Count);
+            Assert.True(df.Columns["Int"].ElementwiseEquals(newDf.Columns["Int"]).All());
+            Assert.True(df.Columns["Double"].ElementwiseEquals(newDf.Columns["Double"]).All());
+        }
+
+        [Fact]
+        public void TestDataFrameFromIDataView_SelectRows()
+        {
+            DataFrame df = DataFrameTests.MakeDataFrameWithAllColumnTypes(10, withNulls: false);
+            df.Columns.Remove("Char"); // Because chars are returned as uint16 by IDataView, so end up comparing CharDataFrameColumn to UInt16DataFrameColumn and fail asserts
+            IDataView dfAsIDataView = df;
+            DataFrame newDf = dfAsIDataView.ToDataFrame(5);
+            Assert.Equal(5, newDf.Rows.Count);
+            Assert.Equal(dfAsIDataView.Schema.Count, newDf.Columns.Count);
+            for (int i = 0; i < df.Columns.Count; i++)
+            {
+                Assert.True(df.Columns[i].ElementwiseEquals(newDf.Columns[i]).All());
+            }
+        }
+
+        [Fact]
+        public void TestDataFrameFromIDataView_SelectColumnsAndRows()
+        {
+            DataFrame df = DataFrameTests.MakeDataFrameWithAllColumnTypes(10, withNulls: false);
+            IDataView dfAsIDataView = df;
+            DataFrame newDf = dfAsIDataView.ToDataFrame(5, "Int", "Double");
+            Assert.Equal(5, newDf.Rows.Count);
+            Assert.Equal(2, newDf.Columns.Count);
+            Assert.True(df.Columns["Int"].ElementwiseEquals(newDf.Columns["Int"]).All());
+            Assert.True(df.Columns["Double"].ElementwiseEquals(newDf.Columns["Double"]).All());
+        }
     }
 }