Removing support for deprecated pandas SparseDataFrame

.travis.yml

@@ -15,7 +15,6 @@ env:
   - TEST_DIR=/tmp/test_dir/
   - MODULE=openml
   matrix:
-  - DISTRIB="conda" PYTHON_VERSION="3.5" SKLEARN_VERSION="0.21.2"
   - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.21.2" TEST_DIST="true"
   - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.21.2" RUN_FLAKE8="true" SKIP_TESTS="true"
   - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.21.2" COVERAGE="true" DOCPUSH="true"

appveyor.yml

@@ -5,10 +5,10 @@ environment:
 #     CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\scikit-learn-contrib\\run_with_env.cmd"
 
  matrix:
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5"
+    - PYTHON: "C:\\Python3-x64"
+      PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
-      MINICONDA: "C:\\Miniconda35-x64"
+      MINICONDA: "C:\\Miniconda36-x64"
 
 matrix:
     fast_finish: true

doc/progress.rst

@@ -10,11 +10,14 @@ Changelog
 ~~~~~~
 
 * FIX #873: Fixes an issue which resulted in incorrect URLs when printing OpenML objects after
-  switching the server
+  switching the server.
 * FIX #885: Logger no longer registered by default. Added utility functions to easily register
   logging to console and file.
 * MAINT #767: Source distribution installation is now unit-tested.
+* MAINT #836: OpenML supports only pandas version 1.0.0 or above to match the removal
+  of SparseDataFrame support.
 * MAINT #865: OpenML no longer bundles test files in the source distribution.
+* MAINT #897: Dropping support for Python 3.5.
 
 0.10.2
 ~~~~~~

examples/30_extended/create_upload_tutorial.py

@@ -291,7 +291,7 @@
     ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1])
 ))
 column_names = ['input1', 'input2', 'y']
-df = pd.SparseDataFrame(sparse_data, columns=column_names)
+df = pd.DataFrame.sparse.from_spmatrix(sparse_data, columns=column_names)
 print(df.info())
 
 xor_dataset = create_dataset(

examples/30_extended/datasets_tutorial.py

@@ -68,7 +68,7 @@
 # Get the actual data.
 #
 # The dataset can be returned in 2 possible formats: as a NumPy array, a SciPy
-# sparse matrix, or as a Pandas DataFrame (or SparseDataFrame). The format is
+# sparse matrix, or as a Pandas DataFrame. The format is
 # controlled with the parameter ``dataset_format`` which can be either 'array'
 # (default) or 'dataframe'. Let's first build our dataset from a NumPy array
 # and manually create a dataframe.

openml/datasets/dataset.py

@@ -14,6 +14,7 @@
 import pandas as pd
 import scipy.sparse
 from warnings import warn
+from distutils.version import LooseVersion
 
 from openml.base import OpenMLBase
 from .data_feature import OpenMLDataFeature
@@ -509,7 +510,12 @@ def _encode_if_category(column):
                 )
         elif array_format == "dataframe":
             if scipy.sparse.issparse(data):
-                return pd.SparseDataFrame(data, columns=attribute_names)
+                # SparseDataFrame removed in pandas 1.0.0
+                if LooseVersion(pd.__version__) >= "1.0.0":
+                    return pd.DataFrame.sparse.from_spmatrix(data, columns=attribute_names)
+                else:
+                    raise Exception("Current pandas version found {}. OpenML supports pandas "
+                                    "1.0.0 or higher.".format(LooseVersion(pd.__version__)))
             else:
                 return data
         else:
@@ -560,7 +566,7 @@ def get_data(
         dataset_format : string (default='dataframe')
             The format of returned dataset.
             If ``array``, the returned dataset will be a NumPy array or a SciPy sparse matrix.
-            If ``dataframe``, the returned dataset will be a Pandas DataFrame or SparseDataFrame.
+            If ``dataframe``, the returned dataset will be a Pandas DataFrame.
 
         Returns
         -------

openml/datasets/functions.py

@@ -664,7 +664,7 @@ def create_dataset(name, description, creator, contributor,
     class:`openml.OpenMLDataset`
         Dataset description."""
 
-    if isinstance(data, (pd.DataFrame, pd.SparseDataFrame)):
+    if isinstance(data, pd.DataFrame):
         # infer the row id from the index of the dataset
         if row_id_attribute is None:
             row_id_attribute = data.index.name
@@ -676,8 +676,7 @@ def create_dataset(name, description, creator, contributor,
     if attributes == 'auto' or isinstance(attributes, dict):
         if not hasattr(data, "columns"):
             raise ValueError("Automatically inferring attributes requires "
-                             "a pandas DataFrame or SparseDataFrame. "
-                             "A {!r} was given instead.".format(data))
+                             "a pandas DataFrame. A {!r} was given instead.".format(data))
         # infer the type of data for each column of the DataFrame
         attributes_ = attributes_arff_from_df(data)
         if isinstance(attributes, dict):

setup.py

@@ -9,9 +9,9 @@
 with open("openml/__version__.py") as fh:
     version = fh.readlines()[-1].split()[-1].strip("\"'")
 
-if sys.version_info < (3, 5):
+if sys.version_info < (3, 6):
     raise ValueError(
-        'Unsupported Python version {}.{}.{} found. OpenML requires Python 3.5 or higher.'
+        'Unsupported Python version {}.{}.{} found. OpenML requires Python 3.6 or higher.'
         .format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
     )
 
@@ -42,14 +42,14 @@
                      exclude=["*.tests", "*.tests.*", "tests.*", "tests"],
                  ),
                  package_data={'': ['*.txt', '*.md']},
-                 python_requires=">=3.5",
+                 python_requires=">=3.6",
                  install_requires=[
                      'liac-arff>=2.4.0',
                      'xmltodict',
                      'requests',
                      'scikit-learn>=0.18',
                      'python-dateutil',  # Installed through pandas anyway.
-                     'pandas>=0.19.2',
+                     'pandas>=1.0.0',
                      'scipy>=0.13.3',
                      'numpy>=1.6.2'
                  ],
@@ -90,7 +90,5 @@
                               'Operating System :: POSIX',
                               'Operating System :: Unix',
                               'Operating System :: MacOS',
-                              'Programming Language :: Python :: 3',
-                              'Programming Language :: Python :: 3.5',
                               'Programming Language :: Python :: 3.6',
                               'Programming Language :: Python :: 3.7'])

tests/test_datasets/test_dataset.py

@@ -8,6 +8,9 @@
 import pytest
 from scipy import sparse
 
+import unittest
+from distutils.version import LooseVersion
+
 import openml
 from openml.testing import TestBase
 from openml.exceptions import PyOpenMLError
@@ -284,9 +287,14 @@ def test_get_sparse_dataset(self):
         self.assertEqual(len(attribute_names), 20001)
         self.assertTrue(all([isinstance(att, str) for att in attribute_names]))
 
+    @unittest.skipIf(LooseVersion(pd.__version__) < "1.0.0",
+                     reason="SparseDataFrame support removed from pandas 1.0.0 and onwards.")
     def test_get_sparse_dataframe(self):
         rval, *_ = self.sparse_dataset.get_data()
-        self.assertTrue(isinstance(rval, pd.SparseDataFrame))
+        dtypes = all([dtype == pd.SparseDtype(np.float32, fill_value=0.0)
+                      for dtype in rval.dtypes])
+        self.assertTrue(isinstance(rval, pd.DataFrame))
+        self.assertTrue(dtypes)
         self.assertEqual((600, 20001), rval.shape)
 
     def test_get_sparse_dataset_with_rowid(self):

tests/test_datasets/test_dataset_functions.py

@@ -561,12 +561,9 @@ def test_attributes_arff_from_df(self):
                                       ('string', 'STRING'),
                                       ('category', ['A', 'B']),
                                       ('boolean', ['True', 'False'])])
-        # SparseDataFrame case
-        df = pd.SparseDataFrame([[1, 1.0],
-                                 [2, 2.0],
-                                 [0, 0]],
-                                columns=['integer', 'floating'],
-                                default_fill_value=0)
+        # DataFrame with Sparse columns case
+        df = pd.DataFrame({"integer": pd.arrays.SparseArray([1, 2, 0], fill_value=0),
+                           "floating": pd.arrays.SparseArray([1.0, 2.0, 0], fill_value=0.0)})
         df['integer'] = df['integer'].astype(np.int64)
         attributes = attributes_arff_from_df(df)
         self.assertEqual(attributes, [('integer', 'INTEGER'),
@@ -925,15 +922,15 @@ def test_create_dataset_pandas(self):
             "Uploaded ARFF does not match original one"
         )
 
-        # Check that SparseDataFrame are supported properly
+        # Check that DataFrame with Sparse columns are supported properly
         sparse_data = scipy.sparse.coo_matrix((
             [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
             ([0, 1, 1, 2, 2, 3, 3], [0, 1, 2, 0, 2, 0, 1])
         ))
         column_names = ['input1', 'input2', 'y']
-        df = pd.SparseDataFrame(sparse_data, columns=column_names)
+        df = pd.DataFrame.sparse.from_spmatrix(sparse_data, columns=column_names)
         # meta-information
-        description = 'Synthetic dataset created from a Pandas SparseDataFrame'
+        description = 'Synthetic dataset created from a Pandas DataFrame with Sparse columns'
         dataset = openml.datasets.functions.create_dataset(
             name=name,
             description=description,
@@ -963,7 +960,7 @@ def test_create_dataset_pandas(self):
         )
         self.assertEqual(
             _get_online_dataset_format(dataset.id),
-            'sparse_arff',
+            'arff',
             "Wrong format for dataset"
         )