connection.py

# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Google Cloud Bigtable HappyBase connection module."""


import datetime
import warnings

import six

from grpc.beta import interfaces
from grpc.framework.interfaces.face import face

try:
    from happybase.hbase.ttypes import AlreadyExists
except ImportError:
    from gcloud.exceptions import Conflict as AlreadyExists

from gcloud.bigtable.client import Client
from gcloud.bigtable.column_family import GCRuleIntersection
from gcloud.bigtable.column_family import MaxAgeGCRule
from gcloud.bigtable.column_family import MaxVersionsGCRule
from gcloud.bigtable.happybase.table import Table
from gcloud.bigtable.table import Table as _LowLevelTable


# Constants reproduced here for HappyBase compatibility, though values
# are all null.
COMPAT_MODES = None
THRIFT_TRANSPORTS = None
THRIFT_PROTOCOLS = None
DEFAULT_HOST = None
DEFAULT_PORT = None
DEFAULT_TRANSPORT = None
DEFAULT_COMPAT = None
DEFAULT_PROTOCOL = None

_LEGACY_ARGS = frozenset(('host', 'port', 'compat', 'transport', 'protocol'))
_WARN = warnings.warn
_DISABLE_DELETE_MSG = ('The disable argument should not be used in '
                       'delete_table(). Cloud Bigtable has no concept '
                       'of enabled / disabled tables.')


def _get_instance(timeout=None):
    """Gets instance for the default project.

    Creates a client with the inferred credentials and project ID from
    the local environment. Then uses
    :meth:`.bigtable.client.Client.list_instances` to
    get the unique instance owned by the project.

    If the request fails for any reason, or if there isn't exactly one instance
    owned by the project, then this function will fail.

    :type timeout: int
    :param timeout: (Optional) The socket timeout in milliseconds.

    :rtype: :class:`gcloud.bigtable.instance.Instance`
    :returns: The unique instance owned by the project inferred from
              the environment.
    :raises ValueError: if there is a failed location or any number of
                        instances other than one.
    """
    client_kwargs = {'admin': True}
    if timeout is not None:
        client_kwargs['timeout_seconds'] = timeout / 1000.0
    client = Client(**client_kwargs)
    try:
        client.start()
        instances, failed_locations = client.list_instances()
    finally:
        client.stop()

    if len(failed_locations) != 0:
        raise ValueError('Determining instance via ListInstances encountered '
                         'failed locations.')
    if len(instances) == 0:
        raise ValueError('This client doesn\'t have access to any instances.')
    if len(instances) > 1:
        raise ValueError('This client has access to more than one instance. '
                         'Please directly pass the instance you\'d '
                         'like to use.')
    return instances[0]


class Connection(object):
    """Connection to Cloud Bigtable backend.

    .. note::

        If you pass a ``instance``, it will be :meth:`.Instance.copy`-ed before
        being stored on the new connection. This also copies the
        :class:`Client <gcloud.bigtable.client.Client>` that created the
        :class:`Instance <gcloud.bigtable.instance.Instance>` instance and the
        :class:`Credentials <oauth2client.client.Credentials>` stored on the
        client.

    The arguments ``host``, ``port``, ``compat``, ``transport`` and
    ``protocol`` are allowed (as keyword arguments) for compatibility with
    HappyBase. However, they will not be used in any way, and will cause a
    warning if passed.

    :type timeout: int
    :param timeout: (Optional) The socket timeout in milliseconds.

    :type autoconnect: bool
    :param autoconnect: (Optional) Whether the connection should be
                        :meth:`open`-ed during construction.

    :type table_prefix: str
    :param table_prefix: (Optional) Prefix used to construct table names.

    :type table_prefix_separator: str
    :param table_prefix_separator: (Optional) Separator used with
                                   ``table_prefix``. Defaults to ``_``.

    :type instance: :class:`Instance <gcloud.bigtable.instance.Instance>`
    :param instance: (Optional) A Cloud Bigtable instance. The instance also
                    owns a client for making gRPC requests to the Cloud
                    Bigtable API. If not passed in, defaults to creating client
                    with ``admin=True`` and using the ``timeout`` here for the
                    ``timeout_seconds`` argument to the
                    :class:`Client <gcloud.bigtable.client.Client>`
                    constructor. The credentials for the client
                    will be the implicit ones loaded from the environment.
                    Then that client is used to retrieve all the instances
                    owned by the client's project.

    :type kwargs: dict
    :param kwargs: Remaining keyword arguments. Provided for HappyBase
                   compatibility.
    """

    _instance = None

    def __init__(self, timeout=None, autoconnect=True, table_prefix=None,
                 table_prefix_separator='_', instance=None, **kwargs):
        self._handle_legacy_args(kwargs)
        if table_prefix is not None:
            if not isinstance(table_prefix, six.string_types):
                raise TypeError('table_prefix must be a string', 'received',
                                table_prefix, type(table_prefix))

        if not isinstance(table_prefix_separator, six.string_types):
            raise TypeError('table_prefix_separator must be a string',
                            'received', table_prefix_separator,
                            type(table_prefix_separator))

        self.table_prefix = table_prefix
        self.table_prefix_separator = table_prefix_separator

        if instance is None:
            self._instance = _get_instance(timeout=timeout)
        else:
            if timeout is not None:
                raise ValueError('Timeout cannot be used when an existing '
                                 'instance is passed')
            self._instance = instance.copy()

        if autoconnect:
            self.open()

        self._initialized = True

    @staticmethod
    def _handle_legacy_args(arguments_dict):
        """Check legacy HappyBase arguments and warn if set.

        :type arguments_dict: dict
        :param arguments_dict: Unused keyword arguments.

        :raises TypeError: if a keyword other than ``host``, ``port``,
                           ``compat``, ``transport`` or ``protocol`` is used.
        """
        common_args = _LEGACY_ARGS.intersection(six.iterkeys(arguments_dict))
        if common_args:
            all_args = ', '.join(common_args)
            message = ('The HappyBase legacy arguments %s were used. These '
                       'arguments are unused by gcloud.' % (all_args,))
            _WARN(message)
        for arg_name in common_args:
            arguments_dict.pop(arg_name)
        if arguments_dict:
            unexpected_names = arguments_dict.keys()
            raise TypeError('Received unexpected arguments', unexpected_names)

    def open(self):
        """Open the underlying transport to Cloud Bigtable.

        This method opens the underlying HTTP/2 gRPC connection using a
        :class:`Client <gcloud.bigtable.client.Client>` bound to the
        :class:`Instance <gcloud.bigtable.instance.Instance>` owned by
        this connection.
        """
        self._instance._client.start()

    def close(self):
        """Close the underlying transport to Cloud Bigtable.

        This method closes the underlying HTTP/2 gRPC connection using a
        :class:`Client <gcloud.bigtable.client.Client>` bound to the
        :class:`Instance <gcloud.bigtable.instance.Instance>` owned by
        this connection.
        """
        self._instance._client.stop()

    def __del__(self):
        if self._instance is not None:
            self.close()

    def _table_name(self, name):
        """Construct a table name by optionally adding a table name prefix.

        :type name: str
        :param name: The name to have a prefix added to it.

        :rtype: str
        :returns: The prefixed name, if the current connection has a table
                  prefix set.
        """
        if self.table_prefix is None:
            return name

        return self.table_prefix + self.table_prefix_separator + name

    def table(self, name, use_prefix=True):
        """Table factory.

        :type name: str
        :param name: The name of the table to be created.

        :type use_prefix: bool
        :param use_prefix: Whether to use the table prefix (if any).

        :rtype: :class:`Table <gcloud.bigtable.happybase.table.Table>`
        :returns: Table instance owned by this connection.
        """
        if use_prefix:
            name = self._table_name(name)
        return Table(name, self)

    def tables(self):
        """Return a list of table names available to this connection.

        .. note::

            This lists every table in the instance owned by this connection,
            **not** every table that a given user may have access to.

        .. note::

            If ``table_prefix`` is set on this connection, only returns the
            table names which match that prefix.

        :rtype: list
        :returns: List of string table names.
        """
        low_level_table_instances = self._instance.list_tables()
        table_names = [table_instance.table_id
                       for table_instance in low_level_table_instances]

        # Filter using prefix, and strip prefix from names
        if self.table_prefix is not None:
            prefix = self._table_name('')
            offset = len(prefix)
            table_names = [name[offset:] for name in table_names
                           if name.startswith(prefix)]

        return table_names

    def create_table(self, name, families):
        """Create a table.

        .. warning::

            The only column family options from HappyBase that are able to be
            used with Cloud Bigtable are ``max_versions`` and ``time_to_live``.

        .. note::

            This method is **not** atomic. The Cloud Bigtable API separates
            the creation of a table from the creation of column families. Thus
            this method needs to send 1 request for the table creation and 1
            request for each column family. If any of these fails, the method
            will fail, but the progress made towards completion cannot be
            rolled back.

        Values in ``families`` represent column family options. In HappyBase,
        these are dictionaries, corresponding to the ``ColumnDescriptor``
        structure in the Thrift API. The accepted keys are:

        * ``max_versions`` (``int``)
        * ``compression`` (``str``)
        * ``in_memory`` (``bool``)
        * ``bloom_filter_type`` (``str``)
        * ``bloom_filter_vector_size`` (``int``)
        * ``bloom_filter_nb_hashes`` (``int``)
        * ``block_cache_enabled`` (``bool``)
        * ``time_to_live`` (``int``)

        :type name: str
        :param name: The name of the table to be created.

        :type families: dict
        :param families: Dictionary with column family names as keys and column
                         family options as the values. The options can be among

                         * :class:`dict`
                         * :class:`.GarbageCollectionRule`

        :raises TypeError: If ``families`` is not a dictionary.
        :raises ValueError: If ``families`` has no entries.
        :raises AlreadyExists: If creation fails due to an already
                               existing table.
        :raises NetworkError: If creation fails for a reason other than
                              table exists.
        """
        if not isinstance(families, dict):
            raise TypeError('families arg must be a dictionary')

        if not families:
            raise ValueError('Cannot create table %r (no column '
                             'families specified)' % (name,))

        # Parse all keys before making any API requests.
        gc_rule_dict = {}
        for column_family_name, option in families.items():
            if isinstance(column_family_name, six.binary_type):
                column_family_name = column_family_name.decode('utf-8')
            if column_family_name.endswith(':'):
                column_family_name = column_family_name[:-1]
            gc_rule_dict[column_family_name] = _parse_family_option(option)

        # Create table instance and then make API calls.
        name = self._table_name(name)
        low_level_table = _LowLevelTable(name, self._instance)
        try:
            low_level_table.create()
        except face.NetworkError as network_err:
            if network_err.code == interfaces.StatusCode.ALREADY_EXISTS:
                raise AlreadyExists(name)
            else:
                raise

        for column_family_name, gc_rule in gc_rule_dict.items():
            column_family = low_level_table.column_family(
                column_family_name, gc_rule=gc_rule)
            column_family.create()

    def delete_table(self, name, disable=False):
        """Delete the specified table.

        :type name: str
        :param name: The name of the table to be deleted. If ``table_prefix``
                     is set, a prefix will be added to the ``name``.

        :type disable: bool
        :param disable: Whether to first disable the table if needed. This
                        is provided for compatibility with HappyBase, but is
                        not relevant for Cloud Bigtable since it has no concept
                        of enabled / disabled tables.
        """
        if disable:
            _WARN(_DISABLE_DELETE_MSG)

        name = self._table_name(name)
        _LowLevelTable(name, self._instance).delete()

    def enable_table(self, name):
        """Enable the specified table.

        .. warning::

            Cloud Bigtable has no concept of enabled / disabled tables so this
            method does not work. It is provided simply for compatibility.

        :raises: :class:`NotImplementedError <exceptions.NotImplementedError>`
                 always
        """
        raise NotImplementedError('The Cloud Bigtable API has no concept of '
                                  'enabled or disabled tables.')

    def disable_table(self, name):
        """Disable the specified table.

        .. warning::

            Cloud Bigtable has no concept of enabled / disabled tables so this
            method does not work. It is provided simply for compatibility.

        :raises: :class:`NotImplementedError <exceptions.NotImplementedError>`
                 always
        """
        raise NotImplementedError('The Cloud Bigtable API has no concept of '
                                  'enabled or disabled tables.')

    def is_table_enabled(self, name):
        """Return whether the specified table is enabled.

        .. warning::

            Cloud Bigtable has no concept of enabled / disabled tables so this
            method does not work. It is provided simply for compatibility.

        :raises: :class:`NotImplementedError <exceptions.NotImplementedError>`
                 always
        """
        raise NotImplementedError('The Cloud Bigtable API has no concept of '
                                  'enabled or disabled tables.')

    def compact_table(self, name, major=False):
        """Compact the specified table.

        .. warning::

            Cloud Bigtable does not support compacting a table, so this
            method does not work. It is provided simply for compatibility.

        :raises: :class:`NotImplementedError <exceptions.NotImplementedError>`
                 always
        """
        raise NotImplementedError('The Cloud Bigtable API does not support '
                                  'compacting a table.')


def _parse_family_option(option):
    """Parses a column family option into a garbage collection rule.

    .. note::

        If ``option`` is not a dictionary, the type is not checked.
        If ``option`` is :data:`None`, there is nothing to do, since this
        is the correct output.

    :type option: :class:`dict`,
                  :data:`NoneType <types.NoneType>`,
                  :class:`.GarbageCollectionRule`
    :param option: A column family option passes as a dictionary value in
                   :meth:`Connection.create_table`.

    :rtype: :class:`.GarbageCollectionRule`
    :returns: A garbage collection rule parsed from the input.
    """
    result = option
    if isinstance(result, dict):
        if not set(result.keys()) <= set(['max_versions', 'time_to_live']):
            all_keys = ', '.join(repr(key) for key in result.keys())
            warning_msg = ('Cloud Bigtable only supports max_versions and '
                           'time_to_live column family settings. '
                           'Received: %s' % (all_keys,))
            _WARN(warning_msg)

        max_num_versions = result.get('max_versions')
        max_age = None
        if 'time_to_live' in result:
            max_age = datetime.timedelta(seconds=result['time_to_live'])

        versions_rule = age_rule = None
        if max_num_versions is not None:
            versions_rule = MaxVersionsGCRule(max_num_versions)
        if max_age is not None:
            age_rule = MaxAgeGCRule(max_age)

        if versions_rule is None:
            result = age_rule
        else:
            if age_rule is None:
                result = versions_rule
            else:
                result = GCRuleIntersection(rules=[age_rule, versions_rule])

    return result