Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions airflow/providers/apache/hive/hooks/hive.py
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,7 @@ class HiveMetastoreHook(BaseHook):

def __init__(self, metastore_conn_id: str = default_conn_name) -> None:
super().__init__()
self.conn_id = metastore_conn_id
self.conn = self.get_connection(metastore_conn_id)
self.metastore = self.get_metastore_client()

def __getstate__(self) -> Dict[str, Any]:
Expand All @@ -506,9 +506,10 @@ def get_metastore_client(self) -> Any:
from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket, TTransport

conn = self._find_valid_server()
host = self._find_valid_host()
conn = self.conn

if not conn:
if not host:
raise AirflowException("Failed to locate the valid server.")

auth_mechanism = conn.extra_dejson.get('authMechanism', 'NOSASL')
Expand All @@ -517,7 +518,7 @@ def get_metastore_client(self) -> Any:
auth_mechanism = conn.extra_dejson.get('authMechanism', 'GSSAPI')
kerberos_service_name = conn.extra_dejson.get('kerberos_service_name', 'hive')

conn_socket = TSocket.TSocket(conn.host, conn.port)
conn_socket = TSocket.TSocket(host, conn.port)

if conf.get('core', 'security') == 'kerberos' and auth_mechanism == 'GSSAPI':
try:
Expand All @@ -527,7 +528,7 @@ def get_metastore_client(self) -> Any:

def sasl_factory() -> sasl.Client:
sasl_client = sasl.Client()
sasl_client.setAttr("host", conn.host)
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", kerberos_service_name)
sasl_client.init()
return sasl_client
Expand All @@ -542,16 +543,18 @@ def sasl_factory() -> sasl.Client:

return hmsclient.HMSClient(iprot=protocol)

def _find_valid_server(self) -> Any:
conn = self.get_connection(self.conn_id)
host_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.log.info("Trying to connect to %s:%s", conn.host, conn.port)
if host_socket.connect_ex((conn.host, conn.port)) == 0:
self.log.info("Connected to %s:%s", conn.host, conn.port)
host_socket.close()
return conn
else:
self.log.error("Could not connect to %s:%s", conn.host, conn.port)
def _find_valid_host(self) -> Any:
conn = self.conn
hosts = conn.host.split(',')
for host in hosts:
host_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.log.info("Trying to connect to %s:%s", host, conn.port)
if host_socket.connect_ex((host, conn.port)) == 0:
self.log.info("Connected to %s:%s", host, conn.port)
host_socket.close()
return host
else:
self.log.error("Could not connect to %s:%s", host, conn.port)
return None

def get_conn(self) -> Any:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ Configuring the Connection
--------------------------

Host (optional)
The host of your Hive Metastore node.
The host of your Hive Metastore node. It is possible to specify multiple hosts as a comma-separated list.

Port (optional)
Your Hive Metastore port number.
Expand Down
30 changes: 23 additions & 7 deletions tests/providers/apache/hive/hooks/test_hive.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ def setUp(self):
self.table = 'static_babynames_partitioned'
with mock.patch(
'airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_metastore_client'
) as get_metastore_mock:
) as get_metastore_mock, mock.patch(
'airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection'
):
get_metastore_mock.return_value = mock.MagicMock()

self.hook = HiveMetastoreHook()
Expand Down Expand Up @@ -400,19 +402,33 @@ def test_get_max_partition_from_valid_part_specs_return_type(self):
assert isinstance(max_partition, str)

@mock.patch(
"airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection",
return_value=Connection(host="localhost", port=9802),
"airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook._find_valid_host",
return_value="localhost",
)
@mock.patch("airflow.providers.apache.hive.hooks.hive.socket")
def test_error_metastore_client(self, socket_mock, _find_valid_server_mock):
def test_error_metastore_client(self, socket_mock, _find_valid_host_mock):
socket_mock.socket.return_value.connect_ex.return_value = 0
self.hook.get_metastore_client()

@mock.patch(
"airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection",
return_value=Connection(host="metastore1.host,metastore2.host", port=9802),
)
@mock.patch("airflow.providers.apache.hive.hooks.hive.socket")
def test_ha_hosts(self, socket_mock, get_connection_mock):
socket_mock.socket.return_value.connect_ex.return_value = 1
with pytest.raises(AirflowException):
HiveMetastoreHook()
assert socket_mock.socket.call_count == 2

def test_get_conn(self):
with mock.patch(
'airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook._find_valid_server'
) as find_valid_server:
find_valid_server.return_value = mock.MagicMock(return_value={})
'airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook._find_valid_host'
) as find_valid_host, mock.patch(
'airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook.get_connection'
) as get_connection:
find_valid_host.return_value = mock.MagicMock(return_value="")
get_connection.return_value = mock.MagicMock(return_value="")
metastore_hook = HiveMetastoreHook()

assert isinstance(metastore_hook.get_conn(), HMSClient)
Expand Down