From 5cc245b5d3c540023cc312731073e8a5875a4b07 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 20 Jan 2021 20:20:20 +0100 Subject: [PATCH 01/15] Fix S3ToFTPOperator --- .../amazon/aws/transfers/s3_to_ftp.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index 3dfae24afcde3..9e3b11487360e 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -26,12 +26,11 @@ class S3ToFTPOperator(BaseOperator): """ This operator enables the transferring of files from S3 to a FTP server. - :param ftp_conn_id: The ftp connection id. The name or identifier for establishing a connection to the FTP server. :type ftp_conn_id: str - :param ftp_path: The ftp remote path. This is the specified file path for - uploading file to the FTP server. + :param ftp_path: The ftp remote path in where the file will be stored. + The desired filename bust be specified here. :type ftp_path: str :param s3_bucket: The targeted s3 bucket. This is the S3 bucket from where the file is downloaded. @@ -41,17 +40,17 @@ class S3ToFTPOperator(BaseOperator): :type s3_key: str """ - template_fields = ('s3_bucket', 's3_key') + template_fields = ('s3_bucket', 's3_key', 'ftp_path') def __init__( - self, - *, - s3_bucket, - s3_key, - ftp_path, - aws_conn_id='aws_default', - ftp_conn_id='ftp_default', - **kwargs, + self, + *, + s3_bucket, + s3_key, + ftp_path, + aws_conn_id='aws_default', + ftp_conn_id='ftp_default', + **kwargs, ) -> None: super().__init__(**kwargs) self.s3_bucket = s3_bucket @@ -64,8 +63,8 @@ def execute(self, context): s3_hook = S3Hook(self.aws_conn_id) ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id) - s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket) + s3_client = s3_hook.get_conn() with NamedTemporaryFile() as local_tmp_file: - s3_obj.download_fileobj(local_tmp_file) + s3_client.download_file(self.s3_bucket, self.s3_key, local_tmp_file.name) ftp_hook.store_file(self.ftp_path, local_tmp_file.name) From f8c949de7572cc3a3f6a3f0f7133ad01ffeb9dfc Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 20 Jan 2021 20:25:55 +0100 Subject: [PATCH 02/15] remove spaces --- .../providers/amazon/aws/transfers/s3_to_ftp.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index 9e3b11487360e..2d42c630034e6 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -43,14 +43,14 @@ class S3ToFTPOperator(BaseOperator): template_fields = ('s3_bucket', 's3_key', 'ftp_path') def __init__( - self, - *, - s3_bucket, - s3_key, - ftp_path, - aws_conn_id='aws_default', - ftp_conn_id='ftp_default', - **kwargs, + self, + *, + s3_bucket, + s3_key, + ftp_path, + aws_conn_id='aws_default', + ftp_conn_id='ftp_default', + **kwargs, ) -> None: super().__init__(**kwargs) self.s3_bucket = s3_bucket From 7dade6587b3a08d3d6123e147d2eed9c1c3b3d51 Mon Sep 17 00:00:00 2001 From: JavierLopezT Date: Wed, 20 Jan 2021 20:24:54 +0100 Subject: [PATCH 03/15] Update airflow/providers/amazon/aws/transfers/s3_to_ftp.py Co-authored-by: Jarek Potiuk --- airflow/providers/amazon/aws/transfers/s3_to_ftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index 2d42c630034e6..ba8882e9435a4 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -30,7 +30,7 @@ class S3ToFTPOperator(BaseOperator): establishing a connection to the FTP server. :type ftp_conn_id: str :param ftp_path: The ftp remote path in where the file will be stored. - The desired filename bust be specified here. + The desired filename must be specified here. :type ftp_path: str :param s3_bucket: The targeted s3 bucket. This is the S3 bucket from where the file is downloaded. From a1e8fae7440b940c31d5a7a4d262aaccd80919e7 Mon Sep 17 00:00:00 2001 From: JavierLopezT Date: Wed, 20 Jan 2021 22:50:30 +0100 Subject: [PATCH 04/15] Update airflow/providers/amazon/aws/transfers/s3_to_ftp.py Co-authored-by: RosterIn <48057736+RosterIn@users.noreply.github.com> --- airflow/providers/amazon/aws/transfers/s3_to_ftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index ba8882e9435a4..a7be4f2371309 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -30,7 +30,7 @@ class S3ToFTPOperator(BaseOperator): establishing a connection to the FTP server. :type ftp_conn_id: str :param ftp_path: The ftp remote path in where the file will be stored. - The desired filename must be specified here. + The desired file name must be specified here. :type ftp_path: str :param s3_bucket: The targeted s3 bucket. This is the S3 bucket from where the file is downloaded. From 545a980ca8e07faa61f90781a89a7a77c562ad87 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Thu, 21 Jan 2021 16:32:10 +0100 Subject: [PATCH 05/15] fixes --- .../providers/amazon/aws/transfers/test_s3_to_ftp.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py index f8ee5a45ab9fd..1965867168f3b 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py @@ -30,14 +30,15 @@ class TestS3ToFTPOperator(unittest.TestCase): @mock.patch("airflow.providers.ftp.hooks.ftp.FTPHook.store_file") - @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_key") + @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_conn.download_file") @mock.patch("airflow.providers.amazon.aws.transfers.s3_to_ftp.NamedTemporaryFile") - def test_execute(self, mock_local_tmp_file, mock_s3_hook_get_key, mock_ftp_hook_store_file): + def test_execute(self, mock_local_tmp_file, mock_s3_hook_download_file, mock_ftp_hook_store_file): operator = S3ToFTPOperator(task_id=TASK_ID, s3_bucket=BUCKET, s3_key=S3_KEY, ftp_path=FTP_PATH) operator.execute(None) - mock_s3_hook_get_key.assert_called_once_with(operator.s3_key, operator.s3_bucket) - mock_local_tmp_file_value = mock_local_tmp_file.return_value.__enter__.return_value - mock_s3_hook_get_key.return_value.download_fileobj.assert_called_once_with(mock_local_tmp_file_value) + + mock_s3_hook_download_file.assert_called_once_with( + operator.s3_bucket, operator.s3_key, mock_local_tmp_file_value.name + ) mock_ftp_hook_store_file.assert_called_once_with(operator.ftp_path, mock_local_tmp_file_value.name) From 0ae84db658e6cd37989f702c76821505e479563e Mon Sep 17 00:00:00 2001 From: JavierLopezT Date: Fri, 5 Feb 2021 08:03:33 +0100 Subject: [PATCH 06/15] Update airflow/providers/amazon/aws/transfers/s3_to_ftp.py Co-authored-by: Daniel Standish --- airflow/providers/amazon/aws/transfers/s3_to_ftp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index a7be4f2371309..e471fb325b339 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -29,8 +29,7 @@ class S3ToFTPOperator(BaseOperator): :param ftp_conn_id: The ftp connection id. The name or identifier for establishing a connection to the FTP server. :type ftp_conn_id: str - :param ftp_path: The ftp remote path in where the file will be stored. - The desired file name must be specified here. + :param ftp_path: The ftp remote path where the file will be stored, inclusive of filename. :type ftp_path: str :param s3_bucket: The targeted s3 bucket. This is the S3 bucket from where the file is downloaded. From c916a6332902311d02f97ada1a2d2fcbf5d59167 Mon Sep 17 00:00:00 2001 From: JavierLopezT Date: Mon, 8 Feb 2021 15:27:59 +0100 Subject: [PATCH 07/15] Update airflow/providers/amazon/aws/transfers/s3_to_ftp.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kamil BreguĊ‚a --- airflow/providers/amazon/aws/transfers/s3_to_ftp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index e471fb325b339..898494efee929 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -26,6 +26,7 @@ class S3ToFTPOperator(BaseOperator): """ This operator enables the transferring of files from S3 to a FTP server. + :param ftp_conn_id: The ftp connection id. The name or identifier for establishing a connection to the FTP server. :type ftp_conn_id: str From b9d165df0b6b68ce00851adbb89fdb2d5a5306d4 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 10 Feb 2021 18:07:13 +0100 Subject: [PATCH 08/15] test --- tests/providers/amazon/aws/transfers/test_s3_to_ftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py index 1965867168f3b..f52cc8c79e8f2 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py @@ -30,7 +30,7 @@ class TestS3ToFTPOperator(unittest.TestCase): @mock.patch("airflow.providers.ftp.hooks.ftp.FTPHook.store_file") - @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_conn.download_file") + @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_conn().download_file") @mock.patch("airflow.providers.amazon.aws.transfers.s3_to_ftp.NamedTemporaryFile") def test_execute(self, mock_local_tmp_file, mock_s3_hook_download_file, mock_ftp_hook_store_file): operator = S3ToFTPOperator(task_id=TASK_ID, s3_bucket=BUCKET, s3_key=S3_KEY, ftp_path=FTP_PATH) From a9d36f13602b842ac87e7bab1591cf89b564987d Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 24 Feb 2021 14:36:31 +0100 Subject: [PATCH 09/15] mock --- tests/providers/amazon/aws/transfers/test_s3_to_ftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py index f52cc8c79e8f2..1965867168f3b 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py @@ -30,7 +30,7 @@ class TestS3ToFTPOperator(unittest.TestCase): @mock.patch("airflow.providers.ftp.hooks.ftp.FTPHook.store_file") - @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_conn().download_file") + @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_conn.download_file") @mock.patch("airflow.providers.amazon.aws.transfers.s3_to_ftp.NamedTemporaryFile") def test_execute(self, mock_local_tmp_file, mock_s3_hook_download_file, mock_ftp_hook_store_file): operator = S3ToFTPOperator(task_id=TASK_ID, s3_bucket=BUCKET, s3_key=S3_KEY, ftp_path=FTP_PATH) From 694765f184a4594c41a05d777e6f361477480920 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 10 Mar 2021 18:58:00 +0100 Subject: [PATCH 10/15] update test --- .../amazon/aws/transfers/test_s3_to_ftp.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py index 1965867168f3b..b03a224a0f11e 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py @@ -30,15 +30,23 @@ class TestS3ToFTPOperator(unittest.TestCase): @mock.patch("airflow.providers.ftp.hooks.ftp.FTPHook.store_file") - @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_conn.download_file") @mock.patch("airflow.providers.amazon.aws.transfers.s3_to_ftp.NamedTemporaryFile") - def test_execute(self, mock_local_tmp_file, mock_s3_hook_download_file, mock_ftp_hook_store_file): + def test_execute(self, mock_local_tmp_file, mock_ftp_hook_store_file): operator = S3ToFTPOperator(task_id=TASK_ID, s3_bucket=BUCKET, s3_key=S3_KEY, ftp_path=FTP_PATH) operator.execute(None) mock_local_tmp_file_value = mock_local_tmp_file.return_value.__enter__.return_value - mock_s3_hook_download_file.assert_called_once_with( - operator.s3_bucket, operator.s3_key, mock_local_tmp_file_value.name - ) + s3_hook = S3Hook(aws_conn_id=AWS_CONN_ID) + s3_hook.check_for_key = Mock(return_value=True) + s3_obj = Mock() + s3_obj.download_fileobj = Mock(return_value=None) + s3_hook.get_key = Mock(return_value=s3_obj) + key = operator.s3_key + bucket = operator.s3_bucket + s3_hook.download_file(key=key, bucket_name=bucket) + s3_hook.check_for_key.assert_called_once_with(key, bucket) + s3_hook.get_key.assert_called_once_with(key, bucket) + s3_obj.download_fileobj.assert_called_once_with(mock_local_tmp_file_value.name) + mock_ftp_hook_store_file.assert_called_once_with(operator.ftp_path, mock_local_tmp_file_value.name) From 2d51d6cf2b0bdeee9b02bec57ea1d9eb730618a7 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 10 Mar 2021 19:30:33 +0100 Subject: [PATCH 11/15] imports --- tests/providers/amazon/aws/transfers/test_s3_to_ftp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py index b03a224a0f11e..d9e37ed0d5535 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py @@ -16,8 +16,9 @@ # specific language governing permissions and limitations # under the License. import unittest -from unittest import mock +from unittest import mock, Mock +from airflow.provider.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.transfers.s3_to_ftp import S3ToFTPOperator TASK_ID = 'test_s3_to_ftp' From a526a97ce6e54a78244e402db36bdb64e620eb97 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 31 Mar 2021 12:03:54 +0200 Subject: [PATCH 12/15] test --- .../amazon/aws/transfers/test_s3_to_ftp.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py index d9e37ed0d5535..d92829ddcd81e 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py @@ -32,22 +32,18 @@ class TestS3ToFTPOperator(unittest.TestCase): @mock.patch("airflow.providers.ftp.hooks.ftp.FTPHook.store_file") @mock.patch("airflow.providers.amazon.aws.transfers.s3_to_ftp.NamedTemporaryFile") - def test_execute(self, mock_local_tmp_file, mock_ftp_hook_store_file): + @mock.patch('airflow.providers.amazon.aws.transfers.s3_to_ftp.S3Hook') + def test_execute(self, mock_s3_hook, mock_local_tmp_file, mock_ftp_hook_store_file): operator = S3ToFTPOperator(task_id=TASK_ID, s3_bucket=BUCKET, s3_key=S3_KEY, ftp_path=FTP_PATH) operator.execute(None) mock_local_tmp_file_value = mock_local_tmp_file.return_value.__enter__.return_value - - s3_hook = S3Hook(aws_conn_id=AWS_CONN_ID) - s3_hook.check_for_key = Mock(return_value=True) - s3_obj = Mock() - s3_obj.download_fileobj = Mock(return_value=None) - s3_hook.get_key = Mock(return_value=s3_obj) key = operator.s3_key bucket = operator.s3_bucket - s3_hook.download_file(key=key, bucket_name=bucket) - s3_hook.check_for_key.assert_called_once_with(key, bucket) - s3_hook.get_key.assert_called_once_with(key, bucket) - s3_obj.download_fileobj.assert_called_once_with(mock_local_tmp_file_value.name) + + s3_client = mock_s3_hook.get_conn() + + s3_client.download_file(bucket, key, mock_local_tmp_file.name) + s3_client.download_file.assert_called_once_with(bucket, key, mock_local_tmp_file.name) mock_ftp_hook_store_file.assert_called_once_with(operator.ftp_path, mock_local_tmp_file_value.name) From fca490f0aba1e912625c1769f5baa9df3d93044f Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Fri, 11 Jun 2021 19:13:17 +0200 Subject: [PATCH 13/15] ash suggestion --- .../amazon/aws/transfers/s3_to_ftp.py | 24 ++++++++++++------- .../amazon/aws/transfers/test_s3_to_ftp.py | 18 +++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index 898494efee929..dd3782cf7e090 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -27,17 +27,20 @@ class S3ToFTPOperator(BaseOperator): """ This operator enables the transferring of files from S3 to a FTP server. - :param ftp_conn_id: The ftp connection id. The name or identifier for - establishing a connection to the FTP server. - :type ftp_conn_id: str - :param ftp_path: The ftp remote path where the file will be stored, inclusive of filename. - :type ftp_path: str :param s3_bucket: The targeted s3 bucket. This is the S3 bucket from - where the file is downloaded. + where the file is downloaded. :type s3_bucket: str :param s3_key: The targeted s3 key. This is the specified file path for - downloading the file from S3. + downloading the file from S3. :type s3_key: str + :param ftp_path: The ftp remote path. This is the specified file path for + uploading file to the FTP server. + :type ftp_path: str + :param aws_conn_id: reference to a specific AWS connection + :type aws_conn_id: str + :param ftp_conn_id: The ftp connection id. The name or identifier for + establishing a connection to the FTP server. + :type ftp_conn_id: str """ template_fields = ('s3_bucket', 's3_key', 'ftp_path') @@ -63,8 +66,11 @@ def execute(self, context): s3_hook = S3Hook(self.aws_conn_id) ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id) - s3_client = s3_hook.get_conn() + s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket) with NamedTemporaryFile() as local_tmp_file: - s3_client.download_file(self.s3_bucket, self.s3_key, local_tmp_file.name) + self.log.info(f'Downloading file from {self.s3_key}') + s3_obj.download_fileobj(local_tmp_file) + local_tmp_file.seek(0) ftp_hook.store_file(self.ftp_path, local_tmp_file.name) + self.log.info(f'File stored in {self.ftp_path}') diff --git a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py index d92829ddcd81e..f8ee5a45ab9fd 100644 --- a/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py +++ b/tests/providers/amazon/aws/transfers/test_s3_to_ftp.py @@ -16,9 +16,8 @@ # specific language governing permissions and limitations # under the License. import unittest -from unittest import mock, Mock +from unittest import mock -from airflow.provider.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.transfers.s3_to_ftp import S3ToFTPOperator TASK_ID = 'test_s3_to_ftp' @@ -31,19 +30,14 @@ class TestS3ToFTPOperator(unittest.TestCase): @mock.patch("airflow.providers.ftp.hooks.ftp.FTPHook.store_file") + @mock.patch("airflow.providers.amazon.aws.hooks.s3.S3Hook.get_key") @mock.patch("airflow.providers.amazon.aws.transfers.s3_to_ftp.NamedTemporaryFile") - @mock.patch('airflow.providers.amazon.aws.transfers.s3_to_ftp.S3Hook') - def test_execute(self, mock_s3_hook, mock_local_tmp_file, mock_ftp_hook_store_file): + def test_execute(self, mock_local_tmp_file, mock_s3_hook_get_key, mock_ftp_hook_store_file): operator = S3ToFTPOperator(task_id=TASK_ID, s3_bucket=BUCKET, s3_key=S3_KEY, ftp_path=FTP_PATH) operator.execute(None) - mock_local_tmp_file_value = mock_local_tmp_file.return_value.__enter__.return_value - key = operator.s3_key - bucket = operator.s3_bucket - - s3_client = mock_s3_hook.get_conn() - - s3_client.download_file(bucket, key, mock_local_tmp_file.name) - s3_client.download_file.assert_called_once_with(bucket, key, mock_local_tmp_file.name) + mock_s3_hook_get_key.assert_called_once_with(operator.s3_key, operator.s3_bucket) + mock_local_tmp_file_value = mock_local_tmp_file.return_value.__enter__.return_value + mock_s3_hook_get_key.return_value.download_fileobj.assert_called_once_with(mock_local_tmp_file_value) mock_ftp_hook_store_file.assert_called_once_with(operator.ftp_path, mock_local_tmp_file_value.name) From d101b44f3877e48b5c2edbd87bd6366712405169 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Fri, 11 Jun 2021 19:14:03 +0200 Subject: [PATCH 14/15] typo --- airflow/providers/amazon/aws/transfers/s3_to_ftp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index dd3782cf7e090..6d6347649c21b 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -28,13 +28,13 @@ class S3ToFTPOperator(BaseOperator): This operator enables the transferring of files from S3 to a FTP server. :param s3_bucket: The targeted s3 bucket. This is the S3 bucket from - where the file is downloaded. + where the file is downloaded. :type s3_bucket: str :param s3_key: The targeted s3 key. This is the specified file path for - downloading the file from S3. + downloading the file from S3. :type s3_key: str :param ftp_path: The ftp remote path. This is the specified file path for - uploading file to the FTP server. + uploading file to the FTP server. :type ftp_path: str :param aws_conn_id: reference to a specific AWS connection :type aws_conn_id: str From c7f76d0c91032f1358db78cb224b15c1d4f19d9c Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Thu, 17 Jun 2021 10:33:56 +0200 Subject: [PATCH 15/15] pylint --- airflow/providers/amazon/aws/transfers/s3_to_ftp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py index 6d6347649c21b..0cf2125aefd0d 100644 --- a/airflow/providers/amazon/aws/transfers/s3_to_ftp.py +++ b/airflow/providers/amazon/aws/transfers/s3_to_ftp.py @@ -69,8 +69,8 @@ def execute(self, context): s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket) with NamedTemporaryFile() as local_tmp_file: - self.log.info(f'Downloading file from {self.s3_key}') + self.log.info('Downloading file from %s', self.s3_key) s3_obj.download_fileobj(local_tmp_file) local_tmp_file.seek(0) ftp_hook.store_file(self.ftp_path, local_tmp_file.name) - self.log.info(f'File stored in {self.ftp_path}') + self.log.info('File stored in %s', {self.ftp_path})