From 40d4c384d7c2433b241688175264871b70ac98ff Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Wed, 13 Mar 2019 17:07:15 -0700 Subject: [PATCH 1/7] Add S3 SDK sample to list objects --- storage/s3-sdk/README.rst | 30 +++++++++++ storage/s3-sdk/README.rst.in | 3 ++ storage/s3-sdk/list_gcs_objects.py | 68 +++++++++++++++++++++++++ storage/s3-sdk/list_gcs_objects_test.py | 28 ++++++++++ 4 files changed, 129 insertions(+) create mode 100644 storage/s3-sdk/list_gcs_objects.py create mode 100644 storage/s3-sdk/list_gcs_objects_test.py diff --git a/storage/s3-sdk/README.rst b/storage/s3-sdk/README.rst index 9eaddcc6a5f..42cf0f3b7aa 100644 --- a/storage/s3-sdk/README.rst +++ b/storage/s3-sdk/README.rst @@ -79,6 +79,36 @@ To run this sample: +List GCS Objects using S3 SDK ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/s3-sdk/list_gcs_objects.py,storage/s3-sdk/README.rst + + + + +To run this sample: + +.. code-block:: bash + + $ python list_gcs_objects.py + + usage: list_gcs_objects.py [-h] + google_access_key_id google_access_key_secret + bucket_name + + positional arguments: + google_access_key_id Your Cloud Storage HMAC Access Key ID. + google_access_key_secret + Your Cloud Storage HMAC Access Key Secret. + bucket_name Your Cloud Storage bucket name + + optional arguments: + -h, --help show this help message and exit + + + .. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/storage/s3-sdk/README.rst.in b/storage/s3-sdk/README.rst.in index a5031c33e3f..d5b3a2c276e 100644 --- a/storage/s3-sdk/README.rst.in +++ b/storage/s3-sdk/README.rst.in @@ -17,6 +17,9 @@ samples: - name: List GCS Buckets using S3 SDK file: list_gcs_buckets.py show_help: true +- name: List GCS Objects using S3 SDK + file: list_gcs_objects.py + show_help: true cloud_client_library: false diff --git a/storage/s3-sdk/list_gcs_objects.py b/storage/s3-sdk/list_gcs_objects.py new file mode 100644 index 00000000000..25667a5fca7 --- /dev/null +++ b/storage/s3-sdk/list_gcs_objects.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +# [START storage_s3_sdk_list_objects] +import boto3 +from botocore.handlers import set_list_objects_encoding_type_url +boto3.set_stream_logger(name='botocore') + +def list_gcs_buckets(google_access_key_id, google_access_key_secret, bucket_name): + """Lists all GCS buckets using boto3 SDK""" + # Create a new client and do the following: + # 1. Change the endpoint URL to use the + # Google Cloud Storage XML API endpoint. + # 2. Use Cloud Storage HMAC Credentials. + + client = boto3.client("s3", region_name="auto", + endpoint_url="https://storage.googleapis.com", + aws_access_key_id=google_access_key_id, + aws_secret_access_key=google_access_key_secret) + + # Cloud Storage does not support encoding-type and must be removed + # from a ListObjectsRequest made by the client. + # The following class removes encoding-type and is used whenever + # the listObjects() is called using S3 SDK. + client.meta.events.unregister('before-parameter-build.s3.ListObjects', + set_list_objects_encoding_type_url) + + # Call GCS to list objects in bucket_name + print(bucket_name) + response = client.list_objects(Bucket=bucket_name) + + # Print object names + print("Objects:") + for bucket in response["Contents"]: + print(bucket["Key"]) +# [END storage_s3_sdk_list_objects] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("google_access_key_id", + help="Your Cloud Storage HMAC Access Key ID.") + parser.add_argument("google_access_key_secret", + help="Your Cloud Storage HMAC Access Key Secret.") + parser.add_argument('bucket_name', + help="Your Cloud Storage bucket name") + + args = parser.parse_args() + + list_gcs_buckets(google_access_key_id=args.google_access_key_id, + google_access_key_secret=args.google_access_key_secret, + bucket_name=args.bucket_name) diff --git a/storage/s3-sdk/list_gcs_objects_test.py b/storage/s3-sdk/list_gcs_objects_test.py new file mode 100644 index 00000000000..1d29f23a389 --- /dev/null +++ b/storage/s3-sdk/list_gcs_objects_test.py @@ -0,0 +1,28 @@ +# Copyright 2019 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import list_gcs_buckets + +BUCKET = os.environ["GOOGLE_CLOUD_PROJECT_S3_SDK"] +KEY_ID = os.environ["STORAGE_HMAC_ACCESS_KEY_ID"] +SECRET_KEY = os.environ["STORAGE_HMAC_ACCESS_SECRET_KEY"] + + +def test_list_blobs(capsys): + list_gcs_buckets.list_gcs_buckets(google_access_key_id=KEY_ID, + google_access_key_secret=SECRET_KEY) + out, _ = capsys.readouterr() + assert BUCKET in out From 5337e075310fc09be987999fa9f897cf171d0e0c Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Wed, 13 Mar 2019 17:11:09 -0700 Subject: [PATCH 2/7] Fix test and method names --- storage/s3-sdk/list_gcs_objects.py | 7 +++---- storage/s3-sdk/list_gcs_objects_test.py | 9 +++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/storage/s3-sdk/list_gcs_objects.py b/storage/s3-sdk/list_gcs_objects.py index 25667a5fca7..474369277e6 100644 --- a/storage/s3-sdk/list_gcs_objects.py +++ b/storage/s3-sdk/list_gcs_objects.py @@ -18,10 +18,9 @@ # [START storage_s3_sdk_list_objects] import boto3 from botocore.handlers import set_list_objects_encoding_type_url -boto3.set_stream_logger(name='botocore') -def list_gcs_buckets(google_access_key_id, google_access_key_secret, bucket_name): - """Lists all GCS buckets using boto3 SDK""" +def list_gcs_objects(google_access_key_id, google_access_key_secret, bucket_name): + """Lists GCS objects using boto3 SDK""" # Create a new client and do the following: # 1. Change the endpoint URL to use the # Google Cloud Storage XML API endpoint. @@ -63,6 +62,6 @@ def list_gcs_buckets(google_access_key_id, google_access_key_secret, bucket_name args = parser.parse_args() - list_gcs_buckets(google_access_key_id=args.google_access_key_id, + list_gcs_objects(google_access_key_id=args.google_access_key_id, google_access_key_secret=args.google_access_key_secret, bucket_name=args.bucket_name) diff --git a/storage/s3-sdk/list_gcs_objects_test.py b/storage/s3-sdk/list_gcs_objects_test.py index 1d29f23a389..ab915b5ca1e 100644 --- a/storage/s3-sdk/list_gcs_objects_test.py +++ b/storage/s3-sdk/list_gcs_objects_test.py @@ -14,7 +14,7 @@ import os -import list_gcs_buckets +import list_gcs_objects BUCKET = os.environ["GOOGLE_CLOUD_PROJECT_S3_SDK"] KEY_ID = os.environ["STORAGE_HMAC_ACCESS_KEY_ID"] @@ -22,7 +22,8 @@ def test_list_blobs(capsys): - list_gcs_buckets.list_gcs_buckets(google_access_key_id=KEY_ID, - google_access_key_secret=SECRET_KEY) + list_gcs_objects.list_gcs_objects(google_access_key_id=KEY_ID, + google_access_key_secret=SECRET_KEY, + bucket_name=BUCKET) out, _ = capsys.readouterr() - assert BUCKET in out + assert "Objects:" in out From a03b0e3355e1b6f1d52fe1018717f696c4cab9e5 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Wed, 13 Mar 2019 17:12:49 -0700 Subject: [PATCH 3/7] Fix typo in helpful information --- storage/s3-sdk/list_gcs_objects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/s3-sdk/list_gcs_objects.py b/storage/s3-sdk/list_gcs_objects.py index 474369277e6..47623e14221 100644 --- a/storage/s3-sdk/list_gcs_objects.py +++ b/storage/s3-sdk/list_gcs_objects.py @@ -32,9 +32,9 @@ def list_gcs_objects(google_access_key_id, google_access_key_secret, bucket_name aws_secret_access_key=google_access_key_secret) # Cloud Storage does not support encoding-type and must be removed - # from a ListObjectsRequest made by the client. + # from ListObjects made by the client. # The following class removes encoding-type and is used whenever - # the listObjects() is called using S3 SDK. + # the list_objects() is called using boto3 S3 SDK. client.meta.events.unregister('before-parameter-build.s3.ListObjects', set_list_objects_encoding_type_url) From 7bbf92734ca8cdf3a67d57370110af9d72c403c2 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Wed, 13 Mar 2019 20:25:20 -0700 Subject: [PATCH 4/7] Fixing lint issues --- storage/s3-sdk/list_gcs_objects.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/storage/s3-sdk/list_gcs_objects.py b/storage/s3-sdk/list_gcs_objects.py index 47623e14221..6467cd31d3a 100644 --- a/storage/s3-sdk/list_gcs_objects.py +++ b/storage/s3-sdk/list_gcs_objects.py @@ -19,7 +19,9 @@ import boto3 from botocore.handlers import set_list_objects_encoding_type_url -def list_gcs_objects(google_access_key_id, google_access_key_secret, bucket_name): + +def list_gcs_objects(google_access_key_id, google_access_key_secret, + bucket_name): """Lists GCS objects using boto3 SDK""" # Create a new client and do the following: # 1. Change the endpoint URL to use the @@ -32,7 +34,7 @@ def list_gcs_objects(google_access_key_id, google_access_key_secret, bucket_name aws_secret_access_key=google_access_key_secret) # Cloud Storage does not support encoding-type and must be removed - # from ListObjects made by the client. + # from ListObjects made by the client. # The following class removes encoding-type and is used whenever # the list_objects() is called using boto3 S3 SDK. client.meta.events.unregister('before-parameter-build.s3.ListObjects', From 466e610f59d2c223023cc8352816c8a45e8e8234 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 14 Mar 2019 15:01:59 -0700 Subject: [PATCH 5/7] Remove workaround --- storage/s3-sdk/list_gcs_objects.py | 9 ------- storage/signed_urls/generate_signed_urls.py | 27 +++++++++++++++++---- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/storage/s3-sdk/list_gcs_objects.py b/storage/s3-sdk/list_gcs_objects.py index 6467cd31d3a..d593152057a 100644 --- a/storage/s3-sdk/list_gcs_objects.py +++ b/storage/s3-sdk/list_gcs_objects.py @@ -17,7 +17,6 @@ import argparse # [START storage_s3_sdk_list_objects] import boto3 -from botocore.handlers import set_list_objects_encoding_type_url def list_gcs_objects(google_access_key_id, google_access_key_secret, @@ -33,15 +32,7 @@ def list_gcs_objects(google_access_key_id, google_access_key_secret, aws_access_key_id=google_access_key_id, aws_secret_access_key=google_access_key_secret) - # Cloud Storage does not support encoding-type and must be removed - # from ListObjects made by the client. - # The following class removes encoding-type and is used whenever - # the list_objects() is called using boto3 S3 SDK. - client.meta.events.unregister('before-parameter-build.s3.ListObjects', - set_list_objects_encoding_type_url) - # Call GCS to list objects in bucket_name - print(bucket_name) response = client.list_objects(Bucket=bucket_name) # Print object names diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index 7cee36461b6..876eff692d9 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -78,8 +78,10 @@ def generate_signed_url(service_account_file, bucket_name, object_name, ordered_headers = collections.OrderedDict(sorted(headers.items())) for k, v in ordered_headers.items(): lower_k = str(k).lower() - strip_v = str(v).lower() + strip_v = v canonical_headers += '{}:{}\n'.format(lower_k, strip_v) + canonical_headers = "host:storage.googleapis.com\nx-goog-encryption-algorithm:AES56\n" # "x-goog-encryption-key:AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=\nx-goog-encryption-key-sha256:QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=\n" + print(canonical_headers) # [END storage_signed_url_canonical_headers] # [START storage_signed_url_signed_headers] @@ -88,6 +90,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, lower_k = str(k).lower() signed_headers += '{};'.format(lower_k) signed_headers = signed_headers[:-1] # remove trailing ';' + signed_headers = 'host;x-goog-encryption-algorithm' #;x-goog-encryption-key;x-goog-encryption-key-sha256' # [END storage_signed_url_signed_headers] if query_parameters is None: @@ -105,7 +108,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, for k, v in ordered_query_parameters.items(): encoded_k = quote(str(k), safe='') encoded_v = quote(str(v), safe='') - canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) + canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) canonical_query_string = canonical_query_string[:-1] # remove trailing ';' # [END storage_signed_url_canonical_query_parameters] @@ -117,7 +120,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, signed_headers, 'UNSIGNED-PAYLOAD']) # [END storage_signed_url_canonical_request] - + print(canonical_request) # [START storage_signed_url_hash] canonical_request_hash = hashlib.sha256( canonical_request.encode()).hexdigest() @@ -128,6 +131,8 @@ def generate_signed_url(service_account_file, bucket_name, object_name, request_timestamp, credential_scope, canonical_request_hash]) + print(string_to_sign) + print("\n\n\n") # [END storage_signed_url_string_to_sign] # [START storage_signed_url_signer] @@ -159,9 +164,21 @@ def generate_signed_url(service_account_file, bucket_name, object_name, parser.add_argument('expiration', help='Expiration Time.') args = parser.parse_args() + headers = {'X-Goog-Encryption-Key': 'AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=', + 'X-Goog-Encryption-Key-Sha256': 'QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=', + 'X-Goog-Encryption-Algorithm': 'AES256'} + + no_key_headers = {'X-Goog-Encryption-Algorithm': 'AES256'} signed_url = generate_signed_url( service_account_file=args.service_account_file, http_method=args.request_method, bucket_name=args.bucket_name, - object_name=args.object_name, expiration=int(args.expiration)) - + object_name=args.object_name, expiration=int(args.expiration), + headers=headers) + + # importing the requests library + import requests + + # sending get request and saving the response as response object + r = requests.get(url = signed_url, headers=headers) + print(r.text) print(signed_url) From faf0e3cd1602a90adde5cf3b25f36ad16b63c703 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 2 Apr 2019 16:03:40 -0700 Subject: [PATCH 6/7] Remove debug code from generate_signed_urls.py --- storage/signed_urls/generate_signed_urls.py | 27 ++++----------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/storage/signed_urls/generate_signed_urls.py b/storage/signed_urls/generate_signed_urls.py index 876eff692d9..7cee36461b6 100644 --- a/storage/signed_urls/generate_signed_urls.py +++ b/storage/signed_urls/generate_signed_urls.py @@ -78,10 +78,8 @@ def generate_signed_url(service_account_file, bucket_name, object_name, ordered_headers = collections.OrderedDict(sorted(headers.items())) for k, v in ordered_headers.items(): lower_k = str(k).lower() - strip_v = v + strip_v = str(v).lower() canonical_headers += '{}:{}\n'.format(lower_k, strip_v) - canonical_headers = "host:storage.googleapis.com\nx-goog-encryption-algorithm:AES56\n" # "x-goog-encryption-key:AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=\nx-goog-encryption-key-sha256:QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=\n" - print(canonical_headers) # [END storage_signed_url_canonical_headers] # [START storage_signed_url_signed_headers] @@ -90,7 +88,6 @@ def generate_signed_url(service_account_file, bucket_name, object_name, lower_k = str(k).lower() signed_headers += '{};'.format(lower_k) signed_headers = signed_headers[:-1] # remove trailing ';' - signed_headers = 'host;x-goog-encryption-algorithm' #;x-goog-encryption-key;x-goog-encryption-key-sha256' # [END storage_signed_url_signed_headers] if query_parameters is None: @@ -108,7 +105,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, for k, v in ordered_query_parameters.items(): encoded_k = quote(str(k), safe='') encoded_v = quote(str(v), safe='') - canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) + canonical_query_string += '{}={}&'.format(encoded_k, encoded_v) canonical_query_string = canonical_query_string[:-1] # remove trailing ';' # [END storage_signed_url_canonical_query_parameters] @@ -120,7 +117,7 @@ def generate_signed_url(service_account_file, bucket_name, object_name, signed_headers, 'UNSIGNED-PAYLOAD']) # [END storage_signed_url_canonical_request] - print(canonical_request) + # [START storage_signed_url_hash] canonical_request_hash = hashlib.sha256( canonical_request.encode()).hexdigest() @@ -131,8 +128,6 @@ def generate_signed_url(service_account_file, bucket_name, object_name, request_timestamp, credential_scope, canonical_request_hash]) - print(string_to_sign) - print("\n\n\n") # [END storage_signed_url_string_to_sign] # [START storage_signed_url_signer] @@ -164,21 +159,9 @@ def generate_signed_url(service_account_file, bucket_name, object_name, parser.add_argument('expiration', help='Expiration Time.') args = parser.parse_args() - headers = {'X-Goog-Encryption-Key': 'AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=', - 'X-Goog-Encryption-Key-Sha256': 'QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=', - 'X-Goog-Encryption-Algorithm': 'AES256'} - - no_key_headers = {'X-Goog-Encryption-Algorithm': 'AES256'} signed_url = generate_signed_url( service_account_file=args.service_account_file, http_method=args.request_method, bucket_name=args.bucket_name, - object_name=args.object_name, expiration=int(args.expiration), - headers=headers) - - # importing the requests library - import requests - - # sending get request and saving the response as response object - r = requests.get(url = signed_url, headers=headers) - print(r.text) + object_name=args.object_name, expiration=int(args.expiration)) + print(signed_url) From 75c0221941e893a410d85d15441665e3b722f1e0 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 2 Apr 2019 16:07:07 -0700 Subject: [PATCH 7/7] update bucket to blob --- storage/s3-sdk/list_gcs_objects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/s3-sdk/list_gcs_objects.py b/storage/s3-sdk/list_gcs_objects.py index d593152057a..3747a164084 100644 --- a/storage/s3-sdk/list_gcs_objects.py +++ b/storage/s3-sdk/list_gcs_objects.py @@ -37,8 +37,8 @@ def list_gcs_objects(google_access_key_id, google_access_key_secret, # Print object names print("Objects:") - for bucket in response["Contents"]: - print(bucket["Key"]) + for blob in response["Contents"]: + print(blob["Key"]) # [END storage_s3_sdk_list_objects]