refactor(bigquery): add extract table and table cmek operations

bigquery/docs/usage/encryption.rst

@@ -10,15 +10,15 @@ in the BigQuery documentation for more details.
 Create a new table, using a customer-managed encryption key from
 Cloud KMS to encrypt it.
 
-.. literalinclude:: ../snippets.py
+.. literalinclude:: ../samples/create_table_cmek.py
    :language: python
    :dedent: 4
    :start-after: [START bigquery_create_table_cmek]
    :end-before: [END bigquery_create_table_cmek]
 
 Change the key used to encrypt a table.
 
-.. literalinclude:: ../snippets.py
+.. literalinclude:: ../samples/update_table_cmek.py
    :language: python
    :dedent: 4
    :start-after: [START bigquery_update_table_cmek]

bigquery/docs/usage/tables.rst

@@ -169,12 +169,28 @@ Copy a table with the
 Copy table data to Google Cloud Storage with the
 :func:`~google.cloud.bigquery.client.Client.extract_table` method:
 
-.. literalinclude:: ../snippets.py
+.. literalinclude:: ../samples/extract_table.py
    :language: python
    :dedent: 4
    :start-after: [START bigquery_extract_table]
    :end-before: [END bigquery_extract_table]
 
+Copy table data to Google Cloud Storage json file:
+
+.. literalinclude:: ../samples/extract_table_compressed.py
+   :language: python
+   :dedent: 4
+   :start-after: [START bigquery_extract_table_compressed]
+   :end-before: [END bigquery_extract_table_compressed]
+
+Copy table data to Google Cloud Storage compressed file:
+
+.. literalinclude:: ../samples/extract_table_json.py
+   :language: python
+   :dedent: 4
+   :start-after: [START bigquery_extract_table_json]
+   :end-before: [END bigquery_extract_table_json]
+
 Deleting a Table
 ^^^^^^^^^^^^^^^^
 

bigquery/samples/create_table_cmek.py

@@ -0,0 +1,43 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def create_table_cmek(table_id, kms_key_name):
+
+    # [START bigquery_create_table_cmek]
+    from google.cloud import bigquery
+
+    # Construct a BigQuery client object.
+    client = bigquery.Client()
+
+    # TODO(developer): Set table_id to the ID of the table to create.
+    # table_id = "your-project.your_dataset.your_table_name"
+
+    # Set the encryption key to use for the destination.
+    # TODO: Replace this key with a key you have created in KMS.
+    # kms_key_name = "projects/{}/locations/{}/keyRings/{}/cryptoKeys/{}".format(
+    #     "cloud-samples-tests", "us", "test", "test"
+    # )
+
+    table = bigquery.Table(table_id)
+    table.encryption_configuration = bigquery.EncryptionConfiguration(
+        kms_key_name=kms_key_name
+    )
+
+    table = client.create_table(table)  # Make an API request.
+
+    if table.encryption_configuration.kms_key_name == kms_key_name:
+        print("A table created with encryption configuration key")
+
+    # [END bigquery_create_table_cmek]

bigquery/samples/extract_table.py

@@ -0,0 +1,63 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def extract_table(table_id):
+
+    # [START bigquery_extract_table]
+    import time
+
+    from google.cloud import bigquery
+    from google.cloud import storage
+
+    # Construct a BigQuery client object.
+    client = bigquery.Client()
+
+    # Construct a Storage client object.
+    storage_client = storage.Client()
+
+    # TODO(developer): Set table_id to the ID of the model to fetch.
+    # table_id = 'your-project.your_dataset.your_table'
+
+    bucket_name = "extract_shakespeare_{}".format(int(time.time() * 1000))
+    bucket = storage_client.create_bucket(bucket_name)
+
+    destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv")
+
+    table = bigquery.Table(
+        table_id,
+        schema=[
+            bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
+            bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
+        ],
+    )
+    table = client.create_table(table)
+
+    extract_job = client.extract_table(
+        table,
+        destination_uri,
+        # Must match the source table location.
+        location="US",
+    )  # Make an API request.
+    extract_job.result()  # Waits for job to complete.
+
+    print(
+        "Exported {}.{}.{} to {}".format(
+            table.project, table.dataset_id, table.table_id, destination_uri
+        )
+    )
+    # [END bigquery_extract_table]
+
+    blob = bucket.get_blob("shakespeare.csv")
+    return blob, bucket

bigquery/samples/extract_table_compressed.py

@@ -0,0 +1,59 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def extract_table_compressed(table_id):
+
+    # [START bigquery_extract_table_compressed]
+    import time
+
+    from google.cloud import bigquery
+    from google.cloud import storage
+
+    # Construct a BigQuery client object.
+    client = bigquery.Client()
+
+    # Construct a Storage client object.
+    storage_client = storage.Client()
+
+    # TODO(developer): Set table_id to the ID of the model to fetch.
+    # table_id = 'your-project.your_dataset.your_table'
+
+    bucket_name = "extract_shakespeare_compress_{}".format(int(time.time() * 1000))
+    bucket = storage_client.create_bucket(bucket_name)
+
+    destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.csv.gz")
+
+    job_config = bigquery.job.ExtractJobConfig(
+        destination_format=bigquery.Compression.GZIP
+    )
+    extract_job = client.extract_table(
+        table_id,
+        destination_uri,
+        job_config=job_config,
+        # Must match the source table location.
+        location="US",
+    )  # Make an API request.
+    extract_job.result()  # Waits for job to complete.
+
+    table = client.get_table(table_id)
+    print(
+        "Exported {}.{}.{} to {}".format(
+            table.project, table.dataset_id, table.table_id, destination_uri
+        )
+    )
+    # [END bigquery_extract_table_compressed]
+
+    blob = bucket.get_blob("shakespeare.csv.gz")
+    return blob, bucket

bigquery/samples/extract_table_json.py

@@ -0,0 +1,59 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def extract_table_json(table_id):
+
+    # [START bigquery_extract_table_json]
+    import time
+
+    from google.cloud import bigquery
+    from google.cloud import storage
+
+    # Construct a BigQuery client object.
+    client = bigquery.Client()
+
+    # Construct a Storage client object.
+    storage_client = storage.Client()
+
+    # TODO(developer): Set table_id to the ID of the model to fetch.
+    # table_id = 'your-project.your_dataset.your_table'
+
+    bucket_name = "extract_shakespeare_json_{}".format(int(time.time() * 1000))
+    bucket = storage_client.create_bucket(bucket_name)
+
+    destination_uri = "gs://{}/{}".format(bucket_name, "shakespeare.json")
+
+    job_config = bigquery.job.ExtractJobConfig(
+        destination_format=bigquery.DestinationFormat.NEWLINE_DELIMITED_JSON
+    )
+    extract_job = client.extract_table(
+        table_id,
+        destination_uri,
+        job_config=job_config,
+        # Must match the source table location.
+        location="US",
+    )  # Make an API request.
+    extract_job.result()  # Waits for job to complete.
+
+    table = client.get_table(table_id)
+    print(
+        "Exported {}.{}.{} to {}".format(
+            table.project, table.dataset_id, table.table_id, destination_uri
+        )
+    )
+    # [END bigquery_extract_table_json]
+
+    blob = bucket.get_blob("shakespeare.json")
+    return blob, bucket

bigquery/samples/tests/test_create_table_cmek.py

@@ -0,0 +1,22 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import create_table_cmek
+
+
+def test_create_table_cmek(capsys, random_table_id, kms_key_name):
+
+    create_table_cmek.create_table_cmek(random_table_id, kms_key_name)
+    out, err = capsys.readouterr()
+    assert "A table created with encryption configuration key" in out

bigquery/samples/tests/test_extract_table.py

@@ -0,0 +1,27 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import extract_table
+
+
+def test_extract_table(capsys, random_table_id):
+
+    blob, bucket = extract_table.extract_table(random_table_id)
+    out, _ = capsys.readouterr()
+    assert "Exported {} ".format(random_table_id) in out
+    assert blob.exists
+    assert blob.size > 0
+
+    blob.delete()
+    bucket.delete()

bigquery/samples/tests/test_extract_table_compressed.py

@@ -0,0 +1,27 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import extract_table_compressed
+
+
+def test_extract_table_compressed(capsys, table_with_data_id):
+
+    blob, bucket = extract_table_compressed.extract_table_compressed(table_with_data_id)
+    out, _ = capsys.readouterr()
+    assert "Exported {} ".format(table_with_data_id) in out
+    assert blob.exists
+    assert blob.size > 0
+
+    blob.delete()
+    bucket.delete()

bigquery/samples/tests/test_extract_table_json.py

@@ -0,0 +1,27 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import extract_table_json
+
+
+def test_extract_table_json(capsys, table_with_data_id):
+
+    blob, bucket = extract_table_json.extract_table_json(table_with_data_id)
+    out, _ = capsys.readouterr()
+    assert "Exported {} ".format(table_with_data_id) in out
+    assert blob.exists
+    assert blob.size > 0
+
+    blob.delete()
+    bucket.delete()