diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f834a492..015f7d7f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ - Containing a catalog in `schema` is not allowed anymore. - Need to explicitly use `catalog` instead. +## dbt-databricks 1.3.2 (Release TBD) + +### Fixes +- Fix copy into macro when passing `expression_list`. ([#223](https://github.com/databricks/dbt-databricks/pull/223)) + ## dbt-databricks 1.3.1 (November 1, 2022) ### Under the hood diff --git a/dbt/include/databricks/macros/copy_into.sql b/dbt/include/databricks/macros/copy_into.sql index 6376a4b49..fd483311f 100644 --- a/dbt/include/databricks/macros/copy_into.sql +++ b/dbt/include/databricks/macros/copy_into.sql @@ -18,7 +18,7 @@ {%- set source_clause -%} {%- if expression_list -%} - select {{ expression_list }} from '{{ source }}' + ( select {{ expression_list }} from '{{ source }}' ) {%- else -%} '{{ source }}' {%- endif -%} diff --git a/tests/integration/copy_into/models/expected_target_with_expression_list.sql b/tests/integration/copy_into/models/expected_target_with_expression_list.sql new file mode 100644 index 000000000..6bca6c058 --- /dev/null +++ b/tests/integration/copy_into/models/expected_target_with_expression_list.sql @@ -0,0 +1,7 @@ +{{config(materialized='table')}} + +select * from values + (0, 'Zero', '2022-01-01'), + (1, 'Alice', null), + (2, 'Bob', null) + as t(id, name, date) diff --git a/tests/integration/copy_into/models/target_with_expression_list.sql b/tests/integration/copy_into/models/target_with_expression_list.sql new file mode 100644 index 000000000..c937d5184 --- /dev/null +++ b/tests/integration/copy_into/models/target_with_expression_list.sql @@ -0,0 +1,4 @@ +{{config(materialized='table')}} + +select * from values + (0, 'Zero', '2022-01-01') as t(id, name, date) diff --git a/tests/integration/copy_into/test_copy_into.py b/tests/integration/copy_into/test_copy_into.py index 4a5f617fe..d265be092 100644 --- a/tests/integration/copy_into/test_copy_into.py +++ b/tests/integration/copy_into/test_copy_into.py @@ -11,6 +11,17 @@ mergeSchema: 'true' """ +args_with_expression_list = """ +target_table: target_with_expression_list +source: {source_path} +expression_list: 'id, name' +file_format: parquet +format_options: + mergeSchema: 'true' +copy_options: + mergeSchema: 'true' +""" + class TestCopyInto(DBTIntegrationTest): @property @@ -21,7 +32,7 @@ def schema(self): def models(self): return "models" - def test_copy_into(self): + def prepare(self): self.run_dbt(["run"]) # Get the location of the source table. rows = self.run_sql("describe table extended {database_schema}.source", fetch="all") @@ -31,6 +42,11 @@ def test_copy_into(self): path = row.data_type if path is None: raise Exception("No location found for the source table") + return path + + def test_copy_into(self): + path = self.prepare() + self.run_dbt( [ "run-operation", @@ -41,6 +57,25 @@ def test_copy_into(self): ) self.assertTablesEqual("target", "expected_target") + def test_copy_into_with_expression_list(self): + path = self.prepare() + + self.run_dbt( + [ + "run-operation", + "databricks_copy_into", + "--args", + args_with_expression_list.format(source_path=path), + ] + ) + self.assertTablesEqual( + "target_with_expression_list", "expected_target_with_expression_list" + ) + @use_profile("databricks_cluster") - def test_databricks_cluster(self): + def test_copy_into_databricks_cluster(self): self.test_copy_into() + + @use_profile("databricks_cluster") + def test_copy_into_with_expression_list_databricks_cluster(self): + self.test_copy_into_with_expression_list()