Skip to content

Commit 78f32cb

Browse files
author
Benjamin E. Coe
authored
tests: sample integration tests relied on deleted dataset (#334)
1 parent 5b673ce commit 78f32cb

2 files changed

Lines changed: 19 additions & 96 deletions

File tree

‎dlp/system-test/deid.test.js‎

Lines changed: 3 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Copyright 2018, Google, Inc.
2+
* Copyright 2018 Google LLC
33
* Licensed under the Apache License, Version 2.0 (the "License");
44
* you may not use this file except in compliance with the License.
55
* You may obtain a copy of the License at
@@ -26,12 +26,8 @@ const cmd = 'node deid.js';
2626
const harmfulString = 'My SSN is 372819127';
2727
const harmlessString = 'My favorite color is blue';
2828
const surrogateType = 'SSN_TOKEN';
29-
let labeledFPEString;
30-
const wrappedKey = process.env.DLP_DEID_WRAPPED_KEY;
31-
const keyName = process.env.DLP_DEID_KEY_NAME;
3229
const csvFile = 'resources/dates.csv';
3330
const tempOutputFile = path.join(__dirname, 'temp.result.csv');
34-
const csvContextField = 'name';
3531
const dateShiftAmount = 30;
3632
const dateFields = 'birth_date register_date';
3733

@@ -53,48 +49,17 @@ describe('deid', () => {
5349
});
5450

5551
// deidentify_fpe
56-
it('should FPE encrypt sensitive data in a string', () => {
57-
const output = execSync(
58-
`${cmd} deidFpe "${harmfulString}" ${wrappedKey} ${keyName} -a NUMERIC`
59-
);
60-
assert.match(output, /My SSN is \d{9}/);
61-
assert.notInclude(output, harmfulString);
62-
});
63-
64-
it('should use surrogate info types in FPE encryption', () => {
65-
const output = execSync(
66-
`${cmd} deidFpe "${harmfulString}" ${wrappedKey} ${keyName} -a NUMERIC -s ${surrogateType}`
67-
);
68-
assert.match(output, /My SSN is SSN_TOKEN\(9\):\d{9}/);
69-
labeledFPEString = output;
70-
});
71-
72-
it('should ignore insensitive data when FPE encrypting a string', () => {
73-
const output = execSync(
74-
`${cmd} deidFpe "${harmlessString}" ${wrappedKey} ${keyName}`
75-
);
76-
assert.include(output, harmlessString);
77-
});
78-
7952
it('should handle FPE encryption errors', () => {
8053
const output = execSync(
81-
`${cmd} deidFpe "${harmfulString}" ${wrappedKey} BAD_KEY_NAME`
54+
`${cmd} deidFpe "${harmfulString}" BAD_KEY_NAME BAD_KEY_NAME`
8255
);
8356
assert.match(output, /Error in deidentifyWithFpe/);
8457
});
8558

8659
// reidentify_fpe
87-
it('should FPE decrypt surrogate-typed sensitive data in a string', () => {
88-
assert.ok(labeledFPEString, 'Verify that FPE encryption succeeded.');
89-
const output = execSync(
90-
`${cmd} reidFpe "${labeledFPEString}" ${surrogateType} ${wrappedKey} ${keyName} -a NUMERIC`
91-
);
92-
assert.include(output, harmfulString);
93-
});
94-
9560
it('should handle FPE decryption errors', () => {
9661
const output = execSync(
97-
`${cmd} reidFpe "${harmfulString}" ${surrogateType} ${wrappedKey} BAD_KEY_NAME -a NUMERIC`
62+
`${cmd} reidFpe "${harmfulString}" ${surrogateType} BAD_KEY_NAME BAD_KEY_NAME -a NUMERIC`
9863
);
9964
assert.match(output, /Error in reidentifyWithFpe/);
10065
});
@@ -115,30 +80,6 @@ describe('deid', () => {
11580
);
11681
});
11782

118-
it('should date-shift a CSV file using a context field', () => {
119-
const outputCsvFile = 'dates-context.actual.csv';
120-
const expectedCsvFile =
121-
'system-test/resources/date-shift-context.expected.csv';
122-
const output = execSync(
123-
`${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName} -w ${wrappedKey}`
124-
);
125-
assert.include(
126-
output,
127-
`Successfully saved date-shift output to ${outputCsvFile}`
128-
);
129-
assert.include(
130-
fs.readFileSync(outputCsvFile).toString(),
131-
fs.readFileSync(expectedCsvFile).toString()
132-
);
133-
});
134-
135-
it('should require all-or-none of {contextField, wrappedKey, keyName}', () => {
136-
const output = execSync(
137-
`${cmd} deidDateShift "${csvFile}" "${tempOutputFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName}`
138-
);
139-
assert.match(output, /You must set either ALL or NONE of/);
140-
});
141-
14283
it('should handle date-shift errors', () => {
14384
const output = execSync(
14485
`${cmd} deidDateShift "${csvFile}" "${tempOutputFile}" ${dateShiftAmount} ${dateShiftAmount}`

‎dlp/system-test/risk.test.js‎

Lines changed: 16 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Copyright 2018, Google, Inc.
2+
* Copyright 2018 Google LLC
33
* Licensed under the Apache License, Version 2.0 (the "License");
44
* you may not use this file except in compliance with the License.
55
* You may obtain a copy of the License at
@@ -30,12 +30,19 @@ const execSync = cmd => {
3030
const cmd = 'node risk.js';
3131
const dataset = 'integration_tests_dlp';
3232
const uniqueField = 'Name';
33-
const repeatedField = 'Mystery';
3433
const numericField = 'Age';
35-
const stringBooleanField = 'Gender';
3634
const testProjectId = process.env.GCLOUD_PROJECT;
3735
const pubsub = new PubSub();
3836

37+
/*
38+
* The tests in this file rely on a table in BigQuery entitled
39+
* "integration_tests_dlp.harmful" with the following fields:
40+
*
41+
* Age NUMERIC NULLABLE
42+
* Name STRING NULLABLE
43+
*
44+
* Insert into this table a few rows of Age/Name pairs.
45+
*/
3946
describe('risk', () => {
4047
// Create new custom topic/subscription
4148
let topic, subscription;
@@ -57,8 +64,9 @@ describe('risk', () => {
5764
const output = execSync(
5865
`${cmd} numerical ${dataset} harmful ${numericField} ${topicName} ${subscriptionName} -p ${testProjectId}`
5966
);
60-
assert.match(output, /Value at 0% quantile: \d{2}/);
61-
assert.match(output, /Value at \d{2}% quantile: \d{2}/);
67+
console.info(output);
68+
assert.match(output, /Value at 0% quantile:/);
69+
assert.match(output, /Value at \d+% quantile:/);
6270
});
6371

6472
it('should handle numerical risk analysis errors', () => {
@@ -95,15 +103,7 @@ describe('risk', () => {
95103
const output = execSync(
96104
`${cmd} kAnonymity ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}`
97105
);
98-
assert.match(output, /Quasi-ID values: \{\d{2}\}/);
99-
assert.match(output, /Class size: \d/);
100-
});
101-
102-
it('should perform k-anonymity analysis on multiple fields', () => {
103-
const output = execSync(
104-
`${cmd} kAnonymity ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} ${repeatedField} -p ${testProjectId}`
105-
);
106-
assert.match(output, /Quasi-ID values: \{\d{2}, \d{4} \d{4} \d{4} \d{4}\}/);
106+
assert.match(output, /Quasi-ID values:/);
107107
assert.match(output, /Class size: \d/);
108108
});
109109

@@ -124,15 +124,6 @@ describe('risk', () => {
124124
assert.match(output, /Values: \d{2}/);
125125
});
126126

127-
it('should perform k-map analysis on multiple fields', () => {
128-
const output = execSync(
129-
`${cmd} kMap ${dataset} harmful ${topicName} ${subscriptionName} ${numericField} ${stringBooleanField} -t AGE GENDER -p ${testProjectId}`
130-
);
131-
assert.match(output, /Anonymity range: \[\d+, \d+\]/);
132-
assert.match(output, /Size: \d/);
133-
assert.match(output, /Values: \d{2} Female/);
134-
});
135-
136127
it('should handle k-map analysis errors', () => {
137128
const output = execSync(
138129
`${cmd} kMap ${dataset} nonexistent ${topicName} ${subscriptionName} ${numericField} -t AGE -p ${testProjectId}`
@@ -153,18 +144,9 @@ describe('risk', () => {
153144
const output = execSync(
154145
`${cmd} lDiversity ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} ${numericField} -p ${testProjectId}`
155146
);
156-
assert.match(output, /Quasi-ID values: \{\d{2}\}/);
157-
assert.match(output, /Class size: \d/);
158-
assert.match(output, /Sensitive value James occurs \d time\(s\)/);
159-
});
160-
161-
it('should perform l-diversity analysis on multiple fields', () => {
162-
const output = execSync(
163-
`${cmd} lDiversity ${dataset} harmful ${uniqueField} ${topicName} ${subscriptionName} ${numericField} ${repeatedField} -p ${testProjectId}`
164-
);
165-
assert.match(output, /Quasi-ID values: \{\d{2}, \d{4} \d{4} \d{4} \d{4}\}/);
147+
assert.match(output, /Quasi-ID values:/);
166148
assert.match(output, /Class size: \d/);
167-
assert.match(output, /Sensitive value James occurs \d time\(s\)/);
149+
assert.match(output, /Sensitive value/);
168150
});
169151

170152
it('should handle l-diversity analysis errors', () => {

0 commit comments

Comments
 (0)