Skip to content

sqlparser: Remove unneeded escaping (#16255) #172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
meiji163 merged 2 commits into from
Jul 23, 2025
Merged

sqlparser: Remove unneeded escaping (#16255) #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a919b4d
sqlparser: Remove unneeded escaping (#16255)
dbussink Jun 24, 2024
6a99578
fix escape in testPlayerCopyVarcharPKCaseInsensitive
meiji163 Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions go/sqltypes/value.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -849,7 +849,25 @@ var SQLEncodeMap [256]byte
// SQLDecodeMap is the reverse of SQLEncodeMap
var SQLDecodeMap [256]byte

// encodeRef is a map of characters we use for escaping.
// This doesn't include double quotes since we don't need
// to escape that, as we always generate single quoted strings.
var encodeRef = map[byte]byte{
'\x00': '0',
'\'': '\'',
'\b': 'b',
'\n': 'n',
'\r': 'r',
'\t': 't',
26: 'Z', // ctl-Z
'\\': '\\',
}

// decodeRef is a map of characters we use for unescaping.
// We do need all characters here, since we do accept
// escaped double quotes in single quote strings and
// double quoted strings.
var decodeRef = map[byte]byte{
'\x00': '0',
'\'': '\'',
'"': '"',
Expand All @@ -869,6 +887,11 @@ func init() {
for i := range SQLEncodeMap {
if to, ok := encodeRef[byte(i)]; ok {
SQLEncodeMap[byte(i)] = to
}
}

for i := range SQLDecodeMap {
if to, ok := decodeRef[byte(i)]; ok {
SQLDecodeMap[to] = byte(i)
}
}
Expand Down
2 changes: 1 addition & 1 deletion go/sqltypes/value_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ func TestEncode(t *testing.T) {
outASCII: "'Zm9v'",
}, {
in: TestValue(VarChar, "\x00'\"\b\n\r\t\x1A\\"),
outSQL: "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'",
outSQL: "'\\0\\'\"\\b\\n\\r\\t\\Z\\\\'",
outASCII: "'ACciCAoNCRpc'",
}, {
in: TestValue(Bit, "a"),
Expand Down
4 changes: 2 additions & 2 deletions go/vt/binlog/binlogplayer/binlog_player_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ func applyEvents(blp *BinlogPlayer) func() error {
func TestCreateVReplicationKeyRange(t *testing.T) {
want := "insert into _vt.vreplication " +
"(workflow, source, pos, max_tps, max_replication_lag, time_updated, transaction_timestamp, state, db_name, workflow_type, workflow_sub_type, defer_secondary_keys) " +
`values ('Resharding', 'keyspace:\"ks\" shard:\"0\" key_range:{end:\"\\x80\"}', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false)`
`values ('Resharding', 'keyspace:"ks" shard:"0" key_range:{end:"\\x80"}', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false)`

bls := binlogdatapb.BinlogSource{
Keyspace: "ks",
Expand All @@ -401,7 +401,7 @@ func TestCreateVReplicationKeyRange(t *testing.T) {
func TestCreateVReplicationTables(t *testing.T) {
want := "insert into _vt.vreplication " +
"(workflow, source, pos, max_tps, max_replication_lag, time_updated, transaction_timestamp, state, db_name, workflow_type, workflow_sub_type, defer_secondary_keys) " +
`values ('Resharding', 'keyspace:\"ks\" shard:\"0\" tables:\"a\" tables:\"b\"', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false)`
`values ('Resharding', 'keyspace:"ks" shard:"0" tables:"a" tables:"b"', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false)`

bls := binlogdatapb.BinlogSource{
Keyspace: "ks",
Expand Down
4 changes: 3 additions & 1 deletion go/vt/sqlparser/ast_format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,9 @@ func (idx *IndexDefinition) Format(buf *TrackedBuffer) {
buf.astPrintf(idx, ")")

for _, opt := range idx.Options {
buf.astPrintf(idx, " %s", opt.Name)
if opt.Name != "" {
buf.astPrintf(idx, " %s", opt.Name)
}
if opt.String != "" {
buf.astPrintf(idx, " %#s", opt.String)
} else if opt.Value != nil {
Expand Down
170 changes: 88 additions & 82 deletions go/vt/sqlparser/parse_test.go
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion go/vt/sqlparser/parsed_query_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func TestGenerateQuery(t *testing.T) {
"v1": sqltypes.ValueBindVariable(sqltypes.MakeTrusted(querypb.Type_JSON, []byte(`{"key": "value"}`))),
"v2": sqltypes.ValueBindVariable(sqltypes.MakeTrusted(querypb.Type_RAW, []byte(`json_object("k", "v")`))),
},
output: `insert into t values ('{\"key\": \"value\"}', json_object("k", "v"))`,
output: `insert into t values ('{"key": "value"}', json_object("k", "v"))`,
}, {
desc: "list bind vars 0 arguments",
query: "select * from a where id in ::vals",
Expand Down
20 changes: 10 additions & 10 deletions go/vt/sqlparser/testdata/select_cases.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4286,7 +4286,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"xt indexes"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"xt indexes\"' in boolean mode)
select * from t1 where match(a, b) against ('"xt indexes"' in boolean mode)
END
INPUT
select max(value) from t1 AS m LEFT JOIN t2 AS c1 ON m.c1id = c1.id AND c1.active = 'Yes' LEFT JOIN t3 AS c2 ON m.c2id = c2.id AND c2.active = 'Yes' WHERE m.pid=1 AND (c1.id IS NOT NULL OR c2.id IS NOT NULL);
Expand Down Expand Up @@ -5660,7 +5660,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text search" "now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text search\" \"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"text search" "now support"' in boolean mode)
END
INPUT
select insert('hello', 1, -18446744073709551616, 'hi');
Expand Down Expand Up @@ -8048,7 +8048,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text i"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text i\"' in boolean mode)
select * from t1 where match(a, b) against ('"text i"' in boolean mode)
END
INPUT
select column_name,data_type,CHARACTER_OCTET_LENGTH, CHARACTER_MAXIMUM_LENGTH from information_schema.columns where table_name='t1' order by column_name;
Expand Down Expand Up @@ -8144,7 +8144,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"support now"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"support now\"' in boolean mode)
select * from t1 where match(a, b) against ('"support now"' in boolean mode)
END
INPUT
select hex(inet_aton('127.1.1'));
Expand Down Expand Up @@ -12212,7 +12212,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text search" +"now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text search\" +\"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"text search" +"now support"' in boolean mode)
END
INPUT
select trigger_name from information_schema.triggers where event_object_table='t1';
Expand Down Expand Up @@ -14750,7 +14750,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST('"space model' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"space model' in boolean mode)
select * from t1 where match(a, b) against ('"space model' in boolean mode)
END
INPUT
select @ujis4 = CONVERT(@utf84 USING ujis);
Expand Down Expand Up @@ -14906,7 +14906,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text search" -"now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text search\" -\"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"text search" -"now support"' in boolean mode)
END
INPUT
select _rowid,t1._rowid,skey,sval from t1;
Expand Down Expand Up @@ -17612,7 +17612,7 @@ INPUT
select 'aaa','aa''a',"aa""a";
END
OUTPUT
select 'aaa', 'aa\'a', 'aa\"a' from dual
select 'aaa', 'aa\'a', 'aa"a' from dual
END
INPUT
select cast('18446744073709551615' as signed);
Expand Down Expand Up @@ -18542,7 +18542,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"now support"' in boolean mode)
END
INPUT
select date_add("1997-12-31 23:59:59",INTERVAL "10000:99:99" HOUR_SECOND);
Expand Down Expand Up @@ -22148,7 +22148,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"Now sUPPort"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"Now sUPPort\"' in boolean mode)
select * from t1 where match(a, b) against ('"Now sUPPort"' in boolean mode)
END
INPUT
select sum(all a),count(all a),avg(all a),std(all a),variance(all a),bit_or(all a),bit_and(all a),min(all a),max(all a),min(all c),max(all c) from t1;
Expand Down
40 changes: 20 additions & 20 deletions go/vt/vtctl/workflow/materializer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2453,11 +2453,11 @@ func TestMaterializerOneToOne(t *testing.T) {
insertPrefix+
`\(`+
`'workflow', `+
(`'keyspace:\\"sourceks\\" shard:\\"0\\" `+
(`'keyspace:"sourceks" shard:"0" `+
`filter:{`+
`rules:{match:\\"t1\\" filter:\\"select.*t1\\"} `+
`rules:{match:\\"t2\\" filter:\\"select.*t3\\"} `+
`rules:{match:\\"t4\\"}`+
`rules:{match:"t1" filter:"select.*t1"} `+
`rules:{match:"t2" filter:"select.*t3"} `+
`rules:{match:"t4"}`+
`}', `)+
`'', [0-9]*, [0-9]*, 'zone1', 'primary,rdonly', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false`+
`\)`+eol,
Expand Down Expand Up @@ -2495,9 +2495,9 @@ func TestMaterializerManyToOne(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`\('workflow', 'keyspace:\\"sourceks\\" shard:\\"-80\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1\\"} rules:{match:\\"t2\\" filter:\\"select.*t3\\"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
`\('workflow', 'keyspace:"sourceks" shard:"-80" filter:{rules:{match:"t1" filter:"select.*t1"} rules:{match:"t2" filter:"select.*t3"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
`, `+
`\('workflow', 'keyspace:\\"sourceks\\" shard:\\"80-\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1\\"} rules:{match:\\"t2\\" filter:\\"select.*t3\\"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
`\('workflow', 'keyspace:"sourceks" shard:"80-" filter:{rules:{match:"t1" filter:"select.*t1"} rules:{match:"t2" filter:"select.*t3"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
eol,
&sqltypes.Result{},
)
Expand Down Expand Up @@ -2551,13 +2551,13 @@ func TestMaterializerOneToMany(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*-80.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*-80.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(
210,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*80-.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*80-.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
Expand Down Expand Up @@ -2611,15 +2611,15 @@ func TestMaterializerManyToMany(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`.*shard:\\"-40\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*-80.*`+
`.*shard:\\"40-\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*-80.*`,
`.*shard:"-40" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*-80.*`+
`.*shard:"40-" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*-80.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(
210,
insertPrefix+
`.*shard:\\"-40\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*80-.*`+
`.*shard:\\"40-\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*80-.*`,
`.*shard:"-40" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*80-.*`+
`.*shard:"40-" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1.*targetks\.xxhash.*80-.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
Expand Down Expand Up @@ -2675,13 +2675,13 @@ func TestMaterializerMulticolumnVindex(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*-80.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*-80.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(
210,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*80-.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*80-.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
Expand Down Expand Up @@ -2720,7 +2720,7 @@ func TestMaterializerDeploySchema(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`\('workflow', 'keyspace:\\"sourceks\\" shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1\\"} rules:{match:\\"t2\\" filter:\\"select.*t3\\"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
`\('workflow', 'keyspace:"sourceks" shard:"0" filter:{rules:{match:"t1" filter:"select.*t1"} rules:{match:"t2" filter:"select.*t3"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
eol,
&sqltypes.Result{},
)
Expand Down Expand Up @@ -2761,7 +2761,7 @@ func TestMaterializerCopySchema(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`\('workflow', 'keyspace:\\"sourceks\\" shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1\\"} rules:{match:\\"t2\\" filter:\\"select.*t3\\"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
`\('workflow', 'keyspace:"sourceks" shard:"0" filter:{rules:{match:"t1" filter:"select.*t1"} rules:{match:"t2" filter:"select.*t3"}}', '', [0-9]*, [0-9]*, '', '', [0-9]*, 0, 'Stopped', 'vt_targetks', 0, 0, false\)`+
eol,
&sqltypes.Result{},
)
Expand Down Expand Up @@ -2821,13 +2821,13 @@ func TestMaterializerExplicitColumns(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*-80.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*-80.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(
210,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*80-.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c1, c2.*targetks\.region.*80-.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
Expand Down Expand Up @@ -2884,13 +2884,13 @@ func TestMaterializerRenamedColumns(t *testing.T) {
env.tmc.expectVRQuery(
200,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c3, c4.*targetks\.region.*-80.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c3, c4.*targetks\.region.*-80.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(
210,
insertPrefix+
`.*shard:\\"0\\" filter:{rules:{match:\\"t1\\" filter:\\"select.*t1 where in_keyrange\(c3, c4.*targetks\.region.*80-.*`,
`.*shard:"0" filter:{rules:{match:"t1" filter:"select.*t1 where in_keyrange\(c3, c4.*targetks\.region.*80-.*`,
&sqltypes.Result{},
)
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vtctl/workflow/resharder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ func TestReshardCreate(t *testing.T) {
env.tmc.expectVRQuery(
tabletUID,
insertPrefix+
`\('`+workflowName+`', 'keyspace:\\"`+targetKeyspaceName+`\\" shard:\\"0\\" filter:{rules:{match:\\"/.*\\" filter:\\"`+target+`\\"}}', '', [0-9]*, [0-9]*, '`+
`\('`+workflowName+`', 'keyspace:"`+targetKeyspaceName+`" shard:"0" filter:{rules:{match:"/.*" filter:"`+target+`"}}', '', [0-9]*, [0-9]*, '`+
env.cell+`', '`+tabletTypesStr+`', [0-9]*, 0, 'Stopped', 'vt_`+targetKeyspaceName+`', 4, 0, false\)`+eol,
&sqltypes.Result{},
)
Expand Down
Loading
Loading