Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -134,21 +134,21 @@ public ArrayNode filterDataNew(
}

Map<String, DataType> schema = generateSchema(items, dataTypeConversionMap);
String tableName = generateTable(schema);

// insert the data
insertAllData(tableName, items, schema, dataTypeConversionMap);

// Filter the data
List<Map<String, Object>> finalResults =
executeFilterQueryNew(tableName, schema, uqiDataFilterParams, dataTypeConversionMap);
validateProjectionColumns(uqiDataFilterParams.getProjectionColumns(), schema);
validateSortByColumns(uqiDataFilterParams.getSortBy(), schema);

// Now that the data has been filtered. Clean Up. Drop the table
dropTable(tableName);
String tableName = generateTable(schema);
try {
insertAllData(tableName, items, schema, dataTypeConversionMap);

ArrayNode finalResultsNode = objectMapper.valueToTree(finalResults);
List<Map<String, Object>> finalResults =
executeFilterQueryNew(tableName, schema, uqiDataFilterParams, dataTypeConversionMap);

return finalResultsNode;
return objectMapper.valueToTree(finalResults);
} finally {
dropTable(tableName);
}
}

private List<Map<String, Object>> executeFilterQueryNew(
Expand Down Expand Up @@ -277,6 +277,48 @@ private void addPaginationCondition(
values.add(new PreparedStatementValueDTO(offset, DataType.INTEGER));
}

private void validateProjectionColumns(List<String> projectionColumns, Map<String, DataType> schema) {
if (CollectionUtils.isEmpty(projectionColumns)) {
return;
}
for (String columnName : projectionColumns) {
if (!schema.containsKey(columnName)) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
columnName + " not found in the known column names: " + schema.keySet());
}
}
}

private void validateSortByColumns(List<Map<String, String>> sortBy, Map<String, DataType> schema) {
if (isSortConditionEmpty(sortBy)) {
return;
}
for (Map<String, String> sortCondition : sortBy) {
String columnName = sortCondition.get(SORT_BY_COLUMN_NAME_KEY);
if (!isBlank(columnName) && !schema.containsKey(columnName)) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
columnName + " not found in the known column names: " + schema.keySet());
}
if (!isBlank(columnName)) {
String order = sortCondition.get(SORT_BY_TYPE_KEY);
if (isBlank(order)) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
"Sort order is required for column: " + columnName);
}
try {
SortType.valueOf(order.toUpperCase());
} catch (IllegalArgumentException e) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
"Unsupported sort order '" + order + "' for column: " + columnName);
}
}
}
}

/**
* Display only those columns that the user has chosen to display.
* E.g. if the projectionColumns is a list that contains ["ID, Name"], then this method will add the following
Expand All @@ -286,10 +328,15 @@ private void addPaginationCondition(
* @param projectionColumns - list of columns that need to be displayed
* @param tableName - table name in database
*/
private static String escapeBacktickIdentifier(String identifier) {
return identifier.replace("`", "``");
}

private void addProjectionCondition(StringBuilder sb, List<String> projectionColumns, String tableName) {
if (!CollectionUtils.isEmpty(projectionColumns)) {
sb.append("SELECT");
projectionColumns.stream().forEach(columnName -> sb.append(" `" + columnName + "`,"));
projectionColumns.stream()
.forEach(columnName -> sb.append(" `" + escapeBacktickIdentifier(columnName) + "`,"));

sb.setLength(sb.length() - 1);
sb.append(" FROM " + tableName);
Expand Down Expand Up @@ -337,7 +384,7 @@ private void addSortCondition(StringBuilder sb, List<Map<String, String>> sortBy
+ "to parse the type of sort condition. Please reach out to Appsmith customer support "
+ "to resolve this.");
}
sb.append(" `" + columnName + "` " + sortType + ",");
sb.append(" `" + escapeBacktickIdentifier(columnName) + "` " + sortType + ",");
});

sb.setLength(sb.length() - 1);
Expand Down Expand Up @@ -615,10 +662,10 @@ public Map<String, DataType> generateSchema(ArrayNode items, Map<DataType, DataT
.takeWhile(x -> fieldNamesIterator.hasNext())
.map(n -> fieldNamesIterator.next())
.map(name -> {
if (name.contains("\"") || name.contains("\'")) {
if (name.contains("\"") || name.contains("\'") || name.contains("`")) {
throw new AppsmithPluginException(
AppsmithPluginError.PLUGIN_EXECUTE_ARGUMENT_ERROR,
"\' or \" are unsupported symbols in column names for filtering. Caused by column name : "
"', \" or ` are unsupported symbols in column names for filtering. Caused by column name : "
+ name);
}
return name;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1336,4 +1336,157 @@ public void testFilterEmptyAndNonEmptyCondition() {
fail(e.getMessage());
}
}

private static final String SIMPLE_DATA = "[\n" + " {\n"
+ " \"id\": 1,\n"
+ " \"email\": \"user@example.com\",\n"
+ " \"name\": \"Alice\"\n"
+ " },\n"
+ " {\n"
+ " \"id\": 2,\n"
+ " \"email\": \"admin@example.com\",\n"
+ " \"name\": \"Bob\"\n"
+ " }\n"
+ "]";

@Test
public void testProjectionWithInvalidColumnName_throwsException() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<String> projectionColumns = List.of("id", "nonExistentColumn");

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, projectionColumns, null, null));
});
assertThat(exception.getMessage()).contains("nonExistentColumn");
assertThat(exception.getMessage()).contains("not found in the known column names");
}

@Test
public void testSortByWithInvalidColumnName_throwsException() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<Map<String, String>> sortBy =
List.of(Map.of(SORT_BY_COLUMN_NAME_KEY, "nonExistentColumn", SORT_BY_TYPE_KEY, VALUE_DESCENDING));

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, null, sortBy, null));
});
assertThat(exception.getMessage()).contains("nonExistentColumn");
assertThat(exception.getMessage()).contains("not found in the known column names");
}

@Test
public void testProjectionWithBacktickInjection_throwsException() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<String> maliciousProjection = List.of("id` , FILE_READ('/etc/passwd') AS leaked , `id");

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, maliciousProjection, null, null));
});
assertThat(exception.getMessage()).contains("not found in the known column names");
}

@Test
public void testSortByWithBacktickInjection_throwsException() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<Map<String, String>> maliciousSortBy = List.of(Map.of(
SORT_BY_COLUMN_NAME_KEY,
"id` , FILE_READ('/etc/passwd') AS leaked , `id",
SORT_BY_TYPE_KEY,
VALUE_DESCENDING));

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, null, maliciousSortBy, null));
});
assertThat(exception.getMessage()).contains("not found in the known column names");
}

@Test
public void testSchemaRejectsBackticksInColumnNames() {
String dataWithBacktickColumn =
"[\n" + " {\n" + " \"normal_col\": \"value1\",\n" + " \"bad`col\": \"value2\"\n" + " }\n" + "]";

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
ArrayNode items = (ArrayNode) objectMapper.readTree(dataWithBacktickColumn);
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, null, null, null));
});
assertThat(exception.getMessage()).contains("unsupported symbols in column names");
}

@Test
public void testProjectionWithSqlExpressionInjection_throwsException() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<String> maliciousProjection = List.of("(SELECT CURRENT_TIMESTAMP)");

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, maliciousProjection, null, null));
});
assertThat(exception.getMessage()).contains("not found in the known column names");
}

@Test
public void testValidProjectionStillWorks() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<String> projectionColumns = List.of("id", "email");

ArrayNode result =
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, projectionColumns, null, null));

assertEquals(2, result.size());
List<String> returnedColumns = new ArrayList<>();
result.get(0).fieldNames().forEachRemaining(returnedColumns::add);
assertEquals(List.of("id", "email"), returnedColumns);
}

@Test
public void testValidSortByStillWorks() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<Map<String, String>> sortBy =
List.of(Map.of(SORT_BY_COLUMN_NAME_KEY, "id", SORT_BY_TYPE_KEY, VALUE_DESCENDING));

ArrayNode result = filterDataService.filterDataNew(items, new UQIDataFilterParams(null, null, sortBy, null));

assertEquals(2, result.size());
assertEquals(2, result.get(0).get("id").asInt());
assertEquals(1, result.get(1).get("id").asInt());
}

@Test
public void testSortByWithNullOrder_throwsException() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
Map<String, String> sortCondition = new HashMap<>();
sortCondition.put(SORT_BY_COLUMN_NAME_KEY, "id");
sortCondition.put(SORT_BY_TYPE_KEY, null);
List<Map<String, String>> sortBy = List.of(sortCondition);

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, null, sortBy, null));
});
assertThat(exception.getMessage()).contains("Sort order is required for column");
}

@Test
public void testSortByWithInvalidOrder_throwsException() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<Map<String, String>> sortBy =
List.of(Map.of(SORT_BY_COLUMN_NAME_KEY, "id", SORT_BY_TYPE_KEY, "INVALID_ORDER"));

AppsmithPluginException exception = assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, null, sortBy, null));
});
assertThat(exception.getMessage()).contains("Unsupported sort order");
}

@Test
public void testInvalidProjectionDoesNotLeakTable() throws IOException {
ArrayNode items = (ArrayNode) objectMapper.readTree(SIMPLE_DATA);
List<String> badProjection = List.of("nonExistentColumn");

assertThrows(AppsmithPluginException.class, () -> {
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, badProjection, null, null));
});

ArrayNode result =
filterDataService.filterDataNew(items, new UQIDataFilterParams(null, List.of("id"), null, null));
assertEquals(2, result.size());
}
}
Loading