Skip to content

Commit c402f9d

Browse files
committed
support orc reading based on index
1 parent 698e479 commit c402f9d

1 file changed

Lines changed: 30 additions & 6 deletions

File tree

native-engine/datafusion-ext-plans/src/orc_exec.rs

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -304,12 +304,36 @@ impl SchemaAdapter {
304304
let mut projection = Vec::with_capacity(self.table_schema.fields().len());
305305
let mut field_mappings = vec![None; self.table_schema.fields().len()];
306306

307-
for named_column in orc_file_meta.root_data_type().children() {
308-
if let Some((table_idx, _table_field)) =
309-
self.table_schema.fields().find(named_column.name())
310-
{
311-
field_mappings[table_idx] = Some(projection.len());
312-
projection.push(named_column.data_type().column_index());
307+
let file_named_columns = orc_file_meta.root_data_type().children();
308+
if file_named_columns
309+
.iter()
310+
.all(|named_col| named_col.name().starts_with("_col"))
311+
{
312+
let table_schema_fields = self.table_schema.fields();
313+
assert!(
314+
file_named_columns.len() <= table_schema_fields.len(),
315+
"The given data schema {:?} (length:{}) has fewer {} fields than \
316+
the actual ORC physical schema {:?} (length:{})",
317+
table_schema_fields,
318+
table_schema_fields.len(),
319+
file_named_columns.len() - table_schema_fields.len(),
320+
file_named_columns,
321+
file_named_columns.len()
322+
);
323+
for (col_idx, named_column) in file_named_columns.iter().enumerate() {
324+
if col_idx < table_schema_fields.len() {
325+
field_mappings[col_idx] = Some(projection.len());
326+
projection.push(named_column.data_type().column_index());
327+
}
328+
}
329+
} else {
330+
for named_column in file_named_columns {
331+
if let Some((table_idx, _table_field)) =
332+
self.table_schema.fields().find(named_column.name())
333+
{
334+
field_mappings[table_idx] = Some(projection.len());
335+
projection.push(named_column.data_type().column_index());
336+
}
313337
}
314338
}
315339

0 commit comments

Comments
 (0)