docs: auto generate metrics documentation

Author: aaron-ang

.github/workflows/rust.yml

@@ -708,6 +708,11 @@ jobs:
           # If you encounter an error, run './dev/update_function_docs.sh' and commit
           ./dev/update_function_docs.sh
           git diff --exit-code
+      - name: Check if metrics.md has been modified
+        run: |
+          # If you encounter an error, run './dev/update_metric_docs.sh' and commit
+          ./dev/update_metric_docs.sh
+          git diff --exit-code
 
   # Verify MSRV for the crates which are directly used by other projects:
   # - datafusion

Cargo.lock

@@ -129,6 +129,7 @@ datafusion-datasource-parquet = { workspace = true, optional = true }
 datafusion-execution = { workspace = true }
 datafusion-expr = { workspace = true, default-features = false }
 datafusion-expr-common = { workspace = true }
+datafusion-doc = { workspace = true }
 datafusion-functions = { workspace = true }
 datafusion-functions-aggregate = { workspace = true }
 datafusion-functions-nested = { workspace = true, default-features = false, optional = true }
@@ -167,7 +168,6 @@ ctor = { workspace = true }
 dashmap = "6.1.0"
 datafusion-doc = { workspace = true }
 datafusion-functions-window-common = { workspace = true }
-datafusion-macros = { workspace = true }
 datafusion-physical-optimizer = { workspace = true }
 doc-comment = { workspace = true }
 env_logger = { workspace = true }

datafusion/core/Cargo.toml

@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Print metrics documentation collected via `DocumentedMetrics`/`DocumentedExec`.
+//! Called from doc generation scripts to refresh `docs/source/user-guide/metrics.md`.
+
+use std::{collections::HashSet, fs, path::PathBuf};
+
+use datafusion_doc::metric_doc_sections::{
+    ExecDoc, MetricDoc, MetricDocPosition, exec_docs, metric_docs,
+};
+use datafusion_execution as _; // Link metrics defined in execution crate.
+use datafusion_physical_plan as _; // Link metrics and execs defined in physical plan.
+
+const LICENSE_HEADER: &str = "<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  \"License\"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->";
+
+const INTRO: &str = "DataFusion operators expose runtime metrics so you can understand where time is spent and how much data flows through the pipeline. See more in [EXPLAIN ANALYZE](sql/explain.md#explain-analyze).";
+
+fn main() -> std::io::Result<()> {
+    let mut content = String::new();
+    content.push_str(LICENSE_HEADER);
+    content.push_str("\n\n# Metrics\n\n");
+    content.push_str(INTRO);
+    content.push_str("\n\n");
+
+    let mut metrics: Vec<&MetricDoc> = metric_docs().collect();
+    metrics.sort_by(|a, b| a.name.cmp(b.name));
+
+    let mut execs: Vec<&ExecDoc> = exec_docs().collect();
+    execs.sort_by(|a, b| a.name.cmp(b.name));
+
+    let common: Vec<&MetricDoc> = metrics
+        .iter()
+        .copied()
+        .filter(|m| m.position == MetricDocPosition::Common)
+        .collect();
+
+    // Collect names of common metric types for filtering embedded fields
+    let common_metric_names: HashSet<&str> = common.iter().map(|m| m.name).collect();
+
+    if !common.is_empty() {
+        content.push_str("## Common Metrics\n\n");
+        for metric in common {
+            render_metric_doc(&mut content, metric, 3, &common_metric_names);
+        }
+    }
+
+    if !execs.is_empty() {
+        content.push_str("## Operator-specific Metrics\n\n");
+        for exec in execs {
+            render_exec_doc(&mut content, exec, &common_metric_names);
+        }
+    }
+
+    let path = output_path();
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    fs::write(path, content)
+}
+
+fn render_exec_doc(
+    out: &mut String,
+    exec: &ExecDoc,
+    common_metric_names: &HashSet<&str>,
+) {
+    out.push_str(&heading(3, exec.name));
+    out.push_str("\n\n");
+
+    if let Some(doc) = summarize(exec.doc) {
+        if !doc.is_empty() {
+            out.push_str(&sanitize(doc));
+            out.push_str("\n\n");
+        }
+    }
+
+    // Filter to operator-specific metrics only (common metrics are documented separately)
+    let mut metrics: Vec<&MetricDoc> = exec
+        .metrics
+        .iter()
+        .copied()
+        .filter(|metric| metric.position != MetricDocPosition::Common)
+        .collect();
+    metrics.sort_by(|a, b| a.name.cmp(b.name));
+
+    if metrics.is_empty() {
+        out.push_str("_No operator-specific metrics documented._\n\n");
+    } else {
+        for metric in metrics {
+            render_metric_doc(out, metric, 4, common_metric_names);
+        }
+    }
+}
+
+fn render_metric_doc(
+    out: &mut String,
+    metric: &MetricDoc,
+    heading_level: usize,
+    common_metric_names: &HashSet<&str>,
+) {
+    out.push_str(&heading(heading_level, metric.name));
+    out.push_str("\n\n");
+
+    if let Some(doc) = summarize(metric.doc) {
+        if !doc.is_empty() {
+            out.push_str(&sanitize(doc));
+            out.push_str("\n\n");
+        }
+    }
+
+    // Filter out fields whose type is a common metric (documented separately)
+    let fields: Vec<_> = metric
+        .fields
+        .iter()
+        .filter(|field| !common_metric_names.contains(field.type_name))
+        .collect();
+
+    if fields.is_empty() {
+        out.push_str("_No metrics documented._\n\n");
+        return;
+    }
+
+    out.push_str("| Metric | Description |\n");
+    out.push_str("| --- | --- |\n");
+    for field in fields {
+        out.push_str(&format!("| {} | {} |\n", field.name, sanitize(field.doc)));
+    }
+    out.push('\n');
+}
+
+fn heading(level: usize, title: &str) -> String {
+    format!("{} {}", "#".repeat(level), title)
+}
+
+fn summarize(doc: &str) -> Option<&str> {
+    let trimmed = doc.trim();
+    if trimmed.is_empty() {
+        return None;
+    }
+    trimmed.split("\n\n").next().map(str::trim)
+}
+
+fn sanitize(doc: &str) -> String {
+    doc.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+fn output_path() -> PathBuf {
+    let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    manifest_dir
+        .parent()
+        .and_then(|p| p.parent())
+        .unwrap_or(&manifest_dir)
+        .join("docs/source/user-guide/metrics.md")
+}

datafusion/core/src/bin/print_metric_docs.rs

@@ -28,6 +28,9 @@ license = { workspace = true }
 authors = { workspace = true }
 rust-version = { workspace = true }
 
+[dependencies]
+inventory = "0.3.15"
+
 [package.metadata.docs.rs]
 all-features = true
 

datafusion/doc/Cargo.toml

@@ -23,10 +23,12 @@
 )]
 #![cfg_attr(docsrs, feature(doc_cfg))]
 
+mod metrics;
 mod udaf;
 mod udf;
 mod udwf;
 
+pub use metrics::metric_doc_sections;
 pub use udaf::aggregate_doc_sections;
 pub use udf::scalar_doc_sections;
 pub use udwf::window_doc_sections;