Applying queue with SparkSubmitOperator is ignored from 2.6.0 onwards #38461

from airflow.providers.apache.spark.operators.spark_submit import SparkSubmitOperator
(...)
class CustomSparkSubmitOperator(SparkSubmitOperator):
    """Extended spark submit operator."""

    template_fields = list(SparkSubmitOperator.template_fields)
    template_fields.append("_java_class")

def spark(
    self,
    application,
    java_class,
    task_id,
    priority_weight=1,
    alert_cc=None,
    auto_tags=None,
    **task_kwargs,
):
    """Create a task that runs a spark job."""
    _LOGGER.debug(f"Creating task: {task_id}")

    env = get_env()
    default_bp_config_path = (
        f"/data/business-parameters/business-parameters-master/data/all.{env}.conf"
    )
    extra_java_options = nlstrip(
        f"""
        -DApplication={application}
        (...)
        """
    )

    logstore_options = (
        f"{{{{ params.get('s3a_logstore_class', '{DEFAULT_LOGSTORE}') }}}}"
    )

    jar_path = f"{JAR_FOLDER}{{{{ params.jar_name }}}}"

    conf = {
        "spark.driver.memory": "{{ params.driver_memory }}",
        (...)
    }

    task = CustomSparkSubmitOperator(
        task_id=task_id,
        java_class=java_class,
        conn_id=f"spark_{env}_client",
        priority_weight=priority_weight,
        queue="airflow_spark",
        application=jar_path,
        verbose=True,
        spark_binary=SPARK_BINARY_PATH,
        conf=conf,
        env_vars=get_task_env(None),
        **get_task_kwargs(
            task_kwargs,
            alert_cc=alert_cc,
            dag=self.default_dag,
        ),
    )
    task.auto_tags = ["spark"]
    if auto_tags:
        task.auto_tags.extend(auto_tags)
    return task

/data/airflow/env/bin/pip list | grep providers
apache-airflow-providers-amazon          8.19.0
apache-airflow-providers-apache-spark    4.5.0
apache-airflow-providers-celery          3.6.1
apache-airflow-providers-common-io       1.3.0
apache-airflow-providers-common-sql      1.11.1
apache-airflow-providers-elasticsearch   5.3.3
apache-airflow-providers-ftp             3.7.0
apache-airflow-providers-http            4.10.0
apache-airflow-providers-imap            3.5.0
apache-airflow-providers-postgres        5.10.2
apache-airflow-providers-redis           3.6.0
apache-airflow-providers-slack           8.6.1
apache-airflow-providers-smtp            1.6.1
apache-airflow-providers-sqlite          3.7.1
apache-airflow-providers-ssh             3.10.1