From 1bf63a0c14a035f844fc7cd321376e19e01d214c Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 14 May 2024 14:49:35 +0300 Subject: [PATCH 1/9] AIP-21: rename files, emit deprecation warning --- airflow/providers/yandex/hooks/dataproc.py | 35 ++ .../yandex/hooks/yandexcloud_dataproc.py | 26 +- airflow/providers/yandex/hooks/yq.py | 15 +- .../providers/yandex/operators/dataproc.py | 535 ++++++++++++++++++ .../yandex/operators/yandexcloud_dataproc.py | 521 +---------------- airflow/providers/yandex/provider.yaml | 4 +- generated/provider_dependencies.json | 25 +- generated/provider_metadata.json | 404 ++----------- .../run_provider_yaml_files_check.py | 2 + ...ndexcloud_dataproc.py => test_dataproc.py} | 2 +- tests/providers/yandex/hooks/test_yandex.py | 14 +- tests/providers/yandex/hooks/test_yq.py | 21 +- ...ndexcloud_dataproc.py => test_dataproc.py} | 13 +- .../yandex/example_yandexcloud_dataproc.py | 2 +- ...xample_yandexcloud_dataproc_lightweight.py | 2 +- 15 files changed, 657 insertions(+), 964 deletions(-) create mode 100644 airflow/providers/yandex/hooks/dataproc.py create mode 100644 airflow/providers/yandex/operators/dataproc.py rename tests/providers/yandex/hooks/{test_yandexcloud_dataproc.py => test_dataproc.py} (99%) rename tests/providers/yandex/operators/{test_yandexcloud_dataproc.py => test_dataproc.py} (97%) diff --git a/airflow/providers/yandex/hooks/dataproc.py b/airflow/providers/yandex/hooks/dataproc.py new file mode 100644 index 0000000000000..9b1862205e466 --- /dev/null +++ b/airflow/providers/yandex/hooks/dataproc.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook + + +class DataprocHook(YandexCloudBaseHook): + """ + A base hook for Yandex.Cloud Data Proc. + + :param yandex_conn_id: The connection ID to use when fetching connection info. + """ + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.cluster_id = None + self.client = self.sdk.wrappers.Dataproc( + default_folder_id=self.default_folder_id, + default_public_ssh_key=self.default_public_ssh_key, + ) diff --git a/airflow/providers/yandex/hooks/yandexcloud_dataproc.py b/airflow/providers/yandex/hooks/yandexcloud_dataproc.py index 9b1862205e466..739438d0c2850 100644 --- a/airflow/providers/yandex/hooks/yandexcloud_dataproc.py +++ b/airflow/providers/yandex/hooks/yandexcloud_dataproc.py @@ -1,3 +1,4 @@ +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,22 +15,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - -from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook +"""This module is deprecated. Please use :mod:`airflow.providers.yandex.hooks.dataproc` instead.""" +from __future__ import annotations -class DataprocHook(YandexCloudBaseHook): - """ - A base hook for Yandex.Cloud Data Proc. +import warnings - :param yandex_conn_id: The connection ID to use when fetching connection info. - """ +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.providers.yandex.hooks.dataproc import * # noqa: F403 - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.cluster_id = None - self.client = self.sdk.wrappers.Dataproc( - default_folder_id=self.default_folder_id, - default_public_ssh_key=self.default_public_ssh_key, - ) +warnings.warn( + "This module is deprecated. Please use `airflow.providers.yandex.hooks.dataproc` instead.", + AirflowProviderDeprecationWarning, + stacklevel=2, +) diff --git a/airflow/providers/yandex/hooks/yq.py b/airflow/providers/yandex/hooks/yq.py index 37f7550df623a..c62595fe0a2da 100644 --- a/airflow/providers/yandex/hooks/yq.py +++ b/airflow/providers/yandex/hooks/yq.py @@ -19,9 +19,7 @@ from datetime import timedelta from typing import Any -import yandexcloud -import yandexcloud._auth_fabric as auth_fabric -from yandex.cloud.iam.v1.iam_token_service_pb2_grpc import IamTokenServiceStub +import yandexcloud.auth as yc_auth from yandex_query_client import YQHttpClient, YQHttpClientConfig from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook @@ -100,13 +98,4 @@ def _get_iam_token(self) -> str: if iam_token is not None: return iam_token - service_account_key = self.credentials.get("service_account_key") - # if service_account_key is None metadata server will be used - token_requester = auth_fabric.get_auth_token_requester(service_account_key=service_account_key) - - if service_account_key is None: - return token_requester.get_token() - - sdk = yandexcloud.SDK() - client = sdk.client(IamTokenServiceStub) - return client.Create(token_requester.get_token_request()).iam_token + return yc_auth.get_auth_token(service_account_key=self.credentials.get("service_account_key")) diff --git a/airflow/providers/yandex/operators/dataproc.py b/airflow/providers/yandex/operators/dataproc.py new file mode 100644 index 0000000000000..49bf136c3faeb --- /dev/null +++ b/airflow/providers/yandex/operators/dataproc.py @@ -0,0 +1,535 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import warnings +from dataclasses import dataclass +from typing import TYPE_CHECKING, Iterable, Sequence + +from airflow.exceptions import AirflowProviderDeprecationWarning +from airflow.models import BaseOperator +from airflow.providers.yandex.hooks.yandexcloud_dataproc import DataprocHook + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +@dataclass +class InitializationAction: + """Data for initialization action to be run at start of DataProc cluster.""" + + uri: str # Uri of the executable file + args: Sequence[str] # Arguments to the initialization action + timeout: int # Execution timeout + + +class DataprocCreateClusterOperator(BaseOperator): + """Creates Yandex.Cloud Data Proc cluster. + + :param folder_id: ID of the folder in which cluster should be created. + :param cluster_name: Cluster name. Must be unique inside the folder. + :param cluster_description: Cluster description. + :param cluster_image_version: Cluster image version. Use default. + :param ssh_public_keys: List of SSH public keys that will be deployed to created compute instances. + :param subnet_id: ID of the subnetwork. All Data Proc cluster nodes will use one subnetwork. + :param services: List of services that will be installed to the cluster. Possible options: + HDFS, YARN, MAPREDUCE, HIVE, TEZ, ZOOKEEPER, HBASE, SQOOP, FLUME, SPARK, SPARK, ZEPPELIN, OOZIE + :param s3_bucket: Yandex.Cloud S3 bucket to store cluster logs. + Jobs will not work if the bucket is not specified. + :param zone: Availability zone to create cluster in. + Currently there are ru-central1-a, ru-central1-b and ru-central1-c. + :param service_account_id: Service account id for the cluster. + Service account can be created inside the folder. + :param masternode_resource_preset: Resources preset (CPU+RAM configuration) + for the primary node of the cluster. + :param masternode_disk_size: Masternode storage size in GiB. + :param masternode_disk_type: Masternode storage type. Possible options: network-ssd, network-hdd. + :param datanode_resource_preset: Resources preset (CPU+RAM configuration) + for the data nodes of the cluster. + :param datanode_disk_size: Datanodes storage size in GiB. + :param datanode_disk_type: Datanodes storage type. Possible options: network-ssd, network-hdd. + :param computenode_resource_preset: Resources preset (CPU+RAM configuration) + for the compute nodes of the cluster. + :param computenode_disk_size: Computenodes storage size in GiB. + :param computenode_disk_type: Computenodes storage type. Possible options: network-ssd, network-hdd. + :param connection_id: ID of the Yandex.Cloud Airflow connection. + :param computenode_max_count: Maximum number of nodes of compute autoscaling subcluster. + :param computenode_warmup_duration: The warmup time of the instance in seconds. During this time, + traffic is sent to the instance, + but instance metrics are not collected. In seconds. + :param computenode_stabilization_duration: Minimum amount of time in seconds for monitoring before + Instance Groups can reduce the number of instances in the group. + During this time, the group size doesn't decrease, + even if the new metric values indicate that it should. In seconds. + :param computenode_preemptible: Preemptible instances are stopped at least once every 24 hours, + and can be stopped at any time if their resources are needed by Compute. + :param computenode_cpu_utilization_target: Defines an autoscaling rule + based on the average CPU utilization of the instance group. + in percents. 10-100. + By default is not set and default autoscaling strategy is used. + :param computenode_decommission_timeout: Timeout to gracefully decommission nodes during downscaling. + In seconds + :param properties: Properties passed to main node software. + Docs: https://cloud.yandex.com/docs/data-proc/concepts/settings-list + :param enable_ui_proxy: Enable UI Proxy feature for forwarding Hadoop components web interfaces + Docs: https://cloud.yandex.com/docs/data-proc/concepts/ui-proxy + :param host_group_ids: Dedicated host groups to place VMs of cluster on. + Docs: https://cloud.yandex.com/docs/compute/concepts/dedicated-host + :param security_group_ids: User security groups. + Docs: https://cloud.yandex.com/docs/data-proc/concepts/network#security-groups + :param log_group_id: Id of log group to write logs. By default logs will be sent to default log group. + To disable cloud log sending set cluster property dataproc:disable_cloud_logging = true + Docs: https://cloud.yandex.com/docs/data-proc/concepts/logs + :param initialization_actions: Set of init-actions to run when cluster starts. + Docs: https://cloud.yandex.com/docs/data-proc/concepts/init-action + :param labels: Cluster labels as key:value pairs. No more than 64 per resource. + Docs: https://cloud.yandex.com/docs/resource-manager/concepts/labels + """ + + def __init__( + self, + *, + folder_id: str | None = None, + cluster_name: str | None = None, + cluster_description: str | None = "", + cluster_image_version: str | None = None, + ssh_public_keys: str | Iterable[str] | None = None, + subnet_id: str | None = None, + services: Iterable[str] = ("HDFS", "YARN", "MAPREDUCE", "HIVE", "SPARK"), + s3_bucket: str | None = None, + zone: str = "ru-central1-b", + service_account_id: str | None = None, + masternode_resource_preset: str | None = None, + masternode_disk_size: int | None = None, + masternode_disk_type: str | None = None, + datanode_resource_preset: str | None = None, + datanode_disk_size: int | None = None, + datanode_disk_type: str | None = None, + datanode_count: int = 1, + computenode_resource_preset: str | None = None, + computenode_disk_size: int | None = None, + computenode_disk_type: str | None = None, + computenode_count: int = 0, + computenode_max_hosts_count: int | None = None, + computenode_measurement_duration: int | None = None, + computenode_warmup_duration: int | None = None, + computenode_stabilization_duration: int | None = None, + computenode_preemptible: bool = False, + computenode_cpu_utilization_target: int | None = None, + computenode_decommission_timeout: int | None = None, + connection_id: str | None = None, + properties: dict[str, str] | None = None, + enable_ui_proxy: bool = False, + host_group_ids: Iterable[str] | None = None, + security_group_ids: Iterable[str] | None = None, + log_group_id: str | None = None, + initialization_actions: Iterable[InitializationAction] | None = None, + labels: dict[str, str] | None = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.folder_id = folder_id + self.yandex_conn_id = connection_id + self.cluster_name = cluster_name + self.cluster_description = cluster_description + self.cluster_image_version = cluster_image_version + self.ssh_public_keys = ssh_public_keys + self.subnet_id = subnet_id + self.services = services + self.s3_bucket = s3_bucket + self.zone = zone + self.service_account_id = service_account_id + self.masternode_resource_preset = masternode_resource_preset + self.masternode_disk_size = masternode_disk_size + self.masternode_disk_type = masternode_disk_type + self.datanode_resource_preset = datanode_resource_preset + self.datanode_disk_size = datanode_disk_size + self.datanode_disk_type = datanode_disk_type + self.datanode_count = datanode_count + self.computenode_resource_preset = computenode_resource_preset + self.computenode_disk_size = computenode_disk_size + self.computenode_disk_type = computenode_disk_type + self.computenode_count = computenode_count + self.computenode_max_hosts_count = computenode_max_hosts_count + self.computenode_measurement_duration = computenode_measurement_duration + self.computenode_warmup_duration = computenode_warmup_duration + self.computenode_stabilization_duration = computenode_stabilization_duration + self.computenode_preemptible = computenode_preemptible + self.computenode_cpu_utilization_target = computenode_cpu_utilization_target + self.computenode_decommission_timeout = computenode_decommission_timeout + self.properties = properties + self.enable_ui_proxy = enable_ui_proxy + self.host_group_ids = host_group_ids + self.security_group_ids = security_group_ids + self.log_group_id = log_group_id + self.initialization_actions = initialization_actions + self.labels = labels + + self.hook: DataprocHook | None = None + + def execute(self, context: Context) -> dict: + self.hook = DataprocHook( + yandex_conn_id=self.yandex_conn_id, + ) + operation_result = self.hook.client.create_cluster( + folder_id=self.folder_id, + cluster_name=self.cluster_name, + cluster_description=self.cluster_description, + cluster_image_version=self.cluster_image_version, + ssh_public_keys=self.ssh_public_keys, + subnet_id=self.subnet_id, + services=self.services, + s3_bucket=self.s3_bucket, + zone=self.zone, + service_account_id=self.service_account_id or self.hook.default_service_account_id, + masternode_resource_preset=self.masternode_resource_preset, + masternode_disk_size=self.masternode_disk_size, + masternode_disk_type=self.masternode_disk_type, + datanode_resource_preset=self.datanode_resource_preset, + datanode_disk_size=self.datanode_disk_size, + datanode_disk_type=self.datanode_disk_type, + datanode_count=self.datanode_count, + computenode_resource_preset=self.computenode_resource_preset, + computenode_disk_size=self.computenode_disk_size, + computenode_disk_type=self.computenode_disk_type, + computenode_count=self.computenode_count, + computenode_max_hosts_count=self.computenode_max_hosts_count, + computenode_measurement_duration=self.computenode_measurement_duration, + computenode_warmup_duration=self.computenode_warmup_duration, + computenode_stabilization_duration=self.computenode_stabilization_duration, + computenode_preemptible=self.computenode_preemptible, + computenode_cpu_utilization_target=self.computenode_cpu_utilization_target, + computenode_decommission_timeout=self.computenode_decommission_timeout, + properties=self.properties, + enable_ui_proxy=self.enable_ui_proxy, + host_group_ids=self.host_group_ids, + security_group_ids=self.security_group_ids, + log_group_id=self.log_group_id, + labels=self.labels, + initialization_actions=self.initialization_actions + and [ + self.hook.sdk.wrappers.InitializationAction( + uri=init_action.uri, + args=init_action.args, + timeout=init_action.timeout, + ) + for init_action in self.initialization_actions + ], + ) + cluster_id = operation_result.response.id + + context["task_instance"].xcom_push(key="cluster_id", value=cluster_id) + # Deprecated + context["task_instance"].xcom_push(key="yandexcloud_connection_id", value=self.yandex_conn_id) + return cluster_id + + @property + def cluster_id(self): + return self.output + + +class DataprocBaseOperator(BaseOperator): + """Base class for DataProc operators working with given cluster. + + :param connection_id: ID of the Yandex.Cloud Airflow connection. + :param cluster_id: ID of the cluster to remove. (templated) + """ + + template_fields: Sequence[str] = ("cluster_id",) + + def __init__(self, *, yandex_conn_id: str | None = None, cluster_id: str | None = None, **kwargs) -> None: + super().__init__(**kwargs) + self.cluster_id = cluster_id + self.yandex_conn_id = yandex_conn_id + + def _setup(self, context: Context) -> DataprocHook: + if self.cluster_id is None: + self.cluster_id = context["task_instance"].xcom_pull(key="cluster_id") + if self.yandex_conn_id is None: + xcom_yandex_conn_id = context["task_instance"].xcom_pull(key="yandexcloud_connection_id") + if xcom_yandex_conn_id: + warnings.warn( + "Implicit pass of `yandex_conn_id` is deprecated, please pass it explicitly", + AirflowProviderDeprecationWarning, + stacklevel=2, + ) + self.yandex_conn_id = xcom_yandex_conn_id + + return DataprocHook(yandex_conn_id=self.yandex_conn_id) + + def execute(self, context: Context): + raise NotImplementedError() + + +class DataprocDeleteClusterOperator(DataprocBaseOperator): + """Deletes Yandex.Cloud Data Proc cluster. + + :param connection_id: ID of the Yandex.Cloud Airflow connection. + :param cluster_id: ID of the cluster to remove. (templated) + """ + + def __init__(self, *, connection_id: str | None = None, cluster_id: str | None = None, **kwargs) -> None: + super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) + + def execute(self, context: Context) -> None: + hook = self._setup(context) + hook.client.delete_cluster(self.cluster_id) + + +class DataprocCreateHiveJobOperator(DataprocBaseOperator): + """Runs Hive job in Data Proc cluster. + + :param query: Hive query. + :param query_file_uri: URI of the script that contains Hive queries. Can be placed in HDFS or S3. + :param properties: A mapping of property names to values, used to configure Hive. + :param script_variables: Mapping of query variable names to values. + :param continue_on_failure: Whether to continue executing queries if a query fails. + :param name: Name of the job. Used for labeling. + :param cluster_id: ID of the cluster to run job in. + Will try to take the ID from Dataproc Hook object if it's specified. (templated) + :param connection_id: ID of the Yandex.Cloud Airflow connection. + """ + + def __init__( + self, + *, + query: str | None = None, + query_file_uri: str | None = None, + script_variables: dict[str, str] | None = None, + continue_on_failure: bool = False, + properties: dict[str, str] | None = None, + name: str = "Hive job", + cluster_id: str | None = None, + connection_id: str | None = None, + **kwargs, + ) -> None: + super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) + self.query = query + self.query_file_uri = query_file_uri + self.script_variables = script_variables + self.continue_on_failure = continue_on_failure + self.properties = properties + self.name = name + + def execute(self, context: Context) -> None: + hook = self._setup(context) + hook.client.create_hive_job( + query=self.query, + query_file_uri=self.query_file_uri, + script_variables=self.script_variables, + continue_on_failure=self.continue_on_failure, + properties=self.properties, + name=self.name, + cluster_id=self.cluster_id, + ) + + +class DataprocCreateMapReduceJobOperator(DataprocBaseOperator): + """Runs Mapreduce job in Data Proc cluster. + + :param main_jar_file_uri: URI of jar file with job. + Can be placed in HDFS or S3. Can be specified instead of main_class. + :param main_class: Name of the main class of the job. Can be specified instead of main_jar_file_uri. + :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. + :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. + :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. + :param properties: Properties for the job. + :param args: Arguments to be passed to the job. + :param name: Name of the job. Used for labeling. + :param cluster_id: ID of the cluster to run job in. + Will try to take the ID from Dataproc Hook object if it's specified. (templated) + :param connection_id: ID of the Yandex.Cloud Airflow connection. + """ + + def __init__( + self, + *, + main_class: str | None = None, + main_jar_file_uri: str | None = None, + jar_file_uris: Iterable[str] | None = None, + archive_uris: Iterable[str] | None = None, + file_uris: Iterable[str] | None = None, + args: Iterable[str] | None = None, + properties: dict[str, str] | None = None, + name: str = "Mapreduce job", + cluster_id: str | None = None, + connection_id: str | None = None, + **kwargs, + ) -> None: + super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) + self.main_class = main_class + self.main_jar_file_uri = main_jar_file_uri + self.jar_file_uris = jar_file_uris + self.archive_uris = archive_uris + self.file_uris = file_uris + self.args = args + self.properties = properties + self.name = name + + def execute(self, context: Context) -> None: + hook = self._setup(context) + hook.client.create_mapreduce_job( + main_class=self.main_class, + main_jar_file_uri=self.main_jar_file_uri, + jar_file_uris=self.jar_file_uris, + archive_uris=self.archive_uris, + file_uris=self.file_uris, + args=self.args, + properties=self.properties, + name=self.name, + cluster_id=self.cluster_id, + ) + + +class DataprocCreateSparkJobOperator(DataprocBaseOperator): + """Runs Spark job in Data Proc cluster. + + :param main_jar_file_uri: URI of jar file with job. Can be placed in HDFS or S3. + :param main_class: Name of the main class of the job. + :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. + :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. + :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. + :param properties: Properties for the job. + :param args: Arguments to be passed to the job. + :param name: Name of the job. Used for labeling. + :param cluster_id: ID of the cluster to run job in. + Will try to take the ID from Dataproc Hook object if it's specified. (templated) + :param connection_id: ID of the Yandex.Cloud Airflow connection. + :param packages: List of maven coordinates of jars to include on the driver and executor classpaths. + :param repositories: List of additional remote repositories to search for the maven coordinates + given with --packages. + :param exclude_packages: List of groupId:artifactId, to exclude while resolving the dependencies + provided in --packages to avoid dependency conflicts. + """ + + def __init__( + self, + *, + main_class: str | None = None, + main_jar_file_uri: str | None = None, + jar_file_uris: Iterable[str] | None = None, + archive_uris: Iterable[str] | None = None, + file_uris: Iterable[str] | None = None, + args: Iterable[str] | None = None, + properties: dict[str, str] | None = None, + name: str = "Spark job", + cluster_id: str | None = None, + connection_id: str | None = None, + packages: Iterable[str] | None = None, + repositories: Iterable[str] | None = None, + exclude_packages: Iterable[str] | None = None, + **kwargs, + ) -> None: + super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) + self.main_class = main_class + self.main_jar_file_uri = main_jar_file_uri + self.jar_file_uris = jar_file_uris + self.archive_uris = archive_uris + self.file_uris = file_uris + self.args = args + self.properties = properties + self.name = name + self.packages = packages + self.repositories = repositories + self.exclude_packages = exclude_packages + + def execute(self, context: Context) -> None: + hook = self._setup(context) + hook.client.create_spark_job( + main_class=self.main_class, + main_jar_file_uri=self.main_jar_file_uri, + jar_file_uris=self.jar_file_uris, + archive_uris=self.archive_uris, + file_uris=self.file_uris, + args=self.args, + properties=self.properties, + packages=self.packages, + repositories=self.repositories, + exclude_packages=self.exclude_packages, + name=self.name, + cluster_id=self.cluster_id, + ) + + +class DataprocCreatePysparkJobOperator(DataprocBaseOperator): + """Runs Pyspark job in Data Proc cluster. + + :param main_python_file_uri: URI of python file with job. Can be placed in HDFS or S3. + :param python_file_uris: URIs of python files used in the job. Can be placed in HDFS or S3. + :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. + :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. + :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. + :param properties: Properties for the job. + :param args: Arguments to be passed to the job. + :param name: Name of the job. Used for labeling. + :param cluster_id: ID of the cluster to run job in. + Will try to take the ID from Dataproc Hook object if it's specified. (templated) + :param connection_id: ID of the Yandex.Cloud Airflow connection. + :param packages: List of maven coordinates of jars to include on the driver and executor classpaths. + :param repositories: List of additional remote repositories to search for the maven coordinates + given with --packages. + :param exclude_packages: List of groupId:artifactId, to exclude while resolving the dependencies + provided in --packages to avoid dependency conflicts. + """ + + def __init__( + self, + *, + main_python_file_uri: str | None = None, + python_file_uris: Iterable[str] | None = None, + jar_file_uris: Iterable[str] | None = None, + archive_uris: Iterable[str] | None = None, + file_uris: Iterable[str] | None = None, + args: Iterable[str] | None = None, + properties: dict[str, str] | None = None, + name: str = "Pyspark job", + cluster_id: str | None = None, + connection_id: str | None = None, + packages: Iterable[str] | None = None, + repositories: Iterable[str] | None = None, + exclude_packages: Iterable[str] | None = None, + **kwargs, + ) -> None: + super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) + self.main_python_file_uri = main_python_file_uri + self.python_file_uris = python_file_uris + self.jar_file_uris = jar_file_uris + self.archive_uris = archive_uris + self.file_uris = file_uris + self.args = args + self.properties = properties + self.name = name + self.packages = packages + self.repositories = repositories + self.exclude_packages = exclude_packages + + def execute(self, context: Context) -> None: + hook = self._setup(context) + hook.client.create_pyspark_job( + main_python_file_uri=self.main_python_file_uri, + python_file_uris=self.python_file_uris, + jar_file_uris=self.jar_file_uris, + archive_uris=self.archive_uris, + file_uris=self.file_uris, + args=self.args, + properties=self.properties, + packages=self.packages, + repositories=self.repositories, + exclude_packages=self.exclude_packages, + name=self.name, + cluster_id=self.cluster_id, + ) diff --git a/airflow/providers/yandex/operators/yandexcloud_dataproc.py b/airflow/providers/yandex/operators/yandexcloud_dataproc.py index 49bf136c3faeb..1f7db5d512c77 100644 --- a/airflow/providers/yandex/operators/yandexcloud_dataproc.py +++ b/airflow/providers/yandex/operators/yandexcloud_dataproc.py @@ -14,522 +14,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""This module is deprecated. Please use :mod:`airflow.providers.yandex.operators.dataproc` instead.""" + from __future__ import annotations import warnings -from dataclasses import dataclass -from typing import TYPE_CHECKING, Iterable, Sequence from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.models import BaseOperator -from airflow.providers.yandex.hooks.yandexcloud_dataproc import DataprocHook - -if TYPE_CHECKING: - from airflow.utils.context import Context - - -@dataclass -class InitializationAction: - """Data for initialization action to be run at start of DataProc cluster.""" - - uri: str # Uri of the executable file - args: Sequence[str] # Arguments to the initialization action - timeout: int # Execution timeout - - -class DataprocCreateClusterOperator(BaseOperator): - """Creates Yandex.Cloud Data Proc cluster. - - :param folder_id: ID of the folder in which cluster should be created. - :param cluster_name: Cluster name. Must be unique inside the folder. - :param cluster_description: Cluster description. - :param cluster_image_version: Cluster image version. Use default. - :param ssh_public_keys: List of SSH public keys that will be deployed to created compute instances. - :param subnet_id: ID of the subnetwork. All Data Proc cluster nodes will use one subnetwork. - :param services: List of services that will be installed to the cluster. Possible options: - HDFS, YARN, MAPREDUCE, HIVE, TEZ, ZOOKEEPER, HBASE, SQOOP, FLUME, SPARK, SPARK, ZEPPELIN, OOZIE - :param s3_bucket: Yandex.Cloud S3 bucket to store cluster logs. - Jobs will not work if the bucket is not specified. - :param zone: Availability zone to create cluster in. - Currently there are ru-central1-a, ru-central1-b and ru-central1-c. - :param service_account_id: Service account id for the cluster. - Service account can be created inside the folder. - :param masternode_resource_preset: Resources preset (CPU+RAM configuration) - for the primary node of the cluster. - :param masternode_disk_size: Masternode storage size in GiB. - :param masternode_disk_type: Masternode storage type. Possible options: network-ssd, network-hdd. - :param datanode_resource_preset: Resources preset (CPU+RAM configuration) - for the data nodes of the cluster. - :param datanode_disk_size: Datanodes storage size in GiB. - :param datanode_disk_type: Datanodes storage type. Possible options: network-ssd, network-hdd. - :param computenode_resource_preset: Resources preset (CPU+RAM configuration) - for the compute nodes of the cluster. - :param computenode_disk_size: Computenodes storage size in GiB. - :param computenode_disk_type: Computenodes storage type. Possible options: network-ssd, network-hdd. - :param connection_id: ID of the Yandex.Cloud Airflow connection. - :param computenode_max_count: Maximum number of nodes of compute autoscaling subcluster. - :param computenode_warmup_duration: The warmup time of the instance in seconds. During this time, - traffic is sent to the instance, - but instance metrics are not collected. In seconds. - :param computenode_stabilization_duration: Minimum amount of time in seconds for monitoring before - Instance Groups can reduce the number of instances in the group. - During this time, the group size doesn't decrease, - even if the new metric values indicate that it should. In seconds. - :param computenode_preemptible: Preemptible instances are stopped at least once every 24 hours, - and can be stopped at any time if their resources are needed by Compute. - :param computenode_cpu_utilization_target: Defines an autoscaling rule - based on the average CPU utilization of the instance group. - in percents. 10-100. - By default is not set and default autoscaling strategy is used. - :param computenode_decommission_timeout: Timeout to gracefully decommission nodes during downscaling. - In seconds - :param properties: Properties passed to main node software. - Docs: https://cloud.yandex.com/docs/data-proc/concepts/settings-list - :param enable_ui_proxy: Enable UI Proxy feature for forwarding Hadoop components web interfaces - Docs: https://cloud.yandex.com/docs/data-proc/concepts/ui-proxy - :param host_group_ids: Dedicated host groups to place VMs of cluster on. - Docs: https://cloud.yandex.com/docs/compute/concepts/dedicated-host - :param security_group_ids: User security groups. - Docs: https://cloud.yandex.com/docs/data-proc/concepts/network#security-groups - :param log_group_id: Id of log group to write logs. By default logs will be sent to default log group. - To disable cloud log sending set cluster property dataproc:disable_cloud_logging = true - Docs: https://cloud.yandex.com/docs/data-proc/concepts/logs - :param initialization_actions: Set of init-actions to run when cluster starts. - Docs: https://cloud.yandex.com/docs/data-proc/concepts/init-action - :param labels: Cluster labels as key:value pairs. No more than 64 per resource. - Docs: https://cloud.yandex.com/docs/resource-manager/concepts/labels - """ - - def __init__( - self, - *, - folder_id: str | None = None, - cluster_name: str | None = None, - cluster_description: str | None = "", - cluster_image_version: str | None = None, - ssh_public_keys: str | Iterable[str] | None = None, - subnet_id: str | None = None, - services: Iterable[str] = ("HDFS", "YARN", "MAPREDUCE", "HIVE", "SPARK"), - s3_bucket: str | None = None, - zone: str = "ru-central1-b", - service_account_id: str | None = None, - masternode_resource_preset: str | None = None, - masternode_disk_size: int | None = None, - masternode_disk_type: str | None = None, - datanode_resource_preset: str | None = None, - datanode_disk_size: int | None = None, - datanode_disk_type: str | None = None, - datanode_count: int = 1, - computenode_resource_preset: str | None = None, - computenode_disk_size: int | None = None, - computenode_disk_type: str | None = None, - computenode_count: int = 0, - computenode_max_hosts_count: int | None = None, - computenode_measurement_duration: int | None = None, - computenode_warmup_duration: int | None = None, - computenode_stabilization_duration: int | None = None, - computenode_preemptible: bool = False, - computenode_cpu_utilization_target: int | None = None, - computenode_decommission_timeout: int | None = None, - connection_id: str | None = None, - properties: dict[str, str] | None = None, - enable_ui_proxy: bool = False, - host_group_ids: Iterable[str] | None = None, - security_group_ids: Iterable[str] | None = None, - log_group_id: str | None = None, - initialization_actions: Iterable[InitializationAction] | None = None, - labels: dict[str, str] | None = None, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.folder_id = folder_id - self.yandex_conn_id = connection_id - self.cluster_name = cluster_name - self.cluster_description = cluster_description - self.cluster_image_version = cluster_image_version - self.ssh_public_keys = ssh_public_keys - self.subnet_id = subnet_id - self.services = services - self.s3_bucket = s3_bucket - self.zone = zone - self.service_account_id = service_account_id - self.masternode_resource_preset = masternode_resource_preset - self.masternode_disk_size = masternode_disk_size - self.masternode_disk_type = masternode_disk_type - self.datanode_resource_preset = datanode_resource_preset - self.datanode_disk_size = datanode_disk_size - self.datanode_disk_type = datanode_disk_type - self.datanode_count = datanode_count - self.computenode_resource_preset = computenode_resource_preset - self.computenode_disk_size = computenode_disk_size - self.computenode_disk_type = computenode_disk_type - self.computenode_count = computenode_count - self.computenode_max_hosts_count = computenode_max_hosts_count - self.computenode_measurement_duration = computenode_measurement_duration - self.computenode_warmup_duration = computenode_warmup_duration - self.computenode_stabilization_duration = computenode_stabilization_duration - self.computenode_preemptible = computenode_preemptible - self.computenode_cpu_utilization_target = computenode_cpu_utilization_target - self.computenode_decommission_timeout = computenode_decommission_timeout - self.properties = properties - self.enable_ui_proxy = enable_ui_proxy - self.host_group_ids = host_group_ids - self.security_group_ids = security_group_ids - self.log_group_id = log_group_id - self.initialization_actions = initialization_actions - self.labels = labels - - self.hook: DataprocHook | None = None - - def execute(self, context: Context) -> dict: - self.hook = DataprocHook( - yandex_conn_id=self.yandex_conn_id, - ) - operation_result = self.hook.client.create_cluster( - folder_id=self.folder_id, - cluster_name=self.cluster_name, - cluster_description=self.cluster_description, - cluster_image_version=self.cluster_image_version, - ssh_public_keys=self.ssh_public_keys, - subnet_id=self.subnet_id, - services=self.services, - s3_bucket=self.s3_bucket, - zone=self.zone, - service_account_id=self.service_account_id or self.hook.default_service_account_id, - masternode_resource_preset=self.masternode_resource_preset, - masternode_disk_size=self.masternode_disk_size, - masternode_disk_type=self.masternode_disk_type, - datanode_resource_preset=self.datanode_resource_preset, - datanode_disk_size=self.datanode_disk_size, - datanode_disk_type=self.datanode_disk_type, - datanode_count=self.datanode_count, - computenode_resource_preset=self.computenode_resource_preset, - computenode_disk_size=self.computenode_disk_size, - computenode_disk_type=self.computenode_disk_type, - computenode_count=self.computenode_count, - computenode_max_hosts_count=self.computenode_max_hosts_count, - computenode_measurement_duration=self.computenode_measurement_duration, - computenode_warmup_duration=self.computenode_warmup_duration, - computenode_stabilization_duration=self.computenode_stabilization_duration, - computenode_preemptible=self.computenode_preemptible, - computenode_cpu_utilization_target=self.computenode_cpu_utilization_target, - computenode_decommission_timeout=self.computenode_decommission_timeout, - properties=self.properties, - enable_ui_proxy=self.enable_ui_proxy, - host_group_ids=self.host_group_ids, - security_group_ids=self.security_group_ids, - log_group_id=self.log_group_id, - labels=self.labels, - initialization_actions=self.initialization_actions - and [ - self.hook.sdk.wrappers.InitializationAction( - uri=init_action.uri, - args=init_action.args, - timeout=init_action.timeout, - ) - for init_action in self.initialization_actions - ], - ) - cluster_id = operation_result.response.id - - context["task_instance"].xcom_push(key="cluster_id", value=cluster_id) - # Deprecated - context["task_instance"].xcom_push(key="yandexcloud_connection_id", value=self.yandex_conn_id) - return cluster_id - - @property - def cluster_id(self): - return self.output - - -class DataprocBaseOperator(BaseOperator): - """Base class for DataProc operators working with given cluster. - - :param connection_id: ID of the Yandex.Cloud Airflow connection. - :param cluster_id: ID of the cluster to remove. (templated) - """ - - template_fields: Sequence[str] = ("cluster_id",) - - def __init__(self, *, yandex_conn_id: str | None = None, cluster_id: str | None = None, **kwargs) -> None: - super().__init__(**kwargs) - self.cluster_id = cluster_id - self.yandex_conn_id = yandex_conn_id - - def _setup(self, context: Context) -> DataprocHook: - if self.cluster_id is None: - self.cluster_id = context["task_instance"].xcom_pull(key="cluster_id") - if self.yandex_conn_id is None: - xcom_yandex_conn_id = context["task_instance"].xcom_pull(key="yandexcloud_connection_id") - if xcom_yandex_conn_id: - warnings.warn( - "Implicit pass of `yandex_conn_id` is deprecated, please pass it explicitly", - AirflowProviderDeprecationWarning, - stacklevel=2, - ) - self.yandex_conn_id = xcom_yandex_conn_id - - return DataprocHook(yandex_conn_id=self.yandex_conn_id) - - def execute(self, context: Context): - raise NotImplementedError() - - -class DataprocDeleteClusterOperator(DataprocBaseOperator): - """Deletes Yandex.Cloud Data Proc cluster. - - :param connection_id: ID of the Yandex.Cloud Airflow connection. - :param cluster_id: ID of the cluster to remove. (templated) - """ - - def __init__(self, *, connection_id: str | None = None, cluster_id: str | None = None, **kwargs) -> None: - super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) - - def execute(self, context: Context) -> None: - hook = self._setup(context) - hook.client.delete_cluster(self.cluster_id) - - -class DataprocCreateHiveJobOperator(DataprocBaseOperator): - """Runs Hive job in Data Proc cluster. - - :param query: Hive query. - :param query_file_uri: URI of the script that contains Hive queries. Can be placed in HDFS or S3. - :param properties: A mapping of property names to values, used to configure Hive. - :param script_variables: Mapping of query variable names to values. - :param continue_on_failure: Whether to continue executing queries if a query fails. - :param name: Name of the job. Used for labeling. - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if it's specified. (templated) - :param connection_id: ID of the Yandex.Cloud Airflow connection. - """ - - def __init__( - self, - *, - query: str | None = None, - query_file_uri: str | None = None, - script_variables: dict[str, str] | None = None, - continue_on_failure: bool = False, - properties: dict[str, str] | None = None, - name: str = "Hive job", - cluster_id: str | None = None, - connection_id: str | None = None, - **kwargs, - ) -> None: - super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) - self.query = query - self.query_file_uri = query_file_uri - self.script_variables = script_variables - self.continue_on_failure = continue_on_failure - self.properties = properties - self.name = name - - def execute(self, context: Context) -> None: - hook = self._setup(context) - hook.client.create_hive_job( - query=self.query, - query_file_uri=self.query_file_uri, - script_variables=self.script_variables, - continue_on_failure=self.continue_on_failure, - properties=self.properties, - name=self.name, - cluster_id=self.cluster_id, - ) - - -class DataprocCreateMapReduceJobOperator(DataprocBaseOperator): - """Runs Mapreduce job in Data Proc cluster. - - :param main_jar_file_uri: URI of jar file with job. - Can be placed in HDFS or S3. Can be specified instead of main_class. - :param main_class: Name of the main class of the job. Can be specified instead of main_jar_file_uri. - :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. - :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. - :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. - :param properties: Properties for the job. - :param args: Arguments to be passed to the job. - :param name: Name of the job. Used for labeling. - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if it's specified. (templated) - :param connection_id: ID of the Yandex.Cloud Airflow connection. - """ - - def __init__( - self, - *, - main_class: str | None = None, - main_jar_file_uri: str | None = None, - jar_file_uris: Iterable[str] | None = None, - archive_uris: Iterable[str] | None = None, - file_uris: Iterable[str] | None = None, - args: Iterable[str] | None = None, - properties: dict[str, str] | None = None, - name: str = "Mapreduce job", - cluster_id: str | None = None, - connection_id: str | None = None, - **kwargs, - ) -> None: - super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) - self.main_class = main_class - self.main_jar_file_uri = main_jar_file_uri - self.jar_file_uris = jar_file_uris - self.archive_uris = archive_uris - self.file_uris = file_uris - self.args = args - self.properties = properties - self.name = name - - def execute(self, context: Context) -> None: - hook = self._setup(context) - hook.client.create_mapreduce_job( - main_class=self.main_class, - main_jar_file_uri=self.main_jar_file_uri, - jar_file_uris=self.jar_file_uris, - archive_uris=self.archive_uris, - file_uris=self.file_uris, - args=self.args, - properties=self.properties, - name=self.name, - cluster_id=self.cluster_id, - ) - - -class DataprocCreateSparkJobOperator(DataprocBaseOperator): - """Runs Spark job in Data Proc cluster. - - :param main_jar_file_uri: URI of jar file with job. Can be placed in HDFS or S3. - :param main_class: Name of the main class of the job. - :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. - :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. - :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. - :param properties: Properties for the job. - :param args: Arguments to be passed to the job. - :param name: Name of the job. Used for labeling. - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if it's specified. (templated) - :param connection_id: ID of the Yandex.Cloud Airflow connection. - :param packages: List of maven coordinates of jars to include on the driver and executor classpaths. - :param repositories: List of additional remote repositories to search for the maven coordinates - given with --packages. - :param exclude_packages: List of groupId:artifactId, to exclude while resolving the dependencies - provided in --packages to avoid dependency conflicts. - """ - - def __init__( - self, - *, - main_class: str | None = None, - main_jar_file_uri: str | None = None, - jar_file_uris: Iterable[str] | None = None, - archive_uris: Iterable[str] | None = None, - file_uris: Iterable[str] | None = None, - args: Iterable[str] | None = None, - properties: dict[str, str] | None = None, - name: str = "Spark job", - cluster_id: str | None = None, - connection_id: str | None = None, - packages: Iterable[str] | None = None, - repositories: Iterable[str] | None = None, - exclude_packages: Iterable[str] | None = None, - **kwargs, - ) -> None: - super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) - self.main_class = main_class - self.main_jar_file_uri = main_jar_file_uri - self.jar_file_uris = jar_file_uris - self.archive_uris = archive_uris - self.file_uris = file_uris - self.args = args - self.properties = properties - self.name = name - self.packages = packages - self.repositories = repositories - self.exclude_packages = exclude_packages - - def execute(self, context: Context) -> None: - hook = self._setup(context) - hook.client.create_spark_job( - main_class=self.main_class, - main_jar_file_uri=self.main_jar_file_uri, - jar_file_uris=self.jar_file_uris, - archive_uris=self.archive_uris, - file_uris=self.file_uris, - args=self.args, - properties=self.properties, - packages=self.packages, - repositories=self.repositories, - exclude_packages=self.exclude_packages, - name=self.name, - cluster_id=self.cluster_id, - ) - - -class DataprocCreatePysparkJobOperator(DataprocBaseOperator): - """Runs Pyspark job in Data Proc cluster. - - :param main_python_file_uri: URI of python file with job. Can be placed in HDFS or S3. - :param python_file_uris: URIs of python files used in the job. Can be placed in HDFS or S3. - :param file_uris: URIs of files used in the job. Can be placed in HDFS or S3. - :param archive_uris: URIs of archive files used in the job. Can be placed in HDFS or S3. - :param jar_file_uris: URIs of JAR files used in the job. Can be placed in HDFS or S3. - :param properties: Properties for the job. - :param args: Arguments to be passed to the job. - :param name: Name of the job. Used for labeling. - :param cluster_id: ID of the cluster to run job in. - Will try to take the ID from Dataproc Hook object if it's specified. (templated) - :param connection_id: ID of the Yandex.Cloud Airflow connection. - :param packages: List of maven coordinates of jars to include on the driver and executor classpaths. - :param repositories: List of additional remote repositories to search for the maven coordinates - given with --packages. - :param exclude_packages: List of groupId:artifactId, to exclude while resolving the dependencies - provided in --packages to avoid dependency conflicts. - """ - - def __init__( - self, - *, - main_python_file_uri: str | None = None, - python_file_uris: Iterable[str] | None = None, - jar_file_uris: Iterable[str] | None = None, - archive_uris: Iterable[str] | None = None, - file_uris: Iterable[str] | None = None, - args: Iterable[str] | None = None, - properties: dict[str, str] | None = None, - name: str = "Pyspark job", - cluster_id: str | None = None, - connection_id: str | None = None, - packages: Iterable[str] | None = None, - repositories: Iterable[str] | None = None, - exclude_packages: Iterable[str] | None = None, - **kwargs, - ) -> None: - super().__init__(yandex_conn_id=connection_id, cluster_id=cluster_id, **kwargs) - self.main_python_file_uri = main_python_file_uri - self.python_file_uris = python_file_uris - self.jar_file_uris = jar_file_uris - self.archive_uris = archive_uris - self.file_uris = file_uris - self.args = args - self.properties = properties - self.name = name - self.packages = packages - self.repositories = repositories - self.exclude_packages = exclude_packages +from airflow.providers.yandex.operators.dataproc import * # noqa: F403 - def execute(self, context: Context) -> None: - hook = self._setup(context) - hook.client.create_pyspark_job( - main_python_file_uri=self.main_python_file_uri, - python_file_uris=self.python_file_uris, - jar_file_uris=self.jar_file_uris, - archive_uris=self.archive_uris, - file_uris=self.file_uris, - args=self.args, - properties=self.properties, - packages=self.packages, - repositories=self.repositories, - exclude_packages=self.exclude_packages, - name=self.name, - cluster_id=self.cluster_id, - ) +warnings.warn( + "This module is deprecated. Please use `airflow.providers.yandex.operators.dataproc` instead.", + AirflowProviderDeprecationWarning, + stacklevel=2, +) diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index efbf4bba93c0c..b00c1aa8a9535 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -50,7 +50,7 @@ versions: dependencies: - apache-airflow>=2.7.0 - - yandexcloud>=0.228.0 + - yandexcloud>=0.278.0 - yandex-query-client>=0.1.4 integrations: @@ -76,6 +76,7 @@ integrations: operators: - integration-name: Yandex.Cloud Dataproc python-modules: + - airflow.providers.yandex.operators.dataproc - airflow.providers.yandex.operators.yandexcloud_dataproc - integration-name: Yandex.Cloud YQ @@ -88,6 +89,7 @@ hooks: - airflow.providers.yandex.hooks.yandex - integration-name: Yandex.Cloud Dataproc python-modules: + - airflow.providers.yandex.hooks.dataproc - airflow.providers.yandex.hooks.yandexcloud_dataproc - integration-name: Yandex.Cloud YQ python-modules: diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index d2c0896ba8961..6c5da7756cc4e 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -37,7 +37,8 @@ "jsonpath_ng>=1.5.3", "redshift_connector>=2.0.918", "sqlalchemy_redshift>=0.8.6", - "watchtower>=2.0.1,<4" + "watchtower>=2.0.1,<4", + "xmlsec<1.3.14" ], "devel-deps": [ "aiobotocore>=2.7.0", @@ -166,17 +167,6 @@ "excluded-python-versions": [], "state": "ready" }, - "apache.iceberg": { - "deps": [ - "apache-airflow>=2.7.0" - ], - "devel-deps": [ - "pyiceberg>=0.5.0" - ], - "cross-providers-deps": [], - "excluded-python-versions": [], - "state": "ready" - }, "apache.impala": { "deps": [ "apache-airflow>=2.7.0", @@ -262,7 +252,7 @@ "apprise": { "deps": [ "apache-airflow>=2.7.0", - "apprise>=1.8.0" + "apprise" ], "devel-deps": [], "cross-providers-deps": [], @@ -534,7 +524,7 @@ "gcsfs>=2023.10.0", "google-ads>=23.1.0", "google-analytics-admin", - "google-api-core>=2.11.0,!=2.16.0,!=2.18.0", + "google-api-core>=2.11.0,!=2.16.0", "google-api-python-client>=1.6.0", "google-auth-httplib2>=0.0.1", "google-auth>=1.0.0", @@ -1119,15 +1109,12 @@ }, "tabular": { "deps": [ - "apache-airflow-providers-apache-iceberg", "apache-airflow>=2.7.0" ], "devel-deps": [ "pyiceberg>=0.5.0" ], - "cross-providers-deps": [ - "apache.iceberg" - ], + "cross-providers-deps": [], "excluded-python-versions": [], "state": "ready" }, @@ -1199,7 +1186,7 @@ "deps": [ "apache-airflow>=2.7.0", "yandex-query-client>=0.1.4", - "yandexcloud>=0.228.0" + "yandexcloud>=0.278.0" ], "devel-deps": [], "cross-providers-deps": [], diff --git a/generated/provider_metadata.json b/generated/provider_metadata.json index 38735d62ca18e..9550d4d9013e1 100644 --- a/generated/provider_metadata.json +++ b/generated/provider_metadata.json @@ -73,12 +73,8 @@ "date_released": "2024-01-27T18:56:21Z" }, "3.7.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "3.8.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "alibaba": { @@ -159,12 +155,8 @@ "date_released": "2024-01-27T18:56:21Z" }, "2.7.3": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "2.8.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "amazon": { @@ -385,12 +377,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "8.20.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "8.21.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.beam": { @@ -507,12 +495,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "5.6.3": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "5.7.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.cassandra": { @@ -581,12 +565,8 @@ "date_released": "2023-12-27T23:07:26Z" }, "3.4.2": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.1", "date_released": "2024-04-13T18:19:54Z" - }, - "3.5.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.drill": { @@ -669,10 +649,6 @@ "2.6.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:35Z" - }, - "2.7.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.druid": { @@ -783,10 +759,6 @@ "3.9.0": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "3.10.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.flink": { @@ -821,10 +793,6 @@ "1.3.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "1.4.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.hdfs": { @@ -915,10 +883,6 @@ "4.3.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "4.4.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.hive": { @@ -1083,12 +1047,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "8.0.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "8.1.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.impala": { @@ -1123,10 +1083,6 @@ "1.3.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "1.4.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.kafka": { @@ -1157,10 +1113,6 @@ "1.3.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" - }, - "1.4.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.kylin": { @@ -1219,10 +1171,6 @@ "3.5.0": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-31T10:31:13Z" - }, - "3.6.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.livy": { @@ -1321,10 +1269,6 @@ "3.7.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "3.8.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:19Z" } }, "apache.pig": { @@ -1383,10 +1327,6 @@ "4.3.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "4.4.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:21Z" } }, "apache.pinot": { @@ -1473,10 +1413,6 @@ "4.3.1": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "4.4.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "apache.spark": { @@ -1593,12 +1529,8 @@ "date_released": "2024-01-27T18:56:21Z" }, "4.7.2": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "4.8.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "apprise": { @@ -1629,10 +1561,6 @@ "1.2.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "1.3.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "arangodb": { @@ -1675,10 +1603,6 @@ "2.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" - }, - "2.5.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "asana": { @@ -1737,10 +1661,6 @@ "2.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" - }, - "2.5.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "atlassian.jira": { @@ -1787,10 +1707,6 @@ "2.5.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-01-27T18:56:21Z" - }, - "2.6.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "celery": { @@ -1891,12 +1807,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "3.6.2": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "3.7.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "cloudant": { @@ -1955,10 +1867,6 @@ "3.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:21Z" } }, "cncf.kubernetes": { @@ -2179,12 +2087,8 @@ "date_released": "2024-04-13T18:19:54Z" }, "8.1.1": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-21T07:55:14Z" - }, - "8.2.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "cohere": { @@ -2205,12 +2109,8 @@ "date_released": "2024-01-27T18:56:22Z" }, "1.1.3": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "1.2.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "common.io": { @@ -2235,7 +2135,7 @@ "date_released": "2024-02-15T15:48:36Z" }, "1.3.1": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" } }, @@ -2341,12 +2241,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "1.12.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "1.13.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "databricks": { @@ -2487,12 +2383,8 @@ "date_released": "2024-02-15T15:48:36Z" }, "6.3.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-21T07:55:14Z" - }, - "6.4.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "datadog": { @@ -2559,10 +2451,6 @@ "3.5.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" - }, - "3.6.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:21Z" } }, "dbt.cloud": { @@ -2659,12 +2547,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "3.7.1": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "3.8.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:21Z" } }, "dingding": { @@ -2723,10 +2607,6 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "discord": { @@ -2797,10 +2677,6 @@ "3.6.0": { "associated_airflow_version": "2.8.2", "date_released": "2024-01-27T18:56:22Z" - }, - "3.7.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:19Z" } }, "docker": { @@ -2953,12 +2829,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "3.10.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "3.11.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:21Z" } }, "elasticsearch": { @@ -3103,12 +2975,8 @@ "date_released": "2024-02-15T15:48:36Z" }, "5.3.4": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "5.4.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "exasol": { @@ -3223,10 +3091,6 @@ "4.4.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "4.5.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "fab": { @@ -3247,12 +3111,8 @@ "date_released": "2024-04-13T18:19:54Z" }, "1.0.4": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.9.0", "date_released": "2024-04-21T07:55:14Z" - }, - "1.1.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "facebook": { @@ -3327,10 +3187,6 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "ftp": { @@ -3423,12 +3279,8 @@ "date_released": "2023-12-12T07:17:12Z" }, "3.8.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.0", "date_released": "2024-04-13T18:19:54Z" - }, - "3.9.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "github": { @@ -3487,10 +3339,6 @@ "2.5.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" - }, - "2.6.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "google": { @@ -3707,7 +3555,7 @@ "date_released": "2024-03-08T10:41:38Z" }, "10.17.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" } }, @@ -3775,10 +3623,6 @@ "3.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "hashicorp": { @@ -3885,10 +3729,6 @@ "3.6.4": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "3.7.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "http": { @@ -4005,12 +3845,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "4.10.1": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "4.11.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "imap": { @@ -4093,10 +3929,6 @@ "3.5.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "3.6.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "influxdb": { @@ -4155,10 +3987,6 @@ "2.4.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "2.5.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "jdbc": { @@ -4245,10 +4073,6 @@ "4.2.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "4.3.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "jenkins": { @@ -4339,10 +4163,6 @@ "3.5.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" - }, - "3.6.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:21Z" } }, "microsoft.azure": { @@ -4547,7 +4367,7 @@ "date_released": "2024-03-08T10:41:38Z" }, "10.0.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" } }, @@ -4639,10 +4459,6 @@ "3.6.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "3.7.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:21Z" } }, "microsoft.psrp": { @@ -4711,12 +4527,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "2.6.1": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "2.7.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "microsoft.winrm": { @@ -4791,10 +4603,6 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:21Z" } }, "mongo": { @@ -4877,10 +4685,6 @@ "4.0.0": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "4.1.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "mysql": { @@ -5015,10 +4819,6 @@ "5.5.4": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "5.6.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "neo4j": { @@ -5097,10 +4897,6 @@ "3.5.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "3.6.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:21Z" } }, "odbc": { @@ -5185,12 +4981,8 @@ "date_released": "2024-01-27T18:56:22Z" }, "4.5.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "4.6.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "openai": { @@ -5205,10 +4997,6 @@ "1.1.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "1.2.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "openfaas": { @@ -5263,10 +5051,6 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "openlineage": { @@ -5319,12 +5103,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "1.7.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "1.7.1": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "opensearch": { @@ -5343,10 +5123,6 @@ "1.1.2": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "1.2.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:21Z" } }, "opsgenie": { @@ -5421,10 +5197,6 @@ "5.5.0": { "associated_airflow_version": "2.8.1", "date_released": "2024-01-10T18:50:01Z" - }, - "5.6.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "oracle": { @@ -5535,10 +5307,6 @@ "3.9.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "3.10.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "pagerduty": { @@ -5617,10 +5385,6 @@ "3.6.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "3.7.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:21Z" } }, "papermill": { @@ -5701,12 +5465,8 @@ "date_released": "2024-01-27T18:56:22Z" }, "3.6.2": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "3.7.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "pgvector": { @@ -5717,10 +5477,6 @@ "1.1.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" - }, - "1.2.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "pinecone": { @@ -5869,10 +5625,6 @@ "5.10.2": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "5.11.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:21Z" } }, "presto": { @@ -5995,20 +5747,12 @@ "5.4.2": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "5.5.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:21Z" } }, "qdrant": { "1.0.0": { "associated_airflow_version": "2.9.0", "date_released": "2024-02-15T15:48:36Z" - }, - "1.1.0": { - "associated_airflow_version": "2.9.0", - "date_released": "2024-05-06T08:35:20Z" } }, "redis": { @@ -6085,12 +5829,8 @@ "date_released": "2024-01-10T18:50:01Z" }, "3.6.1": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.1", "date_released": "2024-04-13T18:19:54Z" - }, - "3.7.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "salesforce": { @@ -6201,10 +5941,6 @@ "5.6.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "5.7.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "samba": { @@ -6273,12 +6009,8 @@ "date_released": "2024-01-10T18:50:01Z" }, "4.6.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.1", "date_released": "2024-04-13T18:19:54Z" - }, - "4.7.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "segment": { @@ -6333,10 +6065,6 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:13Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "sendgrid": { @@ -6399,10 +6127,6 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:13Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "sftp": { @@ -6539,12 +6263,8 @@ "date_released": "2024-02-15T15:48:36Z" }, "4.9.1": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "4.10.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "singularity": { @@ -6607,10 +6327,6 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:13Z" - }, - "3.5.0": { - "associated_airflow_version": "2.8.0", - "date_released": "2024-05-06T08:35:20Z" } }, "slack": { @@ -6731,12 +6447,8 @@ "date_released": "2024-02-15T15:48:36Z" }, "8.6.2": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "8.7.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "smtp": { @@ -6787,10 +6499,6 @@ "1.6.1": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "1.7.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:19Z" } }, "snowflake": { @@ -6967,12 +6675,8 @@ "date_released": "2024-02-15T15:48:36Z" }, "5.4.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-13T18:19:54Z" - }, - "5.5.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "sqlite": { @@ -7071,10 +6775,6 @@ "3.7.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "3.8.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "ssh": { @@ -7197,10 +6897,6 @@ "3.10.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "3.11.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:21Z" } }, "tableau": { @@ -7291,10 +6987,6 @@ "4.4.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "4.5.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "tabular": { @@ -7329,10 +7021,6 @@ "1.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" - }, - "1.5.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:20Z" } }, "telegram": { @@ -7407,20 +7095,12 @@ "4.4.0": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "4.5.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "teradata": { "2.0.0": { "associated_airflow_version": "2.9.0", "date_released": "2024-02-27T05:42:18Z" - }, - "2.1.0": { - "associated_airflow_version": "2.9.0", - "date_released": "2024-05-06T08:35:20Z" } }, "trino": { @@ -7551,10 +7231,6 @@ "5.6.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" - }, - "5.7.0": { - "associated_airflow_version": "2.8.3", - "date_released": "2024-05-06T08:35:20Z" } }, "vertica": { @@ -7645,10 +7321,6 @@ "3.7.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" - }, - "3.8.0": { - "associated_airflow_version": "2.8.2", - "date_released": "2024-05-06T08:35:20Z" } }, "weaviate": { @@ -7681,12 +7353,8 @@ "date_released": "2024-03-08T10:41:38Z" }, "1.3.4": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.3", "date_released": "2024-04-13T18:19:54Z" - }, - "1.4.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "yandex": { @@ -7767,12 +7435,8 @@ "date_released": "2024-02-15T15:48:36Z" }, "3.10.0": { - "associated_airflow_version": "2.9.1", + "associated_airflow_version": "2.8.2", "date_released": "2024-04-16T07:26:28Z" - }, - "3.11.0": { - "associated_airflow_version": "2.9.1", - "date_released": "2024-05-06T08:35:20Z" } }, "zendesk": { @@ -7843,10 +7507,6 @@ "4.6.0": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-31T10:31:13Z" - }, - "4.7.0": { - "associated_airflow_version": "2.8.1", - "date_released": "2024-05-06T08:35:21Z" } } } diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 4f748461475cf..d3cf8b1e2e706 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -51,6 +51,8 @@ "airflow.providers.cncf.kubernetes.triggers.kubernetes_pod", "airflow.providers.cncf.kubernetes.operators.kubernetes_pod", "airflow.providers.tabular.hooks.tabular", + "airflow.providers.yandex.hooks.yandexcloud_dataproc", + "airflow.providers.yandex.operators.yandexcloud_dataproc", ] KNOWN_DEPRECATED_CLASSES = [ diff --git a/tests/providers/yandex/hooks/test_yandexcloud_dataproc.py b/tests/providers/yandex/hooks/test_dataproc.py similarity index 99% rename from tests/providers/yandex/hooks/test_yandexcloud_dataproc.py rename to tests/providers/yandex/hooks/test_dataproc.py index 5629a3678e600..0665b17b47047 100644 --- a/tests/providers/yandex/hooks/test_yandexcloud_dataproc.py +++ b/tests/providers/yandex/hooks/test_dataproc.py @@ -20,7 +20,7 @@ from unittest import mock from airflow.models import Connection -from airflow.providers.yandex.hooks.yandexcloud_dataproc import DataprocHook +from airflow.providers.yandex.hooks.dataproc import DataprocHook # Airflow connection with type "yandexcloud" must be created CONNECTION_ID = "yandexcloud_default" diff --git a/tests/providers/yandex/hooks/test_yandex.py b/tests/providers/yandex/hooks/test_yandex.py index ea885c6b3f95f..ae6dac568bc06 100644 --- a/tests/providers/yandex/hooks/test_yandex.py +++ b/tests/providers/yandex/hooks/test_yandex.py @@ -40,7 +40,7 @@ def test_client_created_without_exceptions(self, mock_get_credentials, mock_get_ mock_get_connection["extra_dejson"] = "sds" mock_get_connection.extra_dejson = '{"extras": "extra"}' mock_get_connection.return_value = mock.Mock( - connection_id="yandexcloud_default", extra_dejson=extra_dejson + yandex_conn_id="yandexcloud_default", extra_dejson=extra_dejson ) mock_get_credentials.return_value = {"token": 122323} @@ -54,7 +54,7 @@ def test_client_created_without_exceptions(self, mock_get_credentials, mock_get_ @mock.patch("airflow.hooks.base.BaseHook.get_connection") @mock.patch("airflow.providers.yandex.utils.credentials.get_credentials") def test_provider_user_agent(self, mock_get_credentials, mock_get_connection): - mock_get_connection.return_value = mock.Mock(connection_id="yandexcloud_default", extra_dejson="{}") + mock_get_connection.return_value = mock.Mock(yandex_conn_id="yandexcloud_default", extra_dejson="{}") mock_get_credentials.return_value = {"token": 122323} sdk_prefix = "MyAirflow" @@ -65,7 +65,7 @@ def test_provider_user_agent(self, mock_get_credentials, mock_get_connection): @mock.patch("airflow.hooks.base.BaseHook.get_connection") @mock.patch("airflow.providers.yandex.utils.credentials.get_credentials") def test_sdk_user_agent(self, mock_get_credentials, mock_get_connection): - mock_get_connection.return_value = mock.Mock(connection_id="yandexcloud_default", extra_dejson="{}") + mock_get_connection.return_value = mock.Mock(yandex_conn_id="yandexcloud_default", extra_dejson="{}") mock_get_credentials.return_value = {"token": 122323} sdk_prefix = "MyAirflow" @@ -97,7 +97,7 @@ def test_get_endpoint_specified(self, mock_get_credentials, mock_get_connection) extra_dejson = {"endpoint": "my_endpoint", "something_else": "some_value"} mock_get_connection.return_value = mock.Mock( - connection_id="yandexcloud_default", extra_dejson=extra_dejson + yandex_conn_id="yandexcloud_default", extra_dejson=extra_dejson ) mock_get_credentials.return_value = {"token": 122323} @@ -117,7 +117,7 @@ def test_get_endpoint_unspecified(self, mock_get_credentials, mock_get_connectio extra_dejson = {"something_else": "some_value"} mock_get_connection.return_value = mock.Mock( - connection_id="yandexcloud_default", extra_dejson=extra_dejson + yandex_conn_id="yandexcloud_default", extra_dejson=extra_dejson ) mock_get_credentials.return_value = {"token": 122323} @@ -140,7 +140,7 @@ def test__get_field(self, mock_get_connection): mock_get_connection["extra_dejson"] = "sds" mock_get_connection.extra_dejson = '{"extras": "extra"}' mock_get_connection.return_value = mock.Mock( - connection_id="yandexcloud_default", extra_dejson=extra_dejson + yandex_conn_id="yandexcloud_default", extra_dejson=extra_dejson ) hook = YandexCloudBaseHook( @@ -163,7 +163,7 @@ def test__get_field_extras_not_found(self, get_connection_mock): get_connection_mock["extra_dejson"] = "sds" get_connection_mock.extra_dejson = '{"extras": "extra"}' get_connection_mock.return_value = mock.Mock( - connection_id="yandexcloud_default", extra_dejson=extra_dejson + yandex_conn_id="yandexcloud_default", extra_dejson=extra_dejson ) hook = YandexCloudBaseHook() diff --git a/tests/providers/yandex/hooks/test_yq.py b/tests/providers/yandex/hooks/test_yq.py index 2864642c0eca1..71723d97b8914 100644 --- a/tests/providers/yandex/hooks/test_yq.py +++ b/tests/providers/yandex/hooks/test_yq.py @@ -31,20 +31,11 @@ SERVICE_ACCOUNT_AUTH_KEY_JSON = """{"id":"my_id", "service_account_id":"my_sa1", "private_key":"my_pk"}""" -class FakeTokenRequester: - def get_token(self) -> str: - return IAM_TOKEN - - def get_token_request(self) -> str: - return "my_dummy_request" - - class TestYandexCloudYqHook: def _init_hook(self): with mock.patch("airflow.hooks.base.BaseHook.get_connection") as mock_get_connection: mock_get_connection.return_value = self.connection - with mock.patch("airflow.providers.yandex.hooks.yq.yandexcloud.SDK.client"): - self.hook = YQHook(default_folder_id="my_folder_id") + self.hook = YQHook(default_folder_id="my_folder_id") def setup_method(self): self.connection = Connection(extra={"service_account_json": SERVICE_ACCOUNT_AUTH_KEY_JSON}) @@ -74,8 +65,8 @@ def test_oauth_token_usage(self): m.assert_called_once_with("query1") @responses.activate() - @mock.patch("yandexcloud._auth_fabric.get_auth_token_requester", return_value=FakeTokenRequester()) - def test_metadata_token_usage(self, mock_get_auth_token_requester): + @mock.patch("yandexcloud.auth.get_auth_token", return_value=IAM_TOKEN) + def test_metadata_token_usage(self, mock_get_auth_token): responses.post( "https://api.yandex-query.cloud.yandex.net/api/fq/v1/queries", match=[ @@ -94,8 +85,8 @@ def test_metadata_token_usage(self, mock_get_auth_token_requester): assert query_id == "query1" @mock.patch("yandexcloud._auth_fabric.__validate_service_account_key") - @mock.patch("yandexcloud._auth_fabric.get_auth_token_requester", return_value=FakeTokenRequester()) - def test_select_results(self, mock_get_auth_token_requester, mock_validate): + @mock.patch("yandexcloud.auth.get_auth_token", return_value=IAM_TOKEN) + def test_select_results(self, mock_get_auth_token, mock_validate): with mock.patch.multiple( "yandex_query_client.YQHttpClient", create_query=mock.DEFAULT, @@ -107,7 +98,7 @@ def test_select_results(self, mock_get_auth_token_requester, mock_validate): ) as mocks: self._init_hook() mock_validate.assert_called() - mock_get_auth_token_requester.assert_called_once_with( + mock_get_auth_token.assert_called_once_with( service_account_key=json.loads(SERVICE_ACCOUNT_AUTH_KEY_JSON) ) diff --git a/tests/providers/yandex/operators/test_yandexcloud_dataproc.py b/tests/providers/yandex/operators/test_dataproc.py similarity index 97% rename from tests/providers/yandex/operators/test_yandexcloud_dataproc.py rename to tests/providers/yandex/operators/test_dataproc.py index 0a055dd3778c7..a4b1242871e5c 100644 --- a/tests/providers/yandex/operators/test_yandexcloud_dataproc.py +++ b/tests/providers/yandex/operators/test_dataproc.py @@ -20,7 +20,7 @@ from unittest.mock import MagicMock, call, patch from airflow.models.dag import DAG -from airflow.providers.yandex.operators.yandexcloud_dataproc import ( +from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, DataprocCreateHiveJobOperator, DataprocCreateMapReduceJobOperator, @@ -161,6 +161,7 @@ def test_create_hive_job_operator(self, mock_create_hive_job, *_): operator = DataprocCreateHiveJobOperator( task_id="create_hive_job", query="SELECT 1;", + connection_id=CONNECTION_ID, ) context = {"task_instance": MagicMock()} context["task_instance"].xcom_pull.return_value = "my_cluster_id" @@ -169,10 +170,10 @@ def test_create_hive_job_operator(self, mock_create_hive_job, *_): context["task_instance"].xcom_pull.assert_has_calls( [ call(key="cluster_id"), - call(key="yandexcloud_connection_id"), ] ) + #mock_create_hive_job.assert_called_once() mock_create_hive_job.assert_called_once_with( cluster_id="my_cluster_id", continue_on_failure=False, @@ -180,7 +181,7 @@ def test_create_hive_job_operator(self, mock_create_hive_job, *_): properties=None, query="SELECT 1;", query_file_uri=None, - script_variables=None, + script_variables=None ) @patch("airflow.providers.yandex.utils.credentials.get_credentials") @@ -189,6 +190,7 @@ def test_create_hive_job_operator(self, mock_create_hive_job, *_): def test_create_mapreduce_job_operator(self, mock_create_mapreduce_job, *_): operator = DataprocCreateMapReduceJobOperator( task_id="run_mapreduce_job", + connection_id=CONNECTION_ID, main_class="org.apache.hadoop.streaming.HadoopStreaming", file_uris=[ "s3a://some-in-bucket/jobs/sources/mapreduce-001/mapper.py", @@ -219,7 +221,6 @@ def test_create_mapreduce_job_operator(self, mock_create_mapreduce_job, *_): context["task_instance"].xcom_pull.assert_has_calls( [ call(key="cluster_id"), - call(key="yandexcloud_connection_id"), ] ) @@ -259,6 +260,7 @@ def test_create_mapreduce_job_operator(self, mock_create_mapreduce_job, *_): def test_create_spark_job_operator(self, mock_create_spark_job, *_): operator = DataprocCreateSparkJobOperator( task_id="create_spark_job", + connection_id=CONNECTION_ID, main_jar_file_uri="s3a://data-proc-public/jobs/sources/java/dataproc-examples-1.0.jar", main_class="ru.yandex.cloud.dataproc.examples.PopulationSparkJob", file_uris=[ @@ -288,7 +290,6 @@ def test_create_spark_job_operator(self, mock_create_spark_job, *_): context["task_instance"].xcom_pull.assert_has_calls( [ call(key="cluster_id"), - call(key="yandexcloud_connection_id"), ] ) @@ -321,6 +322,7 @@ def test_create_spark_job_operator(self, mock_create_spark_job, *_): def test_create_pyspark_job_operator(self, mock_create_pyspark_job, *_): operator = DataprocCreatePysparkJobOperator( task_id="create_pyspark_job", + connection_id=CONNECTION_ID, main_python_file_uri="s3a://some-in-bucket/jobs/sources/pyspark-001/main.py", python_file_uris=[ "s3a://some-in-bucket/jobs/sources/pyspark-001/geonames.py", @@ -351,7 +353,6 @@ def test_create_pyspark_job_operator(self, mock_create_pyspark_job, *_): context["task_instance"].xcom_pull.assert_has_calls( [ call(key="cluster_id"), - call(key="yandexcloud_connection_id"), ] ) diff --git a/tests/system/providers/yandex/example_yandexcloud_dataproc.py b/tests/system/providers/yandex/example_yandexcloud_dataproc.py index cfae4e94e0aab..7ff4aa541dd54 100644 --- a/tests/system/providers/yandex/example_yandexcloud_dataproc.py +++ b/tests/system/providers/yandex/example_yandexcloud_dataproc.py @@ -20,7 +20,7 @@ from datetime import datetime from airflow import DAG -from airflow.providers.yandex.operators.yandexcloud_dataproc import ( +from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, DataprocCreateHiveJobOperator, DataprocCreateMapReduceJobOperator, diff --git a/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py b/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py index fa5a3c758bc1f..475bc789ec2b8 100644 --- a/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py +++ b/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py @@ -19,7 +19,7 @@ from datetime import datetime from airflow import DAG -from airflow.providers.yandex.operators.yandexcloud_dataproc import ( +from airflow.providers.yandex.operators.dataproc import ( DataprocCreateClusterOperator, DataprocCreateSparkJobOperator, DataprocDeleteClusterOperator, From 53ecd3801d908cfb9f7ec7412b6aca66865c3f5f Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 14 May 2024 15:23:43 +0300 Subject: [PATCH 2/9] revert most of the changes --- generated/provider_dependencies.json | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 6c5da7756cc4e..f10e93060e96c 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -37,8 +37,7 @@ "jsonpath_ng>=1.5.3", "redshift_connector>=2.0.918", "sqlalchemy_redshift>=0.8.6", - "watchtower>=2.0.1,<4", - "xmlsec<1.3.14" + "watchtower>=2.0.1,<4" ], "devel-deps": [ "aiobotocore>=2.7.0", @@ -167,6 +166,17 @@ "excluded-python-versions": [], "state": "ready" }, + "apache.iceberg": { + "deps": [ + "apache-airflow>=2.7.0" + ], + "devel-deps": [ + "pyiceberg>=0.5.0" + ], + "cross-providers-deps": [], + "excluded-python-versions": [], + "state": "ready" + }, "apache.impala": { "deps": [ "apache-airflow>=2.7.0", @@ -252,7 +262,7 @@ "apprise": { "deps": [ "apache-airflow>=2.7.0", - "apprise" + "apprise>=1.8.0" ], "devel-deps": [], "cross-providers-deps": [], @@ -524,7 +534,7 @@ "gcsfs>=2023.10.0", "google-ads>=23.1.0", "google-analytics-admin", - "google-api-core>=2.11.0,!=2.16.0", + "google-api-core>=2.11.0,!=2.16.0,!=2.18.0", "google-api-python-client>=1.6.0", "google-auth-httplib2>=0.0.1", "google-auth>=1.0.0", @@ -1109,12 +1119,15 @@ }, "tabular": { "deps": [ + "apache-airflow-providers-apache-iceberg", "apache-airflow>=2.7.0" ], "devel-deps": [ "pyiceberg>=0.5.0" ], - "cross-providers-deps": [], + "cross-providers-deps": [ + "apache.iceberg" + ], "excluded-python-versions": [], "state": "ready" }, From 34b7578e580e4dcd6e87bb9ffdf8920b00b8946c Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 14 May 2024 15:47:40 +0300 Subject: [PATCH 3/9] fix static checks --- tests/always/test_project_structure.py | 2 ++ tests/providers/yandex/operators/test_dataproc.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index 7ad127ac3b370..80444b9907285 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -159,6 +159,8 @@ def test_providers_modules_should_have_tests(self): "tests/providers/redis/sensors/test_redis_key.py", "tests/providers/slack/notifications/test_slack_notifier.py", "tests/providers/snowflake/triggers/test_snowflake_trigger.py", + "tests/providers/yandex/hooks/test_yandexcloud_dataproc.py", + "tests/providers/yandex/operators/test_yandexcloud_dataproc.py", ] # TODO: Should we extend this test to cover other directories? diff --git a/tests/providers/yandex/operators/test_dataproc.py b/tests/providers/yandex/operators/test_dataproc.py index a4b1242871e5c..9711ba55eec0f 100644 --- a/tests/providers/yandex/operators/test_dataproc.py +++ b/tests/providers/yandex/operators/test_dataproc.py @@ -173,7 +173,6 @@ def test_create_hive_job_operator(self, mock_create_hive_job, *_): ] ) - #mock_create_hive_job.assert_called_once() mock_create_hive_job.assert_called_once_with( cluster_id="my_cluster_id", continue_on_failure=False, @@ -181,7 +180,7 @@ def test_create_hive_job_operator(self, mock_create_hive_job, *_): properties=None, query="SELECT 1;", query_file_uri=None, - script_variables=None + script_variables=None, ) @patch("airflow.providers.yandex.utils.credentials.get_credentials") From 7e7e884dc927398e5e5c4def9c6feb748c7447f6 Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 14 May 2024 15:56:03 +0300 Subject: [PATCH 4/9] remove first line --- airflow/providers/yandex/hooks/yandexcloud_dataproc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/yandex/hooks/yandexcloud_dataproc.py b/airflow/providers/yandex/hooks/yandexcloud_dataproc.py index 739438d0c2850..6256769c9230f 100644 --- a/airflow/providers/yandex/hooks/yandexcloud_dataproc.py +++ b/airflow/providers/yandex/hooks/yandexcloud_dataproc.py @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information From 8448e6965d458085d73876c848fad88f2d6c1c2d Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 14 May 2024 15:57:03 +0300 Subject: [PATCH 5/9] revert file --- generated/provider_metadata.json | 404 ++++++++++++++++++++++++++++--- 1 file changed, 372 insertions(+), 32 deletions(-) diff --git a/generated/provider_metadata.json b/generated/provider_metadata.json index 9550d4d9013e1..38735d62ca18e 100644 --- a/generated/provider_metadata.json +++ b/generated/provider_metadata.json @@ -73,8 +73,12 @@ "date_released": "2024-01-27T18:56:21Z" }, "3.7.0": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.8.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "alibaba": { @@ -155,8 +159,12 @@ "date_released": "2024-01-27T18:56:21Z" }, "2.7.3": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "2.8.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "amazon": { @@ -377,8 +385,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "8.20.0": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "8.21.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.beam": { @@ -495,8 +507,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "5.6.3": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "5.7.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.cassandra": { @@ -565,8 +581,12 @@ "date_released": "2023-12-27T23:07:26Z" }, "3.4.2": { - "associated_airflow_version": "2.8.1", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.5.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.drill": { @@ -649,6 +669,10 @@ "2.6.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:35Z" + }, + "2.7.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.druid": { @@ -759,6 +783,10 @@ "3.9.0": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "3.10.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.flink": { @@ -793,6 +821,10 @@ "1.3.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "1.4.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.hdfs": { @@ -883,6 +915,10 @@ "4.3.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "4.4.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.hive": { @@ -1047,8 +1083,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "8.0.0": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "8.1.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.impala": { @@ -1083,6 +1123,10 @@ "1.3.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "1.4.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.kafka": { @@ -1113,6 +1157,10 @@ "1.3.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" + }, + "1.4.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.kylin": { @@ -1171,6 +1219,10 @@ "3.5.0": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-31T10:31:13Z" + }, + "3.6.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.livy": { @@ -1269,6 +1321,10 @@ "3.7.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "3.8.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:19Z" } }, "apache.pig": { @@ -1327,6 +1383,10 @@ "4.3.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "4.4.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:21Z" } }, "apache.pinot": { @@ -1413,6 +1473,10 @@ "4.3.1": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "4.4.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "apache.spark": { @@ -1529,8 +1593,12 @@ "date_released": "2024-01-27T18:56:21Z" }, "4.7.2": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "4.8.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "apprise": { @@ -1561,6 +1629,10 @@ "1.2.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "1.3.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "arangodb": { @@ -1603,6 +1675,10 @@ "2.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" + }, + "2.5.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "asana": { @@ -1661,6 +1737,10 @@ "2.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" + }, + "2.5.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "atlassian.jira": { @@ -1707,6 +1787,10 @@ "2.5.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-01-27T18:56:21Z" + }, + "2.6.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "celery": { @@ -1807,8 +1891,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "3.6.2": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.7.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "cloudant": { @@ -1867,6 +1955,10 @@ "3.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:26Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:21Z" } }, "cncf.kubernetes": { @@ -2087,8 +2179,12 @@ "date_released": "2024-04-13T18:19:54Z" }, "8.1.1": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-21T07:55:14Z" + }, + "8.2.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "cohere": { @@ -2109,8 +2205,12 @@ "date_released": "2024-01-27T18:56:22Z" }, "1.1.3": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "1.2.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "common.io": { @@ -2135,7 +2235,7 @@ "date_released": "2024-02-15T15:48:36Z" }, "1.3.1": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" } }, @@ -2241,8 +2341,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "1.12.0": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "1.13.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "databricks": { @@ -2383,8 +2487,12 @@ "date_released": "2024-02-15T15:48:36Z" }, "6.3.0": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-21T07:55:14Z" + }, + "6.4.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "datadog": { @@ -2451,6 +2559,10 @@ "3.5.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" + }, + "3.6.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:21Z" } }, "dbt.cloud": { @@ -2547,8 +2659,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "3.7.1": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.8.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:21Z" } }, "dingding": { @@ -2607,6 +2723,10 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "discord": { @@ -2677,6 +2797,10 @@ "3.6.0": { "associated_airflow_version": "2.8.2", "date_released": "2024-01-27T18:56:22Z" + }, + "3.7.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:19Z" } }, "docker": { @@ -2829,8 +2953,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "3.10.0": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.11.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:21Z" } }, "elasticsearch": { @@ -2975,8 +3103,12 @@ "date_released": "2024-02-15T15:48:36Z" }, "5.3.4": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "5.4.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "exasol": { @@ -3091,6 +3223,10 @@ "4.4.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "4.5.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "fab": { @@ -3111,8 +3247,12 @@ "date_released": "2024-04-13T18:19:54Z" }, "1.0.4": { - "associated_airflow_version": "2.9.0", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-21T07:55:14Z" + }, + "1.1.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "facebook": { @@ -3187,6 +3327,10 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "ftp": { @@ -3279,8 +3423,12 @@ "date_released": "2023-12-12T07:17:12Z" }, "3.8.0": { - "associated_airflow_version": "2.8.0", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.9.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "github": { @@ -3339,6 +3487,10 @@ "2.5.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" + }, + "2.6.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "google": { @@ -3555,7 +3707,7 @@ "date_released": "2024-03-08T10:41:38Z" }, "10.17.0": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" } }, @@ -3623,6 +3775,10 @@ "3.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "hashicorp": { @@ -3729,6 +3885,10 @@ "3.6.4": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "3.7.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "http": { @@ -3845,8 +4005,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "4.10.1": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "4.11.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "imap": { @@ -3929,6 +4093,10 @@ "3.5.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "3.6.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "influxdb": { @@ -3987,6 +4155,10 @@ "2.4.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "2.5.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "jdbc": { @@ -4073,6 +4245,10 @@ "4.2.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "4.3.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "jenkins": { @@ -4163,6 +4339,10 @@ "3.5.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" + }, + "3.6.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:21Z" } }, "microsoft.azure": { @@ -4367,7 +4547,7 @@ "date_released": "2024-03-08T10:41:38Z" }, "10.0.0": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" } }, @@ -4459,6 +4639,10 @@ "3.6.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "3.7.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:21Z" } }, "microsoft.psrp": { @@ -4527,8 +4711,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "2.6.1": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "2.7.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "microsoft.winrm": { @@ -4603,6 +4791,10 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:21Z" } }, "mongo": { @@ -4685,6 +4877,10 @@ "4.0.0": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "4.1.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "mysql": { @@ -4819,6 +5015,10 @@ "5.5.4": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "5.6.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "neo4j": { @@ -4897,6 +5097,10 @@ "3.5.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "3.6.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:21Z" } }, "odbc": { @@ -4981,8 +5185,12 @@ "date_released": "2024-01-27T18:56:22Z" }, "4.5.0": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "4.6.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "openai": { @@ -4997,6 +5205,10 @@ "1.1.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "1.2.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "openfaas": { @@ -5051,6 +5263,10 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "openlineage": { @@ -5103,8 +5319,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "1.7.0": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "1.7.1": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "opensearch": { @@ -5123,6 +5343,10 @@ "1.1.2": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "1.2.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:21Z" } }, "opsgenie": { @@ -5197,6 +5421,10 @@ "5.5.0": { "associated_airflow_version": "2.8.1", "date_released": "2024-01-10T18:50:01Z" + }, + "5.6.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "oracle": { @@ -5307,6 +5535,10 @@ "3.9.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "3.10.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "pagerduty": { @@ -5385,6 +5617,10 @@ "3.6.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "3.7.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:21Z" } }, "papermill": { @@ -5465,8 +5701,12 @@ "date_released": "2024-01-27T18:56:22Z" }, "3.6.2": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.7.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "pgvector": { @@ -5477,6 +5717,10 @@ "1.1.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:12Z" + }, + "1.2.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "pinecone": { @@ -5625,6 +5869,10 @@ "5.10.2": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "5.11.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:21Z" } }, "presto": { @@ -5747,12 +5995,20 @@ "5.4.2": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "5.5.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:21Z" } }, "qdrant": { "1.0.0": { "associated_airflow_version": "2.9.0", "date_released": "2024-02-15T15:48:36Z" + }, + "1.1.0": { + "associated_airflow_version": "2.9.0", + "date_released": "2024-05-06T08:35:20Z" } }, "redis": { @@ -5829,8 +6085,12 @@ "date_released": "2024-01-10T18:50:01Z" }, "3.6.1": { - "associated_airflow_version": "2.8.1", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "3.7.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "salesforce": { @@ -5941,6 +6201,10 @@ "5.6.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "5.7.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "samba": { @@ -6009,8 +6273,12 @@ "date_released": "2024-01-10T18:50:01Z" }, "4.6.0": { - "associated_airflow_version": "2.8.1", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "4.7.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "segment": { @@ -6065,6 +6333,10 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:13Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "sendgrid": { @@ -6127,6 +6399,10 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:13Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "sftp": { @@ -6263,8 +6539,12 @@ "date_released": "2024-02-15T15:48:36Z" }, "4.9.1": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "4.10.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "singularity": { @@ -6327,6 +6607,10 @@ "3.4.0": { "associated_airflow_version": "2.8.0", "date_released": "2023-12-12T07:17:13Z" + }, + "3.5.0": { + "associated_airflow_version": "2.8.0", + "date_released": "2024-05-06T08:35:20Z" } }, "slack": { @@ -6447,8 +6731,12 @@ "date_released": "2024-02-15T15:48:36Z" }, "8.6.2": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "8.7.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "smtp": { @@ -6499,6 +6787,10 @@ "1.6.1": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "1.7.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:19Z" } }, "snowflake": { @@ -6675,8 +6967,12 @@ "date_released": "2024-02-15T15:48:36Z" }, "5.4.0": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "5.5.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "sqlite": { @@ -6775,6 +7071,10 @@ "3.7.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "3.8.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "ssh": { @@ -6897,6 +7197,10 @@ "3.10.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "3.11.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:21Z" } }, "tableau": { @@ -6987,6 +7291,10 @@ "4.4.2": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "4.5.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "tabular": { @@ -7021,6 +7329,10 @@ "1.4.1": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-27T23:07:27Z" + }, + "1.5.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:20Z" } }, "telegram": { @@ -7095,12 +7407,20 @@ "4.4.0": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "4.5.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "teradata": { "2.0.0": { "associated_airflow_version": "2.9.0", "date_released": "2024-02-27T05:42:18Z" + }, + "2.1.0": { + "associated_airflow_version": "2.9.0", + "date_released": "2024-05-06T08:35:20Z" } }, "trino": { @@ -7231,6 +7551,10 @@ "5.6.3": { "associated_airflow_version": "2.8.3", "date_released": "2024-03-08T10:41:38Z" + }, + "5.7.0": { + "associated_airflow_version": "2.8.3", + "date_released": "2024-05-06T08:35:20Z" } }, "vertica": { @@ -7321,6 +7645,10 @@ "3.7.1": { "associated_airflow_version": "2.8.2", "date_released": "2024-02-15T15:48:36Z" + }, + "3.8.0": { + "associated_airflow_version": "2.8.2", + "date_released": "2024-05-06T08:35:20Z" } }, "weaviate": { @@ -7353,8 +7681,12 @@ "date_released": "2024-03-08T10:41:38Z" }, "1.3.4": { - "associated_airflow_version": "2.8.3", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-13T18:19:54Z" + }, + "1.4.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "yandex": { @@ -7435,8 +7767,12 @@ "date_released": "2024-02-15T15:48:36Z" }, "3.10.0": { - "associated_airflow_version": "2.8.2", + "associated_airflow_version": "2.9.1", "date_released": "2024-04-16T07:26:28Z" + }, + "3.11.0": { + "associated_airflow_version": "2.9.1", + "date_released": "2024-05-06T08:35:20Z" } }, "zendesk": { @@ -7507,6 +7843,10 @@ "4.6.0": { "associated_airflow_version": "2.8.1", "date_released": "2023-12-31T10:31:13Z" + }, + "4.7.0": { + "associated_airflow_version": "2.8.1", + "date_released": "2024-05-06T08:35:21Z" } } } From adaef1a4767222c19f72ae3826f8ed0ffe28ac64 Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 14 May 2024 17:49:47 +0300 Subject: [PATCH 6/9] fix static checks --- airflow/providers/yandex/operators/dataproc.py | 2 +- airflow/providers/yandex/provider.yaml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/airflow/providers/yandex/operators/dataproc.py b/airflow/providers/yandex/operators/dataproc.py index 49bf136c3faeb..94bf096b663b5 100644 --- a/airflow/providers/yandex/operators/dataproc.py +++ b/airflow/providers/yandex/operators/dataproc.py @@ -22,7 +22,7 @@ from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.models import BaseOperator -from airflow.providers.yandex.hooks.yandexcloud_dataproc import DataprocHook +from airflow.providers.yandex.hooks.dataproc import DataprocHook if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index b00c1aa8a9535..764a08586f943 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -77,8 +77,6 @@ operators: - integration-name: Yandex.Cloud Dataproc python-modules: - airflow.providers.yandex.operators.dataproc - - airflow.providers.yandex.operators.yandexcloud_dataproc - - integration-name: Yandex.Cloud YQ python-modules: - airflow.providers.yandex.operators.yq @@ -90,7 +88,6 @@ hooks: - integration-name: Yandex.Cloud Dataproc python-modules: - airflow.providers.yandex.hooks.dataproc - - airflow.providers.yandex.hooks.yandexcloud_dataproc - integration-name: Yandex.Cloud YQ python-modules: - airflow.providers.yandex.hooks.yq From 8f9e15112bd10c6c2cc7f15b74b81200b9945bcc Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Wed, 15 May 2024 12:00:38 +0300 Subject: [PATCH 7/9] revert --- tests/always/test_project_structure.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index 80444b9907285..7ad127ac3b370 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -159,8 +159,6 @@ def test_providers_modules_should_have_tests(self): "tests/providers/redis/sensors/test_redis_key.py", "tests/providers/slack/notifications/test_slack_notifier.py", "tests/providers/snowflake/triggers/test_snowflake_trigger.py", - "tests/providers/yandex/hooks/test_yandexcloud_dataproc.py", - "tests/providers/yandex/operators/test_yandexcloud_dataproc.py", ] # TODO: Should we extend this test to cover other directories? From b9dc8e67bc15a09aca9aaff4236d221644078a6c Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Wed, 15 May 2024 13:19:39 +0300 Subject: [PATCH 8/9] remove 4 ignores --- tests/deprecations_ignore.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/deprecations_ignore.yml b/tests/deprecations_ignore.yml index 1a8c8223f68a0..0f49f80e440a3 100644 --- a/tests/deprecations_ignore.yml +++ b/tests/deprecations_ignore.yml @@ -692,8 +692,4 @@ - tests/providers/weaviate/operators/test_weaviate.py::TestWeaviateIngestOperator::test_execute_with_input_json - tests/providers/yandex/hooks/test_yandex.py::TestYandexHook::test_provider_user_agent - tests/providers/yandex/hooks/test_yandex.py::TestYandexHook::test_backcompat_prefix_works -- tests/providers/yandex/operators/test_yandexcloud_dataproc.py::TestDataprocClusterCreateOperator::test_create_hive_job_operator -- tests/providers/yandex/operators/test_yandexcloud_dataproc.py::TestDataprocClusterCreateOperator::test_create_mapreduce_job_operator -- tests/providers/yandex/operators/test_yandexcloud_dataproc.py::TestDataprocClusterCreateOperator::test_create_spark_job_operator -- tests/providers/yandex/operators/test_yandexcloud_dataproc.py::TestDataprocClusterCreateOperator::test_create_pyspark_job_operator - tests/providers/yandex/secrets/test_lockbox.py::TestLockboxSecretBackend::test_yandex_lockbox_secret_backend_get_connection_from_json From e7f2a933aaa091717f296dbcdff7604db03bc5d3 Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Wed, 15 May 2024 15:09:32 +0300 Subject: [PATCH 9/9] ignore missing tests for yandexcloud deprecated modules --- tests/always/test_project_structure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index 7ad127ac3b370..80444b9907285 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -159,6 +159,8 @@ def test_providers_modules_should_have_tests(self): "tests/providers/redis/sensors/test_redis_key.py", "tests/providers/slack/notifications/test_slack_notifier.py", "tests/providers/snowflake/triggers/test_snowflake_trigger.py", + "tests/providers/yandex/hooks/test_yandexcloud_dataproc.py", + "tests/providers/yandex/operators/test_yandexcloud_dataproc.py", ] # TODO: Should we extend this test to cover other directories?