From f3cde712105235ce44031be2dc06c140641a8656 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Sat, 9 Nov 2024 17:29:20 +0800 Subject: [PATCH 01/10] AIP-84 Get Config --- .../endpoints/config_endpoint.py | 3 + .../api_fastapi/core_api/datamodels/config.py | 64 ++++++ .../core_api/openapi/v1-generated.yaml | 110 ++++++++++ .../core_api/routes/public/__init__.py | 2 + .../core_api/routes/public/config.py | 122 +++++++++++ airflow/ui/openapi-gen/queries/common.ts | 129 ++++++++++++ airflow/ui/openapi-gen/queries/prefetch.ts | 151 ++++++++++++++ airflow/ui/openapi-gen/queries/queries.ts | 196 ++++++++++++++++++ airflow/ui/openapi-gen/queries/suspense.ts | 196 ++++++++++++++++++ .../ui/openapi-gen/requests/services.gen.ts | 66 ++++++ airflow/ui/openapi-gen/requests/types.gen.ts | 133 ++++++++++++ 11 files changed, 1172 insertions(+) create mode 100644 airflow/api_fastapi/core_api/datamodels/config.py create mode 100644 airflow/api_fastapi/core_api/routes/public/config.py diff --git a/airflow/api_connexion/endpoints/config_endpoint.py b/airflow/api_connexion/endpoints/config_endpoint.py index cbb8acdfce166..079fde54637d1 100644 --- a/airflow/api_connexion/endpoints/config_endpoint.py +++ b/airflow/api_connexion/endpoints/config_endpoint.py @@ -25,6 +25,7 @@ from airflow.api_connexion.schemas.config_schema import Config, ConfigOption, ConfigSection, config_schema from airflow.configuration import conf from airflow.settings import json +from airflow.utils.api_migration import mark_fastapi_migration_done LINE_SEP = "\n" # `\n` cannot appear in f-strings @@ -65,6 +66,7 @@ def _config_to_json(config: Config) -> str: return json.dumps(config_schema.dump(config), indent=4) +@mark_fastapi_migration_done @security.requires_access_configuration("GET") def get_config(*, section: str | None = None) -> Response: """Get current configuration.""" @@ -102,6 +104,7 @@ def get_config(*, section: str | None = None) -> Response: ) +@mark_fastapi_migration_done @security.requires_access_configuration("GET") def get_value(*, section: str, option: str) -> Response: serializer = { diff --git a/airflow/api_fastapi/core_api/datamodels/config.py b/airflow/api_fastapi/core_api/datamodels/config.py new file mode 100644 index 0000000000000..0160faa500d62 --- /dev/null +++ b/airflow/api_fastapi/core_api/datamodels/config.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pydantic import BaseModel + + +class ConfigOption(BaseModel): + """Config option.""" + + key: str + value: str | tuple[str, str] + + @property + def text_format(self): + if isinstance(self.value, tuple): + return f"{self.key} = {self.value[0]} {self.value[1]}" + return f"{self.key} = {self.value}" + + +class ConfigSection(BaseModel): + """Config Section Schema.""" + + name: str + options: list[ConfigOption] + + @property + def text_format(self): + """ + Convert the config section to text format. + + Example: + ``` + [section_name] + key1 = value1 + key2 = value2 + ``` + """ + return f"[{self.name}]\n" + "\n".join(option.text_format for option in self.options) + + +class Config(BaseModel): + """List of config sections with their options.""" + + sections: list[ConfigSection] + + @property + def text_format(self): + # convert all config sections to text + return "\n".join(section.text_format for section in self.sections) diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index 11c701f953e9b..ef113f299bc06 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -2091,6 +2091,116 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/config/: + get: + tags: + - Config + summary: Get Config + operationId: get_config + parameters: + - name: section + in: query + required: true + schema: + type: string + title: Section + - name: content-type + in: header + required: true + schema: + enum: + - application/json + - text/plain + type: string + title: Content-Type + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /public/config/section/{section}/option/{option}: + get: + tags: + - Config + summary: Get Config Value + operationId: get_config_value + parameters: + - name: section + in: path + required: true + schema: + type: string + title: Section + - name: option + in: path + required: true + schema: + type: string + title: Option + - name: content-type + in: header + required: true + schema: + enum: + - application/json + - text/plain + type: string + title: Content-Type + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /public/dags/tags: get: tags: diff --git a/airflow/api_fastapi/core_api/routes/public/__init__.py b/airflow/api_fastapi/core_api/routes/public/__init__.py index e85d17ae4ca84..ae4035f6d3532 100644 --- a/airflow/api_fastapi/core_api/routes/public/__init__.py +++ b/airflow/api_fastapi/core_api/routes/public/__init__.py @@ -23,6 +23,7 @@ from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc from airflow.api_fastapi.core_api.routes.public.assets import assets_router from airflow.api_fastapi.core_api.routes.public.backfills import backfills_router +from airflow.api_fastapi.core_api.routes.public.config import config_router from airflow.api_fastapi.core_api.routes.public.connections import connections_router from airflow.api_fastapi.core_api.routes.public.dag_run import dag_run_router from airflow.api_fastapi.core_api.routes.public.dag_sources import dag_sources_router @@ -54,6 +55,7 @@ authenticated_router.include_router(dag_run_router) authenticated_router.include_router(dag_sources_router) authenticated_router.include_router(dag_stats_router) +authenticated_router.include_router(config_router) authenticated_router.include_router(dag_warning_router) authenticated_router.include_router(dags_router) authenticated_router.include_router(event_logs_router) diff --git a/airflow/api_fastapi/core_api/routes/public/config.py b/airflow/api_fastapi/core_api/routes/public/config.py new file mode 100644 index 0000000000000..790ec6cfffaba --- /dev/null +++ b/airflow/api_fastapi/core_api/routes/public/config.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Literal + +from fastapi import Header, HTTPException, status +from fastapi.responses import Response + +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.datamodels.config import ( + Config, + ConfigOption, + ConfigSection, +) +from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc +from airflow.configuration import conf + +config_router = AirflowRouter(tags=["Config"], prefix="/config") + + +@config_router.get( + "/", + responses=create_openapi_http_exception_doc( + [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN, status.HTTP_404_NOT_FOUND] + ), +) +def get_config( + section: str, + content_type: Literal["application/json", "text/plain"] = Header(...), +) -> Response: + if conf.get("webserver", "expose_config").lower() == "non-sensitive-only": + expose_config = True + display_sensitive = False + else: + expose_config = conf.getboolean("webserver", "expose_config") + display_sensitive = True + + if not expose_config: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Your Airflow administrator chose not to expose the configuration, most likely for security reasons.", + ) + if section and not conf.has_section(section): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Section {section} not found.", + ) + conf_dict = conf.as_dict(display_source=False, display_sensitive=display_sensitive) + + if section: + conf_section_value = conf_dict[section] + conf_dict.clear() + conf_dict[section] = conf_section_value + + config = Config( + sections=[ + ConfigSection( + name=section, options=[ConfigOption(key=key, value=value) for key, value in options.items()] + ) + for section, options in conf_dict.items() + ] + ) + + return Response( + content=config.model_dump_json() if content_type == "application/json" else config.text_format, + headers={"Content-Type": content_type}, + ) + + +@config_router.get( + "/section/{section}/option/{option}", + responses=create_openapi_http_exception_doc( + [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN, status.HTTP_404_NOT_FOUND] + ), +) +def get_config_value( + section: str, + option: str, + content_type: Literal["application/json", "text/plain"] = Header(...), +) -> Response: + if conf.get("webserver", "expose_config").lower() == "non-sensitive-only": + expose_config = True + else: + expose_config = conf.getboolean("webserver", "expose_config") + + if not expose_config: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Your Airflow administrator chose not to expose the configuration, most likely for security reasons.", + ) + + if not conf.has_option(section, option): + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Option [{section}/{option}] not found.", + ) + + if (section.lower(), option.lower()) in conf.sensitive_config_values: + value = "< hidden >" + else: + value = conf.get(section, option) + + config = Config(sections=[ConfigSection(name=section, options=[ConfigOption(key=option, value=value)])]) + return Response( + content=config.model_dump_json() if content_type == "application/json" else config.text_format, + headers={"Content-Type": content_type}, + ) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 3d52ef341f31e..1498850542d13 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -4,6 +4,7 @@ import { UseQueryResult } from "@tanstack/react-query"; import { AssetService, BackfillService, + ConfigService, ConnectionService, DagRunService, DagService, @@ -570,6 +571,134 @@ export const UseDagServiceGetDagDetailsKeyFn = ( }, queryKey?: Array, ) => [useDagServiceGetDagDetailsKey, ...(queryKey ?? [{ dagId }])]; +export type ConnectionServiceGetConnectionDefaultResponse = Awaited< + ReturnType +>; +export type ConnectionServiceGetConnectionQueryResult< + TData = ConnectionServiceGetConnectionDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useConnectionServiceGetConnectionKey = + "ConnectionServiceGetConnection"; +export const UseConnectionServiceGetConnectionKeyFn = ( + { + connectionId, + }: { + connectionId: string; + }, + queryKey?: Array, +) => [ + useConnectionServiceGetConnectionKey, + ...(queryKey ?? [{ connectionId }]), +]; +export type ConnectionServiceGetConnectionsDefaultResponse = Awaited< + ReturnType +>; +export type ConnectionServiceGetConnectionsQueryResult< + TData = ConnectionServiceGetConnectionsDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useConnectionServiceGetConnectionsKey = + "ConnectionServiceGetConnections"; +export const UseConnectionServiceGetConnectionsKeyFn = ( + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, + queryKey?: Array, +) => [ + useConnectionServiceGetConnectionsKey, + ...(queryKey ?? [{ limit, offset, orderBy }]), +]; +export type ConfigServiceGetConfigDefaultResponse = Awaited< + ReturnType +>; +export type ConfigServiceGetConfigQueryResult< + TData = ConfigServiceGetConfigDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useConfigServiceGetConfigKey = "ConfigServiceGetConfig"; +export const UseConfigServiceGetConfigKeyFn = ( + { + contentType, + section, + }: { + contentType: "application/json" | "text/plain"; + section: string; + }, + queryKey?: Array, +) => [ + useConfigServiceGetConfigKey, + ...(queryKey ?? [{ contentType, section }]), +]; +export type ConfigServiceGetConfigValueDefaultResponse = Awaited< + ReturnType +>; +export type ConfigServiceGetConfigValueQueryResult< + TData = ConfigServiceGetConfigValueDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useConfigServiceGetConfigValueKey = "ConfigServiceGetConfigValue"; +export const UseConfigServiceGetConfigValueKeyFn = ( + { + contentType, + option, + section, + }: { + contentType: "application/json" | "text/plain"; + option: string; + section: string; + }, + queryKey?: Array, +) => [ + useConfigServiceGetConfigValueKey, + ...(queryKey ?? [{ contentType, option, section }]), +]; +export type DagRunServiceGetDagRunDefaultResponse = Awaited< + ReturnType +>; +export type DagRunServiceGetDagRunQueryResult< + TData = DagRunServiceGetDagRunDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useDagRunServiceGetDagRunKey = "DagRunServiceGetDagRun"; +export const UseDagRunServiceGetDagRunKeyFn = ( + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, + queryKey?: Array, +) => [useDagRunServiceGetDagRunKey, ...(queryKey ?? [{ dagId, dagRunId }])]; +export type DagSourceServiceGetDagSourceDefaultResponse = Awaited< + ReturnType +>; +export type DagSourceServiceGetDagSourceQueryResult< + TData = DagSourceServiceGetDagSourceDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useDagSourceServiceGetDagSourceKey = + "DagSourceServiceGetDagSource"; +export const UseDagSourceServiceGetDagSourceKeyFn = ( + { + accept, + fileToken, + }: { + accept?: string; + fileToken: string; + }, + queryKey?: Array, +) => [ + useDagSourceServiceGetDagSourceKey, + ...(queryKey ?? [{ accept, fileToken }]), +]; export type EventLogServiceGetEventLogDefaultResponse = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index baf49a781b5c5..44ff5421fed3c 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -4,6 +4,7 @@ import { type QueryClient } from "@tanstack/react-query"; import { AssetService, BackfillService, + ConfigService, ConnectionService, DagRunService, DagService, @@ -732,6 +733,156 @@ export const prefetchUseDagServiceGetDagDetails = ( queryKey: Common.UseDagServiceGetDagDetailsKeyFn({ dagId }), queryFn: () => DagService.getDagDetails({ dagId }), }); +/** + * Get Connection + * Get a connection entry. + * @param data The data for the request. + * @param data.connectionId + * @returns ConnectionResponse Successful Response + * @throws ApiError + */ +export const prefetchUseConnectionServiceGetConnection = ( + queryClient: QueryClient, + { + connectionId, + }: { + connectionId: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseConnectionServiceGetConnectionKeyFn({ connectionId }), + queryFn: () => ConnectionService.getConnection({ connectionId }), + }); +/** + * Get Connections + * Get all connection entries. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @param data.orderBy + * @returns ConnectionCollectionResponse Successful Response + * @throws ApiError + */ +export const prefetchUseConnectionServiceGetConnections = ( + queryClient: QueryClient, + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseConnectionServiceGetConnectionsKeyFn({ + limit, + offset, + orderBy, + }), + queryFn: () => ConnectionService.getConnections({ limit, offset, orderBy }), + }); +/** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ +export const prefetchUseConfigServiceGetConfig = ( + queryClient: QueryClient, + { + contentType, + section, + }: { + contentType: "application/json" | "text/plain"; + section: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseConfigServiceGetConfigKeyFn({ contentType, section }), + queryFn: () => ConfigService.getConfig({ contentType, section }), + }); +/** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ +export const prefetchUseConfigServiceGetConfigValue = ( + queryClient: QueryClient, + { + contentType, + option, + section, + }: { + contentType: "application/json" | "text/plain"; + option: string; + section: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseConfigServiceGetConfigValueKeyFn({ + contentType, + option, + section, + }), + queryFn: () => + ConfigService.getConfigValue({ contentType, option, section }), + }); +/** + * Get Dag Run + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns DAGRunResponse Successful Response + * @throws ApiError + */ +export const prefetchUseDagRunServiceGetDagRun = ( + queryClient: QueryClient, + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseDagRunServiceGetDagRunKeyFn({ dagId, dagRunId }), + queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }), + }); +/** + * Get Dag Source + * Get source code using file token. + * @param data The data for the request. + * @param data.fileToken + * @param data.accept + * @returns DAGSourceResponse Successful Response + * @throws ApiError + */ +export const prefetchUseDagSourceServiceGetDagSource = ( + queryClient: QueryClient, + { + accept, + fileToken, + }: { + accept?: string; + fileToken: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseDagSourceServiceGetDagSourceKeyFn({ + accept, + fileToken, + }), + queryFn: () => DagSourceService.getDagSource({ accept, fileToken }), + }); /** * Get Event Log * @param data The data for the request. diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 6a228e5f520f5..32d6c4719c887 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -9,6 +9,7 @@ import { import { AssetService, BackfillService, + ConfigService, ConnectionService, DagRunService, DagService, @@ -906,6 +907,201 @@ export const useDagServiceGetDagDetails = < queryFn: () => DagService.getDagDetails({ dagId }) as TData, ...options, }); +/** + * Get Connection + * Get a connection entry. + * @param data The data for the request. + * @param data.connectionId + * @returns ConnectionResponse Successful Response + * @throws ApiError + */ +export const useConnectionServiceGetConnection = < + TData = Common.ConnectionServiceGetConnectionDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + connectionId, + }: { + connectionId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseConnectionServiceGetConnectionKeyFn( + { connectionId }, + queryKey, + ), + queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, + ...options, + }); +/** + * Get Connections + * Get all connection entries. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @param data.orderBy + * @returns ConnectionCollectionResponse Successful Response + * @throws ApiError + */ +export const useConnectionServiceGetConnections = < + TData = Common.ConnectionServiceGetConnectionsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseConnectionServiceGetConnectionsKeyFn( + { limit, offset, orderBy }, + queryKey, + ), + queryFn: () => + ConnectionService.getConnections({ limit, offset, orderBy }) as TData, + ...options, + }); +/** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfig = < + TData = Common.ConfigServiceGetConfigDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + contentType, + section, + }: { + contentType: "application/json" | "text/plain"; + section: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseConfigServiceGetConfigKeyFn( + { contentType, section }, + queryKey, + ), + queryFn: () => ConfigService.getConfig({ contentType, section }) as TData, + ...options, + }); +/** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfigValue = < + TData = Common.ConfigServiceGetConfigValueDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + contentType, + option, + section, + }: { + contentType: "application/json" | "text/plain"; + option: string; + section: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseConfigServiceGetConfigValueKeyFn( + { contentType, option, section }, + queryKey, + ), + queryFn: () => + ConfigService.getConfigValue({ contentType, option, section }) as TData, + ...options, + }); +/** + * Get Dag Run + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns DAGRunResponse Successful Response + * @throws ApiError + */ +export const useDagRunServiceGetDagRun = < + TData = Common.DagRunServiceGetDagRunDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseDagRunServiceGetDagRunKeyFn( + { dagId, dagRunId }, + queryKey, + ), + queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, + ...options, + }); +/** + * Get Dag Source + * Get source code using file token. + * @param data The data for the request. + * @param data.fileToken + * @param data.accept + * @returns DAGSourceResponse Successful Response + * @throws ApiError + */ +export const useDagSourceServiceGetDagSource = < + TData = Common.DagSourceServiceGetDagSourceDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + accept, + fileToken, + }: { + accept?: string; + fileToken: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseDagSourceServiceGetDagSourceKeyFn( + { accept, fileToken }, + queryKey, + ), + queryFn: () => + DagSourceService.getDagSource({ accept, fileToken }) as TData, + ...options, + }); /** * Get Event Log * @param data The data for the request. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index d663543cfd9ca..58366aa8ae2fc 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -4,6 +4,7 @@ import { UseQueryOptions, useSuspenseQuery } from "@tanstack/react-query"; import { AssetService, BackfillService, + ConfigService, ConnectionService, DagRunService, DagService, @@ -888,6 +889,201 @@ export const useDagServiceGetDagDetailsSuspense = < queryFn: () => DagService.getDagDetails({ dagId }) as TData, ...options, }); +/** + * Get Connection + * Get a connection entry. + * @param data The data for the request. + * @param data.connectionId + * @returns ConnectionResponse Successful Response + * @throws ApiError + */ +export const useConnectionServiceGetConnectionSuspense = < + TData = Common.ConnectionServiceGetConnectionDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + connectionId, + }: { + connectionId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseConnectionServiceGetConnectionKeyFn( + { connectionId }, + queryKey, + ), + queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, + ...options, + }); +/** + * Get Connections + * Get all connection entries. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @param data.orderBy + * @returns ConnectionCollectionResponse Successful Response + * @throws ApiError + */ +export const useConnectionServiceGetConnectionsSuspense = < + TData = Common.ConnectionServiceGetConnectionsDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + limit, + offset, + orderBy, + }: { + limit?: number; + offset?: number; + orderBy?: string; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseConnectionServiceGetConnectionsKeyFn( + { limit, offset, orderBy }, + queryKey, + ), + queryFn: () => + ConnectionService.getConnections({ limit, offset, orderBy }) as TData, + ...options, + }); +/** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfigSuspense = < + TData = Common.ConfigServiceGetConfigDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + contentType, + section, + }: { + contentType: "application/json" | "text/plain"; + section: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseConfigServiceGetConfigKeyFn( + { contentType, section }, + queryKey, + ), + queryFn: () => ConfigService.getConfig({ contentType, section }) as TData, + ...options, + }); +/** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfigValueSuspense = < + TData = Common.ConfigServiceGetConfigValueDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + contentType, + option, + section, + }: { + contentType: "application/json" | "text/plain"; + option: string; + section: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseConfigServiceGetConfigValueKeyFn( + { contentType, option, section }, + queryKey, + ), + queryFn: () => + ConfigService.getConfigValue({ contentType, option, section }) as TData, + ...options, + }); +/** + * Get Dag Run + * @param data The data for the request. + * @param data.dagId + * @param data.dagRunId + * @returns DAGRunResponse Successful Response + * @throws ApiError + */ +export const useDagRunServiceGetDagRunSuspense = < + TData = Common.DagRunServiceGetDagRunDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + dagId, + dagRunId, + }: { + dagId: string; + dagRunId: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseDagRunServiceGetDagRunKeyFn( + { dagId, dagRunId }, + queryKey, + ), + queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, + ...options, + }); +/** + * Get Dag Source + * Get source code using file token. + * @param data The data for the request. + * @param data.fileToken + * @param data.accept + * @returns DAGSourceResponse Successful Response + * @throws ApiError + */ +export const useDagSourceServiceGetDagSourceSuspense = < + TData = Common.DagSourceServiceGetDagSourceDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + accept, + fileToken, + }: { + accept?: string; + fileToken: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseDagSourceServiceGetDagSourceKeyFn( + { accept, fileToken }, + queryKey, + ), + queryFn: () => + DagSourceService.getDagSource({ accept, fileToken }) as TData, + ...options, + }); /** * Get Event Log * @param data The data for the request. diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 72ee239974201..5a2428bcdea99 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -51,6 +51,10 @@ import type { GetConnectionsResponse, PostConnectionData, PostConnectionResponse, + GetConfigData, + GetConfigResponse, + GetConfigValueData, + GetConfigValueResponse, TestConnectionData, TestConnectionResponse, GetDagRunData, @@ -1221,6 +1225,68 @@ export class DagService { }); } +export class ConfigService { + /** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ + public static getConfig( + data: GetConfigData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/config/", + headers: { + "content-type": data.contentType, + }, + query: { + section: data.section, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Validation Error", + }, + }); + } + + /** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.contentType + * @returns unknown Successful Response + * @throws ApiError + */ + public static getConfigValue( + data: GetConfigValueData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/config/section/{section}/option/{option}", + path: { + section: data.section, + option: data.option, + }, + headers: { + "content-type": data.contentType, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 422: "Validation Error", + }, + }); + } +} + /** * Get Dag Tags * Get all DAG tags. diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 50fb44a057a25..1e5225a8996ab 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1366,6 +1366,85 @@ export type GetDagDetailsData = { export type GetDagDetailsResponse = DAGDetailsResponse; +export type DeleteConnectionData = { + connectionId: string; +}; + +export type DeleteConnectionResponse = void; + +export type GetConnectionData = { + connectionId: string; +}; + +export type GetConnectionResponse = ConnectionResponse; + +export type PatchConnectionData = { + connectionId: string; + requestBody: ConnectionBody; + updateMask?: Array | null; +}; + +export type PatchConnectionResponse = ConnectionResponse; + +export type GetConnectionsData = { + limit?: number; + offset?: number; + orderBy?: string; +}; + +export type GetConnectionsResponse = ConnectionCollectionResponse; + +export type PostConnectionData = { + requestBody: ConnectionBody; +}; + +export type PostConnectionResponse = ConnectionResponse; + +export type GetConfigData = { + contentType: "application/json" | "text/plain"; + section: string; +}; + +export type GetConfigResponse = unknown; + +export type GetConfigValueData = { + contentType: "application/json" | "text/plain"; + option: string; + section: string; +}; + +export type GetConfigValueResponse = unknown; + +export type GetDagRunData = { + dagId: string; + dagRunId: string; +}; + +export type GetDagRunResponse = DAGRunResponse; + +export type DeleteDagRunData = { + dagId: string; + dagRunId: string; +}; + +export type DeleteDagRunResponse = void; + +export type PatchDagRunData = { + dagId: string; + dagRunId: string; + requestBody: DAGRunPatchBody; + updateMask?: Array | null; +}; + +export type PatchDagRunResponse = DAGRunResponse; + +export type GetDagSourceData = { + accept?: string; + fileToken: string; +}; + +export type GetDagSourceResponse = DAGSourceResponse; + export type GetEventLogData = { eventLogId: number; }; @@ -2551,6 +2630,60 @@ export type $OpenApiTs = { }; }; }; + "/public/config/": { + get: { + req: GetConfigData; + res: { + /** + * Successful Response + */ + 200: unknown; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; + "/public/config/section/{section}/option/{option}": { + get: { + req: GetConfigValueData; + res: { + /** + * Successful Response + */ + 200: unknown; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; "/public/dags/tags": { get: { req: GetDagTagsData; From c8f584a01b7b85ee941424af5a1ab450927af667 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Sat, 9 Nov 2024 20:30:34 +0800 Subject: [PATCH 02/10] Fix section param type and text format --- airflow/api_fastapi/core_api/datamodels/config.py | 2 +- airflow/api_fastapi/core_api/openapi/v1-generated.yaml | 6 ++++-- airflow/api_fastapi/core_api/routes/public/config.py | 2 +- airflow/ui/openapi-gen/queries/common.ts | 2 +- airflow/ui/openapi-gen/queries/prefetch.ts | 4 ++-- airflow/ui/openapi-gen/queries/queries.ts | 4 ++-- airflow/ui/openapi-gen/queries/suspense.ts | 4 ++-- airflow/ui/openapi-gen/requests/services.gen.ts | 7 ++++--- airflow/ui/openapi-gen/requests/types.gen.ts | 2 +- 9 files changed, 18 insertions(+), 15 deletions(-) diff --git a/airflow/api_fastapi/core_api/datamodels/config.py b/airflow/api_fastapi/core_api/datamodels/config.py index 0160faa500d62..0627832e45f4c 100644 --- a/airflow/api_fastapi/core_api/datamodels/config.py +++ b/airflow/api_fastapi/core_api/datamodels/config.py @@ -50,7 +50,7 @@ def text_format(self): key2 = value2 ``` """ - return f"[{self.name}]\n" + "\n".join(option.text_format for option in self.options) + return f"[{self.name}]\n" + "\n".join(option.text_format for option in self.options) + "\n" class Config(BaseModel): diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index ef113f299bc06..497c80bf23f56 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -2100,9 +2100,11 @@ paths: parameters: - name: section in: query - required: true + required: false schema: - type: string + anyOf: + - type: string + - type: 'null' title: Section - name: content-type in: header diff --git a/airflow/api_fastapi/core_api/routes/public/config.py b/airflow/api_fastapi/core_api/routes/public/config.py index 790ec6cfffaba..0aaf9a456c14d 100644 --- a/airflow/api_fastapi/core_api/routes/public/config.py +++ b/airflow/api_fastapi/core_api/routes/public/config.py @@ -40,7 +40,7 @@ ), ) def get_config( - section: str, + section: str | None = None, content_type: Literal["application/json", "text/plain"] = Header(...), ) -> Response: if conf.get("webserver", "expose_config").lower() == "non-sensitive-only": diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 1498850542d13..dee2973d67d8d 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -629,7 +629,7 @@ export const UseConfigServiceGetConfigKeyFn = ( section, }: { contentType: "application/json" | "text/plain"; - section: string; + section?: string; }, queryKey?: Array, ) => [ diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index 44ff5421fed3c..c9b3e530ec1d0 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -786,8 +786,8 @@ export const prefetchUseConnectionServiceGetConnections = ( /** * Get Config * @param data The data for the request. - * @param data.section * @param data.contentType + * @param data.section * @returns unknown Successful Response * @throws ApiError */ @@ -798,7 +798,7 @@ export const prefetchUseConfigServiceGetConfig = ( section, }: { contentType: "application/json" | "text/plain"; - section: string; + section?: string; }, ) => queryClient.prefetchQuery({ diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 32d6c4719c887..18923f97cf3d1 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -975,8 +975,8 @@ export const useConnectionServiceGetConnections = < /** * Get Config * @param data The data for the request. - * @param data.section * @param data.contentType + * @param data.section * @returns unknown Successful Response * @throws ApiError */ @@ -990,7 +990,7 @@ export const useConfigServiceGetConfig = < section, }: { contentType: "application/json" | "text/plain"; - section: string; + section?: string; }, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">, diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index 58366aa8ae2fc..6d8472e351a4e 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -957,8 +957,8 @@ export const useConnectionServiceGetConnectionsSuspense = < /** * Get Config * @param data The data for the request. - * @param data.section * @param data.contentType + * @param data.section * @returns unknown Successful Response * @throws ApiError */ @@ -972,7 +972,7 @@ export const useConfigServiceGetConfigSuspense = < section, }: { contentType: "application/json" | "text/plain"; - section: string; + section?: string; }, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">, diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 5a2428bcdea99..4301ae89c1914 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -1133,9 +1133,7 @@ export class DagWarningService { }, }); } -} -export class DagService { /** * Get Dags * Get all DAGs. @@ -1224,13 +1222,14 @@ export class DagService { }, }); } +} export class ConfigService { /** * Get Config * @param data The data for the request. - * @param data.section * @param data.contentType + * @param data.section * @returns unknown Successful Response * @throws ApiError */ @@ -1287,6 +1286,8 @@ export class ConfigService { } } +export class DagService { + /** * Get Dag Tags * Get all DAG tags. diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 1e5225a8996ab..33290cf88d159 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1402,7 +1402,7 @@ export type PostConnectionResponse = ConnectionResponse; export type GetConfigData = { contentType: "application/json" | "text/plain"; - section: string; + section?: string | null; }; export type GetConfigResponse = unknown; From 7fa4786b022bd9f40e3033fdcb9e62ff45948c5f Mon Sep 17 00:00:00 2001 From: jason810496 Date: Sat, 9 Nov 2024 20:39:55 +0800 Subject: [PATCH 03/10] Add test for Get Config --- .../core_api/routes/public/test_config.py | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 tests/api_fastapi/core_api/routes/public/test_config.py diff --git a/tests/api_fastapi/core_api/routes/public/test_config.py b/tests/api_fastapi/core_api/routes/public/test_config.py new file mode 100644 index 0000000000000..c92582e1e93bb --- /dev/null +++ b/tests/api_fastapi/core_api/routes/public/test_config.py @@ -0,0 +1,317 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import textwrap +from typing import Generator +from unittest.mock import patch + +import pytest + +from tests_common.test_utils.config import conf_vars + +HEADERS_JSON = {"Content-Type": "application/json"} +HEADERS_TEXT = {"Content-Type": "text/plain"} +HEADERS_INVALID = {"Content-Type": "invalid"} +SECTION_CORE = "core" +SECTION_SMTP = "smtp" +SECTION_DATABASE = "database" +SECTION_NOT_EXIST = "not_exist_section" +OPTION_KEY_PARALLELISM = "parallelism" +OPTION_KEY_SMTP_HOST = "smtp_host" +OPTION_KEY_SMTP_MAIL_FROM = "smtp_mail_from" +OPTION_KEY_SQL_ALCHEMY_CONN = "sql_alchemy_conn" +OPTION_VALUE_PARALLELISM = "1024" +OPTION_VALUE_SMTP_HOST = "smtp.example.com" +OPTION_VALUE_SMTP_MAIL_FROM = "airflow@example.com" +OPTION_VALUE_SQL_ALCHEMY_CONN = "mock_sql_alchemy_conn" +OPTION_NOT_EXIST = "not_exist_option" +OPTION_VALUE_SENSITIVE_HIDDEN = "< hidden >" + +MOCK_CONFIG_DICT = { + SECTION_CORE: { + OPTION_KEY_PARALLELISM: OPTION_VALUE_PARALLELISM, + }, + SECTION_SMTP: { + OPTION_KEY_SMTP_HOST: OPTION_VALUE_SMTP_HOST, + OPTION_KEY_SMTP_MAIL_FROM: OPTION_VALUE_SMTP_MAIL_FROM, + }, + SECTION_DATABASE: { + OPTION_KEY_SQL_ALCHEMY_CONN: OPTION_VALUE_SQL_ALCHEMY_CONN, + }, +} +MOCK_CONFIG_OVERRIDE = { + (SECTION_CORE, OPTION_KEY_PARALLELISM): OPTION_VALUE_PARALLELISM, + (SECTION_SMTP, OPTION_KEY_SMTP_HOST): OPTION_VALUE_SMTP_HOST, + (SECTION_SMTP, OPTION_KEY_SMTP_MAIL_FROM): OPTION_VALUE_SMTP_MAIL_FROM, +} + +AIRFLOW_CONFIG_ENABLE_EXPOSE_CONFIG = {("webserver", "expose_config"): "True"} +AIRFLOW_CONFIG_DISABLE_EXPOSE_CONFIG = {("webserver", "expose_config"): "False"} +FORBIDDEN_RESPONSE = { + "detail": "Your Airflow administrator chose not to expose the configuration, most likely for security reasons." +} + + +class TestConfigEndpoint: + @pytest.fixture(autouse=True) + def setup(self) -> Generator[None, None, None]: + with conf_vars(AIRFLOW_CONFIG_ENABLE_EXPOSE_CONFIG | MOCK_CONFIG_OVERRIDE): + # since the endpoint calls `conf_dict.clear()` to remove extra keys, + # use `new` instead of `return_value` to avoid side effects + with patch( + "airflow.api_fastapi.core_api.routes.public.config.conf.as_dict", + new=lambda **_: MOCK_CONFIG_DICT.copy(), + ): + yield + + +class TestGetConfig(TestConfigEndpoint): + @pytest.mark.parametrize( + "section, headers, expected_status_code, expected_response", + [ + ( + None, + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + { + "name": SECTION_SMTP, + "options": [ + {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, + {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, + ], + }, + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SQL_ALCHEMY_CONN}, + ], + }, + ], + }, + ), + ( + None, + HEADERS_TEXT, + 200, + textwrap.dedent( + f"""\ + [{SECTION_CORE}] + {OPTION_KEY_PARALLELISM} = {OPTION_VALUE_PARALLELISM} + + [{SECTION_SMTP}] + {OPTION_KEY_SMTP_HOST} = {OPTION_VALUE_SMTP_HOST} + {OPTION_KEY_SMTP_MAIL_FROM} = {OPTION_VALUE_SMTP_MAIL_FROM} + + [{SECTION_DATABASE}] + {OPTION_KEY_SQL_ALCHEMY_CONN} = {OPTION_VALUE_SQL_ALCHEMY_CONN} + """ + ), + ), + ( + None, + HEADERS_INVALID, + 422, + { + "detail": [ + { + "type": "literal_error", + "loc": ["header", "content-type"], + "msg": "Input should be 'application/json' or 'text/plain'", + "input": "invalid", + "ctx": {"expected": "'application/json' or 'text/plain'"}, + } + ] + }, + ), + ( + SECTION_CORE, + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + ], + }, + ), + ( + SECTION_SMTP, + HEADERS_TEXT, + 200, + textwrap.dedent( + f"""\ + [{SECTION_SMTP}] + {OPTION_KEY_SMTP_HOST} = {OPTION_VALUE_SMTP_HOST} + {OPTION_KEY_SMTP_MAIL_FROM} = {OPTION_VALUE_SMTP_MAIL_FROM} + """ + ), + ), + ( + SECTION_DATABASE, + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SQL_ALCHEMY_CONN}, + ], + }, + ], + }, + ), + (None, HEADERS_JSON, 403, FORBIDDEN_RESPONSE), + (SECTION_CORE, HEADERS_JSON, 403, FORBIDDEN_RESPONSE), + (SECTION_NOT_EXIST, HEADERS_JSON, 404, {"detail": f"Section {SECTION_NOT_EXIST} not found."}), + ], + ) + def test_get_config(self, test_client, section, headers, expected_status_code, expected_response): + query_params = {"section": section} if section else None + if expected_status_code == 403: + with conf_vars(AIRFLOW_CONFIG_DISABLE_EXPOSE_CONFIG): + response = test_client.get("/public/config/", headers=headers, params=query_params) + else: + response = test_client.get("/public/config/", headers=headers, params=query_params) + assert response.status_code == expected_status_code + if response.status_code != 200 or headers == HEADERS_JSON: + assert response.json() == expected_response + else: + assert response.text == expected_response + + +class TestGetConfigValue(TestConfigEndpoint): + @pytest.mark.parametrize( + "section, option, headers, expected_status_code, expected_response", + [ + ( + SECTION_CORE, + OPTION_KEY_PARALLELISM, + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + ], + }, + ), + ( + SECTION_SMTP, + OPTION_KEY_SMTP_HOST, + HEADERS_TEXT, + 200, + textwrap.dedent( + f"""\ + [{SECTION_SMTP}] + {OPTION_KEY_SMTP_HOST} = {OPTION_VALUE_SMTP_HOST} + """ + ), + ), + ( + SECTION_SMTP, + OPTION_KEY_SMTP_MAIL_FROM, + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_SMTP, + "options": [ + {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, + ], + }, + ], + }, + ), + ( + SECTION_DATABASE, + OPTION_KEY_SQL_ALCHEMY_CONN, + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SENSITIVE_HIDDEN}, + ], + }, + ], + }, + ), + ( + SECTION_DATABASE, + OPTION_KEY_SQL_ALCHEMY_CONN, + HEADERS_TEXT, + 200, + textwrap.dedent( + f"""\ + [{SECTION_DATABASE}] + {OPTION_KEY_SQL_ALCHEMY_CONN} = {OPTION_VALUE_SENSITIVE_HIDDEN} + """ + ), + ), + (SECTION_CORE, OPTION_KEY_PARALLELISM, HEADERS_JSON, 403, FORBIDDEN_RESPONSE), + ( + SECTION_NOT_EXIST, + OPTION_KEY_PARALLELISM, + HEADERS_JSON, + 404, + {"detail": f"Option [{SECTION_NOT_EXIST}/{OPTION_KEY_PARALLELISM}] not found."}, + ), + ( + SECTION_CORE, + OPTION_NOT_EXIST, + HEADERS_JSON, + 404, + {"detail": f"Option [{SECTION_CORE}/{OPTION_NOT_EXIST}] not found."}, + ), + ], + ) + def test_get_config_value( + self, test_client, section, option, headers, expected_status_code, expected_response + ): + if expected_status_code == 403: + with conf_vars(AIRFLOW_CONFIG_DISABLE_EXPOSE_CONFIG): + response = test_client.get( + f"/public/config/section/{section}/option/{option}", headers=headers + ) + else: + response = test_client.get(f"/public/config/section/{section}/option/{option}", headers=headers) + assert response.status_code == expected_status_code + if response.status_code != 200 or headers == HEADERS_JSON: + assert response.json() == expected_response + else: + assert response.text == expected_response From 97ebfd89e6941f75683356f549f90c219382ee41 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Sun, 10 Nov 2024 14:21:53 +0800 Subject: [PATCH 04/10] Consolidate redundant for check expose config --- .../core_api/routes/public/config.py | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/airflow/api_fastapi/core_api/routes/public/config.py b/airflow/api_fastapi/core_api/routes/public/config.py index 0aaf9a456c14d..ece1d8b8c0fcd 100644 --- a/airflow/api_fastapi/core_api/routes/public/config.py +++ b/airflow/api_fastapi/core_api/routes/public/config.py @@ -30,6 +30,24 @@ from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc from airflow.configuration import conf + +def _check_expose_config() -> bool: + display_sensitive: bool | None = None + if conf.get("webserver", "expose_config").lower() == "non-sensitive-only": + expose_config = True + display_sensitive = False + else: + expose_config = conf.getboolean("webserver", "expose_config") + display_sensitive = True + + if not expose_config: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Your Airflow administrator chose not to expose the configuration, most likely for security reasons.", + ) + return display_sensitive + + config_router = AirflowRouter(tags=["Config"], prefix="/config") @@ -43,18 +61,8 @@ def get_config( section: str | None = None, content_type: Literal["application/json", "text/plain"] = Header(...), ) -> Response: - if conf.get("webserver", "expose_config").lower() == "non-sensitive-only": - expose_config = True - display_sensitive = False - else: - expose_config = conf.getboolean("webserver", "expose_config") - display_sensitive = True + display_sensitive = _check_expose_config() - if not expose_config: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Your Airflow administrator chose not to expose the configuration, most likely for security reasons.", - ) if section and not conf.has_section(section): raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -93,16 +101,7 @@ def get_config_value( option: str, content_type: Literal["application/json", "text/plain"] = Header(...), ) -> Response: - if conf.get("webserver", "expose_config").lower() == "non-sensitive-only": - expose_config = True - else: - expose_config = conf.getboolean("webserver", "expose_config") - - if not expose_config: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Your Airflow administrator chose not to expose the configuration, most likely for security reasons.", - ) + _check_expose_config() if not conf.has_option(section, option): raise HTTPException( From f3d550592e549be6e3f41708a0a7e77ee918412f Mon Sep 17 00:00:00 2001 From: jason810496 Date: Fri, 15 Nov 2024 18:25:24 +0800 Subject: [PATCH 05/10] Add `Accept` header depends for Json and Text --- airflow/api_fastapi/common/headers.py | 48 +++++++++++++++++++++++++++ airflow/api_fastapi/common/types.py | 8 +++++ 2 files changed, 56 insertions(+) create mode 100644 airflow/api_fastapi/common/headers.py diff --git a/airflow/api_fastapi/common/headers.py b/airflow/api_fastapi/common/headers.py new file mode 100644 index 0000000000000..0c9924c19b78b --- /dev/null +++ b/airflow/api_fastapi/common/headers.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from fastapi import Depends, Header, HTTPException, status +from typing_extensions import Annotated + +from airflow.api_fastapi.common.types import Mimetype + + +def header_accept_json_or_text_depends( + accept: Annotated[ + str, + Header( + json_schema_extra={ + "type": "string", + "enum": [Mimetype.JSON, Mimetype.TEXT], + } + ), + ] = Mimetype.ANY, +) -> Mimetype: + if accept.startswith(Mimetype.ANY): + return Mimetype.ANY + if accept.startswith(Mimetype.JSON): + return Mimetype.JSON + if accept.startswith(Mimetype.TEXT): + return Mimetype.TEXT + raise HTTPException( + status_code=status.HTTP_406_NOT_ACCEPTABLE, + detail="Only application/json or text/plain is supported", + ) + + +HeaderAcceptJsonOrText = Annotated[Mimetype, Depends(header_accept_json_or_text_depends)] diff --git a/airflow/api_fastapi/common/types.py b/airflow/api_fastapi/common/types.py index ab10a21c97002..e4ffbafc5cf86 100644 --- a/airflow/api_fastapi/common/types.py +++ b/airflow/api_fastapi/common/types.py @@ -18,6 +18,7 @@ from datetime import timedelta from typing import Annotated +from enum import Enum from pydantic import AfterValidator, AliasGenerator, AwareDatetime, BaseModel, BeforeValidator, ConfigDict @@ -56,3 +57,10 @@ class TimeDelta(BaseModel): TimeDeltaWithValidation = Annotated[TimeDelta, BeforeValidator(_validate_timedelta_field)] + +class Mimetype(str, Enum): + """Mimetype for the `Content-Type` header.""" + + TEXT = "text/plain" + JSON = "application/json" + ANY = "*/*" From 33232a1e322da5ada72c9faeb17181b9908c9a78 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Fri, 15 Nov 2024 18:29:20 +0800 Subject: [PATCH 06/10] Refactor config, dag_source with common header --- .../core_api/openapi/v1-generated.yaml | 117 +++++++++-- .../core_api/routes/public/config.py | 76 +++++-- .../core_api/routes/public/dag_sources.py | 22 +-- airflow/ui/openapi-gen/queries/common.ts | 101 +--------- airflow/ui/openapi-gen/queries/prefetch.ts | 125 ++---------- airflow/ui/openapi-gen/queries/queries.ts | 162 ++------------- airflow/ui/openapi-gen/queries/suspense.ts | 155 ++------------- .../ui/openapi-gen/requests/schemas.gen.ts | 70 +++++++ .../ui/openapi-gen/requests/services.gen.ts | 143 +++++++------- airflow/ui/openapi-gen/requests/types.gen.ts | 186 ++++++++---------- .../core_api/routes/public/test_config.py | 55 ++++-- .../routes/public/test_dag_sources.py | 7 +- 12 files changed, 484 insertions(+), 735 deletions(-) diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index 497c80bf23f56..c25d9697f8fd6 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -2106,21 +2106,33 @@ paths: - type: string - type: 'null' title: Section - - name: content-type + - name: accept in: header - required: true + required: false schema: + type: string enum: - application/json - text/plain - type: string - title: Content-Type + default: '*/*' + title: Accept responses: '200': description: Successful Response content: application/json: - schema: {} + schema: + $ref: '#/components/schemas/Config' + text/plain: + schema: + type: string + example: '[core] + + dags_folder = /opt/airflow/dags + + base_log_folder = /opt/airflow/logs + + ' '401': content: application/json: @@ -2139,6 +2151,12 @@ paths: schema: $ref: '#/components/schemas/HTTPExceptionResponse' description: Not Found + '406': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Acceptable '422': description: Validation Error content: @@ -2164,21 +2182,33 @@ paths: schema: type: string title: Option - - name: content-type + - name: accept in: header - required: true + required: false schema: + type: string enum: - application/json - text/plain - type: string - title: Content-Type + default: '*/*' + title: Accept responses: '200': description: Successful Response content: application/json: - schema: {} + schema: + $ref: '#/components/schemas/Config' + text/plain: + schema: + type: string + example: '[core] + + dags_folder = /opt/airflow/dags + + base_log_folder = /opt/airflow/logs + + ' '401': content: application/json: @@ -2197,6 +2227,12 @@ paths: schema: $ref: '#/components/schemas/HTTPExceptionResponse' description: Not Found + '406': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Acceptable '422': description: Validation Error content: @@ -2396,7 +2432,17 @@ paths: required: true schema: type: string - title: Dag Id + title: File Token + - name: accept + in: header + required: false + schema: + type: string + enum: + - application/json + - text/plain + default: '*/*' + title: Accept responses: '200': description: Successful Response @@ -4705,6 +4751,55 @@ components: - status title: BaseInfoSchema description: Base status field for metadatabase and scheduler. + Config: + properties: + sections: + items: + $ref: '#/components/schemas/ConfigSection' + type: array + title: Sections + type: object + required: + - sections + title: Config + description: List of config sections with their options. + ConfigOption: + properties: + key: + type: string + title: Key + value: + anyOf: + - type: string + - prefixItems: + - type: string + - type: string + type: array + maxItems: 2 + minItems: 2 + title: Value + type: object + required: + - key + - value + title: ConfigOption + description: Config option. + ConfigSection: + properties: + name: + type: string + title: Name + options: + items: + $ref: '#/components/schemas/ConfigOption' + type: array + title: Options + type: object + required: + - name + - options + title: ConfigSection + description: Config Section Schema. ConnectionBody: properties: connection_id: diff --git a/airflow/api_fastapi/core_api/routes/public/config.py b/airflow/api_fastapi/core_api/routes/public/config.py index ece1d8b8c0fcd..669cb8eef935e 100644 --- a/airflow/api_fastapi/core_api/routes/public/config.py +++ b/airflow/api_fastapi/core_api/routes/public/config.py @@ -16,12 +16,14 @@ # under the License. from __future__ import annotations -from typing import Literal +import textwrap -from fastapi import Header, HTTPException, status +from fastapi import HTTPException, status from fastapi.responses import Response +from airflow.api_fastapi.common.headers import HeaderAcceptJsonOrText from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.common.types import Mimetype from airflow.api_fastapi.core_api.datamodels.config import ( Config, ConfigOption, @@ -30,6 +32,21 @@ from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc from airflow.configuration import conf +text_example_response = { + Mimetype.TEXT: { + "schema": { + "type": "string", + "example": textwrap.dedent( + """\ + [core] + dags_folder = /opt/airflow/dags + base_log_folder = /opt/airflow/logs + """ + ), + } + } +} + def _check_expose_config() -> bool: display_sensitive: bool | None = None @@ -48,19 +65,34 @@ def _check_expose_config() -> bool: return display_sensitive +def _response_based_on_accept(accept: Mimetype, config: Config): + if accept == Mimetype.TEXT: + return Response(content=config.text_format, media_type=Mimetype.TEXT) + return config + + config_router = AirflowRouter(tags=["Config"], prefix="/config") @config_router.get( "/", - responses=create_openapi_http_exception_doc( - [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN, status.HTTP_404_NOT_FOUND] - ), + responses={ + **create_openapi_http_exception_doc( + [ + status.HTTP_401_UNAUTHORIZED, + status.HTTP_403_FORBIDDEN, + status.HTTP_404_NOT_FOUND, + status.HTTP_406_NOT_ACCEPTABLE, + ] + ), + "200": {"description": "Successful Response", "content": text_example_response}, + }, + response_model=Config, ) def get_config( + accept: HeaderAcceptJsonOrText, section: str | None = None, - content_type: Literal["application/json", "text/plain"] = Header(...), -) -> Response: +): display_sensitive = _check_expose_config() if section and not conf.has_section(section): @@ -83,24 +115,29 @@ def get_config( for section, options in conf_dict.items() ] ) - - return Response( - content=config.model_dump_json() if content_type == "application/json" else config.text_format, - headers={"Content-Type": content_type}, - ) + return _response_based_on_accept(accept, config) @config_router.get( "/section/{section}/option/{option}", - responses=create_openapi_http_exception_doc( - [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN, status.HTTP_404_NOT_FOUND] - ), + responses={ + **create_openapi_http_exception_doc( + [ + status.HTTP_401_UNAUTHORIZED, + status.HTTP_403_FORBIDDEN, + status.HTTP_404_NOT_FOUND, + status.HTTP_406_NOT_ACCEPTABLE, + ] + ), + "200": {"description": "Successful Response", "content": text_example_response}, + }, + response_model=Config, ) def get_config_value( section: str, option: str, - content_type: Literal["application/json", "text/plain"] = Header(...), -) -> Response: + accept: HeaderAcceptJsonOrText, +): _check_expose_config() if not conf.has_option(section, option): @@ -115,7 +152,4 @@ def get_config_value( value = conf.get(section, option) config = Config(sections=[ConfigSection(name=section, options=[ConfigOption(key=option, value=value)])]) - return Response( - content=config.model_dump_json() if content_type == "application/json" else config.text_format, - headers={"Content-Type": content_type}, - ) + return _response_based_on_accept(accept, config) diff --git a/airflow/api_fastapi/core_api/routes/public/dag_sources.py b/airflow/api_fastapi/core_api/routes/public/dag_sources.py index fbe30620228b5..d4fbd774b51e0 100644 --- a/airflow/api_fastapi/core_api/routes/public/dag_sources.py +++ b/airflow/api_fastapi/core_api/routes/public/dag_sources.py @@ -18,21 +18,20 @@ from typing import Annotated -from fastapi import Depends, Header, HTTPException, Request, Response, status +from fastapi import Depends, HTTPException, Request, Response, status +from itsdangerous import BadSignature, URLSafeSerializer from sqlalchemy.orm import Session from airflow.api_fastapi.common.db.common import get_session +from airflow.api_fastapi.common.headers import HeaderAcceptJsonOrText from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.common.types import Mimetype from airflow.api_fastapi.core_api.datamodels.dag_sources import DAGSourceResponse from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc from airflow.models.dag_version import DagVersion dag_sources_router = AirflowRouter(tags=["DagSource"], prefix="/dagSources") -mime_type_text = "text/plain" -mime_type_json = "application/json" -mime_type_any = "*/*" - @dag_sources_router.get( "/{dag_id}", @@ -47,17 +46,16 @@ "200": { "description": "Successful Response", "content": { - mime_type_text: {"schema": {"type": "string", "example": "dag code"}}, + Mimetype.TEXT: {"schema": {"type": "string", "example": "dag code"}}, }, }, }, response_model=DAGSourceResponse, ) def get_dag_source( + accept: HeaderAcceptJsonOrText, dag_id: str, session: Annotated[Session, Depends(get_session)], - request: Request, - accept: Annotated[str, Header()] = mime_type_any, version_number: int | None = None, ): """Get source code using file token.""" @@ -72,8 +70,6 @@ def get_dag_source( version_number = dag_version.version_number dag_source_model = DAGSourceResponse(dag_id=dag_id, content=dag_source, version_number=version_number) - if accept.startswith(mime_type_text): - return Response(dag_source_model.content, media_type=mime_type_text) - if accept.startswith(mime_type_json) or accept.startswith(mime_type_any): - return dag_source_model - raise HTTPException(status.HTTP_406_NOT_ACCEPTABLE, "Content not available for Accept header") + if accept == Mimetype.TEXT: + return Response(dag_source_model.content, media_type=Mimetype.TEXT) + return dag_source_model diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index dee2973d67d8d..a3ecb44c94645 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -571,50 +571,6 @@ export const UseDagServiceGetDagDetailsKeyFn = ( }, queryKey?: Array, ) => [useDagServiceGetDagDetailsKey, ...(queryKey ?? [{ dagId }])]; -export type ConnectionServiceGetConnectionDefaultResponse = Awaited< - ReturnType ->; -export type ConnectionServiceGetConnectionQueryResult< - TData = ConnectionServiceGetConnectionDefaultResponse, - TError = unknown, -> = UseQueryResult; -export const useConnectionServiceGetConnectionKey = - "ConnectionServiceGetConnection"; -export const UseConnectionServiceGetConnectionKeyFn = ( - { - connectionId, - }: { - connectionId: string; - }, - queryKey?: Array, -) => [ - useConnectionServiceGetConnectionKey, - ...(queryKey ?? [{ connectionId }]), -]; -export type ConnectionServiceGetConnectionsDefaultResponse = Awaited< - ReturnType ->; -export type ConnectionServiceGetConnectionsQueryResult< - TData = ConnectionServiceGetConnectionsDefaultResponse, - TError = unknown, -> = UseQueryResult; -export const useConnectionServiceGetConnectionsKey = - "ConnectionServiceGetConnections"; -export const UseConnectionServiceGetConnectionsKeyFn = ( - { - limit, - offset, - orderBy, - }: { - limit?: number; - offset?: number; - orderBy?: string; - } = {}, - queryKey?: Array, -) => [ - useConnectionServiceGetConnectionsKey, - ...(queryKey ?? [{ limit, offset, orderBy }]), -]; export type ConfigServiceGetConfigDefaultResponse = Awaited< ReturnType >; @@ -625,17 +581,14 @@ export type ConfigServiceGetConfigQueryResult< export const useConfigServiceGetConfigKey = "ConfigServiceGetConfig"; export const UseConfigServiceGetConfigKeyFn = ( { - contentType, + accept, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; section?: string; - }, + } = {}, queryKey?: Array, -) => [ - useConfigServiceGetConfigKey, - ...(queryKey ?? [{ contentType, section }]), -]; +) => [useConfigServiceGetConfigKey, ...(queryKey ?? [{ accept, section }])]; export type ConfigServiceGetConfigValueDefaultResponse = Awaited< ReturnType >; @@ -646,58 +599,18 @@ export type ConfigServiceGetConfigValueQueryResult< export const useConfigServiceGetConfigValueKey = "ConfigServiceGetConfigValue"; export const UseConfigServiceGetConfigValueKeyFn = ( { - contentType, + accept, option, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; option: string; section: string; }, queryKey?: Array, ) => [ useConfigServiceGetConfigValueKey, - ...(queryKey ?? [{ contentType, option, section }]), -]; -export type DagRunServiceGetDagRunDefaultResponse = Awaited< - ReturnType ->; -export type DagRunServiceGetDagRunQueryResult< - TData = DagRunServiceGetDagRunDefaultResponse, - TError = unknown, -> = UseQueryResult; -export const useDagRunServiceGetDagRunKey = "DagRunServiceGetDagRun"; -export const UseDagRunServiceGetDagRunKeyFn = ( - { - dagId, - dagRunId, - }: { - dagId: string; - dagRunId: string; - }, - queryKey?: Array, -) => [useDagRunServiceGetDagRunKey, ...(queryKey ?? [{ dagId, dagRunId }])]; -export type DagSourceServiceGetDagSourceDefaultResponse = Awaited< - ReturnType ->; -export type DagSourceServiceGetDagSourceQueryResult< - TData = DagSourceServiceGetDagSourceDefaultResponse, - TError = unknown, -> = UseQueryResult; -export const useDagSourceServiceGetDagSourceKey = - "DagSourceServiceGetDagSource"; -export const UseDagSourceServiceGetDagSourceKeyFn = ( - { - accept, - fileToken, - }: { - accept?: string; - fileToken: string; - }, - queryKey?: Array, -) => [ - useDagSourceServiceGetDagSourceKey, - ...(queryKey ?? [{ accept, fileToken }]), + ...(queryKey ?? [{ accept, option, section }]), ]; export type EventLogServiceGetEventLogDefaultResponse = Awaited< ReturnType diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index c9b3e530ec1d0..4bb1edbec655e 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -733,155 +733,56 @@ export const prefetchUseDagServiceGetDagDetails = ( queryKey: Common.UseDagServiceGetDagDetailsKeyFn({ dagId }), queryFn: () => DagService.getDagDetails({ dagId }), }); -/** - * Get Connection - * Get a connection entry. - * @param data The data for the request. - * @param data.connectionId - * @returns ConnectionResponse Successful Response - * @throws ApiError - */ -export const prefetchUseConnectionServiceGetConnection = ( - queryClient: QueryClient, - { - connectionId, - }: { - connectionId: string; - }, -) => - queryClient.prefetchQuery({ - queryKey: Common.UseConnectionServiceGetConnectionKeyFn({ connectionId }), - queryFn: () => ConnectionService.getConnection({ connectionId }), - }); -/** - * Get Connections - * Get all connection entries. - * @param data The data for the request. - * @param data.limit - * @param data.offset - * @param data.orderBy - * @returns ConnectionCollectionResponse Successful Response - * @throws ApiError - */ -export const prefetchUseConnectionServiceGetConnections = ( - queryClient: QueryClient, - { - limit, - offset, - orderBy, - }: { - limit?: number; - offset?: number; - orderBy?: string; - } = {}, -) => - queryClient.prefetchQuery({ - queryKey: Common.UseConnectionServiceGetConnectionsKeyFn({ - limit, - offset, - orderBy, - }), - queryFn: () => ConnectionService.getConnections({ limit, offset, orderBy }), - }); /** * Get Config * @param data The data for the request. - * @param data.contentType * @param data.section - * @returns unknown Successful Response + * @param data.accept + * @returns Config Successful Response * @throws ApiError */ export const prefetchUseConfigServiceGetConfig = ( queryClient: QueryClient, { - contentType, + accept, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; section?: string; - }, + } = {}, ) => queryClient.prefetchQuery({ - queryKey: Common.UseConfigServiceGetConfigKeyFn({ contentType, section }), - queryFn: () => ConfigService.getConfig({ contentType, section }), + queryKey: Common.UseConfigServiceGetConfigKeyFn({ accept, section }), + queryFn: () => ConfigService.getConfig({ accept, section }), }); /** * Get Config Value * @param data The data for the request. * @param data.section * @param data.option - * @param data.contentType - * @returns unknown Successful Response + * @param data.accept + * @returns Config Successful Response * @throws ApiError */ export const prefetchUseConfigServiceGetConfigValue = ( queryClient: QueryClient, { - contentType, + accept, option, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; option: string; section: string; }, ) => queryClient.prefetchQuery({ queryKey: Common.UseConfigServiceGetConfigValueKeyFn({ - contentType, + accept, option, section, }), - queryFn: () => - ConfigService.getConfigValue({ contentType, option, section }), - }); -/** - * Get Dag Run - * @param data The data for the request. - * @param data.dagId - * @param data.dagRunId - * @returns DAGRunResponse Successful Response - * @throws ApiError - */ -export const prefetchUseDagRunServiceGetDagRun = ( - queryClient: QueryClient, - { - dagId, - dagRunId, - }: { - dagId: string; - dagRunId: string; - }, -) => - queryClient.prefetchQuery({ - queryKey: Common.UseDagRunServiceGetDagRunKeyFn({ dagId, dagRunId }), - queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }), - }); -/** - * Get Dag Source - * Get source code using file token. - * @param data The data for the request. - * @param data.fileToken - * @param data.accept - * @returns DAGSourceResponse Successful Response - * @throws ApiError - */ -export const prefetchUseDagSourceServiceGetDagSource = ( - queryClient: QueryClient, - { - accept, - fileToken, - }: { - accept?: string; - fileToken: string; - }, -) => - queryClient.prefetchQuery({ - queryKey: Common.UseDagSourceServiceGetDagSourceKeyFn({ - accept, - fileToken, - }), - queryFn: () => DagSourceService.getDagSource({ accept, fileToken }), + queryFn: () => ConfigService.getConfigValue({ accept, option, section }), }); /** * Get Event Log diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 18923f97cf3d1..0c6e6c2693e22 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -907,77 +907,12 @@ export const useDagServiceGetDagDetails = < queryFn: () => DagService.getDagDetails({ dagId }) as TData, ...options, }); -/** - * Get Connection - * Get a connection entry. - * @param data The data for the request. - * @param data.connectionId - * @returns ConnectionResponse Successful Response - * @throws ApiError - */ -export const useConnectionServiceGetConnection = < - TData = Common.ConnectionServiceGetConnectionDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - connectionId, - }: { - connectionId: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useQuery({ - queryKey: Common.UseConnectionServiceGetConnectionKeyFn( - { connectionId }, - queryKey, - ), - queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, - ...options, - }); -/** - * Get Connections - * Get all connection entries. - * @param data The data for the request. - * @param data.limit - * @param data.offset - * @param data.orderBy - * @returns ConnectionCollectionResponse Successful Response - * @throws ApiError - */ -export const useConnectionServiceGetConnections = < - TData = Common.ConnectionServiceGetConnectionsDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - limit, - offset, - orderBy, - }: { - limit?: number; - offset?: number; - orderBy?: string; - } = {}, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useQuery({ - queryKey: Common.UseConnectionServiceGetConnectionsKeyFn( - { limit, offset, orderBy }, - queryKey, - ), - queryFn: () => - ConnectionService.getConnections({ limit, offset, orderBy }) as TData, - ...options, - }); /** * Get Config * @param data The data for the request. - * @param data.contentType * @param data.section - * @returns unknown Successful Response + * @param data.accept + * @returns Config Successful Response * @throws ApiError */ export const useConfigServiceGetConfig = < @@ -986,21 +921,21 @@ export const useConfigServiceGetConfig = < TQueryKey extends Array = unknown[], >( { - contentType, + accept, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; section?: string; - }, + } = {}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">, ) => useQuery({ queryKey: Common.UseConfigServiceGetConfigKeyFn( - { contentType, section }, + { accept, section }, queryKey, ), - queryFn: () => ConfigService.getConfig({ contentType, section }) as TData, + queryFn: () => ConfigService.getConfig({ accept, section }) as TData, ...options, }); /** @@ -1008,8 +943,8 @@ export const useConfigServiceGetConfig = < * @param data The data for the request. * @param data.section * @param data.option - * @param data.contentType - * @returns unknown Successful Response + * @param data.accept + * @returns Config Successful Response * @throws ApiError */ export const useConfigServiceGetConfigValue = < @@ -1018,11 +953,11 @@ export const useConfigServiceGetConfigValue = < TQueryKey extends Array = unknown[], >( { - contentType, + accept, option, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; option: string; section: string; }, @@ -1031,75 +966,11 @@ export const useConfigServiceGetConfigValue = < ) => useQuery({ queryKey: Common.UseConfigServiceGetConfigValueKeyFn( - { contentType, option, section }, - queryKey, - ), - queryFn: () => - ConfigService.getConfigValue({ contentType, option, section }) as TData, - ...options, - }); -/** - * Get Dag Run - * @param data The data for the request. - * @param data.dagId - * @param data.dagRunId - * @returns DAGRunResponse Successful Response - * @throws ApiError - */ -export const useDagRunServiceGetDagRun = < - TData = Common.DagRunServiceGetDagRunDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - dagId, - dagRunId, - }: { - dagId: string; - dagRunId: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useQuery({ - queryKey: Common.UseDagRunServiceGetDagRunKeyFn( - { dagId, dagRunId }, - queryKey, - ), - queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, - ...options, - }); -/** - * Get Dag Source - * Get source code using file token. - * @param data The data for the request. - * @param data.fileToken - * @param data.accept - * @returns DAGSourceResponse Successful Response - * @throws ApiError - */ -export const useDagSourceServiceGetDagSource = < - TData = Common.DagSourceServiceGetDagSourceDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - accept, - fileToken, - }: { - accept?: string; - fileToken: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useQuery({ - queryKey: Common.UseDagSourceServiceGetDagSourceKeyFn( - { accept, fileToken }, + { accept, option, section }, queryKey, ), queryFn: () => - DagSourceService.getDagSource({ accept, fileToken }) as TData, + ConfigService.getConfigValue({ accept, option, section }) as TData, ...options, }); /** @@ -3093,6 +2964,7 @@ export const useDagRunServiceDeleteDagRun = < * Delete the specific DAG. * @param data The data for the request. * @param data.dagId + * @param data.accept * @returns unknown Successful Response * @throws ApiError */ @@ -3106,6 +2978,7 @@ export const useDagServiceDeleteDag = < TData, TError, { + accept?: "application/json" | "text/plain"; dagId: string; }, TContext @@ -3117,12 +2990,13 @@ export const useDagServiceDeleteDag = < TData, TError, { + accept?: "application/json" | "text/plain"; dagId: string; }, TContext >({ - mutationFn: ({ dagId }) => - DagService.deleteDag({ dagId }) as unknown as Promise, + mutationFn: ({ accept, dagId }) => + DagService.deleteDag({ accept, dagId }) as unknown as Promise, ...options, }); /** diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index 6d8472e351a4e..b4e88046a38ce 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -889,77 +889,12 @@ export const useDagServiceGetDagDetailsSuspense = < queryFn: () => DagService.getDagDetails({ dagId }) as TData, ...options, }); -/** - * Get Connection - * Get a connection entry. - * @param data The data for the request. - * @param data.connectionId - * @returns ConnectionResponse Successful Response - * @throws ApiError - */ -export const useConnectionServiceGetConnectionSuspense = < - TData = Common.ConnectionServiceGetConnectionDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - connectionId, - }: { - connectionId: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useSuspenseQuery({ - queryKey: Common.UseConnectionServiceGetConnectionKeyFn( - { connectionId }, - queryKey, - ), - queryFn: () => ConnectionService.getConnection({ connectionId }) as TData, - ...options, - }); -/** - * Get Connections - * Get all connection entries. - * @param data The data for the request. - * @param data.limit - * @param data.offset - * @param data.orderBy - * @returns ConnectionCollectionResponse Successful Response - * @throws ApiError - */ -export const useConnectionServiceGetConnectionsSuspense = < - TData = Common.ConnectionServiceGetConnectionsDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - limit, - offset, - orderBy, - }: { - limit?: number; - offset?: number; - orderBy?: string; - } = {}, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useSuspenseQuery({ - queryKey: Common.UseConnectionServiceGetConnectionsKeyFn( - { limit, offset, orderBy }, - queryKey, - ), - queryFn: () => - ConnectionService.getConnections({ limit, offset, orderBy }) as TData, - ...options, - }); /** * Get Config * @param data The data for the request. - * @param data.contentType * @param data.section - * @returns unknown Successful Response + * @param data.accept + * @returns Config Successful Response * @throws ApiError */ export const useConfigServiceGetConfigSuspense = < @@ -968,21 +903,21 @@ export const useConfigServiceGetConfigSuspense = < TQueryKey extends Array = unknown[], >( { - contentType, + accept, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; section?: string; - }, + } = {}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">, ) => useSuspenseQuery({ queryKey: Common.UseConfigServiceGetConfigKeyFn( - { contentType, section }, + { accept, section }, queryKey, ), - queryFn: () => ConfigService.getConfig({ contentType, section }) as TData, + queryFn: () => ConfigService.getConfig({ accept, section }) as TData, ...options, }); /** @@ -990,8 +925,8 @@ export const useConfigServiceGetConfigSuspense = < * @param data The data for the request. * @param data.section * @param data.option - * @param data.contentType - * @returns unknown Successful Response + * @param data.accept + * @returns Config Successful Response * @throws ApiError */ export const useConfigServiceGetConfigValueSuspense = < @@ -1000,11 +935,11 @@ export const useConfigServiceGetConfigValueSuspense = < TQueryKey extends Array = unknown[], >( { - contentType, + accept, option, section, }: { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; option: string; section: string; }, @@ -1013,75 +948,11 @@ export const useConfigServiceGetConfigValueSuspense = < ) => useSuspenseQuery({ queryKey: Common.UseConfigServiceGetConfigValueKeyFn( - { contentType, option, section }, - queryKey, - ), - queryFn: () => - ConfigService.getConfigValue({ contentType, option, section }) as TData, - ...options, - }); -/** - * Get Dag Run - * @param data The data for the request. - * @param data.dagId - * @param data.dagRunId - * @returns DAGRunResponse Successful Response - * @throws ApiError - */ -export const useDagRunServiceGetDagRunSuspense = < - TData = Common.DagRunServiceGetDagRunDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - dagId, - dagRunId, - }: { - dagId: string; - dagRunId: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useSuspenseQuery({ - queryKey: Common.UseDagRunServiceGetDagRunKeyFn( - { dagId, dagRunId }, - queryKey, - ), - queryFn: () => DagRunService.getDagRun({ dagId, dagRunId }) as TData, - ...options, - }); -/** - * Get Dag Source - * Get source code using file token. - * @param data The data for the request. - * @param data.fileToken - * @param data.accept - * @returns DAGSourceResponse Successful Response - * @throws ApiError - */ -export const useDagSourceServiceGetDagSourceSuspense = < - TData = Common.DagSourceServiceGetDagSourceDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - accept, - fileToken, - }: { - accept?: string; - fileToken: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useSuspenseQuery({ - queryKey: Common.UseDagSourceServiceGetDagSourceKeyFn( - { accept, fileToken }, + { accept, option, section }, queryKey, ), queryFn: () => - DagSourceService.getDagSource({ accept, fileToken }) as TData, + ConfigService.getConfigValue({ accept, option, section }) as TData, ...options, }); /** diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 2e4a9cbc832c4..0f08034c533dd 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -459,6 +459,76 @@ export const $BaseInfoSchema = { description: "Base status field for metadatabase and scheduler.", } as const; +export const $Config = { + properties: { + sections: { + items: { + $ref: "#/components/schemas/ConfigSection", + }, + type: "array", + title: "Sections", + }, + }, + type: "object", + required: ["sections"], + title: "Config", + description: "List of config sections with their options.", +} as const; + +export const $ConfigOption = { + properties: { + key: { + type: "string", + title: "Key", + }, + value: { + anyOf: [ + { + type: "string", + }, + { + prefixItems: [ + { + type: "string", + }, + { + type: "string", + }, + ], + type: "array", + maxItems: 2, + minItems: 2, + }, + ], + title: "Value", + }, + }, + type: "object", + required: ["key", "value"], + title: "ConfigOption", + description: "Config option.", +} as const; + +export const $ConfigSection = { + properties: { + name: { + type: "string", + title: "Name", + }, + options: { + items: { + $ref: "#/components/schemas/ConfigOption", + }, + type: "array", + title: "Options", + }, + }, + type: "object", + required: ["name", "options"], + title: "ConfigSection", + description: "Config Section Schema.", +} as const; + export const $ConnectionBody = { properties: { connection_id: { diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 4301ae89c1914..18774cee8a45c 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -51,10 +51,6 @@ import type { GetConnectionsResponse, PostConnectionData, PostConnectionResponse, - GetConfigData, - GetConfigResponse, - GetConfigValueData, - GetConfigValueResponse, TestConnectionData, TestConnectionResponse, GetDagRunData, @@ -87,6 +83,10 @@ import type { DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, + GetConfigData, + GetConfigResponse, + GetConfigValueData, + GetConfigValueResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, @@ -1133,7 +1133,9 @@ export class DagWarningService { }, }); } +} +export class DagService { /** * Get Dags * Get all DAGs. @@ -1222,71 +1224,6 @@ export class DagWarningService { }, }); } -} - -export class ConfigService { - /** - * Get Config - * @param data The data for the request. - * @param data.contentType - * @param data.section - * @returns unknown Successful Response - * @throws ApiError - */ - public static getConfig( - data: GetConfigData, - ): CancelablePromise { - return __request(OpenAPI, { - method: "GET", - url: "/public/config/", - headers: { - "content-type": data.contentType, - }, - query: { - section: data.section, - }, - errors: { - 401: "Unauthorized", - 403: "Forbidden", - 404: "Not Found", - 422: "Validation Error", - }, - }); - } - - /** - * Get Config Value - * @param data The data for the request. - * @param data.section - * @param data.option - * @param data.contentType - * @returns unknown Successful Response - * @throws ApiError - */ - public static getConfigValue( - data: GetConfigValueData, - ): CancelablePromise { - return __request(OpenAPI, { - method: "GET", - url: "/public/config/section/{section}/option/{option}", - path: { - section: data.section, - option: data.option, - }, - headers: { - "content-type": data.contentType, - }, - errors: { - 401: "Unauthorized", - 403: "Forbidden", - 404: "Not Found", - 422: "Validation Error", - }, - }); - } -} - -export class DagService { /** * Get Dag Tags @@ -1383,6 +1320,7 @@ export class DagService { * Delete the specific DAG. * @param data The data for the request. * @param data.dagId + * @param data.accept * @returns unknown Successful Response * @throws ApiError */ @@ -1395,6 +1333,9 @@ export class DagService { path: { dag_id: data.dagId, }, + headers: { + accept: data.accept, + }, errors: { 400: "Bad Request", 401: "Unauthorized", @@ -1433,6 +1374,70 @@ export class DagService { } } +export class ConfigService { + /** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ + public static getConfig( + data: GetConfigData = {}, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/config/", + headers: { + accept: data.accept, + }, + query: { + section: data.section, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 406: "Not Acceptable", + 422: "Validation Error", + }, + }); + } + + /** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ + public static getConfigValue( + data: GetConfigValueData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/config/section/{section}/option/{option}", + path: { + section: data.section, + option: data.option, + }, + headers: { + accept: data.accept, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 406: "Not Acceptable", + 422: "Validation Error", + }, + }); + } +} + export class EventLogService { /** * Get Event Log diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 33290cf88d159..cf68c1bf92a05 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -128,6 +128,29 @@ export type BaseInfoSchema = { status: string | null; }; +/** + * List of config sections with their options. + */ +export type Config = { + sections: Array; +}; + +/** + * Config option. + */ +export type ConfigOption = { + key: string; + value: string | [string, string]; +}; + +/** + * Config Section Schema. + */ +export type ConfigSection = { + name: string; + options: Array; +}; + /** * Connection Serializer for requests body. */ @@ -1355,6 +1378,7 @@ export type PatchDagData = { export type PatchDagResponse = DAGResponse; export type DeleteDagData = { + accept?: "application/json" | "text/plain"; dagId: string; }; @@ -1366,84 +1390,20 @@ export type GetDagDetailsData = { export type GetDagDetailsResponse = DAGDetailsResponse; -export type DeleteConnectionData = { - connectionId: string; -}; - -export type DeleteConnectionResponse = void; - -export type GetConnectionData = { - connectionId: string; -}; - -export type GetConnectionResponse = ConnectionResponse; - -export type PatchConnectionData = { - connectionId: string; - requestBody: ConnectionBody; - updateMask?: Array | null; -}; - -export type PatchConnectionResponse = ConnectionResponse; - -export type GetConnectionsData = { - limit?: number; - offset?: number; - orderBy?: string; -}; - -export type GetConnectionsResponse = ConnectionCollectionResponse; - -export type PostConnectionData = { - requestBody: ConnectionBody; -}; - -export type PostConnectionResponse = ConnectionResponse; - export type GetConfigData = { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; section?: string | null; }; -export type GetConfigResponse = unknown; +export type GetConfigResponse = Config; export type GetConfigValueData = { - contentType: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain"; option: string; section: string; }; -export type GetConfigValueResponse = unknown; - -export type GetDagRunData = { - dagId: string; - dagRunId: string; -}; - -export type GetDagRunResponse = DAGRunResponse; - -export type DeleteDagRunData = { - dagId: string; - dagRunId: string; -}; - -export type DeleteDagRunResponse = void; - -export type PatchDagRunData = { - dagId: string; - dagRunId: string; - requestBody: DAGRunPatchBody; - updateMask?: Array | null; -}; - -export type PatchDagRunResponse = DAGRunResponse; - -export type GetDagSourceData = { - accept?: string; - fileToken: string; -}; - -export type GetDagSourceResponse = DAGSourceResponse; +export type GetConfigValueResponse = Config; export type GetEventLogData = { eventLogId: number; @@ -2630,14 +2590,14 @@ export type $OpenApiTs = { }; }; }; - "/public/config/": { + "/public/dags/tags": { get: { - req: GetConfigData; + req: GetDagTagsData; res: { /** * Successful Response */ - 200: unknown; + 200: DAGTagCollectionResponse; /** * Unauthorized */ @@ -2646,10 +2606,6 @@ export type $OpenApiTs = { * Forbidden */ 403: HTTPExceptionResponse; - /** - * Not Found - */ - 404: HTTPExceptionResponse; /** * Validation Error */ @@ -2657,14 +2613,18 @@ export type $OpenApiTs = { }; }; }; - "/public/config/section/{section}/option/{option}": { + "/public/dags/{dag_id}": { get: { - req: GetConfigValueData; + req: GetDagData; res: { /** * Successful Response */ - 200: unknown; + 200: DAGResponse; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; /** * Unauthorized */ @@ -2678,20 +2638,22 @@ export type $OpenApiTs = { */ 404: HTTPExceptionResponse; /** - * Validation Error + * Unprocessable Entity */ - 422: HTTPValidationError; + 422: HTTPExceptionResponse; }; }; - }; - "/public/dags/tags": { - get: { - req: GetDagTagsData; + patch: { + req: PatchDagData; res: { /** * Successful Response */ - 200: DAGTagCollectionResponse; + 200: DAGResponse; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; /** * Unauthorized */ @@ -2700,21 +2662,23 @@ export type $OpenApiTs = { * Forbidden */ 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; /** * Validation Error */ 422: HTTPValidationError; }; }; - }; - "/public/dags/{dag_id}": { - get: { - req: GetDagData; + delete: { + req: DeleteDagData; res: { /** * Successful Response */ - 200: DAGResponse; + 200: unknown; /** * Bad Request */ @@ -2737,13 +2701,15 @@ export type $OpenApiTs = { 422: HTTPExceptionResponse; }; }; - patch: { - req: PatchDagData; + }; + "/public/dags/{dag_id}/details": { + get: { + req: GetDagDetailsData; res: { /** * Successful Response */ - 200: DAGResponse; + 200: DAGDetailsResponse; /** * Bad Request */ @@ -2766,17 +2732,15 @@ export type $OpenApiTs = { 422: HTTPValidationError; }; }; - delete: { - req: DeleteDagData; + }; + "/public/config/": { + get: { + req: GetConfigData; res: { /** * Successful Response */ - 200: unknown; - /** - * Bad Request - */ - 400: HTTPExceptionResponse; + 200: Config; /** * Unauthorized */ @@ -2790,24 +2754,24 @@ export type $OpenApiTs = { */ 404: HTTPExceptionResponse; /** - * Unprocessable Entity + * Not Acceptable */ - 422: HTTPExceptionResponse; + 406: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; }; }; }; - "/public/dags/{dag_id}/details": { + "/public/config/section/{section}/option/{option}": { get: { - req: GetDagDetailsData; + req: GetConfigValueData; res: { /** * Successful Response */ - 200: DAGDetailsResponse; - /** - * Bad Request - */ - 400: HTTPExceptionResponse; + 200: Config; /** * Unauthorized */ @@ -2820,6 +2784,10 @@ export type $OpenApiTs = { * Not Found */ 404: HTTPExceptionResponse; + /** + * Not Acceptable + */ + 406: HTTPExceptionResponse; /** * Validation Error */ diff --git a/tests/api_fastapi/core_api/routes/public/test_config.py b/tests/api_fastapi/core_api/routes/public/test_config.py index c92582e1e93bb..456172005aeba 100644 --- a/tests/api_fastapi/core_api/routes/public/test_config.py +++ b/tests/api_fastapi/core_api/routes/public/test_config.py @@ -24,9 +24,10 @@ from tests_common.test_utils.config import conf_vars -HEADERS_JSON = {"Content-Type": "application/json"} -HEADERS_TEXT = {"Content-Type": "text/plain"} -HEADERS_INVALID = {"Content-Type": "invalid"} +HEADERS_JSON = {"Accept": "application/json"} +HEADERS_TEXT = {"Accept": "text/plain"} +HEADERS_INVALID = {"Accept": "invalid"} +HEADERS_JSON_UTF8 = {"Accept": "application/json; charset=utf-8"} SECTION_CORE = "core" SECTION_SMTP = "smtp" SECTION_DATABASE = "database" @@ -112,6 +113,34 @@ class TestGetConfig(TestConfigEndpoint): ], }, ), + ( + None, + HEADERS_JSON_UTF8, + 200, + { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + { + "name": SECTION_SMTP, + "options": [ + {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, + {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, + ], + }, + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SQL_ALCHEMY_CONN}, + ], + }, + ], + }, + ), ( None, HEADERS_TEXT, @@ -133,18 +162,8 @@ class TestGetConfig(TestConfigEndpoint): ( None, HEADERS_INVALID, - 422, - { - "detail": [ - { - "type": "literal_error", - "loc": ["header", "content-type"], - "msg": "Input should be 'application/json' or 'text/plain'", - "input": "invalid", - "ctx": {"expected": "'application/json' or 'text/plain'"}, - } - ] - }, + 406, + {"detail": "Only application/json or text/plain is supported"}, ), ( SECTION_CORE, @@ -201,10 +220,10 @@ def test_get_config(self, test_client, section, headers, expected_status_code, e else: response = test_client.get("/public/config/", headers=headers, params=query_params) assert response.status_code == expected_status_code - if response.status_code != 200 or headers == HEADERS_JSON: - assert response.json() == expected_response - else: + if headers == HEADERS_TEXT: assert response.text == expected_response + else: + assert response.json() == expected_response class TestGetConfigValue(TestConfigEndpoint): diff --git a/tests/api_fastapi/core_api/routes/public/test_dag_sources.py b/tests/api_fastapi/core_api/routes/public/test_dag_sources.py index d8f81482cfbff..a5cd52782ca72 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dag_sources.py +++ b/tests/api_fastapi/core_api/routes/public/test_dag_sources.py @@ -74,11 +74,14 @@ def test_should_respond_200_text(self, test_client, test_dag): json.loads(response.content.decode()) assert response.headers["Content-Type"].startswith("text/plain") - def test_should_respond_200_json(self, test_client, test_dag): + @pytest.mark.parametrize( + "headers", [{"Accept": "application/json"}, {"Accept": "application/json; charset=utf-8"}, {}] + ) + def test_should_respond_200_json(self, test_client,test_dag, headers): dag_content = self._get_dag_file_code(test_dag.fileloc) response: Response = test_client.get( f"{API_PREFIX}/{TEST_DAG_ID}", - headers={"Accept": "application/json"}, + headers=headers, ) assert isinstance(response, Response) assert 200 == response.status_code From a7e16e59ed26889827bbdff00f51c7d50fc17691 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Sat, 16 Nov 2024 02:12:35 +0800 Subject: [PATCH 07/10] Refactor test_config with non-sensitive-only case --- airflow/api_fastapi/common/types.py | 3 +- .../core_api/openapi/v1-generated.yaml | 311 +++++++++--------- airflow/ui/openapi-gen/queries/common.ts | 84 ++--- airflow/ui/openapi-gen/queries/prefetch.ts | 104 +++--- airflow/ui/openapi-gen/queries/queries.ts | 141 ++++---- airflow/ui/openapi-gen/queries/suspense.ts | 134 ++++---- .../ui/openapi-gen/requests/services.gen.ts | 140 ++++---- airflow/ui/openapi-gen/requests/types.gen.ts | 157 +++++---- .../core_api/routes/public/test_config.py | 128 ++++++- 9 files changed, 656 insertions(+), 546 deletions(-) diff --git a/airflow/api_fastapi/common/types.py b/airflow/api_fastapi/common/types.py index e4ffbafc5cf86..4a41f4f0ceed8 100644 --- a/airflow/api_fastapi/common/types.py +++ b/airflow/api_fastapi/common/types.py @@ -17,8 +17,8 @@ from __future__ import annotations from datetime import timedelta -from typing import Annotated from enum import Enum +from typing import Annotated from pydantic import AfterValidator, AliasGenerator, AwareDatetime, BaseModel, BeforeValidator, ConfigDict @@ -58,6 +58,7 @@ class TimeDelta(BaseModel): TimeDeltaWithValidation = Annotated[TimeDelta, BeforeValidator(_validate_timedelta_field)] + class Mimetype(str, Enum): """Mimetype for the `Content-Type` header.""" diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index c25d9697f8fd6..ee16e21e3c0b8 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -1683,6 +1683,9 @@ paths: required: false schema: type: string + enum: + - application/json + - text/plain default: '*/*' title: Accept responses: @@ -1785,6 +1788,154 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/config/: + get: + tags: + - Config + summary: Get Config + operationId: get_config + parameters: + - name: section + in: query + required: false + schema: + anyOf: + - type: string + - type: 'null' + title: Section + - name: accept + in: header + required: false + schema: + type: string + enum: + - application/json + - text/plain + default: '*/*' + title: Accept + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Config' + text/plain: + schema: + type: string + example: '[core] + + dags_folder = /opt/airflow/dags + + base_log_folder = /opt/airflow/logs + + ' + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '406': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Acceptable + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /public/config/section/{section}/option/{option}: + get: + tags: + - Config + summary: Get Config Value + operationId: get_config_value + parameters: + - name: section + in: path + required: true + schema: + type: string + title: Section + - name: option + in: path + required: true + schema: + type: string + title: Option + - name: accept + in: header + required: false + schema: + type: string + enum: + - application/json + - text/plain + default: '*/*' + title: Accept + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Config' + text/plain: + schema: + type: string + example: '[core] + + dags_folder = /opt/airflow/dags + + base_log_folder = /opt/airflow/logs + + ' + '401': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Unauthorized + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Forbidden + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '406': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Acceptable + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /public/dagWarnings: get: tags: @@ -2091,154 +2242,6 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' - /public/config/: - get: - tags: - - Config - summary: Get Config - operationId: get_config - parameters: - - name: section - in: query - required: false - schema: - anyOf: - - type: string - - type: 'null' - title: Section - - name: accept - in: header - required: false - schema: - type: string - enum: - - application/json - - text/plain - default: '*/*' - title: Accept - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Config' - text/plain: - schema: - type: string - example: '[core] - - dags_folder = /opt/airflow/dags - - base_log_folder = /opt/airflow/logs - - ' - '401': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Unauthorized - '403': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Forbidden - '404': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Not Found - '406': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Not Acceptable - '422': - description: Validation Error - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' - /public/config/section/{section}/option/{option}: - get: - tags: - - Config - summary: Get Config Value - operationId: get_config_value - parameters: - - name: section - in: path - required: true - schema: - type: string - title: Section - - name: option - in: path - required: true - schema: - type: string - title: Option - - name: accept - in: header - required: false - schema: - type: string - enum: - - application/json - - text/plain - default: '*/*' - title: Accept - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Config' - text/plain: - schema: - type: string - example: '[core] - - dags_folder = /opt/airflow/dags - - base_log_folder = /opt/airflow/logs - - ' - '401': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Unauthorized - '403': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Forbidden - '404': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Not Found - '406': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPExceptionResponse' - description: Not Acceptable - '422': - description: Validation Error - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' /public/dags/tags: get: tags: @@ -2432,17 +2435,7 @@ paths: required: true schema: type: string - title: File Token - - name: accept - in: header - required: false - schema: - type: string - enum: - - application/json - - text/plain - default: '*/*' - title: Accept + title: Dag Id responses: '200': description: Successful Response diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index a3ecb44c94645..2a0f20ba17d21 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -411,7 +411,7 @@ export const UseDagSourceServiceGetDagSourceKeyFn = ( dagId, versionNumber, }: { - accept?: string; + accept?: "application/json" | "text/plain"; dagId: string; versionNumber?: number; }, @@ -436,6 +436,47 @@ export const UseDagStatsServiceGetDagStatsKeyFn = ( } = {}, queryKey?: Array, ) => [useDagStatsServiceGetDagStatsKey, ...(queryKey ?? [{ dagIds }])]; +export type ConfigServiceGetConfigDefaultResponse = Awaited< + ReturnType +>; +export type ConfigServiceGetConfigQueryResult< + TData = ConfigServiceGetConfigDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useConfigServiceGetConfigKey = "ConfigServiceGetConfig"; +export const UseConfigServiceGetConfigKeyFn = ( + { + accept, + section, + }: { + accept?: "application/json" | "text/plain"; + section?: string; + } = {}, + queryKey?: Array, +) => [useConfigServiceGetConfigKey, ...(queryKey ?? [{ accept, section }])]; +export type ConfigServiceGetConfigValueDefaultResponse = Awaited< + ReturnType +>; +export type ConfigServiceGetConfigValueQueryResult< + TData = ConfigServiceGetConfigValueDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useConfigServiceGetConfigValueKey = "ConfigServiceGetConfigValue"; +export const UseConfigServiceGetConfigValueKeyFn = ( + { + accept, + option, + section, + }: { + accept?: "application/json" | "text/plain"; + option: string; + section: string; + }, + queryKey?: Array, +) => [ + useConfigServiceGetConfigValueKey, + ...(queryKey ?? [{ accept, option, section }]), +]; export type DagWarningServiceListDagWarningsDefaultResponse = Awaited< ReturnType >; @@ -571,47 +612,6 @@ export const UseDagServiceGetDagDetailsKeyFn = ( }, queryKey?: Array, ) => [useDagServiceGetDagDetailsKey, ...(queryKey ?? [{ dagId }])]; -export type ConfigServiceGetConfigDefaultResponse = Awaited< - ReturnType ->; -export type ConfigServiceGetConfigQueryResult< - TData = ConfigServiceGetConfigDefaultResponse, - TError = unknown, -> = UseQueryResult; -export const useConfigServiceGetConfigKey = "ConfigServiceGetConfig"; -export const UseConfigServiceGetConfigKeyFn = ( - { - accept, - section, - }: { - accept?: "application/json" | "text/plain"; - section?: string; - } = {}, - queryKey?: Array, -) => [useConfigServiceGetConfigKey, ...(queryKey ?? [{ accept, section }])]; -export type ConfigServiceGetConfigValueDefaultResponse = Awaited< - ReturnType ->; -export type ConfigServiceGetConfigValueQueryResult< - TData = ConfigServiceGetConfigValueDefaultResponse, - TError = unknown, -> = UseQueryResult; -export const useConfigServiceGetConfigValueKey = "ConfigServiceGetConfigValue"; -export const UseConfigServiceGetConfigValueKeyFn = ( - { - accept, - option, - section, - }: { - accept?: "application/json" | "text/plain"; - option: string; - section: string; - }, - queryKey?: Array, -) => [ - useConfigServiceGetConfigValueKey, - ...(queryKey ?? [{ accept, option, section }]), -]; export type EventLogServiceGetEventLogDefaultResponse = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index 4bb1edbec655e..a4a57bf377406 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -509,7 +509,7 @@ export const prefetchUseDagSourceServiceGetDagSource = ( dagId, versionNumber, }: { - accept?: string; + accept?: "application/json" | "text/plain"; dagId: string; versionNumber?: number; }, @@ -543,6 +543,57 @@ export const prefetchUseDagStatsServiceGetDagStats = ( queryKey: Common.UseDagStatsServiceGetDagStatsKeyFn({ dagIds }), queryFn: () => DagStatsService.getDagStats({ dagIds }), }); +/** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ +export const prefetchUseConfigServiceGetConfig = ( + queryClient: QueryClient, + { + accept, + section, + }: { + accept?: "application/json" | "text/plain"; + section?: string; + } = {}, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseConfigServiceGetConfigKeyFn({ accept, section }), + queryFn: () => ConfigService.getConfig({ accept, section }), + }); +/** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ +export const prefetchUseConfigServiceGetConfigValue = ( + queryClient: QueryClient, + { + accept, + option, + section, + }: { + accept?: "application/json" | "text/plain"; + option: string; + section: string; + }, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseConfigServiceGetConfigValueKeyFn({ + accept, + option, + section, + }), + queryFn: () => ConfigService.getConfigValue({ accept, option, section }), + }); /** * List Dag Warnings * Get a list of DAG warnings. @@ -733,57 +784,6 @@ export const prefetchUseDagServiceGetDagDetails = ( queryKey: Common.UseDagServiceGetDagDetailsKeyFn({ dagId }), queryFn: () => DagService.getDagDetails({ dagId }), }); -/** - * Get Config - * @param data The data for the request. - * @param data.section - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ -export const prefetchUseConfigServiceGetConfig = ( - queryClient: QueryClient, - { - accept, - section, - }: { - accept?: "application/json" | "text/plain"; - section?: string; - } = {}, -) => - queryClient.prefetchQuery({ - queryKey: Common.UseConfigServiceGetConfigKeyFn({ accept, section }), - queryFn: () => ConfigService.getConfig({ accept, section }), - }); -/** - * Get Config Value - * @param data The data for the request. - * @param data.section - * @param data.option - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ -export const prefetchUseConfigServiceGetConfigValue = ( - queryClient: QueryClient, - { - accept, - option, - section, - }: { - accept?: "application/json" | "text/plain"; - option: string; - section: string; - }, -) => - queryClient.prefetchQuery({ - queryKey: Common.UseConfigServiceGetConfigValueKeyFn({ - accept, - option, - section, - }), - queryFn: () => ConfigService.getConfigValue({ accept, option, section }), - }); /** * Get Event Log * @param data The data for the request. diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 0c6e6c2693e22..cf79dc49c178e 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -642,7 +642,7 @@ export const useDagSourceServiceGetDagSource = < dagId, versionNumber, }: { - accept?: string; + accept?: "application/json" | "text/plain"; dagId: string; versionNumber?: number; }, @@ -684,6 +684,72 @@ export const useDagStatsServiceGetDagStats = < queryFn: () => DagStatsService.getDagStats({ dagIds }) as TData, ...options, }); +/** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfig = < + TData = Common.ConfigServiceGetConfigDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + accept, + section, + }: { + accept?: "application/json" | "text/plain"; + section?: string; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseConfigServiceGetConfigKeyFn( + { accept, section }, + queryKey, + ), + queryFn: () => ConfigService.getConfig({ accept, section }) as TData, + ...options, + }); +/** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfigValue = < + TData = Common.ConfigServiceGetConfigValueDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + accept, + option, + section, + }: { + accept?: "application/json" | "text/plain"; + option: string; + section: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseConfigServiceGetConfigValueKeyFn( + { accept, option, section }, + queryKey, + ), + queryFn: () => + ConfigService.getConfigValue({ accept, option, section }) as TData, + ...options, + }); /** * List Dag Warnings * Get a list of DAG warnings. @@ -907,72 +973,6 @@ export const useDagServiceGetDagDetails = < queryFn: () => DagService.getDagDetails({ dagId }) as TData, ...options, }); -/** - * Get Config - * @param data The data for the request. - * @param data.section - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ -export const useConfigServiceGetConfig = < - TData = Common.ConfigServiceGetConfigDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - accept, - section, - }: { - accept?: "application/json" | "text/plain"; - section?: string; - } = {}, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useQuery({ - queryKey: Common.UseConfigServiceGetConfigKeyFn( - { accept, section }, - queryKey, - ), - queryFn: () => ConfigService.getConfig({ accept, section }) as TData, - ...options, - }); -/** - * Get Config Value - * @param data The data for the request. - * @param data.section - * @param data.option - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ -export const useConfigServiceGetConfigValue = < - TData = Common.ConfigServiceGetConfigValueDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - accept, - option, - section, - }: { - accept?: "application/json" | "text/plain"; - option: string; - section: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useQuery({ - queryKey: Common.UseConfigServiceGetConfigValueKeyFn( - { accept, option, section }, - queryKey, - ), - queryFn: () => - ConfigService.getConfigValue({ accept, option, section }) as TData, - ...options, - }); /** * Get Event Log * @param data The data for the request. @@ -2964,7 +2964,6 @@ export const useDagRunServiceDeleteDagRun = < * Delete the specific DAG. * @param data The data for the request. * @param data.dagId - * @param data.accept * @returns unknown Successful Response * @throws ApiError */ @@ -2978,7 +2977,6 @@ export const useDagServiceDeleteDag = < TData, TError, { - accept?: "application/json" | "text/plain"; dagId: string; }, TContext @@ -2990,13 +2988,12 @@ export const useDagServiceDeleteDag = < TData, TError, { - accept?: "application/json" | "text/plain"; dagId: string; }, TContext >({ - mutationFn: ({ accept, dagId }) => - DagService.deleteDag({ accept, dagId }) as unknown as Promise, + mutationFn: ({ dagId }) => + DagService.deleteDag({ dagId }) as unknown as Promise, ...options, }); /** diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index b4e88046a38ce..778aaa848e054 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -624,7 +624,7 @@ export const useDagSourceServiceGetDagSourceSuspense = < dagId, versionNumber, }: { - accept?: string; + accept?: "application/json" | "text/plain"; dagId: string; versionNumber?: number; }, @@ -666,6 +666,72 @@ export const useDagStatsServiceGetDagStatsSuspense = < queryFn: () => DagStatsService.getDagStats({ dagIds }) as TData, ...options, }); +/** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfigSuspense = < + TData = Common.ConfigServiceGetConfigDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + accept, + section, + }: { + accept?: "application/json" | "text/plain"; + section?: string; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseConfigServiceGetConfigKeyFn( + { accept, section }, + queryKey, + ), + queryFn: () => ConfigService.getConfig({ accept, section }) as TData, + ...options, + }); +/** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ +export const useConfigServiceGetConfigValueSuspense = < + TData = Common.ConfigServiceGetConfigValueDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + accept, + option, + section, + }: { + accept?: "application/json" | "text/plain"; + option: string; + section: string; + }, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseConfigServiceGetConfigValueKeyFn( + { accept, option, section }, + queryKey, + ), + queryFn: () => + ConfigService.getConfigValue({ accept, option, section }) as TData, + ...options, + }); /** * List Dag Warnings * Get a list of DAG warnings. @@ -889,72 +955,6 @@ export const useDagServiceGetDagDetailsSuspense = < queryFn: () => DagService.getDagDetails({ dagId }) as TData, ...options, }); -/** - * Get Config - * @param data The data for the request. - * @param data.section - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ -export const useConfigServiceGetConfigSuspense = < - TData = Common.ConfigServiceGetConfigDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - accept, - section, - }: { - accept?: "application/json" | "text/plain"; - section?: string; - } = {}, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useSuspenseQuery({ - queryKey: Common.UseConfigServiceGetConfigKeyFn( - { accept, section }, - queryKey, - ), - queryFn: () => ConfigService.getConfig({ accept, section }) as TData, - ...options, - }); -/** - * Get Config Value - * @param data The data for the request. - * @param data.section - * @param data.option - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ -export const useConfigServiceGetConfigValueSuspense = < - TData = Common.ConfigServiceGetConfigValueDefaultResponse, - TError = unknown, - TQueryKey extends Array = unknown[], ->( - { - accept, - option, - section, - }: { - accept?: "application/json" | "text/plain"; - option: string; - section: string; - }, - queryKey?: TQueryKey, - options?: Omit, "queryKey" | "queryFn">, -) => - useSuspenseQuery({ - queryKey: Common.UseConfigServiceGetConfigValueKeyFn( - { accept, option, section }, - queryKey, - ), - queryFn: () => - ConfigService.getConfigValue({ accept, option, section }) as TData, - ...options, - }); /** * Get Event Log * @param data The data for the request. diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 18774cee8a45c..6689fce3a837d 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -67,6 +67,10 @@ import type { GetDagSourceResponse, GetDagStatsData, GetDagStatsResponse, + GetConfigData, + GetConfigResponse, + GetConfigValueData, + GetConfigValueResponse, ListDagWarningsData, ListDagWarningsResponse, GetDagsData, @@ -83,10 +87,6 @@ import type { DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, - GetConfigData, - GetConfigResponse, - GetConfigValueData, - GetConfigValueResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, @@ -1100,6 +1100,70 @@ export class DagStatsService { } } +export class ConfigService { + /** + * Get Config + * @param data The data for the request. + * @param data.section + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ + public static getConfig( + data: GetConfigData = {}, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/config/", + headers: { + accept: data.accept, + }, + query: { + section: data.section, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 406: "Not Acceptable", + 422: "Validation Error", + }, + }); + } + + /** + * Get Config Value + * @param data The data for the request. + * @param data.section + * @param data.option + * @param data.accept + * @returns Config Successful Response + * @throws ApiError + */ + public static getConfigValue( + data: GetConfigValueData, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/config/section/{section}/option/{option}", + path: { + section: data.section, + option: data.option, + }, + headers: { + accept: data.accept, + }, + errors: { + 401: "Unauthorized", + 403: "Forbidden", + 404: "Not Found", + 406: "Not Acceptable", + 422: "Validation Error", + }, + }); + } +} + export class DagWarningService { /** * List Dag Warnings @@ -1320,7 +1384,6 @@ export class DagService { * Delete the specific DAG. * @param data The data for the request. * @param data.dagId - * @param data.accept * @returns unknown Successful Response * @throws ApiError */ @@ -1333,9 +1396,6 @@ export class DagService { path: { dag_id: data.dagId, }, - headers: { - accept: data.accept, - }, errors: { 400: "Bad Request", 401: "Unauthorized", @@ -1374,70 +1434,6 @@ export class DagService { } } -export class ConfigService { - /** - * Get Config - * @param data The data for the request. - * @param data.section - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ - public static getConfig( - data: GetConfigData = {}, - ): CancelablePromise { - return __request(OpenAPI, { - method: "GET", - url: "/public/config/", - headers: { - accept: data.accept, - }, - query: { - section: data.section, - }, - errors: { - 401: "Unauthorized", - 403: "Forbidden", - 404: "Not Found", - 406: "Not Acceptable", - 422: "Validation Error", - }, - }); - } - - /** - * Get Config Value - * @param data The data for the request. - * @param data.section - * @param data.option - * @param data.accept - * @returns Config Successful Response - * @throws ApiError - */ - public static getConfigValue( - data: GetConfigValueData, - ): CancelablePromise { - return __request(OpenAPI, { - method: "GET", - url: "/public/config/section/{section}/option/{option}", - path: { - section: data.section, - option: data.option, - }, - headers: { - accept: data.accept, - }, - errors: { - 401: "Unauthorized", - 403: "Forbidden", - 404: "Not Found", - 406: "Not Acceptable", - 422: "Validation Error", - }, - }); - } -} - export class EventLogService { /** * Get Event Log diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index cf68c1bf92a05..16620de30c6b7 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1301,7 +1301,7 @@ export type ClearDagRunResponse = | DAGRunResponse; export type GetDagSourceData = { - accept?: string; + accept?: "application/json" | "text/plain"; dagId: string; versionNumber?: number | null; }; @@ -1314,6 +1314,21 @@ export type GetDagStatsData = { export type GetDagStatsResponse = DagStatsCollectionResponse; +export type GetConfigData = { + accept?: "application/json" | "text/plain"; + section?: string | null; +}; + +export type GetConfigResponse = Config; + +export type GetConfigValueData = { + accept?: "application/json" | "text/plain"; + option: string; + section: string; +}; + +export type GetConfigValueResponse = Config; + export type ListDagWarningsData = { dagId?: string | null; limit?: number; @@ -1378,7 +1393,6 @@ export type PatchDagData = { export type PatchDagResponse = DAGResponse; export type DeleteDagData = { - accept?: "application/json" | "text/plain"; dagId: string; }; @@ -1390,21 +1404,6 @@ export type GetDagDetailsData = { export type GetDagDetailsResponse = DAGDetailsResponse; -export type GetConfigData = { - accept?: "application/json" | "text/plain"; - section?: string | null; -}; - -export type GetConfigResponse = Config; - -export type GetConfigValueData = { - accept?: "application/json" | "text/plain"; - option: string; - section: string; -}; - -export type GetConfigValueResponse = Config; - export type GetEventLogData = { eventLogId: number; }; @@ -2515,6 +2514,68 @@ export type $OpenApiTs = { }; }; }; + "/public/config/": { + get: { + req: GetConfigData; + res: { + /** + * Successful Response + */ + 200: Config; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Not Acceptable + */ + 406: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; + "/public/config/section/{section}/option/{option}": { + get: { + req: GetConfigValueData; + res: { + /** + * Successful Response + */ + 200: Config; + /** + * Unauthorized + */ + 401: HTTPExceptionResponse; + /** + * Forbidden + */ + 403: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Not Acceptable + */ + 406: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; "/public/dagWarnings": { get: { req: ListDagWarningsData; @@ -2733,68 +2794,6 @@ export type $OpenApiTs = { }; }; }; - "/public/config/": { - get: { - req: GetConfigData; - res: { - /** - * Successful Response - */ - 200: Config; - /** - * Unauthorized - */ - 401: HTTPExceptionResponse; - /** - * Forbidden - */ - 403: HTTPExceptionResponse; - /** - * Not Found - */ - 404: HTTPExceptionResponse; - /** - * Not Acceptable - */ - 406: HTTPExceptionResponse; - /** - * Validation Error - */ - 422: HTTPValidationError; - }; - }; - }; - "/public/config/section/{section}/option/{option}": { - get: { - req: GetConfigValueData; - res: { - /** - * Successful Response - */ - 200: Config; - /** - * Unauthorized - */ - 401: HTTPExceptionResponse; - /** - * Forbidden - */ - 403: HTTPExceptionResponse; - /** - * Not Found - */ - 404: HTTPExceptionResponse; - /** - * Not Acceptable - */ - 406: HTTPExceptionResponse; - /** - * Validation Error - */ - 422: HTTPValidationError; - }; - }; - }; "/public/eventLogs/{event_log_id}": { get: { req: GetEventLogData; diff --git a/tests/api_fastapi/core_api/routes/public/test_config.py b/tests/api_fastapi/core_api/routes/public/test_config.py index 456172005aeba..bd33d499e3452 100644 --- a/tests/api_fastapi/core_api/routes/public/test_config.py +++ b/tests/api_fastapi/core_api/routes/public/test_config.py @@ -39,7 +39,7 @@ OPTION_VALUE_PARALLELISM = "1024" OPTION_VALUE_SMTP_HOST = "smtp.example.com" OPTION_VALUE_SMTP_MAIL_FROM = "airflow@example.com" -OPTION_VALUE_SQL_ALCHEMY_CONN = "mock_sql_alchemy_conn" +OPTION_VALUE_SQL_ALCHEMY_CONN = "sqlite:///example.db" OPTION_NOT_EXIST = "not_exist_option" OPTION_VALUE_SENSITIVE_HIDDEN = "< hidden >" @@ -55,6 +55,18 @@ OPTION_KEY_SQL_ALCHEMY_CONN: OPTION_VALUE_SQL_ALCHEMY_CONN, }, } +MOCK_CONFIG_DICT_SENSITIVE_HIDDEN = { + SECTION_CORE: { + OPTION_KEY_PARALLELISM: OPTION_VALUE_PARALLELISM, + }, + SECTION_SMTP: { + OPTION_KEY_SMTP_HOST: OPTION_VALUE_SMTP_HOST, + OPTION_KEY_SMTP_MAIL_FROM: OPTION_VALUE_SMTP_MAIL_FROM, + }, + SECTION_DATABASE: { + OPTION_KEY_SQL_ALCHEMY_CONN: OPTION_VALUE_SENSITIVE_HIDDEN, + }, +} MOCK_CONFIG_OVERRIDE = { (SECTION_CORE, OPTION_KEY_PARALLELISM): OPTION_VALUE_PARALLELISM, (SECTION_SMTP, OPTION_KEY_SMTP_HOST): OPTION_VALUE_SMTP_HOST, @@ -63,6 +75,7 @@ AIRFLOW_CONFIG_ENABLE_EXPOSE_CONFIG = {("webserver", "expose_config"): "True"} AIRFLOW_CONFIG_DISABLE_EXPOSE_CONFIG = {("webserver", "expose_config"): "False"} +AIRFLOW_CONFIG_NON_SENSITIVE_ONLY_CONFIG = {("webserver", "expose_config"): "non-sensitive-only"} FORBIDDEN_RESPONSE = { "detail": "Your Airflow administrator chose not to expose the configuration, most likely for security reasons." } @@ -74,9 +87,16 @@ def setup(self) -> Generator[None, None, None]: with conf_vars(AIRFLOW_CONFIG_ENABLE_EXPOSE_CONFIG | MOCK_CONFIG_OVERRIDE): # since the endpoint calls `conf_dict.clear()` to remove extra keys, # use `new` instead of `return_value` to avoid side effects + def _mock_conf_as_dict(display_sensitive: bool, **_): + return ( + MOCK_CONFIG_DICT_SENSITIVE_HIDDEN.copy() + if not display_sensitive + else MOCK_CONFIG_DICT.copy() + ) + with patch( "airflow.api_fastapi.core_api.routes.public.config.conf.as_dict", - new=lambda **_: MOCK_CONFIG_DICT.copy(), + new=_mock_conf_as_dict, ): yield @@ -225,6 +245,66 @@ def test_get_config(self, test_client, section, headers, expected_status_code, e else: assert response.json() == expected_response + @pytest.mark.parametrize( + "headers, expected_status_code, expected_response", + [ + ( + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + { + "name": SECTION_SMTP, + "options": [ + {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, + {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, + ], + }, + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SENSITIVE_HIDDEN}, + ], + }, + ], + }, + ), + ( + HEADERS_TEXT, + 200, + textwrap.dedent( + f"""\ + [{SECTION_CORE}] + {OPTION_KEY_PARALLELISM} = {OPTION_VALUE_PARALLELISM} + + [{SECTION_SMTP}] + {OPTION_KEY_SMTP_HOST} = {OPTION_VALUE_SMTP_HOST} + {OPTION_KEY_SMTP_MAIL_FROM} = {OPTION_VALUE_SMTP_MAIL_FROM} + + [{SECTION_DATABASE}] + {OPTION_KEY_SQL_ALCHEMY_CONN} = {OPTION_VALUE_SENSITIVE_HIDDEN} + """ + ), + ), + ], + ) + def test_get_config_non_sensitive_only( + self, test_client, headers, expected_status_code, expected_response + ): + with conf_vars(AIRFLOW_CONFIG_NON_SENSITIVE_ONLY_CONFIG): + response = test_client.get("/public/config/", headers=headers) + assert response.status_code == expected_status_code + if headers == HEADERS_TEXT: + assert response.text == expected_response + else: + assert response.json() == expected_response + class TestGetConfigValue(TestConfigEndpoint): @pytest.mark.parametrize( @@ -334,3 +414,47 @@ def test_get_config_value( assert response.json() == expected_response else: assert response.text == expected_response + + @pytest.mark.parametrize( + "section, option, headers, expected_status_code, expected_response", + [ + ( + SECTION_DATABASE, + OPTION_KEY_SQL_ALCHEMY_CONN, + HEADERS_JSON, + 200, + { + "sections": [ + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SENSITIVE_HIDDEN}, + ], + }, + ], + }, + ), + ( + SECTION_DATABASE, + OPTION_KEY_SQL_ALCHEMY_CONN, + HEADERS_TEXT, + 200, + textwrap.dedent( + f"""\ + [{SECTION_DATABASE}] + {OPTION_KEY_SQL_ALCHEMY_CONN} = {OPTION_VALUE_SENSITIVE_HIDDEN} + """ + ), + ), + ], + ) + def test_get_config_value_non_sensitive_only( + self, test_client, section, option, headers, expected_status_code, expected_response + ): + with conf_vars(AIRFLOW_CONFIG_NON_SENSITIVE_ONLY_CONFIG): + response = test_client.get(f"/public/config/section/{section}/option/{option}", headers=headers) + assert response.status_code == expected_status_code + if response.status_code != 200 or headers == HEADERS_JSON: + assert response.json() == expected_response + else: + assert response.text == expected_response From 8b39d779bea5f68432e853b71f8f7418e22bbcda Mon Sep 17 00:00:00 2001 From: jason810496 Date: Tue, 19 Nov 2024 11:34:21 +0800 Subject: [PATCH 08/10] Fix OpenAPI schema for Headers, Get Config --- airflow/api_fastapi/common/headers.py | 2 +- .../core_api/openapi/v1-generated.yaml | 10 ++++++++ .../core_api/routes/public/config.py | 25 ++++++++++++++++--- airflow/ui/openapi-gen/queries/common.ts | 6 ++--- airflow/ui/openapi-gen/queries/prefetch.ts | 6 ++--- airflow/ui/openapi-gen/queries/queries.ts | 6 ++--- airflow/ui/openapi-gen/queries/suspense.ts | 6 ++--- airflow/ui/openapi-gen/requests/types.gen.ts | 6 ++--- 8 files changed, 48 insertions(+), 19 deletions(-) diff --git a/airflow/api_fastapi/common/headers.py b/airflow/api_fastapi/common/headers.py index 0c9924c19b78b..7e0bd5aae3d29 100644 --- a/airflow/api_fastapi/common/headers.py +++ b/airflow/api_fastapi/common/headers.py @@ -28,7 +28,7 @@ def header_accept_json_or_text_depends( Header( json_schema_extra={ "type": "string", - "enum": [Mimetype.JSON, Mimetype.TEXT], + "enum": [Mimetype.JSON, Mimetype.TEXT, Mimetype.ANY], } ), ] = Mimetype.ANY, diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index ee16e21e3c0b8..3c1d85761ec94 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -1686,6 +1686,7 @@ paths: enum: - application/json - text/plain + - '*/*' default: '*/*' title: Accept responses: @@ -1811,6 +1812,7 @@ paths: enum: - application/json - text/plain + - '*/*' default: '*/*' title: Accept responses: @@ -1829,6 +1831,13 @@ paths: base_log_folder = /opt/airflow/logs + + [smtp] + + smtp_host = localhost + + smtp_mail_from = airflow@example.com + ' '401': content: @@ -1887,6 +1896,7 @@ paths: enum: - application/json - text/plain + - '*/*' default: '*/*' title: Accept responses: diff --git a/airflow/api_fastapi/core_api/routes/public/config.py b/airflow/api_fastapi/core_api/routes/public/config.py index 669cb8eef935e..eb26cfcd2c02a 100644 --- a/airflow/api_fastapi/core_api/routes/public/config.py +++ b/airflow/api_fastapi/core_api/routes/public/config.py @@ -32,7 +32,7 @@ from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc from airflow.configuration import conf -text_example_response = { +text_example_response_for_get_config_value = { Mimetype.TEXT: { "schema": { "type": "string", @@ -47,6 +47,25 @@ } } +text_example_response_for_get_config = { + Mimetype.TEXT: { + "schema": { + "type": "string", + "example": textwrap.dedent( + """\ + [core] + dags_folder = /opt/airflow/dags + base_log_folder = /opt/airflow/logs + + [smtp] + smtp_host = localhost + smtp_mail_from = airflow@example.com + """ + ), + }, + } +} + def _check_expose_config() -> bool: display_sensitive: bool | None = None @@ -85,7 +104,7 @@ def _response_based_on_accept(accept: Mimetype, config: Config): status.HTTP_406_NOT_ACCEPTABLE, ] ), - "200": {"description": "Successful Response", "content": text_example_response}, + "200": {"description": "Successful Response", "content": text_example_response_for_get_config}, }, response_model=Config, ) @@ -129,7 +148,7 @@ def get_config( status.HTTP_406_NOT_ACCEPTABLE, ] ), - "200": {"description": "Successful Response", "content": text_example_response}, + "200": {"description": "Successful Response", "content": text_example_response_for_get_config_value}, }, response_model=Config, ) diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 2a0f20ba17d21..ea4e65d44c91d 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -411,7 +411,7 @@ export const UseDagSourceServiceGetDagSourceKeyFn = ( dagId, versionNumber, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; dagId: string; versionNumber?: number; }, @@ -449,7 +449,7 @@ export const UseConfigServiceGetConfigKeyFn = ( accept, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; section?: string; } = {}, queryKey?: Array, @@ -468,7 +468,7 @@ export const UseConfigServiceGetConfigValueKeyFn = ( option, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; option: string; section: string; }, diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index a4a57bf377406..273882c787dcb 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -509,7 +509,7 @@ export const prefetchUseDagSourceServiceGetDagSource = ( dagId, versionNumber, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; dagId: string; versionNumber?: number; }, @@ -557,7 +557,7 @@ export const prefetchUseConfigServiceGetConfig = ( accept, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; section?: string; } = {}, ) => @@ -581,7 +581,7 @@ export const prefetchUseConfigServiceGetConfigValue = ( option, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; option: string; section: string; }, diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index cf79dc49c178e..386575204aa0b 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -642,7 +642,7 @@ export const useDagSourceServiceGetDagSource = < dagId, versionNumber, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; dagId: string; versionNumber?: number; }, @@ -701,7 +701,7 @@ export const useConfigServiceGetConfig = < accept, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; section?: string; } = {}, queryKey?: TQueryKey, @@ -734,7 +734,7 @@ export const useConfigServiceGetConfigValue = < option, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; option: string; section: string; }, diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index 778aaa848e054..3d57a08a69e64 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -624,7 +624,7 @@ export const useDagSourceServiceGetDagSourceSuspense = < dagId, versionNumber, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; dagId: string; versionNumber?: number; }, @@ -683,7 +683,7 @@ export const useConfigServiceGetConfigSuspense = < accept, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; section?: string; } = {}, queryKey?: TQueryKey, @@ -716,7 +716,7 @@ export const useConfigServiceGetConfigValueSuspense = < option, section, }: { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; option: string; section: string; }, diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 16620de30c6b7..27e3546e5c7be 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1301,7 +1301,7 @@ export type ClearDagRunResponse = | DAGRunResponse; export type GetDagSourceData = { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; dagId: string; versionNumber?: number | null; }; @@ -1315,14 +1315,14 @@ export type GetDagStatsData = { export type GetDagStatsResponse = DagStatsCollectionResponse; export type GetConfigData = { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; section?: string | null; }; export type GetConfigResponse = Config; export type GetConfigValueData = { - accept?: "application/json" | "text/plain"; + accept?: "application/json" | "text/plain" | "*/*"; option: string; section: string; }; From 41281dca9c508260afe9a2bbac550bd92a2373a0 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Tue, 19 Nov 2024 11:58:25 +0800 Subject: [PATCH 09/10] Refactor test_config - add `HEADERS_NONE` and `HEADERS_ANY` cases - modularize common json response cases - add `_validate_response` common method --- .../core_api/routes/public/test_config.py | 250 +++++++++--------- 1 file changed, 132 insertions(+), 118 deletions(-) diff --git a/tests/api_fastapi/core_api/routes/public/test_config.py b/tests/api_fastapi/core_api/routes/public/test_config.py index bd33d499e3452..76aa5b0b6a531 100644 --- a/tests/api_fastapi/core_api/routes/public/test_config.py +++ b/tests/api_fastapi/core_api/routes/public/test_config.py @@ -24,6 +24,8 @@ from tests_common.test_utils.config import conf_vars +HEADERS_NONE = None +HEADERS_ANY = {"Accept": "*/*"} HEADERS_JSON = {"Accept": "application/json"} HEADERS_TEXT = {"Accept": "text/plain"} HEADERS_INVALID = {"Accept": "invalid"} @@ -80,8 +82,82 @@ "detail": "Your Airflow administrator chose not to expose the configuration, most likely for security reasons." } +GET_CONFIG_ALL_JSON_RESPONSE = { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + { + "name": SECTION_SMTP, + "options": [ + {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, + {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, + ], + }, + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SQL_ALCHEMY_CONN}, + ], + }, + ], +} +GET_CONFIG_NON_SENSITIVE_ONLY_JSON_RESPONSE = { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + { + "name": SECTION_SMTP, + "options": [ + {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, + {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, + ], + }, + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SENSITIVE_HIDDEN}, + ], + }, + ], +} +GET_CONFIG_VALUE_CORE_PARALLELISM_JSON_RESPONSE = { + "sections": [ + { + "name": SECTION_CORE, + "options": [ + {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, + ], + }, + ], +} +GET_CONFIG_VALUE_NON_SENSITIVE_ONLY_DATABASE_SQL_ALCHEMY_CONN_JSON_RESPONSE = { + "sections": [ + { + "name": SECTION_DATABASE, + "options": [ + {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SENSITIVE_HIDDEN}, + ], + }, + ], +} + class TestConfigEndpoint: + def _validate_response(self, headers, expected_response, expected_status_code, response): + assert response.status_code == expected_status_code + if headers == HEADERS_TEXT: + assert response.text == expected_response + else: + assert response.json() == expected_response + @pytest.fixture(autouse=True) def setup(self) -> Generator[None, None, None]: with conf_vars(AIRFLOW_CONFIG_ENABLE_EXPOSE_CONFIG | MOCK_CONFIG_OVERRIDE): @@ -109,58 +185,11 @@ class TestGetConfig(TestConfigEndpoint): None, HEADERS_JSON, 200, - { - "sections": [ - { - "name": SECTION_CORE, - "options": [ - {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, - ], - }, - { - "name": SECTION_SMTP, - "options": [ - {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, - {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, - ], - }, - { - "name": SECTION_DATABASE, - "options": [ - {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SQL_ALCHEMY_CONN}, - ], - }, - ], - }, - ), - ( - None, - HEADERS_JSON_UTF8, - 200, - { - "sections": [ - { - "name": SECTION_CORE, - "options": [ - {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, - ], - }, - { - "name": SECTION_SMTP, - "options": [ - {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, - {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, - ], - }, - { - "name": SECTION_DATABASE, - "options": [ - {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SQL_ALCHEMY_CONN}, - ], - }, - ], - }, + GET_CONFIG_ALL_JSON_RESPONSE, ), + (None, HEADERS_JSON_UTF8, 200, GET_CONFIG_ALL_JSON_RESPONSE), + (None, HEADERS_ANY, 200, GET_CONFIG_ALL_JSON_RESPONSE), + (None, HEADERS_NONE, 200, GET_CONFIG_ALL_JSON_RESPONSE), ( None, HEADERS_TEXT, @@ -239,42 +268,15 @@ def test_get_config(self, test_client, section, headers, expected_status_code, e response = test_client.get("/public/config/", headers=headers, params=query_params) else: response = test_client.get("/public/config/", headers=headers, params=query_params) - assert response.status_code == expected_status_code - if headers == HEADERS_TEXT: - assert response.text == expected_response - else: - assert response.json() == expected_response + self._validate_response(headers, expected_response, expected_status_code, response) @pytest.mark.parametrize( "headers, expected_status_code, expected_response", [ - ( - HEADERS_JSON, - 200, - { - "sections": [ - { - "name": SECTION_CORE, - "options": [ - {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, - ], - }, - { - "name": SECTION_SMTP, - "options": [ - {"key": OPTION_KEY_SMTP_HOST, "value": OPTION_VALUE_SMTP_HOST}, - {"key": OPTION_KEY_SMTP_MAIL_FROM, "value": OPTION_VALUE_SMTP_MAIL_FROM}, - ], - }, - { - "name": SECTION_DATABASE, - "options": [ - {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SENSITIVE_HIDDEN}, - ], - }, - ], - }, - ), + (HEADERS_JSON, 200, GET_CONFIG_NON_SENSITIVE_ONLY_JSON_RESPONSE), + (HEADERS_JSON_UTF8, 200, GET_CONFIG_NON_SENSITIVE_ONLY_JSON_RESPONSE), + (HEADERS_ANY, 200, GET_CONFIG_NON_SENSITIVE_ONLY_JSON_RESPONSE), + (HEADERS_NONE, 200, GET_CONFIG_NON_SENSITIVE_ONLY_JSON_RESPONSE), ( HEADERS_TEXT, 200, @@ -299,11 +301,7 @@ def test_get_config_non_sensitive_only( ): with conf_vars(AIRFLOW_CONFIG_NON_SENSITIVE_ONLY_CONFIG): response = test_client.get("/public/config/", headers=headers) - assert response.status_code == expected_status_code - if headers == HEADERS_TEXT: - assert response.text == expected_response - else: - assert response.json() == expected_response + self._validate_response(headers, expected_response, expected_status_code, response) class TestGetConfigValue(TestConfigEndpoint): @@ -315,16 +313,28 @@ class TestGetConfigValue(TestConfigEndpoint): OPTION_KEY_PARALLELISM, HEADERS_JSON, 200, - { - "sections": [ - { - "name": SECTION_CORE, - "options": [ - {"key": OPTION_KEY_PARALLELISM, "value": OPTION_VALUE_PARALLELISM}, - ], - }, - ], - }, + GET_CONFIG_VALUE_CORE_PARALLELISM_JSON_RESPONSE, + ), + ( + SECTION_CORE, + OPTION_KEY_PARALLELISM, + HEADERS_JSON_UTF8, + 200, + GET_CONFIG_VALUE_CORE_PARALLELISM_JSON_RESPONSE, + ), + ( + SECTION_CORE, + OPTION_KEY_PARALLELISM, + HEADERS_ANY, + 200, + GET_CONFIG_VALUE_CORE_PARALLELISM_JSON_RESPONSE, + ), + ( + SECTION_CORE, + OPTION_KEY_PARALLELISM, + HEADERS_NONE, + 200, + GET_CONFIG_VALUE_CORE_PARALLELISM_JSON_RESPONSE, ), ( SECTION_SMTP, @@ -409,11 +419,7 @@ def test_get_config_value( ) else: response = test_client.get(f"/public/config/section/{section}/option/{option}", headers=headers) - assert response.status_code == expected_status_code - if response.status_code != 200 or headers == HEADERS_JSON: - assert response.json() == expected_response - else: - assert response.text == expected_response + self._validate_response(headers, expected_response, expected_status_code, response) @pytest.mark.parametrize( "section, option, headers, expected_status_code, expected_response", @@ -423,16 +429,28 @@ def test_get_config_value( OPTION_KEY_SQL_ALCHEMY_CONN, HEADERS_JSON, 200, - { - "sections": [ - { - "name": SECTION_DATABASE, - "options": [ - {"key": OPTION_KEY_SQL_ALCHEMY_CONN, "value": OPTION_VALUE_SENSITIVE_HIDDEN}, - ], - }, - ], - }, + GET_CONFIG_VALUE_NON_SENSITIVE_ONLY_DATABASE_SQL_ALCHEMY_CONN_JSON_RESPONSE, + ), + ( + SECTION_DATABASE, + OPTION_KEY_SQL_ALCHEMY_CONN, + HEADERS_JSON_UTF8, + 200, + GET_CONFIG_VALUE_NON_SENSITIVE_ONLY_DATABASE_SQL_ALCHEMY_CONN_JSON_RESPONSE, + ), + ( + SECTION_DATABASE, + OPTION_KEY_SQL_ALCHEMY_CONN, + HEADERS_ANY, + 200, + GET_CONFIG_VALUE_NON_SENSITIVE_ONLY_DATABASE_SQL_ALCHEMY_CONN_JSON_RESPONSE, + ), + ( + SECTION_DATABASE, + OPTION_KEY_SQL_ALCHEMY_CONN, + HEADERS_NONE, + 200, + GET_CONFIG_VALUE_NON_SENSITIVE_ONLY_DATABASE_SQL_ALCHEMY_CONN_JSON_RESPONSE, ), ( SECTION_DATABASE, @@ -453,8 +471,4 @@ def test_get_config_value_non_sensitive_only( ): with conf_vars(AIRFLOW_CONFIG_NON_SENSITIVE_ONLY_CONFIG): response = test_client.get(f"/public/config/section/{section}/option/{option}", headers=headers) - assert response.status_code == expected_status_code - if response.status_code != 200 or headers == HEADERS_JSON: - assert response.json() == expected_response - else: - assert response.text == expected_response + self._validate_response(headers, expected_response, expected_status_code, response) From 022e6434e049ae922c6cc72def751b8183ab0f6a Mon Sep 17 00:00:00 2001 From: jason810496 Date: Wed, 20 Nov 2024 22:46:12 +0800 Subject: [PATCH 10/10] Fix style by ruff format --- airflow/api_fastapi/core_api/routes/public/dag_sources.py | 3 +-- tests/api_fastapi/core_api/routes/public/test_dag_sources.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airflow/api_fastapi/core_api/routes/public/dag_sources.py b/airflow/api_fastapi/core_api/routes/public/dag_sources.py index d4fbd774b51e0..d28b08ea8bc88 100644 --- a/airflow/api_fastapi/core_api/routes/public/dag_sources.py +++ b/airflow/api_fastapi/core_api/routes/public/dag_sources.py @@ -18,8 +18,7 @@ from typing import Annotated -from fastapi import Depends, HTTPException, Request, Response, status -from itsdangerous import BadSignature, URLSafeSerializer +from fastapi import Depends, HTTPException, Response, status from sqlalchemy.orm import Session from airflow.api_fastapi.common.db.common import get_session diff --git a/tests/api_fastapi/core_api/routes/public/test_dag_sources.py b/tests/api_fastapi/core_api/routes/public/test_dag_sources.py index a5cd52782ca72..6c040d57f9cbb 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dag_sources.py +++ b/tests/api_fastapi/core_api/routes/public/test_dag_sources.py @@ -77,7 +77,7 @@ def test_should_respond_200_text(self, test_client, test_dag): @pytest.mark.parametrize( "headers", [{"Accept": "application/json"}, {"Accept": "application/json; charset=utf-8"}, {}] ) - def test_should_respond_200_json(self, test_client,test_dag, headers): + def test_should_respond_200_json(self, test_client, test_dag, headers): dag_content = self._get_dag_file_code(test_dag.fileloc) response: Response = test_client.get( f"{API_PREFIX}/{TEST_DAG_ID}",