From d9e3371dc338b597c406c7fd378734e42ce3ab3c Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 15 Apr 2024 14:35:20 +0100 Subject: [PATCH 01/14] Fix alembic autogeneration and rename mismatching constraints The alembic autogeneration is not working as expected and the tests were detecting it because we use DBs created from the ORM to run tests. When a change is made in the ORM and the ORM is used to initialize the database for tests, the changes in the ORM will appear the same with what is in the migration file. To be sure that both match, we have to compare the database generated using the migration file to the database that could be created from the ORM. To fix this, I added 'use_migration_file' arg to resetdb function and updated the db reset in conftest to use migration file during test db reset. As part of this fix, I also updated mismatching constraint names. The update was done in the migration file instead of the ORM as I take the ORM as the source of truth. New airflow users create their DB from the ORM with the correct naming because we have a naming convention. Old airflow users would have to upgrade to use these names from ORM instead of the reverse. I also removed the `sqlite_sequence` table which is specific to sqlite and not needed for anything. An alternative would be to add `sqlite_autoincrement` to table args in the ORM and migration but this table is not that useful. --- airflow/migrations/env.py | 3 + .../0142_2_10_0_update_missing_constraints.py | 72 +++++ airflow/utils/db.py | 13 +- docs/apache-airflow/img/airflow_erd.sha256 | 3 +- docs/apache-airflow/img/airflow_erd.svg | 4 +- docs/apache-airflow/migrations-ref.rst | 257 +++++++++--------- tests/conftest.py | 2 +- tests/utils/test_db.py | 2 +- 8 files changed, 215 insertions(+), 141 deletions(-) create mode 100644 airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py diff --git a/airflow/migrations/env.py b/airflow/migrations/env.py index c521654c1c709..7a5854bb06ece 100644 --- a/airflow/migrations/env.py +++ b/airflow/migrations/env.py @@ -29,6 +29,9 @@ def include_object(_, name, type_, *args): """Filter objects for autogenerating revisions.""" + # Ignore the sqlite_sequence table, which is an internal SQLite construct + if name == "sqlite_sequence": + return False # Ignore _anything_ to do with Celery, or FlaskSession's tables if type_ == "table" and (name.startswith("celery_") or name == "session"): return False diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py new file mode 100644 index 0000000000000..3c620948cde77 --- /dev/null +++ b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Update missing constraints + +Revision ID: 686269002441 +Revises: 677fdbb7fc54 +Create Date: 2024-04-15 14:19:49.913797 + +""" + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = '686269002441' +down_revision = '677fdbb7fc54' +branch_labels = None +depends_on = None +airflow_version = '2.10.0' + + +def upgrade(): + """Apply Update missing constraints""" + with op.batch_alter_table('connection') as batch_op: + batch_op.drop_constraint('unique_conn_id', type_='unique') + batch_op.create_unique_constraint(batch_op.f('connection_conn_id_uq'), ['conn_id']) + + with op.batch_alter_table('dag') as batch_op: + batch_op.alter_column('max_consecutive_failed_dag_runs', + existing_type=sa.Integer(), + nullable=False) + + with op.batch_alter_table('dag_run') as batch_op: + batch_op.drop_constraint('dag_run_dag_id_execution_date_uq', type_='unique') + batch_op.drop_constraint('dag_run_dag_id_run_id_uq', type_='unique') + batch_op.create_unique_constraint('dag_run_dag_id_execution_date_key', ['dag_id', 'execution_date']) + batch_op.create_unique_constraint('dag_run_dag_id_run_id_key', ['dag_id', 'run_id']) + + +def downgrade(): + """Unapply Update missing constraints""" + with op.batch_alter_table('dag_run', schema=None) as batch_op: + batch_op.drop_constraint('dag_run_dag_id_run_id_key', type_='unique') + batch_op.drop_constraint('dag_run_dag_id_execution_date_key', type_='unique') + batch_op.create_unique_constraint('dag_run_dag_id_run_id_uq', ['dag_id', 'run_id']) + batch_op.create_unique_constraint('dag_run_dag_id_execution_date_uq', ['dag_id', 'execution_date']) + + with op.batch_alter_table('dag', schema=None) as batch_op: + batch_op.alter_column('max_consecutive_failed_dag_runs', + existing_type=sa.INTEGER(), + nullable=True) + + with op.batch_alter_table('connection', schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f('connection_conn_id_uq'), type_='unique') + batch_op.create_unique_constraint('unique_conn_id', ['conn_id']) diff --git a/airflow/utils/db.py b/airflow/utils/db.py index af53c241ed6c0..e9652299f940f 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -744,13 +744,13 @@ def _create_flask_session_tbl(sql_database_uri): @provide_session -def initdb(session: Session = NEW_SESSION, load_connections: bool = True): +def initdb(session: Session = NEW_SESSION, load_connections: bool = True, use_migration_files: bool = False): """Initialize Airflow database.""" import_all_models() db_exists = _get_current_revision(session) - if db_exists: - upgradedb(session=session) + if db_exists or use_migration_files: + upgradedb(session=session, use_migration_files=use_migration_files) else: _create_db_from_orm(session=session) if conf.getboolean("database", "LOAD_DEFAULT_CONNECTIONS") and load_connections: @@ -1558,6 +1558,7 @@ def upgradedb( show_sql_only: bool = False, reserialize_dags: bool = True, session: Session = NEW_SESSION, + use_migration_files: bool = False, ): """ Upgrades the DB. @@ -1614,7 +1615,7 @@ def upgradedb( if errors_seen: exit(1) - if not to_revision and not _get_current_revision(session=session): + if not to_revision and not _get_current_revision(session=session) and not use_migration_files: # Don't load default connections # New DB; initialize and exit initdb(session=session, load_connections=False) @@ -1644,7 +1645,7 @@ def upgradedb( @provide_session -def resetdb(session: Session = NEW_SESSION, skip_init: bool = False): +def resetdb(session: Session = NEW_SESSION, skip_init: bool = False, use_migration_files: bool = False): """Clear out the database.""" if not settings.engine: raise RuntimeError("The settings.engine must be set. This is a critical assertion") @@ -1659,7 +1660,7 @@ def resetdb(session: Session = NEW_SESSION, skip_init: bool = False): drop_airflow_moved_tables(connection) if not skip_init: - initdb(session=session) + initdb(session=session, use_migration_files=use_migration_files) @provide_session diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 132655e3b39f8..672ec612a4859 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1,2 @@ -e816aa2004ece1f8e07f6c5978a690ae7176b81f7d9df89115f3b65dab1c5231 \ No newline at end of file +955a39c49c73c0ef63d55f428840c915438df4fd13abb5bfa2bd9cfe5f7c22ae + diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index fb280ee0ea7fc..019ff78714fa6 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1428,14 +1428,14 @@ task_instance--xcom -0..N +1 1 task_instance--xcom -1 +0..N 1 diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 0961b87f395ee..d858879d545fc 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,77 +39,73 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -| ``677fdbb7fc54`` (head) | ``bff083ad727d`` | ``2.10.0`` | add new executor field to db. | +| ``677fdbb7fc54`` (head) | ``1949afb29106`` | ``2.10.0`` | add new executor field to db | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bff083ad727d`` | ``1949afb29106`` | ``2.9.2`` | Remove ``idx_last_scheduling_decision`` index on | -| | | | last_scheduling_decision in dag_run table. | +| ``1949afb29106`` | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type and encrypt | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1949afb29106`` | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type and encrypt. | +| ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance. | -+---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for | -| | | | mysql. | +| ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | -| | | | table. | +| | | | table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` | add dataset_expression in DagModel. | +| ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` | add dataset_expression in DagModel | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``d75389605139`` | ``1fd565369930`` | ``2.9.0`` | Add run_id to (Audit) log table and increase event name | -| | | | length. | +| | | | length | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``1fd565369930`` | ``88344c1d9134`` | ``2.9.0`` | Add rendered_map_index to TaskInstance. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``88344c1d9134`` | ``10b52ebd31f7`` | ``2.8.1`` | Drop unused TI index. | +| ``88344c1d9134`` | ``10b52ebd31f7`` | ``2.8.1`` | Drop unused TI index | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``10b52ebd31f7`` | ``bd5dfbe21f88`` | ``2.8.0`` | Add processor_subdir to ImportError. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bd5dfbe21f88`` | ``f7bf2a57d0a6`` | ``2.8.0`` | Make connection login/password TEXT. | +| ``bd5dfbe21f88`` | ``f7bf2a57d0a6`` | ``2.8.0`` | Make connection login/password TEXT | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f7bf2a57d0a6`` | ``375a816bbbf4`` | ``2.8.0`` | Add owner_display_name to (Audit) Log table. | +| ``f7bf2a57d0a6`` | ``375a816bbbf4`` | ``2.8.0`` | Add owner_display_name to (Audit) Log table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``375a816bbbf4`` | ``405de8318b3a`` | ``2.8.0`` | add new field 'clear_number' to dagrun. | +| ``375a816bbbf4`` | ``405de8318b3a`` | ``2.8.0`` | add new field 'clear_number' to dagrun | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``405de8318b3a`` | ``788397e78828`` | ``2.7.0`` | add include_deferred column to pool. | +| ``405de8318b3a`` | ``788397e78828`` | ``2.7.0`` | add include_deferred column to pool | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``788397e78828`` | ``937cbd173ca1`` | ``2.7.0`` | Add custom_operator_name column. | +| ``788397e78828`` | ``937cbd173ca1`` | ``2.7.0`` | Add custom_operator_name column | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``937cbd173ca1`` | ``c804e5c76e3e`` | ``2.7.0`` | Add index to task_instance table. | +| ``937cbd173ca1`` | ``c804e5c76e3e`` | ``2.7.0`` | Add index to task_instance table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``c804e5c76e3e`` | ``98ae134e6fff`` | ``2.6.2`` | Add ``onupdate`` cascade to ``task_map`` table. | +| ``c804e5c76e3e`` | ``98ae134e6fff`` | ``2.6.2`` | Add ``onupdate`` cascade to ``task_map`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``98ae134e6fff`` | ``6abdffdd4815`` | ``2.6.0`` | Increase length of user identifier columns in ``ab_user`` | -| | | | and ``ab_register_user`` tables. | +| | | | and ``ab_register_user`` tables | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``6abdffdd4815`` | ``290244fb8b83`` | ``2.6.0`` | add dttm index on log table. | +| ``6abdffdd4815`` | ``290244fb8b83`` | ``2.6.0`` | add dttm index on log table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``290244fb8b83`` | ``1986afd32c1b`` | ``2.5.0`` | Add is_orphaned to DatasetModel. | +| ``290244fb8b83`` | ``1986afd32c1b`` | ``2.5.0`` | Add is_orphaned to DatasetModel | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1986afd32c1b`` | ``ee8d93fcc81e`` | ``2.5.0`` | Add DagRunNote and TaskInstanceNote. | +| ``1986afd32c1b`` | ``ee8d93fcc81e`` | ``2.5.0`` | Add DagRunNote and TaskInstanceNote | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ee8d93fcc81e`` | ``e07f49787c9d`` | ``2.5.0`` | Add updated_at column to DagRun and TaskInstance. | +| ``ee8d93fcc81e`` | ``e07f49787c9d`` | ``2.5.0`` | Add updated_at column to DagRun and TaskInstance | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e07f49787c9d`` | ``b0d31815b5a6`` | ``2.4.3`` | Add case-insensitive unique constraint for username. | +| ``e07f49787c9d`` | ``b0d31815b5a6`` | ``2.4.3`` | Add case-insensitive unique constraint for username | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b0d31815b5a6`` | ``ecb43d2a1842`` | ``2.4.2`` | Add missing auto-increment to columns on FAB tables. | +| ``b0d31815b5a6`` | ``ecb43d2a1842`` | ``2.4.2`` | Add missing auto-increment to columns on FAB tables | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ecb43d2a1842`` | ``1486deb605b4`` | ``2.4.0`` | Add processor_subdir column to DagModel, SerializedDagModel | | | | | and CallbackRequest tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1486deb605b4`` | ``f4ff391becb5`` | ``2.4.0`` | add dag_owner_attributes table. | +| ``1486deb605b4`` | ``f4ff391becb5`` | ``2.4.0`` | add dag_owner_attributes table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f4ff391becb5`` | ``0038cd0c28b4`` | ``2.4.0`` | Remove smart sensors. | +| ``f4ff391becb5`` | ``0038cd0c28b4`` | ``2.4.0`` | Remove smart sensors | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``0038cd0c28b4`` | ``44b7034f6bdc`` | ``2.4.0`` | Add Dataset model. | +| ``0038cd0c28b4`` | ``44b7034f6bdc`` | ``2.4.0`` | Add Dataset model | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``44b7034f6bdc`` | ``424117c37d18`` | ``2.4.0`` | compare types between ORM and DB. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``424117c37d18`` | ``f5fcbda3e651`` | ``2.4.0`` | Add DagWarning model. | +| ``424117c37d18`` | ``f5fcbda3e651`` | ``2.4.0`` | Add DagWarning model | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f5fcbda3e651`` | ``3c94c427fdf6`` | ``2.3.3`` | Add indexes for CASCADE deletes on task_instance. | +| ``f5fcbda3e651`` | ``3c94c427fdf6`` | ``2.3.3`` | Add indexes for CASCADE deletes on task_instance | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``3c94c427fdf6`` | ``1de7bc13c950`` | ``2.3.2`` | Add cascade to dag_tag foreign key. | +| ``3c94c427fdf6`` | ``1de7bc13c950`` | ``2.3.2`` | Add cascade to dag_tag foreign key | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``1de7bc13c950`` | ``b1b348e02d07`` | ``2.3.1`` | Add index for ``event`` column in ``log`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ @@ -117,33 +113,35 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``75d5ed6c2b43`` | ``909884dea523`` | ``2.3.0`` | Add map_index to Log. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``909884dea523`` | ``48925b2719cb`` | ``2.3.0`` | Update migration for FAB tables to add missing constraints. | +| ``909884dea523`` | ``48925b2719cb`` | ``2.3.0`` | Update migration for FAB tables to add missing constraints | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``48925b2719cb`` | ``4eaab2fe6582`` | ``2.3.0`` | Add map_index to TaskFail. | +| ``48925b2719cb`` | ``4eaab2fe6582`` | ``2.3.0`` | Add map_index to TaskFail | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``4eaab2fe6582`` | ``c97c2ab6aa23`` | ``2.3.0`` | Migrate RTIF to use run_id and map_index. | +| ``4eaab2fe6582`` | ``c97c2ab6aa23`` | ``2.3.0`` | Migrate RTIF to use run_id and map_index | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``c97c2ab6aa23`` | ``c306b5b5ae4a`` | ``2.3.0`` | add callback request table. | +| ``c97c2ab6aa23`` | ``c306b5b5ae4a`` | ``2.3.0`` | add callback request table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``c306b5b5ae4a`` | ``a3bcd0914482`` | ``2.3.0`` | Switch XCom table to use ``run_id`` and add ``map_index``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``a3bcd0914482`` | ``e655c0453f75`` | ``2.3.0`` | add data_compressed to serialized_dag. | +| ``a3bcd0914482`` | ``e655c0453f75`` | ``2.3.0`` | add data_compressed to serialized_dag | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e655c0453f75`` | ``f9da662e7089`` | ``2.3.0`` | Add TaskMap and map_index on TaskInstance. | +| ``e655c0453f75`` | ``f9da662e7089`` | ``2.3.0`` | Add ``map_index`` column to TaskInstance to identify task- | +| | | | mapping, and a ``task_map`` table to track mapping values | +| | | | from XCom. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``f9da662e7089`` | ``786e3737b18f`` | ``2.3.0`` | Add ``LogTemplate`` table to track changes to config values | -| | | | ``log_filename_template``. | +| | | | ``log_filename_template`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``786e3737b18f`` | ``5e3ec427fdd3`` | ``2.3.0`` | Add ``timetable_description`` column to DagModel for UI. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``5e3ec427fdd3`` | ``587bdf053233`` | ``2.3.0`` | Increase length of email and username in ``ab_user`` and | -| | | | ``ab_register_user`` table to ``256`` characters. | +| | | | ``ab_register_user`` table to ``256`` characters | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``587bdf053233`` | ``c381b21cb7e4`` | ``2.2.4`` | Add index for ``dag_id`` column in ``job`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``c381b21cb7e4`` | ``be2bfac3da23`` | ``2.2.4`` | Create a ``session`` table to store web session data. | +| ``c381b21cb7e4`` | ``be2bfac3da23`` | ``2.2.4`` | Create a ``session`` table to store web session data | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``be2bfac3da23`` | ``7b2661a43ba3`` | ``2.2.3`` | Add has_import_errors column to DagModel. | +| ``be2bfac3da23`` | ``7b2661a43ba3`` | ``2.2.3`` | Add has_import_errors column to DagModel | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``7b2661a43ba3`` | ``142555e44c17`` | ``2.2.0`` | Change ``TaskInstance`` and ``TaskReschedule`` tables from | | | | | execution_date to run_id. | @@ -151,205 +149,204 @@ Here's the list of all the Database Migrations that are executed via when you ru | ``142555e44c17`` | ``54bebd308c5f`` | ``2.2.0`` | Add data_interval_[start|end] to DagModel and DagRun. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``54bebd308c5f`` | ``30867afad44a`` | ``2.2.0`` | Adds ``trigger`` table and deferrable operator columns to | -| | | | task instance. | +| | | | task instance | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``30867afad44a`` | ``e9304a3141f0`` | ``2.2.0`` | Rename ``concurrency`` column in ``dag`` table to`` | -| | | | max_active_tasks``. | +| | | | max_active_tasks`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e9304a3141f0`` | ``83f031fd9f1c`` | ``2.2.0`` | Make XCom primary key columns non-nullable. | +| ``e9304a3141f0`` | ``83f031fd9f1c`` | ``2.2.0`` | Make XCom primary key columns non-nullable | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``83f031fd9f1c`` | ``ccde3e26fe78`` | ``2.2.0`` | Improve MSSQL compatibility. | +| ``83f031fd9f1c`` | ``ccde3e26fe78`` | ``2.2.0`` | Improve MSSQL compatibility | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ccde3e26fe78`` | ``092435bf5d12`` | ``2.1.4`` | Add index on state, dag_id for queued ``dagrun``. | +| ``ccde3e26fe78`` | ``092435bf5d12`` | ``2.1.4`` | Add index on state, dag_id for queued ``dagrun`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``092435bf5d12`` | ``97cdd93827b8`` | ``2.1.4`` | Add ``max_active_runs`` column to ``dag_model`` table. | +| ``092435bf5d12`` | ``97cdd93827b8`` | ``2.1.4`` | Add ``max_active_runs`` column to ``dag_model`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``97cdd93827b8`` | ``a13f7613ad25`` | ``2.1.3`` | Add ``queued_at`` column in ``dag_run`` table. | +| ``97cdd93827b8`` | ``a13f7613ad25`` | ``2.1.3`` | Add ``queued_at`` column in ``dag_run`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``a13f7613ad25`` | ``e165e7455d70`` | ``2.1.0`` | Resource based permissions for default ``Flask-AppBuilder`` | -| | | | views. | +| | | | views | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e165e7455d70`` | ``90d1635d7b86`` | ``2.1.0`` | Add description field to ``Variable`` model. | +| ``e165e7455d70`` | ``90d1635d7b86`` | ``2.1.0`` | Add description field to ``Variable`` model | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``90d1635d7b86`` | ``2e42bb497a22`` | ``2.1.0`` | Increase maximum length of pool name in ``task_instance`` | -| | | | table to ``256`` characters. | +| | | | table to ``256`` characters | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``2e42bb497a22`` | ``8646922c8a04`` | ``2.0.2`` | Rename ``last_scheduler_run`` column in ``DAG`` table to | -| | | | ``last_parsed_time``. | +| | | | ``last_parsed_time`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8646922c8a04`` | ``449b4072c2da`` | ``2.0.2`` | Change default ``pool_slots`` to ``1``. | +| ``8646922c8a04`` | ``449b4072c2da`` | ``2.0.2`` | Change default ``pool_slots`` to ``1`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``449b4072c2da`` | ``82b7c48c147f`` | ``2.0.2`` | Increase size of ``connection.extra`` field to handle | -| | | | multiple RSA keys. | +| | | | multiple RSA keys | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``82b7c48c147f`` | ``e959f08ac86c`` | ``2.0.1`` | Remove ``can_read`` permission on config resource for | -| | | | ``User`` and ``Viewer`` role. | +| | | | ``User`` and ``Viewer`` role | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e959f08ac86c`` | ``64a7d6477aae`` | ``2.0.0`` | Change field in ``DagCode`` to ``MEDIUMTEXT`` for MySql. | +| ``e959f08ac86c`` | ``64a7d6477aae`` | ``2.0.0`` | Change field in ``DagCode`` to ``MEDIUMTEXT`` for MySql | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``64a7d6477aae`` | ``61ec73d9401f`` | ``2.0.0`` | Fix description field in ``connection`` to be ``text``. | +| ``64a7d6477aae`` | ``61ec73d9401f`` | ``2.0.0`` | Fix description field in ``connection`` to be ``text`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``61ec73d9401f`` | ``2c6edca13270`` | ``2.0.0`` | Add description field to ``connection`` table. | +| ``61ec73d9401f`` | ``2c6edca13270`` | ``2.0.0`` | Add description field to ``connection`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``2c6edca13270`` | ``849da589634d`` | ``2.0.0`` | Resource based permissions. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``849da589634d`` | ``45ba3f1493b9`` | ``2.0.0`` | Prefix DAG permissions. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``45ba3f1493b9`` | ``364159666cbd`` | ``2.0.0`` | add-k8s-yaml-to-rendered-templates. | +| ``45ba3f1493b9`` | ``364159666cbd`` | ``2.0.0`` | add-k8s-yaml-to-rendered-templates | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``364159666cbd`` | ``52d53670a240`` | ``2.0.0`` | Add ``creating_job_id`` to ``DagRun`` table. | +| ``364159666cbd`` | ``52d53670a240`` | ``2.0.0`` | Add ``creating_job_id`` to ``DagRun`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``52d53670a240`` | ``98271e7606e2`` | ``2.0.0`` | fix_mssql_exec_date_rendered_task_instance_fields_for_MSSQL. | +| ``52d53670a240`` | ``98271e7606e2`` | ``2.0.0`` | fix_mssql_exec_date_rendered_task_instance_fields_for_MSSQL | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``98271e7606e2`` | ``bef4f3d11e8b`` | ``2.0.0`` | Add ``scheduling_decision`` to ``DagRun`` and ``DAG``. | +| ``98271e7606e2`` | ``bef4f3d11e8b`` | ``2.0.0`` | Add ``scheduling_decision`` to ``DagRun`` and ``DAG`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bef4f3d11e8b`` | ``e1a11ece99cc`` | ``2.0.0`` | Drop ``KubeResourceVersion`` and ``KubeWorkerId``. | +| ``bef4f3d11e8b`` | ``e1a11ece99cc`` | ``2.0.0`` | Drop ``KubeResourceVersion`` and ``KubeWorkerId`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e1a11ece99cc`` | ``b247b1e3d1ed`` | ``2.0.0`` | Add external executor ID to TI. | +| ``e1a11ece99cc`` | ``b247b1e3d1ed`` | ``2.0.0`` | Add external executor ID to TI | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b247b1e3d1ed`` | ``e38be357a868`` | ``2.0.0`` | Add queued by Job ID to TI. | +| ``b247b1e3d1ed`` | ``e38be357a868`` | ``2.0.0`` | Add queued by Job ID to TI | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e38be357a868`` | ``8d48763f6d53`` | ``2.0.0`` | Add ``sensor_instance`` table. | +| ``e38be357a868`` | ``8d48763f6d53`` | ``2.0.0`` | Add ``sensor_instance`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8d48763f6d53`` | ``8f966b9c467a`` | ``2.0.0`` | Add unique constraint to ``conn_id``. | +| ``8d48763f6d53`` | ``8f966b9c467a`` | ``2.0.0`` | Add unique constraint to ``conn_id`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8f966b9c467a`` | ``3c20cacc0044`` | ``2.0.0`` | Set ``conn_type`` as non-nullable. | +| ``8f966b9c467a`` | ``3c20cacc0044`` | ``2.0.0`` | Set ``conn_type`` as non-nullable | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``3c20cacc0044`` | ``b25a55525161`` | ``2.0.0`` | Add ``run_type`` column in ``dag_run`` table. | +| ``3c20cacc0044`` | ``b25a55525161`` | ``2.0.0`` | Add ``run_type`` column in ``dag_run`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b25a55525161`` | ``bbf4a7ad0465`` | ``2.0.0`` | Increase length of pool name. | +| ``b25a55525161`` | ``bbf4a7ad0465`` | ``2.0.0`` | Increase length of pool name | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bbf4a7ad0465`` | ``cf5dc11e79ad`` | ``2.0.0`` | Remove id column from xcom. | +| ``bbf4a7ad0465`` | ``cf5dc11e79ad`` | ``2.0.0`` | Remove id column from xcom | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``cf5dc11e79ad`` | ``03afc6b6f902`` | ``2.0.0`` | Drop ``user`` and ``chart`` table. | +| ``cf5dc11e79ad`` | ``03afc6b6f902`` | ``2.0.0`` | Drop ``user`` and ``chart`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``03afc6b6f902`` | ``92c57b58940d`` | ``1.10.13`` | Increase length of ``Flask-AppBuilder`` | -| | | | ``ab_view_menu.name`` column. | +| | | | ``ab_view_menu.name`` column | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``92c57b58940d`` | ``da3f683c3a5a`` | ``1.10.13`` | Create FAB Tables. | +| ``92c57b58940d`` | ``da3f683c3a5a`` | ``1.10.13`` | Create FAB Tables | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``da3f683c3a5a`` | ``a66efa278eea`` | ``1.10.12`` | Add ``dag_hash`` Column to ``serialized_dag`` table. | +| ``da3f683c3a5a`` | ``a66efa278eea`` | ``1.10.12`` | Add ``dag_hash`` Column to ``serialized_dag`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``a66efa278eea`` | ``952da73b5eff`` | ``1.10.11`` | Add Precision to ``execution_date`` in | -| | | | ``RenderedTaskInstanceFields`` table. | +| | | | ``RenderedTaskInstanceFields`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``952da73b5eff`` | ``852ae6c715af`` | ``1.10.10`` | Add ``dag_code`` table. | +| ``952da73b5eff`` | ``852ae6c715af`` | ``1.10.10`` | Add ``dag_code`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``852ae6c715af`` | ``a4c2fd67d16b`` | ``1.10.10`` | Add ``RenderedTaskInstanceFields`` table. | +| ``852ae6c715af`` | ``a4c2fd67d16b`` | ``1.10.10`` | Add ``RenderedTaskInstanceFields`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``a4c2fd67d16b`` | ``7939bcff74ba`` | ``1.10.10`` | Add ``pool_slots`` field to ``task_instance``. | +| ``a4c2fd67d16b`` | ``7939bcff74ba`` | ``1.10.10`` | Add ``pool_slots`` field to ``task_instance`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``7939bcff74ba`` | ``fe461863935f`` | ``1.10.8`` | Add ``DagTags`` table. | +| ``7939bcff74ba`` | ``fe461863935f`` | ``1.10.8`` | Add ``DagTags`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``fe461863935f`` | ``08364691d074`` | ``1.10.7`` | Increase length for connection password. | +| ``fe461863935f`` | ``08364691d074`` | ``1.10.7`` | Increase length for connection password | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``08364691d074`` (merge_point) | ``a56c9515abdc``, | ``1.10.7`` | Straighten out the migrations. | +| ``08364691d074`` (merge_point) | ``a56c9515abdc``, | ``1.10.7`` | Straighten out the migrations | | | ``004c1210f153``, | | | | | ``74effc47d867``, | | | | | ``b3b105409875`` | | | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``a56c9515abdc`` | ``c8ffec048a3b`` | ``1.10.3`` | Remove ``dag_stat`` table. | +| ``a56c9515abdc`` | ``c8ffec048a3b`` | ``1.10.3`` | Remove ``dag_stat`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``004c1210f153`` | ``939bb1e647c8`` | ``1.10.4`` | Increase queue name size limit. | +| ``004c1210f153`` | ``939bb1e647c8`` | ``1.10.4`` | Increase queue name size limit | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``74effc47d867`` | ``6e96a59344a4`` | ``1.10.5`` | change datetime to datetime2(6) on MSSQL tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b3b105409875`` | ``d38e04c12aa2`` | ``1.10.7`` | Add ``root_dag_id`` to ``DAG``. | +| ``b3b105409875`` | ``d38e04c12aa2`` | ``1.10.7`` | Add ``root_dag_id`` to ``DAG`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``d38e04c12aa2`` | ``6e96a59344a4`` | ``1.10.7`` | Add ``serialized_dag`` table. | +| ``d38e04c12aa2`` | ``6e96a59344a4`` | ``1.10.7`` | Add ``serialized_dag`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``6e96a59344a4`` (branch_point) | ``939bb1e647c8`` | ``1.10.4`` | Make ``TaskInstance.pool`` not nullable. | +| ``6e96a59344a4`` (branch_point) | ``939bb1e647c8`` | ``1.10.4`` | Make ``TaskInstance.pool`` not nullable | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``939bb1e647c8`` (branch_point) | ``dd4ecb8fbee3`` | ``1.10.3`` | task reschedule foreign key on cascade delete. | +| ``939bb1e647c8`` (branch_point) | ``dd4ecb8fbee3`` | ``1.10.3`` | task reschedule foreign key on cascade delete | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``dd4ecb8fbee3`` | ``c8ffec048a3b`` | ``1.10.3`` | Add schedule interval to dag. | +| ``dd4ecb8fbee3`` | ``c8ffec048a3b`` | ``1.10.3`` | Add schedule interval to dag | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``c8ffec048a3b`` (branch_point) | ``41f5f12752f8`` | ``1.10.3`` | Add ``description`` and ``default_view`` column to ``dag`` | -| | | | table. | +| | | | table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``41f5f12752f8`` | ``03bc53e68815`` | ``1.10.2`` | Add superuser field. | +| ``41f5f12752f8`` | ``03bc53e68815`` | ``1.10.2`` | Add superuser field | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``03bc53e68815`` (merge_point) | ``0a2a5b66e19d``, | ``1.10.2`` | Merge migrations Heads. | | | ``bf00311e1990`` | | | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``0a2a5b66e19d`` | ``9635ae0956e7`` | ``1.10.2`` | Add ``task_reschedule`` table. | +| ``0a2a5b66e19d`` | ``9635ae0956e7`` | ``1.10.2`` | Add ``task_reschedule`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bf00311e1990`` | ``dd25f486b8ea`` | ``1.10.2`` | Add index to ``task_instance`` table. | +| ``bf00311e1990`` | ``dd25f486b8ea`` | ``1.10.2`` | Add index to ``task_instance`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``dd25f486b8ea`` | ``9635ae0956e7`` | ``1.10.2`` | Add index on ``log`` table. | +| ``dd25f486b8ea`` | ``9635ae0956e7`` | ``1.10.2`` | Add index on ``log`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``9635ae0956e7`` (branch_point) | ``856955da8476`` | ``1.10.0`` | Create index on ``task_fail`` table. | +| ``9635ae0956e7`` (branch_point) | ``856955da8476`` | ``1.10.0`` | Create index on ``task_fail`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``856955da8476`` | ``f23433877c24`` | ``1.10.0`` | Fix Sqlite foreign key. | +| ``856955da8476`` | ``f23433877c24`` | ``1.10.0`` | Fix Sqlite foreign key | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f23433877c24`` | ``05f30312d566`` | ``1.10.0`` | Fix MySQL not null constraint. | +| ``f23433877c24`` | ``05f30312d566`` | ``1.10.0`` | Fix MySQL not null constraint | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``05f30312d566`` (merge_point) | ``86770d1215c0``, | ``1.10.0`` | Merge migrations Heads. | +| ``05f30312d566`` (merge_point) | ``86770d1215c0``, | ``1.10.0`` | Merge migrations Heads | | | ``0e2a74e0fc9f`` | | | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``86770d1215c0`` | ``27c6a30d7c24`` | ``1.10.0`` | Add kubernetes scheduler uniqueness. | +| ``86770d1215c0`` | ``27c6a30d7c24`` | ``1.10.0`` | Add kubernetes scheduler uniqueness | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``27c6a30d7c24`` | ``33ae817a1ff4`` | ``1.10.0`` | Add ``executor_config`` column to ``task_instance`` table. | +| ``27c6a30d7c24`` | ``33ae817a1ff4`` | ``1.10.0`` | Add ``executor_config`` column to ``task_instance`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``33ae817a1ff4`` | ``d2ae31099d61`` | ``1.10.0`` | Add Kubernetes resource check-pointing. | +| ``33ae817a1ff4`` | ``d2ae31099d61`` | ``1.10.0`` | Add Kubernetes resource check-pointing | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``0e2a74e0fc9f`` | ``d2ae31099d61`` | ``1.10.0`` | Add time zone awareness. | +| ``0e2a74e0fc9f`` | ``d2ae31099d61`` | ``1.10.0`` | Add time zone awareness | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``d2ae31099d61`` (branch_point) | ``947454bf1dff`` | ``1.8.2`` | Increase text size for MySQL (not relevant for other DBs' | -| | | | text types). | +| | | | text types) | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``947454bf1dff`` | ``bdaa763e6c56`` | ``1.8.2`` | Create index on ``job_id`` column in ``task_instance`` | -| | | | table. | +| ``947454bf1dff`` | ``bdaa763e6c56`` | ``1.8.2`` | Create index on ``job_id`` column in ``task_instance`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bdaa763e6c56`` | ``cc1e65623dc7`` | ``1.8.2`` | Make xcom value column a large binary. | +| ``bdaa763e6c56`` | ``cc1e65623dc7`` | ``1.8.2`` | Make xcom value column a large binary | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``cc1e65623dc7`` | ``127d2bf2dfa7`` | ``1.8.2`` | Add ``max_tries`` column to ``task_instance``. | +| ``cc1e65623dc7`` | ``127d2bf2dfa7`` | ``1.8.2`` | Add ``max_tries`` column to ``task_instance`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``127d2bf2dfa7`` | ``5e7d17757c7a`` | ``1.7.1.3`` | Add ``dag_id``/``state`` index on ``dag_run`` table. | +| ``127d2bf2dfa7`` | ``5e7d17757c7a`` | ``1.7.1.3`` | Add ``dag_id``/``state`` index on ``dag_run`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``5e7d17757c7a`` | ``8504051e801b`` | ``1.7.1.3`` | Add ``pid`` field to ``TaskInstance``. | +| ``5e7d17757c7a`` | ``8504051e801b`` | ``1.7.1.3`` | Add ``pid`` field to ``TaskInstance`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8504051e801b`` | ``4addfa1236f1`` | ``1.7.1.3`` | Add indices on ``xcom`` table. | +| ``8504051e801b`` | ``4addfa1236f1`` | ``1.7.1.3`` | Add indices on ``xcom`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``4addfa1236f1`` | ``f2ca10b85618`` | ``1.7.1.3`` | Add fractional seconds to MySQL tables. | +| ``4addfa1236f1`` | ``f2ca10b85618`` | ``1.7.1.3`` | Add fractional seconds to MySQL tables | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f2ca10b85618`` | ``64de9cddf6c9`` | ``1.7.1.3`` | Add ``dag_stats`` table. | +| ``f2ca10b85618`` | ``64de9cddf6c9`` | ``1.7.1.3`` | Add ``dag_stats`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``64de9cddf6c9`` | ``211e584da130`` | ``1.7.1.3`` | Add ``task_fail`` table. | +| ``64de9cddf6c9`` | ``211e584da130`` | ``1.7.1.3`` | Add ``task_fail`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``211e584da130`` | ``2e82aab8ef20`` | ``1.7.1.3`` | Add TI state index. | +| ``211e584da130`` | ``2e82aab8ef20`` | ``1.7.1.3`` | Add TI state index | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``2e82aab8ef20`` | ``1968acfc09e3`` | ``1.7.1`` | Rename user table. | +| ``2e82aab8ef20`` | ``1968acfc09e3`` | ``1.7.1`` | Rename user table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1968acfc09e3`` | ``bba5a7cfc896`` | ``1.7.0`` | Add ``is_encrypted`` column to variable table. | +| ``1968acfc09e3`` | ``bba5a7cfc896`` | ``1.7.0`` | Add ``is_encrypted`` column to variable table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``bba5a7cfc896`` | ``bbc73705a13e`` | ``1.7.0`` | Add a column to track the encryption state of the 'Extra' | -| | | | field in connection. | +| | | | field in connection | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bbc73705a13e`` | ``4446e08588`` | ``1.7.0`` | Add ``notification_sent`` column to ``sla_miss`` table. | +| ``bbc73705a13e`` | ``4446e08588`` | ``1.7.0`` | Add ``notification_sent`` column to ``sla_miss`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``4446e08588`` | ``561833c1c74b`` | ``1.6.2`` | Add ``start_date`` and ``end_date`` in ``dag_run`` table. | +| ``4446e08588`` | ``561833c1c74b`` | ``1.6.2`` | Add ``start_date`` and ``end_date`` in ``dag_run`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``561833c1c74b`` | ``40e67319e3a9`` | ``1.6.2`` | Add ``password`` column to ``user`` table. | +| ``561833c1c74b`` | ``40e67319e3a9`` | ``1.6.2`` | Add ``password`` column to ``user`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``40e67319e3a9`` | ``2e541a1dcfed`` | ``1.6.0`` | Add ``conf`` column in ``dag_run`` table. | +| ``40e67319e3a9`` | ``2e541a1dcfed`` | ``1.6.0`` | Add ``conf`` column in ``dag_run`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``2e541a1dcfed`` | ``1b38cef5b76e`` | ``1.6.0`` | Change ``task_instance.task_duration`` type to ``FLOAT``. | +| ``2e541a1dcfed`` | ``1b38cef5b76e`` | ``1.6.0`` | Change ``task_instance.task_duration`` type to ``FLOAT`` | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1b38cef5b76e`` | ``502898887f84`` | ``1.6.0`` | Add ``dag_run`` table. | +| ``1b38cef5b76e`` | ``502898887f84`` | ``1.6.0`` | Add ``dag_run`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``502898887f84`` | ``52d714495f0`` | ``1.6.0`` | Adding ``extra`` column to ``Log`` table. | +| ``502898887f84`` | ``52d714495f0`` | ``1.6.0`` | Adding ``extra`` column to ``Log`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``52d714495f0`` | ``338e90f54d61`` | ``1.5.2`` | Add indices in ``job`` table. | +| ``52d714495f0`` | ``338e90f54d61`` | ``1.5.2`` | Add indices in ``job`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``338e90f54d61`` | ``13eb55f81627`` | ``1.5.0`` | Add ``operator`` and ``queued_dttm`` to ``task_instance`` | -| | | | table. | +| | | | table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``13eb55f81627`` | ``1507a7289a2f`` | ``1.5.0`` | Maintain history for compatibility with earlier migrations. | +| ``13eb55f81627`` | ``1507a7289a2f`` | ``1.5.0`` | Maintain history for compatibility with earlier migrations | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1507a7289a2f`` | ``e3a246e0dc1`` | ``1.5.0`` | Add ``is_encrypted`` column in ``connection`` table. | +| ``1507a7289a2f`` | ``e3a246e0dc1`` | ``1.5.0`` | Add ``is_encrypted`` column in ``connection`` table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e3a246e0dc1`` (base) | ``None`` | ``1.5.0`` | current schema. | +| ``e3a246e0dc1`` (base) | ``None`` | ``1.5.0`` | current schema | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ .. End of auto-generated table diff --git a/tests/conftest.py b/tests/conftest.py index 32dbc7ec3ea5f..e0aab5551c34c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -324,7 +324,7 @@ def initial_db_init(): from airflow.www.extensions.init_appbuilder import init_appbuilder from airflow.www.extensions.init_auth_manager import get_auth_manager - db.resetdb() + db.resetdb(use_migration_files=True) db.bootstrap_dagbag() # minimal app to add roles flask_app = Flask(__name__) diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index 657f5fbc7cc20..4f7e1f0fde084 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -233,7 +233,7 @@ def test_resetdb( if skip_init: mock_init.assert_not_called() else: - mock_init.assert_called_once_with(session=session_mock) + mock_init.assert_called_once_with(session=session_mock, use_migration_files=False) def test_alembic_configuration(self): with mock.patch.dict( From 271d34e7b64762fbe8d8899975655416193abc08 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 15 Apr 2024 15:56:24 +0100 Subject: [PATCH 02/14] fixup! Fix alembic autogeneration and rename mismatching constraints --- .../versions/0142_2_10_0_update_missing_constraints.py | 2 +- docs/apache-airflow/img/airflow_erd.svg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py index 3c620948cde77..e517e823517ba 100644 --- a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py @@ -64,7 +64,7 @@ def downgrade(): with op.batch_alter_table('dag', schema=None) as batch_op: batch_op.alter_column('max_consecutive_failed_dag_runs', - existing_type=sa.INTEGER(), + existing_type=sa.Integer(), nullable=True) with op.batch_alter_table('connection', schema=None) as batch_op: diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 019ff78714fa6..fb280ee0ea7fc 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1428,14 +1428,14 @@ task_instance--xcom -1 +0..N 1 task_instance--xcom -0..N +1 1 From 6a119622b78dcb83823438e5953f68fdee838d7b Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 15 Apr 2024 17:48:17 +0100 Subject: [PATCH 03/14] fixup! fixup! Fix alembic autogeneration and rename mismatching constraints --- .../0142_2_10_0_update_missing_constraints.py | 44 ++++++++++++++++--- docs/apache-airflow/img/airflow_erd.svg | 2 +- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py index e517e823517ba..09bc6eea853e4 100644 --- a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py @@ -26,7 +26,7 @@ import sqlalchemy as sa from alembic import op - +from sqlalchemy import literal # revision identifiers, used by Alembic. revision = '686269002441' @@ -42,31 +42,61 @@ def upgrade(): batch_op.drop_constraint('unique_conn_id', type_='unique') batch_op.create_unique_constraint(batch_op.f('connection_conn_id_uq'), ['conn_id']) + max_cons = sa.table('dag', sa.column('max_consecutive_failed_dag_runs')) + op.execute(max_cons.update().values(max_consecutive_failed_dag_runs=literal("0"))) with op.batch_alter_table('dag') as batch_op: - batch_op.alter_column('max_consecutive_failed_dag_runs', - existing_type=sa.Integer(), - nullable=False) + batch_op.alter_column('max_consecutive_failed_dag_runs', existing_type=sa.Integer(), nullable=False) + + with op.batch_alter_table("task_instance") as batch_op: + batch_op.drop_constraint("task_instance_dag_run_fkey", type_="foreignkey") + with op.batch_alter_table("task_reschedule") as batch_op: + batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") with op.batch_alter_table('dag_run') as batch_op: batch_op.drop_constraint('dag_run_dag_id_execution_date_uq', type_='unique') batch_op.drop_constraint('dag_run_dag_id_run_id_uq', type_='unique') batch_op.create_unique_constraint('dag_run_dag_id_execution_date_key', ['dag_id', 'execution_date']) batch_op.create_unique_constraint('dag_run_dag_id_run_id_key', ['dag_id', 'run_id']) + with op.batch_alter_table("task_instance") as batch_op: + batch_op.create_foreign_key("task_instance_dag_run_fkey", "dag_run", ["dag_id", "run_id"], ["dag_id", "run_id"]) + with op.batch_alter_table("task_reschedule") as batch_op: + batch_op.create_foreign_key( + "task_reschedule_dr_fkey", + "dag_run", + ["dag_id", "run_id"], + ["dag_id", "run_id"], + ondelete="CASCADE", + ) def downgrade(): """Unapply Update missing constraints""" - with op.batch_alter_table('dag_run', schema=None) as batch_op: + with op.batch_alter_table("task_instance") as batch_op: + batch_op.drop_constraint("task_instance_dag_run_fkey", type_="foreignkey") + with op.batch_alter_table("task_reschedule") as batch_op: + batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") + with op.batch_alter_table('dag_run') as batch_op: batch_op.drop_constraint('dag_run_dag_id_run_id_key', type_='unique') batch_op.drop_constraint('dag_run_dag_id_execution_date_key', type_='unique') batch_op.create_unique_constraint('dag_run_dag_id_run_id_uq', ['dag_id', 'run_id']) batch_op.create_unique_constraint('dag_run_dag_id_execution_date_uq', ['dag_id', 'execution_date']) - with op.batch_alter_table('dag', schema=None) as batch_op: + with op.batch_alter_table("task_instance") as batch_op: + batch_op.create_foreign_key("task_instance_dag_run_fkey", "dag_run", ["dag_id", "run_id"], ["dag_id", "run_id"]) + with op.batch_alter_table("task_reschedule") as batch_op: + batch_op.create_foreign_key( + "task_reschedule_dr_fkey", + "dag_run", + ["dag_id", "run_id"], + ["dag_id", "run_id"], + ondelete="CASCADE", + ) + + with op.batch_alter_table('dag') as batch_op: batch_op.alter_column('max_consecutive_failed_dag_runs', existing_type=sa.Integer(), nullable=True) - with op.batch_alter_table('connection', schema=None) as batch_op: + with op.batch_alter_table('connection') as batch_op: batch_op.drop_constraint(batch_op.f('connection_conn_id_uq'), type_='unique') batch_op.create_unique_constraint('unique_conn_id', ['conn_id']) diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index fb280ee0ea7fc..6d26c871e9f4f 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1435,7 +1435,7 @@ task_instance--xcom -1 +0..N 1 From a71a5e3bc0dd338dbf8a2e34961d76c95cea6170 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Mon, 15 Apr 2024 18:49:57 +0100 Subject: [PATCH 04/14] fixup! fixup! fixup! Fix alembic autogeneration and rename mismatching constraints --- .../0142_2_10_0_update_missing_constraints.py | 42 ++++--------------- docs/apache-airflow/img/airflow_erd.svg | 2 +- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py index 09bc6eea853e4..8fe6f962f9292 100644 --- a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py @@ -38,8 +38,10 @@ def upgrade(): """Apply Update missing constraints""" + op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") + # Dropping and recreating cause there's no IF NOT EXISTS + op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS connection_conn_id_uq") with op.batch_alter_table('connection') as batch_op: - batch_op.drop_constraint('unique_conn_id', type_='unique') batch_op.create_unique_constraint(batch_op.f('connection_conn_id_uq'), ['conn_id']) max_cons = sa.table('dag', sa.column('max_consecutive_failed_dag_runs')) @@ -52,9 +54,12 @@ def upgrade(): with op.batch_alter_table("task_reschedule") as batch_op: batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_uq") + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_uq") + # below we drop and recreate the constraints because there's no IF NOT EXISTS + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_key") + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_key") with op.batch_alter_table('dag_run') as batch_op: - batch_op.drop_constraint('dag_run_dag_id_execution_date_uq', type_='unique') - batch_op.drop_constraint('dag_run_dag_id_run_id_uq', type_='unique') batch_op.create_unique_constraint('dag_run_dag_id_execution_date_key', ['dag_id', 'execution_date']) batch_op.create_unique_constraint('dag_run_dag_id_run_id_key', ['dag_id', 'run_id']) with op.batch_alter_table("task_instance") as batch_op: @@ -70,33 +75,4 @@ def upgrade(): def downgrade(): - """Unapply Update missing constraints""" - with op.batch_alter_table("task_instance") as batch_op: - batch_op.drop_constraint("task_instance_dag_run_fkey", type_="foreignkey") - with op.batch_alter_table("task_reschedule") as batch_op: - batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") - with op.batch_alter_table('dag_run') as batch_op: - batch_op.drop_constraint('dag_run_dag_id_run_id_key', type_='unique') - batch_op.drop_constraint('dag_run_dag_id_execution_date_key', type_='unique') - batch_op.create_unique_constraint('dag_run_dag_id_run_id_uq', ['dag_id', 'run_id']) - batch_op.create_unique_constraint('dag_run_dag_id_execution_date_uq', ['dag_id', 'execution_date']) - - with op.batch_alter_table("task_instance") as batch_op: - batch_op.create_foreign_key("task_instance_dag_run_fkey", "dag_run", ["dag_id", "run_id"], ["dag_id", "run_id"]) - with op.batch_alter_table("task_reschedule") as batch_op: - batch_op.create_foreign_key( - "task_reschedule_dr_fkey", - "dag_run", - ["dag_id", "run_id"], - ["dag_id", "run_id"], - ondelete="CASCADE", - ) - - with op.batch_alter_table('dag') as batch_op: - batch_op.alter_column('max_consecutive_failed_dag_runs', - existing_type=sa.Integer(), - nullable=True) - - with op.batch_alter_table('connection') as batch_op: - batch_op.drop_constraint(batch_op.f('connection_conn_id_uq'), type_='unique') - batch_op.create_unique_constraint('unique_conn_id', ['conn_id']) + """NO downgrade because this is to make ORM consistent with the database.""" diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 6d26c871e9f4f..019ff78714fa6 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1428,7 +1428,7 @@ task_instance--xcom -0..N +1 1 From 6629fff83bd907ee665e1b4ae1af2a30808e68d0 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Tue, 16 Apr 2024 09:19:32 +0100 Subject: [PATCH 05/14] Fix mysql, sqlite and issue with cascading deletes --- .../0142_2_10_0_update_missing_constraints.py | 35 ++++++++++++++----- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 2 +- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py index 8fe6f962f9292..516f89e70597b 100644 --- a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py @@ -38,9 +38,18 @@ def upgrade(): """Apply Update missing constraints""" - op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") - # Dropping and recreating cause there's no IF NOT EXISTS - op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS connection_conn_id_uq") + conn = op.get_bind() + if conn.dialect.name == "sqlite": + # SQLite does not support DROP CONSTRAINT + return + if conn.dialect.name == "mysql": + op.execute("ALTER TABLE connection DROP INDEX IF EXISTS unique_conn_id") + # Dropping and recreating cause there's no IF NOT EXISTS + op.execute("ALTER TABLE connection DROP INDEX IF EXISTS connection_conn_id_uq") + else: + op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") + # Dropping and recreating cause there's no IF NOT EXISTS + op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS connection_conn_id_uq") with op.batch_alter_table('connection') as batch_op: batch_op.create_unique_constraint(batch_op.f('connection_conn_id_uq'), ['conn_id']) @@ -54,16 +63,24 @@ def upgrade(): with op.batch_alter_table("task_reschedule") as batch_op: batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") - op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_uq") - op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_uq") - # below we drop and recreate the constraints because there's no IF NOT EXISTS - op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_key") - op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_key") + if conn.dialect.name == "mysql": + op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_execution_date_uq") + op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_run_id_uq") + # below we drop and recreate the constraints because there's no IF NOT EXISTS + op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_execution_date_key") + op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_run_id_key") + else: + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_uq") + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_uq") + # below we drop and recreate the constraints because there's no IF NOT EXISTS + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_key") + op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_key") with op.batch_alter_table('dag_run') as batch_op: batch_op.create_unique_constraint('dag_run_dag_id_execution_date_key', ['dag_id', 'execution_date']) batch_op.create_unique_constraint('dag_run_dag_id_run_id_key', ['dag_id', 'run_id']) with op.batch_alter_table("task_instance") as batch_op: - batch_op.create_foreign_key("task_instance_dag_run_fkey", "dag_run", ["dag_id", "run_id"], ["dag_id", "run_id"]) + batch_op.create_foreign_key("task_instance_dag_run_fkey", "dag_run", ["dag_id", "run_id"], ["dag_id", "run_id"], + ondelete="CASCADE") with op.batch_alter_table("task_reschedule") as batch_op: batch_op.create_foreign_key( "task_reschedule_dr_fkey", diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 672ec612a4859..04c33d2ee34d4 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1,2 +1,2 @@ -955a39c49c73c0ef63d55f428840c915438df4fd13abb5bfa2bd9cfe5f7c22ae +0d18bba1555dd5927812aeed569b37eb871d41c79921e510fa3e83f844e0f30e diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 019ff78714fa6..6d26c871e9f4f 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1428,7 +1428,7 @@ task_instance--xcom -1 +0..N 1 From e3742afb4d13641835a9d53c55b84f2484069a5a Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Tue, 16 Apr 2024 09:57:35 +0100 Subject: [PATCH 06/14] fixup! Fix mysql, sqlite and issue with cascading deletes --- .../0142_2_10_0_update_missing_constraints.py | 12 ++++++------ docs/apache-airflow/img/airflow_erd.sha256 | 3 +-- docs/apache-airflow/img/airflow_erd.svg | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py index 516f89e70597b..dab3994dc2995 100644 --- a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py @@ -43,9 +43,9 @@ def upgrade(): # SQLite does not support DROP CONSTRAINT return if conn.dialect.name == "mysql": - op.execute("ALTER TABLE connection DROP INDEX IF EXISTS unique_conn_id") + op.drop_index('unique_conn_id', table_name='connection', if_exists=True) # Dropping and recreating cause there's no IF NOT EXISTS - op.execute("ALTER TABLE connection DROP INDEX IF EXISTS connection_conn_id_uq") + op.drop_index('connection_conn_id_uq', table_name='connection', if_exists=True) else: op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") # Dropping and recreating cause there's no IF NOT EXISTS @@ -64,11 +64,11 @@ def upgrade(): batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") if conn.dialect.name == "mysql": - op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_execution_date_uq") - op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_run_id_uq") + op.drop_index('dag_run_dag_id_execution_date_uq', table_name='dag_run', if_exists=True) + op.drop_index('dag_run_dag_id_run_id_uq', table_name='dag_run', if_exists=True) # below we drop and recreate the constraints because there's no IF NOT EXISTS - op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_execution_date_key") - op.execute("ALTER TABLE dag_run DROP INDEX IF EXISTS dag_run_dag_id_run_id_key") + op.drop_index('dag_run_dag_id_execution_date_key', table_name='dag_run', if_exists=True) + op.drop_index('dag_run_dag_id_run_id_key', table_name='dag_run', if_exists=True) else: op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_uq") op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_uq") diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 04c33d2ee34d4..fba5babee2bd1 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1,2 +1 @@ -0d18bba1555dd5927812aeed569b37eb871d41c79921e510fa3e83f844e0f30e - +320570337a9f2b7f0a118ca22e86633df6d6a1ac5712add285cf2807c0382ef6 diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 6d26c871e9f4f..5fab31c970798 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1421,14 +1421,14 @@ task_instance--xcom -1 +0..N 1 task_instance--xcom -0..N +1 1 From a25203cfb952bf816ec275eadc97ae1ffdc00b36 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Tue, 30 Apr 2024 08:19:36 +0100 Subject: [PATCH 07/14] Fix migration for mysql --- .../0142_2_10_0_update_missing_constraints.py | 81 +++++++++++++++++-- docs/apache-airflow/img/airflow_erd.sha256 | 1 + 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py index dab3994dc2995..c3f8373097d83 100644 --- a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py @@ -43,9 +43,32 @@ def upgrade(): # SQLite does not support DROP CONSTRAINT return if conn.dialect.name == "mysql": - op.drop_index('unique_conn_id', table_name='connection', if_exists=True) - # Dropping and recreating cause there's no IF NOT EXISTS - op.drop_index('connection_conn_id_uq', table_name='connection', if_exists=True) + # TODO: Rewrite these queries to use alembic when lowest MYSQL version supports IF EXISTS + conn.execute(sa.text(""" + set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE + CONSTRAINT_SCHEMA = DATABASE() AND + TABLE_NAME = 'connection' AND + CONSTRAINT_NAME = 'unique_conn_id' AND + CONSTRAINT_TYPE = 'UNIQUE') = true,'ALTER TABLE connection + drop constraint unique_conn_id','select 1'); + + prepare stmt from @var; + execute stmt; + deallocate prepare stmt; + """)) + # Dropping the below and recreating cause there's no IF NOT EXISTS in mysql + conn.execute(sa.text(""" + set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE + CONSTRAINT_SCHEMA = DATABASE() AND + TABLE_NAME = 'connection' AND + CONSTRAINT_NAME = 'connection_conn_id_uq' AND + CONSTRAINT_TYPE = 'UNIQUE') = true,'ALTER TABLE connection + drop constraint connection_conn_id_uq','select 1'); + + prepare stmt from @var; + execute stmt; + deallocate prepare stmt; + """)) else: op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") # Dropping and recreating cause there's no IF NOT EXISTS @@ -64,11 +87,55 @@ def upgrade(): batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") if conn.dialect.name == "mysql": - op.drop_index('dag_run_dag_id_execution_date_uq', table_name='dag_run', if_exists=True) - op.drop_index('dag_run_dag_id_run_id_uq', table_name='dag_run', if_exists=True) + conn.execute(sa.text(""" + set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE + CONSTRAINT_SCHEMA = DATABASE() AND + TABLE_NAME = 'dag_run' AND + CONSTRAINT_NAME = 'dag_run_dag_id_execution_date_uq' AND + CONSTRAINT_TYPE = 'UNIQUE') = true,'ALTER TABLE dag_run + drop constraint dag_run_dag_id_execution_date_uq','select 1'); + + prepare stmt from @var; + execute stmt; + deallocate prepare stmt; + """)) + conn.execute(sa.text(""" + set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE + CONSTRAINT_SCHEMA = DATABASE() AND + TABLE_NAME = 'dag_run' AND + CONSTRAINT_NAME = 'dag_run_dag_id_run_id_uq' AND + CONSTRAINT_TYPE = 'UNIQUE') = true,'ALTER TABLE dag_run + drop constraint dag_run_dag_id_run_id_uq','select 1'); + + prepare stmt from @var; + execute stmt; + deallocate prepare stmt; + """)) # below we drop and recreate the constraints because there's no IF NOT EXISTS - op.drop_index('dag_run_dag_id_execution_date_key', table_name='dag_run', if_exists=True) - op.drop_index('dag_run_dag_id_run_id_key', table_name='dag_run', if_exists=True) + conn.execute(sa.text(""" + set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE + CONSTRAINT_SCHEMA = DATABASE() AND + TABLE_NAME = 'dag_run' AND + CONSTRAINT_NAME = 'dag_run_dag_id_execution_date_key' AND + CONSTRAINT_TYPE = 'UNIQUE') = true,'ALTER TABLE dag_run + drop constraint dag_run_dag_id_execution_date_key','select 1'); + + prepare stmt from @var; + execute stmt; + deallocate prepare stmt; + """)) + conn.execute(sa.text(""" + set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE + CONSTRAINT_SCHEMA = DATABASE() AND + TABLE_NAME = 'dag_run' AND + CONSTRAINT_NAME = 'dag_run_dag_id_run_id_key' AND + CONSTRAINT_TYPE = 'UNIQUE') = true,'ALTER TABLE dag_run + drop constraint dag_run_dag_id_run_id_key','select 1'); + + prepare stmt from @var; + execute stmt; + deallocate prepare stmt; + """)) else: op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_uq") op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_uq") diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index fba5babee2bd1..4a433eea7bf2c 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1,2 @@ 320570337a9f2b7f0a118ca22e86633df6d6a1ac5712add285cf2807c0382ef6 + From f59d9c574ff60540ed70a7259f774b3c9acc543a Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Tue, 30 Apr 2024 08:21:23 +0100 Subject: [PATCH 08/14] Fix clear_number ORM server_default --- airflow/models/dagrun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index 77679f8d9a881..117dd59e288ba 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -147,7 +147,7 @@ class DagRun(Base, LoggingMixin): # Keeps track of the number of times the dagrun had been cleared. # This number is incremented only when the DagRun is re-Queued, # when the DagRun is cleared. - clear_number = Column(Integer, default=0, nullable=False) + clear_number = Column(Integer, default=0, nullable=False, server_default="0") # Remove this `if` after upgrading Sphinx-AutoAPI if not TYPE_CHECKING and "BUILDING_AIRFLOW_DOCS" in os.environ: From 13f2156e9e07702697b6094583579c304908237d Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Wed, 1 May 2024 05:57:34 +0100 Subject: [PATCH 09/14] Fix type change for mysql --- airflow/models/dag.py | 2 +- airflow/models/db_callback_request.py | 4 ++-- airflow/models/errors.py | 2 +- airflow/models/serialized_dag.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 111757b0161b7..3268d1ea37b8a 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -3619,7 +3619,7 @@ class DagModel(Base): # associated zip. fileloc = Column(String(2000)) # The base directory used by Dag Processor that parsed this dag. - processor_subdir = Column(String(2000), nullable=True) + processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) # String representing the owners owners = Column(String(2000)) # Display name of the dag diff --git a/airflow/models/db_callback_request.py b/airflow/models/db_callback_request.py index 764fbe2ff797d..5bc3a20a98f26 100644 --- a/airflow/models/db_callback_request.py +++ b/airflow/models/db_callback_request.py @@ -20,7 +20,7 @@ from importlib import import_module from typing import TYPE_CHECKING -from sqlalchemy import Column, Integer, String +from sqlalchemy import Column, Integer, String, Text from airflow.models.base import Base from airflow.utils import timezone @@ -40,7 +40,7 @@ class DbCallbackRequest(Base): priority_weight = Column(Integer(), nullable=False) callback_data = Column(ExtendedJSON, nullable=False) callback_type = Column(String(20), nullable=False) - processor_subdir = Column(String(2000), nullable=True) + processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) def __init__(self, priority_weight: int, callback: CallbackRequest): self.created_at = timezone.utcnow() diff --git a/airflow/models/errors.py b/airflow/models/errors.py index f891b03d67a15..b862e94f2d9a4 100644 --- a/airflow/models/errors.py +++ b/airflow/models/errors.py @@ -34,7 +34,7 @@ class ParseImportError(Base): timestamp = Column(UtcDateTime) filename = Column(String(1024)) stacktrace = Column(Text) - processor_subdir = Column(String(2000), nullable=True) + processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) def __getattr__(name: str): diff --git a/airflow/models/serialized_dag.py b/airflow/models/serialized_dag.py index cba8d3a1cbf7c..59600f4dee848 100644 --- a/airflow/models/serialized_dag.py +++ b/airflow/models/serialized_dag.py @@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Collection import sqlalchemy_jsonfield -from sqlalchemy import BigInteger, Column, Index, LargeBinary, String, and_, exc, or_, select +from sqlalchemy import BigInteger, Column, Index, LargeBinary, String, Text, and_, exc, or_, select from sqlalchemy.orm import backref, foreign, relationship from sqlalchemy.sql.expression import func, literal @@ -83,7 +83,7 @@ class SerializedDagModel(Base): _data_compressed = Column("data_compressed", LargeBinary, nullable=True) last_updated = Column(UtcDateTime, nullable=False) dag_hash = Column(String(32), nullable=False) - processor_subdir = Column(String(2000), nullable=True) + processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) __table_args__ = (Index("idx_fileloc_hash", fileloc_hash, unique=False),) From a6a6de1c6ac039e59cb840b9c432a55c5009a420 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Thu, 2 May 2024 08:18:12 +0100 Subject: [PATCH 10/14] Fix constraints for sqlite and move migration to 2.9.2 --- ... 0142_2_9_2_update_missing_constraints.py} | 160 ++++++++--- ...43_2_10_0_add_new_executor_field_to_db.py} | 4 +- airflow/utils/db.py | 2 +- docs/apache-airflow/img/airflow_erd.sha256 | 3 +- docs/apache-airflow/img/airflow_erd.svg | 6 +- docs/apache-airflow/migrations-ref.rst | 259 +++++++++--------- 6 files changed, 268 insertions(+), 166 deletions(-) rename airflow/migrations/versions/{0142_2_10_0_update_missing_constraints.py => 0142_2_9_2_update_missing_constraints.py} (56%) rename airflow/migrations/versions/{0142_2_10_0_add_new_executor_field_to_db.py => 0143_2_10_0_add_new_executor_field_to_db.py} (96%) diff --git a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py b/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py similarity index 56% rename from airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py rename to airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py index c3f8373097d83..e8a37bd191241 100644 --- a/airflow/migrations/versions/0142_2_10_0_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py @@ -16,35 +16,35 @@ # specific language governing permissions and limitations # under the License. -"""Update missing constraints +"""Update missing constraints. Revision ID: 686269002441 -Revises: 677fdbb7fc54 +Revises: bff083ad727d Create Date: 2024-04-15 14:19:49.913797 """ +from __future__ import annotations + import sqlalchemy as sa from alembic import op from sqlalchemy import literal # revision identifiers, used by Alembic. -revision = '686269002441' -down_revision = '677fdbb7fc54' +revision = "686269002441" +down_revision = "bff083ad727d" branch_labels = None depends_on = None -airflow_version = '2.10.0' +airflow_version = "2.9.2" def upgrade(): - """Apply Update missing constraints""" + """Apply Update missing constraints.""" conn = op.get_bind() - if conn.dialect.name == "sqlite": - # SQLite does not support DROP CONSTRAINT - return if conn.dialect.name == "mysql": # TODO: Rewrite these queries to use alembic when lowest MYSQL version supports IF EXISTS - conn.execute(sa.text(""" + conn.execute( + sa.text(""" set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = 'connection' AND @@ -55,9 +55,11 @@ def upgrade(): prepare stmt from @var; execute stmt; deallocate prepare stmt; - """)) + """) + ) # Dropping the below and recreating cause there's no IF NOT EXISTS in mysql - conn.execute(sa.text(""" + conn.execute( + sa.text(""" set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = 'connection' AND @@ -68,18 +70,47 @@ def upgrade(): prepare stmt from @var; execute stmt; deallocate prepare stmt; - """)) + """) + ) + elif conn.dialect.name == "sqlite": + # SQLite does not support DROP CONSTRAINT + # We have to recreate the table without the constraint + conn.execute(sa.text("PRAGMA foreign_keys=off")) + conn.execute(sa.text("ALTER TABLE connection RENAME TO connection_old")) + conn.execute( + sa.text(""" + CREATE TABLE "connection" ( + id INTEGER NOT NULL, + conn_id VARCHAR(250) NOT NULL, + conn_type VARCHAR(500) NOT NULL, + host VARCHAR(500), + schema VARCHAR(500), + login TEXT, + password TEXT, + port INTEGER, + extra TEXT, + is_encrypted BOOLEAN, + is_extra_encrypted BOOLEAN, + description VARCHAR(5000), + CONSTRAINT connection_pkey PRIMARY KEY (id), + CONSTRAINT connection_conn_id_uq UNIQUE (conn_id) + ) + """) + ) + conn.execute(sa.text("INSERT INTO connection SELECT * FROM connection_old")) + conn.execute(sa.text("DROP TABLE connection_old")) + conn.execute(sa.text("PRAGMA foreign_keys=on")) else: op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") # Dropping and recreating cause there's no IF NOT EXISTS op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS connection_conn_id_uq") - with op.batch_alter_table('connection') as batch_op: - batch_op.create_unique_constraint(batch_op.f('connection_conn_id_uq'), ['conn_id']) + with op.batch_alter_table("connection") as batch_op: + batch_op.create_unique_constraint(batch_op.f("connection_conn_id_uq"), ["conn_id"]) - max_cons = sa.table('dag', sa.column('max_consecutive_failed_dag_runs')) + max_cons = sa.table("dag", sa.column("max_consecutive_failed_dag_runs")) op.execute(max_cons.update().values(max_consecutive_failed_dag_runs=literal("0"))) - with op.batch_alter_table('dag') as batch_op: - batch_op.alter_column('max_consecutive_failed_dag_runs', existing_type=sa.Integer(), nullable=False) + with op.batch_alter_table("dag") as batch_op: + batch_op.alter_column("max_consecutive_failed_dag_runs", existing_type=sa.Integer(), nullable=False) with op.batch_alter_table("task_instance") as batch_op: batch_op.drop_constraint("task_instance_dag_run_fkey", type_="foreignkey") @@ -87,7 +118,8 @@ def upgrade(): batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") if conn.dialect.name == "mysql": - conn.execute(sa.text(""" + conn.execute( + sa.text(""" set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = 'dag_run' AND @@ -98,8 +130,10 @@ def upgrade(): prepare stmt from @var; execute stmt; deallocate prepare stmt; - """)) - conn.execute(sa.text(""" + """) + ) + conn.execute( + sa.text(""" set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = 'dag_run' AND @@ -110,9 +144,11 @@ def upgrade(): prepare stmt from @var; execute stmt; deallocate prepare stmt; - """)) + """) + ) # below we drop and recreate the constraints because there's no IF NOT EXISTS - conn.execute(sa.text(""" + conn.execute( + sa.text(""" set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = 'dag_run' AND @@ -123,8 +159,10 @@ def upgrade(): prepare stmt from @var; execute stmt; deallocate prepare stmt; - """)) - conn.execute(sa.text(""" + """) + ) + conn.execute( + sa.text(""" set @var=if((SELECT true FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = 'dag_run' AND @@ -135,19 +173,79 @@ def upgrade(): prepare stmt from @var; execute stmt; deallocate prepare stmt; - """)) + """) + ) + elif conn.dialect.name == "sqlite": + # SQLite does not support DROP CONSTRAINT + # We have to recreate the table without the constraint + conn.execute(sa.text("PRAGMA foreign_keys=off")) + conn.execute(sa.text("ALTER TABLE dag_run RENAME TO dag_run_old")) + conn.execute( + sa.text(""" + CREATE TABLE dag_run ( + id INTEGER NOT NULL, + dag_id VARCHAR(250) NOT NULL, + queued_at TIMESTAMP, + execution_date TIMESTAMP NOT NULL, + start_date TIMESTAMP, + end_date TIMESTAMP, + state VARCHAR(50), + run_id VARCHAR(250) NOT NULL, + creating_job_id INTEGER, + external_trigger BOOLEAN, + run_type VARCHAR(50) NOT NULL, + conf BLOB, + data_interval_start TIMESTAMP, + data_interval_end TIMESTAMP, + last_scheduling_decision TIMESTAMP, + dag_hash VARCHAR(32), + log_template_id INTEGER, + updated_at TIMESTAMP, + clear_number INTEGER DEFAULT '0' NOT NULL, + CONSTRAINT dag_run_pkey PRIMARY KEY (id), + CONSTRAINT dag_run_dag_id_execution_date_key UNIQUE (dag_id, execution_date), + CONSTRAINT dag_run_dag_id_run_id_key UNIQUE (dag_id, run_id), + CONSTRAINT task_instance_log_template_id_fkey FOREIGN KEY(log_template_id) REFERENCES log_template (id) ON DELETE NO ACTION + ) + """) + ) + + conn.execute(sa.text("INSERT INTO dag_run SELECT * FROM dag_run_old")) + conn.execute(sa.text("DROP TABLE dag_run_old")) + conn.execute(sa.text("PRAGMA foreign_keys=on")) + with op.batch_alter_table("dag_run") as batch_op: + batch_op.create_index("dag_id_state", ["dag_id", "state"], if_not_exists=True) + batch_op.create_index("idx_dag_run_dag_id", ["dag_id"], if_not_exists=True) + batch_op.create_index( + "idx_dag_run_running_dags", + ["state", "dag_id"], + sqlite_where=sa.text("state='running'"), + if_not_exists=True, + ) + batch_op.create_index( + "idx_dag_run_queued_dags", + ["state", "dag_id"], + sqlite_where=sa.text("state='queued'"), + if_not_exists=True, + ) + else: op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_uq") op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_uq") # below we drop and recreate the constraints because there's no IF NOT EXISTS op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_key") op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_key") - with op.batch_alter_table('dag_run') as batch_op: - batch_op.create_unique_constraint('dag_run_dag_id_execution_date_key', ['dag_id', 'execution_date']) - batch_op.create_unique_constraint('dag_run_dag_id_run_id_key', ['dag_id', 'run_id']) + with op.batch_alter_table("dag_run") as batch_op: + batch_op.create_unique_constraint("dag_run_dag_id_execution_date_key", ["dag_id", "execution_date"]) + batch_op.create_unique_constraint("dag_run_dag_id_run_id_key", ["dag_id", "run_id"]) with op.batch_alter_table("task_instance") as batch_op: - batch_op.create_foreign_key("task_instance_dag_run_fkey", "dag_run", ["dag_id", "run_id"], ["dag_id", "run_id"], - ondelete="CASCADE") + batch_op.create_foreign_key( + "task_instance_dag_run_fkey", + "dag_run", + ["dag_id", "run_id"], + ["dag_id", "run_id"], + ondelete="CASCADE", + ) with op.batch_alter_table("task_reschedule") as batch_op: batch_op.create_foreign_key( "task_reschedule_dr_fkey", diff --git a/airflow/migrations/versions/0142_2_10_0_add_new_executor_field_to_db.py b/airflow/migrations/versions/0143_2_10_0_add_new_executor_field_to_db.py similarity index 96% rename from airflow/migrations/versions/0142_2_10_0_add_new_executor_field_to_db.py rename to airflow/migrations/versions/0143_2_10_0_add_new_executor_field_to_db.py index 8fda89da849ff..5adf5dbb330ec 100644 --- a/airflow/migrations/versions/0142_2_10_0_add_new_executor_field_to_db.py +++ b/airflow/migrations/versions/0143_2_10_0_add_new_executor_field_to_db.py @@ -19,7 +19,7 @@ """add new executor field to db. Revision ID: 677fdbb7fc54 -Revises: 1949afb29106 +Revises: 686269002441 Create Date: 2024-04-01 15:26:59.186579 """ @@ -31,7 +31,7 @@ # revision identifiers, used by Alembic. revision = "677fdbb7fc54" -down_revision = "bff083ad727d" +down_revision = "686269002441" branch_labels = None depends_on = None airflow_version = "2.10.0" diff --git a/airflow/utils/db.py b/airflow/utils/db.py index e9652299f940f..e20836f0e2feb 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -91,7 +91,7 @@ "2.8.0": "10b52ebd31f7", "2.8.1": "88344c1d9134", "2.9.0": "1949afb29106", - "2.9.2": "bff083ad727d", + "2.9.2": "686269002441", "2.10.0": "677fdbb7fc54", } diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 4a433eea7bf2c..b64f914c18eb7 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1,2 +1 @@ -320570337a9f2b7f0a118ca22e86633df6d6a1ac5712add285cf2807c0382ef6 - +8e508a6b71c59805a60b6aebc9cb2aff1d76a16ed0d7419dc3a9a32721ea2ef9 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 5fab31c970798..fb280ee0ea7fc 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1421,21 +1421,21 @@ task_instance--xcom -0..N +1 1 task_instance--xcom -1 +0..N 1 task_instance--xcom -0..N +1 1 diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index d858879d545fc..ee0dd1dd91b5a 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,73 +39,79 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -| ``677fdbb7fc54`` (head) | ``1949afb29106`` | ``2.10.0`` | add new executor field to db | +| ``677fdbb7fc54`` (head) | ``686269002441`` | ``2.10.0`` | add new executor field to db. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1949afb29106`` | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type and encrypt | +| ``686269002441`` | ``bff083ad727d`` | ``2.9.2`` | Update missing constraints. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | +| ``bff083ad727d`` | ``1949afb29106`` | ``2.9.2`` | Remove ``idx_last_scheduling_decision`` index on | +| | | | last_scheduling_decision in dag_run table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +| ``1949afb29106`` | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type and encrypt. | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance. | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for | +| | | | mysql. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | -| | | | table | +| | | | table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` | add dataset_expression in DagModel | +| ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` | add dataset_expression in DagModel. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``d75389605139`` | ``1fd565369930`` | ``2.9.0`` | Add run_id to (Audit) log table and increase event name | -| | | | length | +| | | | length. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``1fd565369930`` | ``88344c1d9134`` | ``2.9.0`` | Add rendered_map_index to TaskInstance. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``88344c1d9134`` | ``10b52ebd31f7`` | ``2.8.1`` | Drop unused TI index | +| ``88344c1d9134`` | ``10b52ebd31f7`` | ``2.8.1`` | Drop unused TI index. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``10b52ebd31f7`` | ``bd5dfbe21f88`` | ``2.8.0`` | Add processor_subdir to ImportError. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bd5dfbe21f88`` | ``f7bf2a57d0a6`` | ``2.8.0`` | Make connection login/password TEXT | +| ``bd5dfbe21f88`` | ``f7bf2a57d0a6`` | ``2.8.0`` | Make connection login/password TEXT. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f7bf2a57d0a6`` | ``375a816bbbf4`` | ``2.8.0`` | Add owner_display_name to (Audit) Log table | +| ``f7bf2a57d0a6`` | ``375a816bbbf4`` | ``2.8.0`` | Add owner_display_name to (Audit) Log table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``375a816bbbf4`` | ``405de8318b3a`` | ``2.8.0`` | add new field 'clear_number' to dagrun | +| ``375a816bbbf4`` | ``405de8318b3a`` | ``2.8.0`` | add new field 'clear_number' to dagrun. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``405de8318b3a`` | ``788397e78828`` | ``2.7.0`` | add include_deferred column to pool | +| ``405de8318b3a`` | ``788397e78828`` | ``2.7.0`` | add include_deferred column to pool. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``788397e78828`` | ``937cbd173ca1`` | ``2.7.0`` | Add custom_operator_name column | +| ``788397e78828`` | ``937cbd173ca1`` | ``2.7.0`` | Add custom_operator_name column. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``937cbd173ca1`` | ``c804e5c76e3e`` | ``2.7.0`` | Add index to task_instance table | +| ``937cbd173ca1`` | ``c804e5c76e3e`` | ``2.7.0`` | Add index to task_instance table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``c804e5c76e3e`` | ``98ae134e6fff`` | ``2.6.2`` | Add ``onupdate`` cascade to ``task_map`` table | +| ``c804e5c76e3e`` | ``98ae134e6fff`` | ``2.6.2`` | Add ``onupdate`` cascade to ``task_map`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``98ae134e6fff`` | ``6abdffdd4815`` | ``2.6.0`` | Increase length of user identifier columns in ``ab_user`` | -| | | | and ``ab_register_user`` tables | +| | | | and ``ab_register_user`` tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``6abdffdd4815`` | ``290244fb8b83`` | ``2.6.0`` | add dttm index on log table | +| ``6abdffdd4815`` | ``290244fb8b83`` | ``2.6.0`` | add dttm index on log table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``290244fb8b83`` | ``1986afd32c1b`` | ``2.5.0`` | Add is_orphaned to DatasetModel | +| ``290244fb8b83`` | ``1986afd32c1b`` | ``2.5.0`` | Add is_orphaned to DatasetModel. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1986afd32c1b`` | ``ee8d93fcc81e`` | ``2.5.0`` | Add DagRunNote and TaskInstanceNote | +| ``1986afd32c1b`` | ``ee8d93fcc81e`` | ``2.5.0`` | Add DagRunNote and TaskInstanceNote. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ee8d93fcc81e`` | ``e07f49787c9d`` | ``2.5.0`` | Add updated_at column to DagRun and TaskInstance | +| ``ee8d93fcc81e`` | ``e07f49787c9d`` | ``2.5.0`` | Add updated_at column to DagRun and TaskInstance. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e07f49787c9d`` | ``b0d31815b5a6`` | ``2.4.3`` | Add case-insensitive unique constraint for username | +| ``e07f49787c9d`` | ``b0d31815b5a6`` | ``2.4.3`` | Add case-insensitive unique constraint for username. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b0d31815b5a6`` | ``ecb43d2a1842`` | ``2.4.2`` | Add missing auto-increment to columns on FAB tables | +| ``b0d31815b5a6`` | ``ecb43d2a1842`` | ``2.4.2`` | Add missing auto-increment to columns on FAB tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ecb43d2a1842`` | ``1486deb605b4`` | ``2.4.0`` | Add processor_subdir column to DagModel, SerializedDagModel | | | | | and CallbackRequest tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1486deb605b4`` | ``f4ff391becb5`` | ``2.4.0`` | add dag_owner_attributes table | +| ``1486deb605b4`` | ``f4ff391becb5`` | ``2.4.0`` | add dag_owner_attributes table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f4ff391becb5`` | ``0038cd0c28b4`` | ``2.4.0`` | Remove smart sensors | +| ``f4ff391becb5`` | ``0038cd0c28b4`` | ``2.4.0`` | Remove smart sensors. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``0038cd0c28b4`` | ``44b7034f6bdc`` | ``2.4.0`` | Add Dataset model | +| ``0038cd0c28b4`` | ``44b7034f6bdc`` | ``2.4.0`` | Add Dataset model. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``44b7034f6bdc`` | ``424117c37d18`` | ``2.4.0`` | compare types between ORM and DB. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``424117c37d18`` | ``f5fcbda3e651`` | ``2.4.0`` | Add DagWarning model | +| ``424117c37d18`` | ``f5fcbda3e651`` | ``2.4.0`` | Add DagWarning model. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f5fcbda3e651`` | ``3c94c427fdf6`` | ``2.3.3`` | Add indexes for CASCADE deletes on task_instance | +| ``f5fcbda3e651`` | ``3c94c427fdf6`` | ``2.3.3`` | Add indexes for CASCADE deletes on task_instance. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``3c94c427fdf6`` | ``1de7bc13c950`` | ``2.3.2`` | Add cascade to dag_tag foreign key | +| ``3c94c427fdf6`` | ``1de7bc13c950`` | ``2.3.2`` | Add cascade to dag_tag foreign key. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``1de7bc13c950`` | ``b1b348e02d07`` | ``2.3.1`` | Add index for ``event`` column in ``log`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ @@ -113,35 +119,33 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``75d5ed6c2b43`` | ``909884dea523`` | ``2.3.0`` | Add map_index to Log. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``909884dea523`` | ``48925b2719cb`` | ``2.3.0`` | Update migration for FAB tables to add missing constraints | +| ``909884dea523`` | ``48925b2719cb`` | ``2.3.0`` | Update migration for FAB tables to add missing constraints. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``48925b2719cb`` | ``4eaab2fe6582`` | ``2.3.0`` | Add map_index to TaskFail | +| ``48925b2719cb`` | ``4eaab2fe6582`` | ``2.3.0`` | Add map_index to TaskFail. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``4eaab2fe6582`` | ``c97c2ab6aa23`` | ``2.3.0`` | Migrate RTIF to use run_id and map_index | +| ``4eaab2fe6582`` | ``c97c2ab6aa23`` | ``2.3.0`` | Migrate RTIF to use run_id and map_index. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``c97c2ab6aa23`` | ``c306b5b5ae4a`` | ``2.3.0`` | add callback request table | +| ``c97c2ab6aa23`` | ``c306b5b5ae4a`` | ``2.3.0`` | add callback request table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``c306b5b5ae4a`` | ``a3bcd0914482`` | ``2.3.0`` | Switch XCom table to use ``run_id`` and add ``map_index``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``a3bcd0914482`` | ``e655c0453f75`` | ``2.3.0`` | add data_compressed to serialized_dag | +| ``a3bcd0914482`` | ``e655c0453f75`` | ``2.3.0`` | add data_compressed to serialized_dag. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e655c0453f75`` | ``f9da662e7089`` | ``2.3.0`` | Add ``map_index`` column to TaskInstance to identify task- | -| | | | mapping, and a ``task_map`` table to track mapping values | -| | | | from XCom. | +| ``e655c0453f75`` | ``f9da662e7089`` | ``2.3.0`` | Add TaskMap and map_index on TaskInstance. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``f9da662e7089`` | ``786e3737b18f`` | ``2.3.0`` | Add ``LogTemplate`` table to track changes to config values | -| | | | ``log_filename_template`` | +| | | | ``log_filename_template``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``786e3737b18f`` | ``5e3ec427fdd3`` | ``2.3.0`` | Add ``timetable_description`` column to DagModel for UI. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``5e3ec427fdd3`` | ``587bdf053233`` | ``2.3.0`` | Increase length of email and username in ``ab_user`` and | -| | | | ``ab_register_user`` table to ``256`` characters | +| | | | ``ab_register_user`` table to ``256`` characters. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``587bdf053233`` | ``c381b21cb7e4`` | ``2.2.4`` | Add index for ``dag_id`` column in ``job`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``c381b21cb7e4`` | ``be2bfac3da23`` | ``2.2.4`` | Create a ``session`` table to store web session data | +| ``c381b21cb7e4`` | ``be2bfac3da23`` | ``2.2.4`` | Create a ``session`` table to store web session data. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``be2bfac3da23`` | ``7b2661a43ba3`` | ``2.2.3`` | Add has_import_errors column to DagModel | +| ``be2bfac3da23`` | ``7b2661a43ba3`` | ``2.2.3`` | Add has_import_errors column to DagModel. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``7b2661a43ba3`` | ``142555e44c17`` | ``2.2.0`` | Change ``TaskInstance`` and ``TaskReschedule`` tables from | | | | | execution_date to run_id. | @@ -149,204 +153,205 @@ Here's the list of all the Database Migrations that are executed via when you ru | ``142555e44c17`` | ``54bebd308c5f`` | ``2.2.0`` | Add data_interval_[start|end] to DagModel and DagRun. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``54bebd308c5f`` | ``30867afad44a`` | ``2.2.0`` | Adds ``trigger`` table and deferrable operator columns to | -| | | | task instance | +| | | | task instance. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``30867afad44a`` | ``e9304a3141f0`` | ``2.2.0`` | Rename ``concurrency`` column in ``dag`` table to`` | -| | | | max_active_tasks`` | +| | | | max_active_tasks``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e9304a3141f0`` | ``83f031fd9f1c`` | ``2.2.0`` | Make XCom primary key columns non-nullable | +| ``e9304a3141f0`` | ``83f031fd9f1c`` | ``2.2.0`` | Make XCom primary key columns non-nullable. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``83f031fd9f1c`` | ``ccde3e26fe78`` | ``2.2.0`` | Improve MSSQL compatibility | +| ``83f031fd9f1c`` | ``ccde3e26fe78`` | ``2.2.0`` | Improve MSSQL compatibility. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ccde3e26fe78`` | ``092435bf5d12`` | ``2.1.4`` | Add index on state, dag_id for queued ``dagrun`` | +| ``ccde3e26fe78`` | ``092435bf5d12`` | ``2.1.4`` | Add index on state, dag_id for queued ``dagrun``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``092435bf5d12`` | ``97cdd93827b8`` | ``2.1.4`` | Add ``max_active_runs`` column to ``dag_model`` table | +| ``092435bf5d12`` | ``97cdd93827b8`` | ``2.1.4`` | Add ``max_active_runs`` column to ``dag_model`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``97cdd93827b8`` | ``a13f7613ad25`` | ``2.1.3`` | Add ``queued_at`` column in ``dag_run`` table | +| ``97cdd93827b8`` | ``a13f7613ad25`` | ``2.1.3`` | Add ``queued_at`` column in ``dag_run`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``a13f7613ad25`` | ``e165e7455d70`` | ``2.1.0`` | Resource based permissions for default ``Flask-AppBuilder`` | -| | | | views | +| | | | views. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e165e7455d70`` | ``90d1635d7b86`` | ``2.1.0`` | Add description field to ``Variable`` model | +| ``e165e7455d70`` | ``90d1635d7b86`` | ``2.1.0`` | Add description field to ``Variable`` model. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``90d1635d7b86`` | ``2e42bb497a22`` | ``2.1.0`` | Increase maximum length of pool name in ``task_instance`` | -| | | | table to ``256`` characters | +| | | | table to ``256`` characters. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``2e42bb497a22`` | ``8646922c8a04`` | ``2.0.2`` | Rename ``last_scheduler_run`` column in ``DAG`` table to | -| | | | ``last_parsed_time`` | +| | | | ``last_parsed_time``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8646922c8a04`` | ``449b4072c2da`` | ``2.0.2`` | Change default ``pool_slots`` to ``1`` | +| ``8646922c8a04`` | ``449b4072c2da`` | ``2.0.2`` | Change default ``pool_slots`` to ``1``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``449b4072c2da`` | ``82b7c48c147f`` | ``2.0.2`` | Increase size of ``connection.extra`` field to handle | -| | | | multiple RSA keys | +| | | | multiple RSA keys. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``82b7c48c147f`` | ``e959f08ac86c`` | ``2.0.1`` | Remove ``can_read`` permission on config resource for | -| | | | ``User`` and ``Viewer`` role | +| | | | ``User`` and ``Viewer`` role. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e959f08ac86c`` | ``64a7d6477aae`` | ``2.0.0`` | Change field in ``DagCode`` to ``MEDIUMTEXT`` for MySql | +| ``e959f08ac86c`` | ``64a7d6477aae`` | ``2.0.0`` | Change field in ``DagCode`` to ``MEDIUMTEXT`` for MySql. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``64a7d6477aae`` | ``61ec73d9401f`` | ``2.0.0`` | Fix description field in ``connection`` to be ``text`` | +| ``64a7d6477aae`` | ``61ec73d9401f`` | ``2.0.0`` | Fix description field in ``connection`` to be ``text``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``61ec73d9401f`` | ``2c6edca13270`` | ``2.0.0`` | Add description field to ``connection`` table | +| ``61ec73d9401f`` | ``2c6edca13270`` | ``2.0.0`` | Add description field to ``connection`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``2c6edca13270`` | ``849da589634d`` | ``2.0.0`` | Resource based permissions. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``849da589634d`` | ``45ba3f1493b9`` | ``2.0.0`` | Prefix DAG permissions. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``45ba3f1493b9`` | ``364159666cbd`` | ``2.0.0`` | add-k8s-yaml-to-rendered-templates | +| ``45ba3f1493b9`` | ``364159666cbd`` | ``2.0.0`` | add-k8s-yaml-to-rendered-templates. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``364159666cbd`` | ``52d53670a240`` | ``2.0.0`` | Add ``creating_job_id`` to ``DagRun`` table | +| ``364159666cbd`` | ``52d53670a240`` | ``2.0.0`` | Add ``creating_job_id`` to ``DagRun`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``52d53670a240`` | ``98271e7606e2`` | ``2.0.0`` | fix_mssql_exec_date_rendered_task_instance_fields_for_MSSQL | +| ``52d53670a240`` | ``98271e7606e2`` | ``2.0.0`` | fix_mssql_exec_date_rendered_task_instance_fields_for_MSSQL. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``98271e7606e2`` | ``bef4f3d11e8b`` | ``2.0.0`` | Add ``scheduling_decision`` to ``DagRun`` and ``DAG`` | +| ``98271e7606e2`` | ``bef4f3d11e8b`` | ``2.0.0`` | Add ``scheduling_decision`` to ``DagRun`` and ``DAG``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bef4f3d11e8b`` | ``e1a11ece99cc`` | ``2.0.0`` | Drop ``KubeResourceVersion`` and ``KubeWorkerId`` | +| ``bef4f3d11e8b`` | ``e1a11ece99cc`` | ``2.0.0`` | Drop ``KubeResourceVersion`` and ``KubeWorkerId``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e1a11ece99cc`` | ``b247b1e3d1ed`` | ``2.0.0`` | Add external executor ID to TI | +| ``e1a11ece99cc`` | ``b247b1e3d1ed`` | ``2.0.0`` | Add external executor ID to TI. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b247b1e3d1ed`` | ``e38be357a868`` | ``2.0.0`` | Add queued by Job ID to TI | +| ``b247b1e3d1ed`` | ``e38be357a868`` | ``2.0.0`` | Add queued by Job ID to TI. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e38be357a868`` | ``8d48763f6d53`` | ``2.0.0`` | Add ``sensor_instance`` table | +| ``e38be357a868`` | ``8d48763f6d53`` | ``2.0.0`` | Add ``sensor_instance`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8d48763f6d53`` | ``8f966b9c467a`` | ``2.0.0`` | Add unique constraint to ``conn_id`` | +| ``8d48763f6d53`` | ``8f966b9c467a`` | ``2.0.0`` | Add unique constraint to ``conn_id``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8f966b9c467a`` | ``3c20cacc0044`` | ``2.0.0`` | Set ``conn_type`` as non-nullable | +| ``8f966b9c467a`` | ``3c20cacc0044`` | ``2.0.0`` | Set ``conn_type`` as non-nullable. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``3c20cacc0044`` | ``b25a55525161`` | ``2.0.0`` | Add ``run_type`` column in ``dag_run`` table | +| ``3c20cacc0044`` | ``b25a55525161`` | ``2.0.0`` | Add ``run_type`` column in ``dag_run`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b25a55525161`` | ``bbf4a7ad0465`` | ``2.0.0`` | Increase length of pool name | +| ``b25a55525161`` | ``bbf4a7ad0465`` | ``2.0.0`` | Increase length of pool name. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bbf4a7ad0465`` | ``cf5dc11e79ad`` | ``2.0.0`` | Remove id column from xcom | +| ``bbf4a7ad0465`` | ``cf5dc11e79ad`` | ``2.0.0`` | Remove id column from xcom. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``cf5dc11e79ad`` | ``03afc6b6f902`` | ``2.0.0`` | Drop ``user`` and ``chart`` table | +| ``cf5dc11e79ad`` | ``03afc6b6f902`` | ``2.0.0`` | Drop ``user`` and ``chart`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``03afc6b6f902`` | ``92c57b58940d`` | ``1.10.13`` | Increase length of ``Flask-AppBuilder`` | -| | | | ``ab_view_menu.name`` column | +| | | | ``ab_view_menu.name`` column. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``92c57b58940d`` | ``da3f683c3a5a`` | ``1.10.13`` | Create FAB Tables | +| ``92c57b58940d`` | ``da3f683c3a5a`` | ``1.10.13`` | Create FAB Tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``da3f683c3a5a`` | ``a66efa278eea`` | ``1.10.12`` | Add ``dag_hash`` Column to ``serialized_dag`` table | +| ``da3f683c3a5a`` | ``a66efa278eea`` | ``1.10.12`` | Add ``dag_hash`` Column to ``serialized_dag`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``a66efa278eea`` | ``952da73b5eff`` | ``1.10.11`` | Add Precision to ``execution_date`` in | -| | | | ``RenderedTaskInstanceFields`` table | +| | | | ``RenderedTaskInstanceFields`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``952da73b5eff`` | ``852ae6c715af`` | ``1.10.10`` | Add ``dag_code`` table | +| ``952da73b5eff`` | ``852ae6c715af`` | ``1.10.10`` | Add ``dag_code`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``852ae6c715af`` | ``a4c2fd67d16b`` | ``1.10.10`` | Add ``RenderedTaskInstanceFields`` table | +| ``852ae6c715af`` | ``a4c2fd67d16b`` | ``1.10.10`` | Add ``RenderedTaskInstanceFields`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``a4c2fd67d16b`` | ``7939bcff74ba`` | ``1.10.10`` | Add ``pool_slots`` field to ``task_instance`` | +| ``a4c2fd67d16b`` | ``7939bcff74ba`` | ``1.10.10`` | Add ``pool_slots`` field to ``task_instance``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``7939bcff74ba`` | ``fe461863935f`` | ``1.10.8`` | Add ``DagTags`` table | +| ``7939bcff74ba`` | ``fe461863935f`` | ``1.10.8`` | Add ``DagTags`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``fe461863935f`` | ``08364691d074`` | ``1.10.7`` | Increase length for connection password | +| ``fe461863935f`` | ``08364691d074`` | ``1.10.7`` | Increase length for connection password. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``08364691d074`` (merge_point) | ``a56c9515abdc``, | ``1.10.7`` | Straighten out the migrations | +| ``08364691d074`` (merge_point) | ``a56c9515abdc``, | ``1.10.7`` | Straighten out the migrations. | | | ``004c1210f153``, | | | | | ``74effc47d867``, | | | | | ``b3b105409875`` | | | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``a56c9515abdc`` | ``c8ffec048a3b`` | ``1.10.3`` | Remove ``dag_stat`` table | +| ``a56c9515abdc`` | ``c8ffec048a3b`` | ``1.10.3`` | Remove ``dag_stat`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``004c1210f153`` | ``939bb1e647c8`` | ``1.10.4`` | Increase queue name size limit | +| ``004c1210f153`` | ``939bb1e647c8`` | ``1.10.4`` | Increase queue name size limit. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``74effc47d867`` | ``6e96a59344a4`` | ``1.10.5`` | change datetime to datetime2(6) on MSSQL tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b3b105409875`` | ``d38e04c12aa2`` | ``1.10.7`` | Add ``root_dag_id`` to ``DAG`` | +| ``b3b105409875`` | ``d38e04c12aa2`` | ``1.10.7`` | Add ``root_dag_id`` to ``DAG``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``d38e04c12aa2`` | ``6e96a59344a4`` | ``1.10.7`` | Add ``serialized_dag`` table | +| ``d38e04c12aa2`` | ``6e96a59344a4`` | ``1.10.7`` | Add ``serialized_dag`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``6e96a59344a4`` (branch_point) | ``939bb1e647c8`` | ``1.10.4`` | Make ``TaskInstance.pool`` not nullable | +| ``6e96a59344a4`` (branch_point) | ``939bb1e647c8`` | ``1.10.4`` | Make ``TaskInstance.pool`` not nullable. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``939bb1e647c8`` (branch_point) | ``dd4ecb8fbee3`` | ``1.10.3`` | task reschedule foreign key on cascade delete | +| ``939bb1e647c8`` (branch_point) | ``dd4ecb8fbee3`` | ``1.10.3`` | task reschedule foreign key on cascade delete. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``dd4ecb8fbee3`` | ``c8ffec048a3b`` | ``1.10.3`` | Add schedule interval to dag | +| ``dd4ecb8fbee3`` | ``c8ffec048a3b`` | ``1.10.3`` | Add schedule interval to dag. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``c8ffec048a3b`` (branch_point) | ``41f5f12752f8`` | ``1.10.3`` | Add ``description`` and ``default_view`` column to ``dag`` | -| | | | table | +| | | | table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``41f5f12752f8`` | ``03bc53e68815`` | ``1.10.2`` | Add superuser field | +| ``41f5f12752f8`` | ``03bc53e68815`` | ``1.10.2`` | Add superuser field. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``03bc53e68815`` (merge_point) | ``0a2a5b66e19d``, | ``1.10.2`` | Merge migrations Heads. | | | ``bf00311e1990`` | | | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``0a2a5b66e19d`` | ``9635ae0956e7`` | ``1.10.2`` | Add ``task_reschedule`` table | +| ``0a2a5b66e19d`` | ``9635ae0956e7`` | ``1.10.2`` | Add ``task_reschedule`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bf00311e1990`` | ``dd25f486b8ea`` | ``1.10.2`` | Add index to ``task_instance`` table | +| ``bf00311e1990`` | ``dd25f486b8ea`` | ``1.10.2`` | Add index to ``task_instance`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``dd25f486b8ea`` | ``9635ae0956e7`` | ``1.10.2`` | Add index on ``log`` table | +| ``dd25f486b8ea`` | ``9635ae0956e7`` | ``1.10.2`` | Add index on ``log`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``9635ae0956e7`` (branch_point) | ``856955da8476`` | ``1.10.0`` | Create index on ``task_fail`` table | +| ``9635ae0956e7`` (branch_point) | ``856955da8476`` | ``1.10.0`` | Create index on ``task_fail`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``856955da8476`` | ``f23433877c24`` | ``1.10.0`` | Fix Sqlite foreign key | +| ``856955da8476`` | ``f23433877c24`` | ``1.10.0`` | Fix Sqlite foreign key. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f23433877c24`` | ``05f30312d566`` | ``1.10.0`` | Fix MySQL not null constraint | +| ``f23433877c24`` | ``05f30312d566`` | ``1.10.0`` | Fix MySQL not null constraint. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``05f30312d566`` (merge_point) | ``86770d1215c0``, | ``1.10.0`` | Merge migrations Heads | +| ``05f30312d566`` (merge_point) | ``86770d1215c0``, | ``1.10.0`` | Merge migrations Heads. | | | ``0e2a74e0fc9f`` | | | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``86770d1215c0`` | ``27c6a30d7c24`` | ``1.10.0`` | Add kubernetes scheduler uniqueness | +| ``86770d1215c0`` | ``27c6a30d7c24`` | ``1.10.0`` | Add kubernetes scheduler uniqueness. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``27c6a30d7c24`` | ``33ae817a1ff4`` | ``1.10.0`` | Add ``executor_config`` column to ``task_instance`` table | +| ``27c6a30d7c24`` | ``33ae817a1ff4`` | ``1.10.0`` | Add ``executor_config`` column to ``task_instance`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``33ae817a1ff4`` | ``d2ae31099d61`` | ``1.10.0`` | Add Kubernetes resource check-pointing | +| ``33ae817a1ff4`` | ``d2ae31099d61`` | ``1.10.0`` | Add Kubernetes resource check-pointing. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``0e2a74e0fc9f`` | ``d2ae31099d61`` | ``1.10.0`` | Add time zone awareness | +| ``0e2a74e0fc9f`` | ``d2ae31099d61`` | ``1.10.0`` | Add time zone awareness. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``d2ae31099d61`` (branch_point) | ``947454bf1dff`` | ``1.8.2`` | Increase text size for MySQL (not relevant for other DBs' | -| | | | text types) | +| | | | text types). | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``947454bf1dff`` | ``bdaa763e6c56`` | ``1.8.2`` | Create index on ``job_id`` column in ``task_instance`` table | +| ``947454bf1dff`` | ``bdaa763e6c56`` | ``1.8.2`` | Create index on ``job_id`` column in ``task_instance`` | +| | | | table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bdaa763e6c56`` | ``cc1e65623dc7`` | ``1.8.2`` | Make xcom value column a large binary | +| ``bdaa763e6c56`` | ``cc1e65623dc7`` | ``1.8.2`` | Make xcom value column a large binary. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``cc1e65623dc7`` | ``127d2bf2dfa7`` | ``1.8.2`` | Add ``max_tries`` column to ``task_instance`` | +| ``cc1e65623dc7`` | ``127d2bf2dfa7`` | ``1.8.2`` | Add ``max_tries`` column to ``task_instance``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``127d2bf2dfa7`` | ``5e7d17757c7a`` | ``1.7.1.3`` | Add ``dag_id``/``state`` index on ``dag_run`` table | +| ``127d2bf2dfa7`` | ``5e7d17757c7a`` | ``1.7.1.3`` | Add ``dag_id``/``state`` index on ``dag_run`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``5e7d17757c7a`` | ``8504051e801b`` | ``1.7.1.3`` | Add ``pid`` field to ``TaskInstance`` | +| ``5e7d17757c7a`` | ``8504051e801b`` | ``1.7.1.3`` | Add ``pid`` field to ``TaskInstance``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8504051e801b`` | ``4addfa1236f1`` | ``1.7.1.3`` | Add indices on ``xcom`` table | +| ``8504051e801b`` | ``4addfa1236f1`` | ``1.7.1.3`` | Add indices on ``xcom`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``4addfa1236f1`` | ``f2ca10b85618`` | ``1.7.1.3`` | Add fractional seconds to MySQL tables | +| ``4addfa1236f1`` | ``f2ca10b85618`` | ``1.7.1.3`` | Add fractional seconds to MySQL tables. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``f2ca10b85618`` | ``64de9cddf6c9`` | ``1.7.1.3`` | Add ``dag_stats`` table | +| ``f2ca10b85618`` | ``64de9cddf6c9`` | ``1.7.1.3`` | Add ``dag_stats`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``64de9cddf6c9`` | ``211e584da130`` | ``1.7.1.3`` | Add ``task_fail`` table | +| ``64de9cddf6c9`` | ``211e584da130`` | ``1.7.1.3`` | Add ``task_fail`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``211e584da130`` | ``2e82aab8ef20`` | ``1.7.1.3`` | Add TI state index | +| ``211e584da130`` | ``2e82aab8ef20`` | ``1.7.1.3`` | Add TI state index. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``2e82aab8ef20`` | ``1968acfc09e3`` | ``1.7.1`` | Rename user table | +| ``2e82aab8ef20`` | ``1968acfc09e3`` | ``1.7.1`` | Rename user table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1968acfc09e3`` | ``bba5a7cfc896`` | ``1.7.0`` | Add ``is_encrypted`` column to variable table | +| ``1968acfc09e3`` | ``bba5a7cfc896`` | ``1.7.0`` | Add ``is_encrypted`` column to variable table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``bba5a7cfc896`` | ``bbc73705a13e`` | ``1.7.0`` | Add a column to track the encryption state of the 'Extra' | -| | | | field in connection | +| | | | field in connection. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``bbc73705a13e`` | ``4446e08588`` | ``1.7.0`` | Add ``notification_sent`` column to ``sla_miss`` table | +| ``bbc73705a13e`` | ``4446e08588`` | ``1.7.0`` | Add ``notification_sent`` column to ``sla_miss`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``4446e08588`` | ``561833c1c74b`` | ``1.6.2`` | Add ``start_date`` and ``end_date`` in ``dag_run`` table | +| ``4446e08588`` | ``561833c1c74b`` | ``1.6.2`` | Add ``start_date`` and ``end_date`` in ``dag_run`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``561833c1c74b`` | ``40e67319e3a9`` | ``1.6.2`` | Add ``password`` column to ``user`` table | +| ``561833c1c74b`` | ``40e67319e3a9`` | ``1.6.2`` | Add ``password`` column to ``user`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``40e67319e3a9`` | ``2e541a1dcfed`` | ``1.6.0`` | Add ``conf`` column in ``dag_run`` table | +| ``40e67319e3a9`` | ``2e541a1dcfed`` | ``1.6.0`` | Add ``conf`` column in ``dag_run`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``2e541a1dcfed`` | ``1b38cef5b76e`` | ``1.6.0`` | Change ``task_instance.task_duration`` type to ``FLOAT`` | +| ``2e541a1dcfed`` | ``1b38cef5b76e`` | ``1.6.0`` | Change ``task_instance.task_duration`` type to ``FLOAT``. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1b38cef5b76e`` | ``502898887f84`` | ``1.6.0`` | Add ``dag_run`` table | +| ``1b38cef5b76e`` | ``502898887f84`` | ``1.6.0`` | Add ``dag_run`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``502898887f84`` | ``52d714495f0`` | ``1.6.0`` | Adding ``extra`` column to ``Log`` table | +| ``502898887f84`` | ``52d714495f0`` | ``1.6.0`` | Adding ``extra`` column to ``Log`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``52d714495f0`` | ``338e90f54d61`` | ``1.5.2`` | Add indices in ``job`` table | +| ``52d714495f0`` | ``338e90f54d61`` | ``1.5.2`` | Add indices in ``job`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``338e90f54d61`` | ``13eb55f81627`` | ``1.5.0`` | Add ``operator`` and ``queued_dttm`` to ``task_instance`` | -| | | | table | +| | | | table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``13eb55f81627`` | ``1507a7289a2f`` | ``1.5.0`` | Maintain history for compatibility with earlier migrations | +| ``13eb55f81627`` | ``1507a7289a2f`` | ``1.5.0`` | Maintain history for compatibility with earlier migrations. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``1507a7289a2f`` | ``e3a246e0dc1`` | ``1.5.0`` | Add ``is_encrypted`` column in ``connection`` table | +| ``1507a7289a2f`` | ``e3a246e0dc1`` | ``1.5.0`` | Add ``is_encrypted`` column in ``connection`` table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``e3a246e0dc1`` (base) | ``None`` | ``1.5.0`` | current schema | +| ``e3a246e0dc1`` (base) | ``None`` | ``1.5.0`` | current schema. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ .. End of auto-generated table From 0b6ae3d2a4f61e4af5185d186bc635063e193313 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Thu, 2 May 2024 17:14:37 +0100 Subject: [PATCH 11/14] Fix sqlite constraints update and ignore session_session_id_uq index --- .../0142_2_9_2_update_missing_constraints.py | 16 ++++++++-------- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- tests/utils/test_db.py | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py b/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py index e8a37bd191241..509cc0a0864d2 100644 --- a/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py @@ -76,10 +76,9 @@ def upgrade(): # SQLite does not support DROP CONSTRAINT # We have to recreate the table without the constraint conn.execute(sa.text("PRAGMA foreign_keys=off")) - conn.execute(sa.text("ALTER TABLE connection RENAME TO connection_old")) conn.execute( sa.text(""" - CREATE TABLE "connection" ( + CREATE TABLE connection_new ( id INTEGER NOT NULL, conn_id VARCHAR(250) NOT NULL, conn_type VARCHAR(500) NOT NULL, @@ -97,8 +96,9 @@ def upgrade(): ) """) ) - conn.execute(sa.text("INSERT INTO connection SELECT * FROM connection_old")) - conn.execute(sa.text("DROP TABLE connection_old")) + conn.execute(sa.text("INSERT INTO connection_new SELECT * FROM connection")) + conn.execute(sa.text("DROP TABLE connection")) + conn.execute(sa.text("ALTER TABLE connection_new RENAME TO connection")) conn.execute(sa.text("PRAGMA foreign_keys=on")) else: op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") @@ -179,10 +179,9 @@ def upgrade(): # SQLite does not support DROP CONSTRAINT # We have to recreate the table without the constraint conn.execute(sa.text("PRAGMA foreign_keys=off")) - conn.execute(sa.text("ALTER TABLE dag_run RENAME TO dag_run_old")) conn.execute( sa.text(""" - CREATE TABLE dag_run ( + CREATE TABLE dag_run_new ( id INTEGER NOT NULL, dag_id VARCHAR(250) NOT NULL, queued_at TIMESTAMP, @@ -210,8 +209,9 @@ def upgrade(): """) ) - conn.execute(sa.text("INSERT INTO dag_run SELECT * FROM dag_run_old")) - conn.execute(sa.text("DROP TABLE dag_run_old")) + conn.execute(sa.text("INSERT INTO dag_run_new SELECT * FROM dag_run")) + conn.execute(sa.text("DROP TABLE dag_run")) + conn.execute(sa.text("ALTER TABLE dag_run_new RENAME TO dag_run")) conn.execute(sa.text("PRAGMA foreign_keys=on")) with op.batch_alter_table("dag_run") as batch_op: batch_op.create_index("dag_id_state", ["dag_id", "state"], if_not_exists=True) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index b64f914c18eb7..9750e7db5d290 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -8e508a6b71c59805a60b6aebc9cb2aff1d76a16ed0d7419dc3a9a32721ea2ef9 \ No newline at end of file +8dbea540d4bb502765e91cc229ddfc2fe1e175ffd8418a6141a6cbb866f30952 \ No newline at end of file diff --git a/tests/utils/test_db.py b/tests/utils/test_db.py index 4f7e1f0fde084..0b4347f589d22 100644 --- a/tests/utils/test_db.py +++ b/tests/utils/test_db.py @@ -84,6 +84,7 @@ def test_database_schema_and_sqlalchemy_model_are_in_sync(self): # Ignore flask-session table/index lambda t: (t[0] == "remove_table" and t[1].name == "session"), lambda t: (t[0] == "remove_index" and t[1].name == "session_id"), + lambda t: (t[0] == "remove_index" and t[1].name == "session_session_id_uq"), # sqlite sequence is used for autoincrementing columns created with `sqlite_autoincrement` option lambda t: (t[0] == "remove_table" and t[1].name == "sqlite_sequence"), ] From 5bdffbc393d02adfe76f80c043aaf025cd3c4166 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 3 May 2024 14:43:37 +0100 Subject: [PATCH 12/14] Fix processor_subdir in the migration file for mysql and make use-migration-files an option in db commands --- airflow/cli/cli_config.py | 10 +++++- airflow/cli/commands/db_command.py | 3 +- ...stency_between_ORM_and_migration_files.py} | 34 ++++++++++++++++++- airflow/models/dag.py | 2 +- airflow/models/db_callback_request.py | 4 +-- airflow/models/errors.py | 2 +- airflow/models/serialized_dag.py | 4 +-- docs/apache-airflow/img/airflow_erd.sha256 | 2 +- docs/apache-airflow/img/airflow_erd.svg | 4 +-- docs/apache-airflow/migrations-ref.rst | 2 +- 10 files changed, 54 insertions(+), 13 deletions(-) rename airflow/migrations/versions/{0142_2_9_2_update_missing_constraints.py => 0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py} (90%) diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index cd2a25b074c80..82b00f0ff2d55 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -691,6 +691,12 @@ def string_lower_type(val): action="store_true", default=False, ) +ARG_DB_USE_MIGRATION_FILES = Arg( + ("-m", "--use-migration-files"), + help="Use migration files to perform migration", + action="store_true", + default=False, +) # webserver ARG_PORT = Arg( @@ -1525,7 +1531,7 @@ class GroupCommand(NamedTuple): name="reset", help="Burn down and rebuild the metadata database", func=lazy_load_command("airflow.cli.commands.db_command.resetdb"), - args=(ARG_YES, ARG_DB_SKIP_INIT, ARG_VERBOSE), + args=(ARG_YES, ARG_DB_SKIP_INIT, ARG_DB_USE_MIGRATION_FILES, ARG_VERBOSE), ), ActionCommand( name="upgrade", @@ -1545,6 +1551,7 @@ class GroupCommand(NamedTuple): ARG_DB_FROM_REVISION, ARG_DB_FROM_VERSION, ARG_DB_RESERIALIZE_DAGS, + ARG_DB_USE_MIGRATION_FILES, ARG_VERBOSE, ), hide=True, @@ -1568,6 +1575,7 @@ class GroupCommand(NamedTuple): ARG_DB_FROM_REVISION, ARG_DB_FROM_VERSION, ARG_DB_RESERIALIZE_DAGS, + ARG_DB_USE_MIGRATION_FILES, ARG_VERBOSE, ), ), diff --git a/airflow/cli/commands/db_command.py b/airflow/cli/commands/db_command.py index 776dabcdead7e..737dfa1ed469b 100644 --- a/airflow/cli/commands/db_command.py +++ b/airflow/cli/commands/db_command.py @@ -63,7 +63,7 @@ def resetdb(args): print(f"DB: {settings.engine.url!r}") if not (args.yes or input("This will drop existing tables if they exist. Proceed? (y/n)").upper() == "Y"): raise SystemExit("Cancelled") - db.resetdb(skip_init=args.skip_init) + db.resetdb(skip_init=args.skip_init, use_migration_files=args.use_migration_files) def upgradedb(args): @@ -142,6 +142,7 @@ def migratedb(args): from_revision=from_revision, show_sql_only=args.show_sql_only, reserialize_dags=args.reserialize_dags, + use_migration_files=args.use_migration_files, ) if not args.show_sql_only: print("Database migrating done!") diff --git a/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py b/airflow/migrations/versions/0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py similarity index 90% rename from airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py rename to airflow/migrations/versions/0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py index 509cc0a0864d2..07cb59842b78f 100644 --- a/airflow/migrations/versions/0142_2_9_2_update_missing_constraints.py +++ b/airflow/migrations/versions/0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Update missing constraints. +"""Fix inconsistency between ORM and migration files. Revision ID: 686269002441 Revises: bff083ad727d @@ -175,6 +175,38 @@ def upgrade(): deallocate prepare stmt; """) ) + with op.batch_alter_table("callback_request", schema=None) as batch_op: + batch_op.alter_column( + "processor_subdir", + existing_type=sa.Text(length=2000), + type_=sa.String(length=2000), + existing_nullable=True, + ) + + with op.batch_alter_table("dag", schema=None) as batch_op: + batch_op.alter_column( + "processor_subdir", + existing_type=sa.Text(length=2000), + type_=sa.String(length=2000), + existing_nullable=True, + ) + + with op.batch_alter_table("import_error", schema=None) as batch_op: + batch_op.alter_column( + "processor_subdir", + existing_type=sa.Text(length=2000), + type_=sa.String(length=2000), + existing_nullable=True, + ) + + with op.batch_alter_table("serialized_dag", schema=None) as batch_op: + batch_op.alter_column( + "processor_subdir", + existing_type=sa.Text(length=2000), + type_=sa.String(length=2000), + existing_nullable=True, + ) + elif conn.dialect.name == "sqlite": # SQLite does not support DROP CONSTRAINT # We have to recreate the table without the constraint diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 3268d1ea37b8a..111757b0161b7 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -3619,7 +3619,7 @@ class DagModel(Base): # associated zip. fileloc = Column(String(2000)) # The base directory used by Dag Processor that parsed this dag. - processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) + processor_subdir = Column(String(2000), nullable=True) # String representing the owners owners = Column(String(2000)) # Display name of the dag diff --git a/airflow/models/db_callback_request.py b/airflow/models/db_callback_request.py index 5bc3a20a98f26..764fbe2ff797d 100644 --- a/airflow/models/db_callback_request.py +++ b/airflow/models/db_callback_request.py @@ -20,7 +20,7 @@ from importlib import import_module from typing import TYPE_CHECKING -from sqlalchemy import Column, Integer, String, Text +from sqlalchemy import Column, Integer, String from airflow.models.base import Base from airflow.utils import timezone @@ -40,7 +40,7 @@ class DbCallbackRequest(Base): priority_weight = Column(Integer(), nullable=False) callback_data = Column(ExtendedJSON, nullable=False) callback_type = Column(String(20), nullable=False) - processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) + processor_subdir = Column(String(2000), nullable=True) def __init__(self, priority_weight: int, callback: CallbackRequest): self.created_at = timezone.utcnow() diff --git a/airflow/models/errors.py b/airflow/models/errors.py index b862e94f2d9a4..f891b03d67a15 100644 --- a/airflow/models/errors.py +++ b/airflow/models/errors.py @@ -34,7 +34,7 @@ class ParseImportError(Base): timestamp = Column(UtcDateTime) filename = Column(String(1024)) stacktrace = Column(Text) - processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) + processor_subdir = Column(String(2000), nullable=True) def __getattr__(name: str): diff --git a/airflow/models/serialized_dag.py b/airflow/models/serialized_dag.py index 59600f4dee848..cba8d3a1cbf7c 100644 --- a/airflow/models/serialized_dag.py +++ b/airflow/models/serialized_dag.py @@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Collection import sqlalchemy_jsonfield -from sqlalchemy import BigInteger, Column, Index, LargeBinary, String, Text, and_, exc, or_, select +from sqlalchemy import BigInteger, Column, Index, LargeBinary, String, and_, exc, or_, select from sqlalchemy.orm import backref, foreign, relationship from sqlalchemy.sql.expression import func, literal @@ -83,7 +83,7 @@ class SerializedDagModel(Base): _data_compressed = Column("data_compressed", LargeBinary, nullable=True) last_updated = Column(UtcDateTime, nullable=False) dag_hash = Column(String(32), nullable=False) - processor_subdir = Column(String(2000).with_variant(Text(length=2000), "mysql"), nullable=True) + processor_subdir = Column(String(2000), nullable=True) __table_args__ = (Index("idx_fileloc_hash", fileloc_hash, unique=False),) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 9750e7db5d290..8ed0236348d59 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -8dbea540d4bb502765e91cc229ddfc2fe1e175ffd8418a6141a6cbb866f30952 \ No newline at end of file +583c80d275d356b870b082ea4525e6c4be69b8fd3797a1404cc8fbf801b08a88 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index fb280ee0ea7fc..7225b0ce3ef15 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -1421,14 +1421,14 @@ task_instance--xcom -1 +0..N 1 task_instance--xcom -0..N +1 1 diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index ee0dd1dd91b5a..eb01d6aec39ed 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -41,7 +41,7 @@ Here's the list of all the Database Migrations that are executed via when you ru +=================================+===================+===================+==============================================================+ | ``677fdbb7fc54`` (head) | ``686269002441`` | ``2.10.0`` | add new executor field to db. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``686269002441`` | ``bff083ad727d`` | ``2.9.2`` | Update missing constraints. | +| ``686269002441`` | ``bff083ad727d`` | ``2.9.2`` | Fix inconsistency between ORM and migration files. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``bff083ad727d`` | ``1949afb29106`` | ``2.9.2`` | Remove ``idx_last_scheduling_decision`` index on | | | | | last_scheduling_decision in dag_run table. | From 4c1b163490cb89f5022eb8ae827f0738bf1a3679 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 3 May 2024 15:32:04 +0100 Subject: [PATCH 13/14] fixup! Fix processor_subdir in the migration file for mysql and make use-migration-files an option in db commands --- tests/cli/commands/test_db_command.py | 66 ++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/tests/cli/commands/test_db_command.py b/tests/cli/commands/test_db_command.py index 3321887287b1a..352292bd61dce 100644 --- a/tests/cli/commands/test_db_command.py +++ b/tests/cli/commands/test_db_command.py @@ -46,12 +46,17 @@ def test_cli_initdb(self, mock_initdb): def test_cli_resetdb(self, mock_resetdb): db_command.resetdb(self.parser.parse_args(["db", "reset", "--yes"])) - mock_resetdb.assert_called_once_with(skip_init=False) + mock_resetdb.assert_called_once_with(skip_init=False, use_migration_files=False) @mock.patch("airflow.cli.commands.db_command.db.resetdb") def test_cli_resetdb_skip_init(self, mock_resetdb): db_command.resetdb(self.parser.parse_args(["db", "reset", "--yes", "--skip-init"])) - mock_resetdb.assert_called_once_with(skip_init=True) + mock_resetdb.assert_called_once_with(skip_init=True, use_migration_files=False) + + @mock.patch("airflow.cli.commands.db_command.db.resetdb") + def test_cli_resetdb_use_migration_files(self, mock_resetdb): + db_command.resetdb(self.parser.parse_args(["db", "reset", "--yes", "--use-migration-files"])) + mock_resetdb.assert_called_once_with(skip_init=False, use_migration_files=True) @mock.patch("airflow.cli.commands.db_command.db.check_migrations") def test_cli_check_migrations(self, mock_wait_for_migrations): @@ -62,36 +67,73 @@ def test_cli_check_migrations(self, mock_wait_for_migrations): @pytest.mark.parametrize( "args, called_with", [ - ([], dict(to_revision=None, from_revision=None, show_sql_only=False)), - (["--show-sql-only"], dict(to_revision=None, from_revision=None, show_sql_only=True)), - (["--to-revision", "abc"], dict(to_revision="abc", from_revision=None, show_sql_only=False)), + ([], dict(to_revision=None, from_revision=None, show_sql_only=False, use_migration_files=False)), + ( + ["--show-sql-only"], + dict(to_revision=None, from_revision=None, show_sql_only=True, use_migration_files=False), + ), + ( + ["--to-revision", "abc"], + dict(to_revision="abc", from_revision=None, show_sql_only=False, use_migration_files=False), + ), ( ["--to-revision", "abc", "--show-sql-only"], - dict(to_revision="abc", from_revision=None, show_sql_only=True), + dict(to_revision="abc", from_revision=None, show_sql_only=True, use_migration_files=False), ), ( ["--to-version", "2.2.2"], - dict(to_revision="7b2661a43ba3", from_revision=None, show_sql_only=False), + dict( + to_revision="7b2661a43ba3", + from_revision=None, + show_sql_only=False, + use_migration_files=False, + ), ), ( ["--to-version", "2.2.2", "--show-sql-only"], - dict(to_revision="7b2661a43ba3", from_revision=None, show_sql_only=True), + dict( + to_revision="7b2661a43ba3", + from_revision=None, + show_sql_only=True, + use_migration_files=False, + ), ), ( ["--to-revision", "abc", "--from-revision", "abc123", "--show-sql-only"], - dict(to_revision="abc", from_revision="abc123", show_sql_only=True), + dict( + to_revision="abc", from_revision="abc123", show_sql_only=True, use_migration_files=False + ), ), ( ["--to-revision", "abc", "--from-version", "2.2.2", "--show-sql-only"], - dict(to_revision="abc", from_revision="7b2661a43ba3", show_sql_only=True), + dict( + to_revision="abc", + from_revision="7b2661a43ba3", + show_sql_only=True, + use_migration_files=False, + ), ), ( ["--to-version", "2.2.4", "--from-revision", "abc123", "--show-sql-only"], - dict(to_revision="587bdf053233", from_revision="abc123", show_sql_only=True), + dict( + to_revision="587bdf053233", + from_revision="abc123", + show_sql_only=True, + use_migration_files=False, + ), ), ( ["--to-version", "2.2.4", "--from-version", "2.2.2", "--show-sql-only"], - dict(to_revision="587bdf053233", from_revision="7b2661a43ba3", show_sql_only=True), + dict( + to_revision="587bdf053233", + from_revision="7b2661a43ba3", + show_sql_only=True, + use_migration_files=False, + ), + ), + ( + ["--use-migration-files", "--show-sql-only"], + dict(to_revision=None, from_revision=None, use_migration_files=True, show_sql_only=True), ), ], ) From 93357c3c9617bfe395f739a7dd664bec9bb597c6 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Fri, 3 May 2024 23:47:55 +0100 Subject: [PATCH 14/14] Apply suggestions from code review Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- ..._9_2_fix_inconsistency_between_ORM_and_migration_files.py | 5 +++++ docs/apache-airflow/img/airflow_erd.sha256 | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/airflow/migrations/versions/0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py b/airflow/migrations/versions/0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py index 07cb59842b78f..3d07b67e3a27b 100644 --- a/airflow/migrations/versions/0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py +++ b/airflow/migrations/versions/0142_2_9_2_fix_inconsistency_between_ORM_and_migration_files.py @@ -104,6 +104,7 @@ def upgrade(): op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS unique_conn_id") # Dropping and recreating cause there's no IF NOT EXISTS op.execute("ALTER TABLE connection DROP CONSTRAINT IF EXISTS connection_conn_id_uq") + with op.batch_alter_table("connection") as batch_op: batch_op.create_unique_constraint(batch_op.f("connection_conn_id_uq"), ["conn_id"]) @@ -114,6 +115,7 @@ def upgrade(): with op.batch_alter_table("task_instance") as batch_op: batch_op.drop_constraint("task_instance_dag_run_fkey", type_="foreignkey") + with op.batch_alter_table("task_reschedule") as batch_op: batch_op.drop_constraint("task_reschedule_dr_fkey", type_="foreignkey") @@ -267,9 +269,11 @@ def upgrade(): # below we drop and recreate the constraints because there's no IF NOT EXISTS op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_execution_date_key") op.execute("ALTER TABLE dag_run DROP CONSTRAINT IF EXISTS dag_run_dag_id_run_id_key") + with op.batch_alter_table("dag_run") as batch_op: batch_op.create_unique_constraint("dag_run_dag_id_execution_date_key", ["dag_id", "execution_date"]) batch_op.create_unique_constraint("dag_run_dag_id_run_id_key", ["dag_id", "run_id"]) + with op.batch_alter_table("task_instance") as batch_op: batch_op.create_foreign_key( "task_instance_dag_run_fkey", @@ -278,6 +282,7 @@ def upgrade(): ["dag_id", "run_id"], ondelete="CASCADE", ) + with op.batch_alter_table("task_reschedule") as batch_op: batch_op.create_foreign_key( "task_reschedule_dr_fkey", diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 8ed0236348d59..464463bb3f4b0 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -583c80d275d356b870b082ea4525e6c4be69b8fd3797a1404cc8fbf801b08a88 \ No newline at end of file +45e8aa557ff6c5c69995915206df2fef17495d5da04453e897cea73a8df0f492 \ No newline at end of file