Skip to content

[Merged by Bors] - Delete DB schema migrations for v11 and earlier #3761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
michaelsproul wants to merge 3 commits into from
Closed

[Merged by Bors] - Delete DB schema migrations for v11 and earlier #3761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
40c9441
Delete DB schema migrations for v11 and earlier
michaelsproul Nov 30, 2022
5106304
Update book
michaelsproul Nov 30, 2022
0f039da
Appease clippy
michaelsproul Nov 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 0 additions & 40 deletions beacon_node/beacon_chain/src/beacon_chain.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -997,46 +997,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(self.store.get_state(state_root, slot)?)
}

/// Run a function with mutable access to a state for `block_root`.
///
/// The primary purpose of this function is to borrow a state with its tree hash cache
/// from the snapshot cache *without moving it*. This means that calls to this function should
/// be kept to an absolute minimum, because holding the snapshot cache lock has the ability
/// to delay block import.
///
/// If there is no appropriate state in the snapshot cache then one will be loaded from disk.
/// If no state is found on disk then `Ok(None)` will be returned.
///
/// The 2nd parameter to the closure is a bool indicating whether the snapshot cache was used,
/// which can inform logging/metrics.
///
/// NOTE: the medium-term plan is to delete this function and the snapshot cache in favour
/// of `tree-states`, where all caches are CoW and everything is good in the world.
pub fn with_mutable_state_for_block<F, V, Payload: ExecPayload<T::EthSpec>>(
&self,
block: &SignedBeaconBlock<T::EthSpec, Payload>,
block_root: Hash256,
f: F,
) -> Result<Option<V>, Error>
where
F: FnOnce(&mut BeaconState<T::EthSpec>, bool) -> Result<V, Error>,
{
if let Some(state) = self
.snapshot_cache
.try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
.ok_or(Error::SnapshotCacheLockTimeout)?
.borrow_unadvanced_state_mut(block_root)
{
let cache_hit = true;
f(state, cache_hit).map(Some)
} else if let Some(mut state) = self.get_state(&block.state_root(), Some(block.slot()))? {
let cache_hit = false;
f(&mut state, cache_hit).map(Some)
} else {
Ok(None)
}
}

/// Return the sync committee at `slot + 1` from the canonical chain.
///
/// This is useful when dealing with sync committee messages, because messages are signed
Expand Down
22 changes: 7 additions & 15 deletions beacon_node/beacon_chain/src/beacon_fork_choice_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ pub fn get_effective_balances<T: EthSpec>(state: &BeaconState<T>) -> Vec<u64> {
}

#[superstruct(
variants(V1, V8),
variants(V8),
variant_attributes(derive(PartialEq, Clone, Debug, Encode, Decode)),
no_enum
)]
Expand All @@ -75,13 +75,11 @@ pub(crate) struct CacheItem {
pub(crate) type CacheItem = CacheItemV8;

#[superstruct(
variants(V1, V8),
variants(V8),
Copy link
Member

@macladson macladson Dec 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to keep the V8 variant here? I suppose it makes it easier for when it is updated in the future since the boilerplate is already there?
Just seems strange to have a V8 variant when there is no longer a V8 schema.

Copy link
Member Author

@michaelsproul michaelsproul Dec 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I was super tempted to delete this as well, but figured it might come in handy for future migrations

The naming thing does turn out a bit strange, because the names are the first version in which the thing is introduced for the reasons discussed here: https://github.com/sigp/lighthouse/blob/stable/beacon_node/beacon_chain/src/schema_change/README.md#naming-conventions

variant_attributes(derive(PartialEq, Clone, Default, Debug, Encode, Decode)),
no_enum
)]
pub struct BalancesCache {
#[superstruct(only(V1))]
pub(crate) items: Vec<CacheItemV1>,
#[superstruct(only(V8))]
pub(crate) items: Vec<CacheItemV8>,
}
Expand Down Expand Up @@ -366,26 +364,20 @@ where
}

/// A container which allows persisting the `BeaconForkChoiceStore` to the on-disk database.
#[superstruct(
variants(V1, V7, V8, V10, V11),
variant_attributes(derive(Encode, Decode)),
no_enum
)]
#[superstruct(variants(V11), variant_attributes(derive(Encode, Decode)), no_enum)]
pub struct PersistedForkChoiceStore {
#[superstruct(only(V1, V7))]
pub balances_cache: BalancesCacheV1,
#[superstruct(only(V8, V10, V11))]
#[superstruct(only(V11))]
pub balances_cache: BalancesCacheV8,
pub time: Slot,
pub finalized_checkpoint: Checkpoint,
pub justified_checkpoint: Checkpoint,
pub justified_balances: Vec<u64>,
pub best_justified_checkpoint: Checkpoint,
#[superstruct(only(V10, V11))]
#[superstruct(only(V11))]
pub unrealized_justified_checkpoint: Checkpoint,
#[superstruct(only(V10, V11))]
#[superstruct(only(V11))]
pub unrealized_finalized_checkpoint: Checkpoint,
#[superstruct(only(V7, V8, V10, V11))]
#[superstruct(only(V11))]
pub proposer_boost_root: Hash256,
#[superstruct(only(V11))]
pub equivocating_indices: BTreeSet<u64>,
Expand Down
23 changes: 2 additions & 21 deletions beacon_node/beacon_chain/src/persisted_fork_choice.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use crate::beacon_fork_choice_store::{
PersistedForkChoiceStoreV1, PersistedForkChoiceStoreV10, PersistedForkChoiceStoreV11,
PersistedForkChoiceStoreV7, PersistedForkChoiceStoreV8,
};
use crate::beacon_fork_choice_store::PersistedForkChoiceStoreV11;
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use store::{DBColumn, Error, StoreItem};
Expand All @@ -10,21 +7,9 @@ use superstruct::superstruct;
// If adding a new version you should update this type alias and fix the breakages.
pub type PersistedForkChoice = PersistedForkChoiceV11;

#[superstruct(
variants(V1, V7, V8, V10, V11),
variant_attributes(derive(Encode, Decode)),
no_enum
)]
#[superstruct(variants(V11), variant_attributes(derive(Encode, Decode)), no_enum)]
pub struct PersistedForkChoice {
pub fork_choice: fork_choice::PersistedForkChoice,
#[superstruct(only(V1))]
pub fork_choice_store: PersistedForkChoiceStoreV1,
#[superstruct(only(V7))]
pub fork_choice_store: PersistedForkChoiceStoreV7,
#[superstruct(only(V8))]
pub fork_choice_store: PersistedForkChoiceStoreV8,
#[superstruct(only(V10))]
pub fork_choice_store: PersistedForkChoiceStoreV10,
#[superstruct(only(V11))]
pub fork_choice_store: PersistedForkChoiceStoreV11,
}
Expand All @@ -47,8 +32,4 @@ macro_rules! impl_store_item {
};
}

impl_store_item!(PersistedForkChoiceV1);
impl_store_item!(PersistedForkChoiceV7);
impl_store_item!(PersistedForkChoiceV8);
impl_store_item!(PersistedForkChoiceV10);
impl_store_item!(PersistedForkChoiceV11);
163 changes: 3 additions & 160 deletions beacon_node/beacon_chain/src/schema_change.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,9 @@
//! Utilities for managing database schema changes.
mod migration_schema_v10;
mod migration_schema_v11;
mod migration_schema_v12;
mod migration_schema_v13;
mod migration_schema_v6;
mod migration_schema_v7;
mod migration_schema_v8;
mod migration_schema_v9;
mod types;

use crate::beacon_chain::{BeaconChainTypes, ETH1_CACHE_DB_KEY, FORK_CHOICE_DB_KEY};
use crate::beacon_chain::{BeaconChainTypes, ETH1_CACHE_DB_KEY};
use crate::eth1_chain::SszEth1;
use crate::persisted_fork_choice::{
PersistedForkChoiceV1, PersistedForkChoiceV10, PersistedForkChoiceV11, PersistedForkChoiceV7,
PersistedForkChoiceV8,
};
use crate::types::ChainSpec;
use slog::{warn, Logger};
use std::sync::Arc;
Expand All @@ -23,6 +12,7 @@ use store::metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION};
use store::{Error as StoreError, StoreItem};

/// Migrate the database from one schema version to another, applying all requisite mutations.
#[allow(clippy::only_used_in_recursion)] // spec is not used but likely to be used in future
pub fn migrate_schema<T: BeaconChainTypes>(
db: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
deposit_contract_deploy_block: u64,
Expand Down Expand Up @@ -62,156 +52,9 @@ pub fn migrate_schema<T: BeaconChainTypes>(
}

//
// Migrations from before SchemaVersion(5) are deprecated.
// Migrations from before SchemaVersion(11) are deprecated.
//

// Migration for adding `execution_status` field to the fork choice store.
(SchemaVersion(5), SchemaVersion(6)) => {
// Database operations to be done atomically
let mut ops = vec![];

// The top-level `PersistedForkChoice` struct is still V1 but will have its internal
// bytes for the fork choice updated to V6.
let fork_choice_opt = db.get_item::<PersistedForkChoiceV1>(&FORK_CHOICE_DB_KEY)?;
if let Some(mut persisted_fork_choice) = fork_choice_opt {
migration_schema_v6::update_execution_statuses::<T>(&mut persisted_fork_choice)
.map_err(StoreError::SchemaMigrationError)?;

// Store the converted fork choice store under the same key.
ops.push(persisted_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
}

db.store_schema_version_atomically(to, ops)?;

Ok(())
}
// 1. Add `proposer_boost_root`.
// 2. Update `justified_epoch` to `justified_checkpoint` and `finalized_epoch` to
// `finalized_checkpoint`.
// 3. This migration also includes a potential update to the justified
// checkpoint in case the fork choice store's justified checkpoint and finalized checkpoint
// combination does not actually exist for any blocks in fork choice. This was possible in
// the consensus spec prior to v1.1.6.
//
// Relevant issues:
//
// https://github.com/sigp/lighthouse/issues/2741
// https://github.com/ethereum/consensus-specs/pull/2727
// https://github.com/ethereum/consensus-specs/pull/2730
(SchemaVersion(6), SchemaVersion(7)) => {
// Database operations to be done atomically
let mut ops = vec![];

let fork_choice_opt = db.get_item::<PersistedForkChoiceV1>(&FORK_CHOICE_DB_KEY)?;
if let Some(persisted_fork_choice_v1) = fork_choice_opt {
// This migrates the `PersistedForkChoiceStore`, adding the `proposer_boost_root` field.
let mut persisted_fork_choice_v7 = persisted_fork_choice_v1.into();

let result = migration_schema_v7::update_fork_choice::<T>(
&mut persisted_fork_choice_v7,
db.clone(),
);

// Fall back to re-initializing fork choice from an anchor state if necessary.
if let Err(e) = result {
warn!(log, "Unable to migrate to database schema 7, re-initializing fork choice"; "error" => ?e);
migration_schema_v7::update_with_reinitialized_fork_choice::<T>(
&mut persisted_fork_choice_v7,
db.clone(),
spec,
)
.map_err(StoreError::SchemaMigrationError)?;
}

// Store the converted fork choice store under the same key.
ops.push(persisted_fork_choice_v7.as_kv_store_op(FORK_CHOICE_DB_KEY));
}

db.store_schema_version_atomically(to, ops)?;

Ok(())
}
// Migration to add an `epoch` key to the fork choice's balances cache.
(SchemaVersion(7), SchemaVersion(8)) => {
let mut ops = vec![];
let fork_choice_opt = db.get_item::<PersistedForkChoiceV7>(&FORK_CHOICE_DB_KEY)?;
if let Some(fork_choice) = fork_choice_opt {
let updated_fork_choice =
migration_schema_v8::update_fork_choice::<T>(fork_choice, db.clone())?;

ops.push(updated_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
}

db.store_schema_version_atomically(to, ops)?;

Ok(())
}
// Upgrade from v8 to v9 to separate the execution payloads into their own column.
(SchemaVersion(8), SchemaVersion(9)) => {
migration_schema_v9::upgrade_to_v9::<T>(db.clone(), log)?;
db.store_schema_version(to)
}
// Downgrade from v9 to v8 to ignore the separation of execution payloads
// NOTE: only works before the Bellatrix fork epoch.
(SchemaVersion(9), SchemaVersion(8)) => {
migration_schema_v9::downgrade_from_v9::<T>(db.clone(), log)?;
db.store_schema_version(to)
}
(SchemaVersion(9), SchemaVersion(10)) => {
let mut ops = vec![];
let fork_choice_opt = db.get_item::<PersistedForkChoiceV8>(&FORK_CHOICE_DB_KEY)?;
if let Some(fork_choice) = fork_choice_opt {
let updated_fork_choice = migration_schema_v10::update_fork_choice(fork_choice)?;

ops.push(updated_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
}

db.store_schema_version_atomically(to, ops)?;

Ok(())
}
(SchemaVersion(10), SchemaVersion(9)) => {
let mut ops = vec![];
let fork_choice_opt = db.get_item::<PersistedForkChoiceV10>(&FORK_CHOICE_DB_KEY)?;
if let Some(fork_choice) = fork_choice_opt {
let updated_fork_choice = migration_schema_v10::downgrade_fork_choice(fork_choice)?;

ops.push(updated_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
}

db.store_schema_version_atomically(to, ops)?;

Ok(())
}
// Upgrade from v10 to v11 adding support for equivocating indices to fork choice.
(SchemaVersion(10), SchemaVersion(11)) => {
let mut ops = vec![];
let fork_choice_opt = db.get_item::<PersistedForkChoiceV10>(&FORK_CHOICE_DB_KEY)?;
if let Some(fork_choice) = fork_choice_opt {
let updated_fork_choice = migration_schema_v11::update_fork_choice(fork_choice);

ops.push(updated_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
}

db.store_schema_version_atomically(to, ops)?;

Ok(())
}
// Downgrade from v11 to v10 removing support for equivocating indices from fork choice.
(SchemaVersion(11), SchemaVersion(10)) => {
let mut ops = vec![];
let fork_choice_opt = db.get_item::<PersistedForkChoiceV11>(&FORK_CHOICE_DB_KEY)?;
if let Some(fork_choice) = fork_choice_opt {
let updated_fork_choice =
migration_schema_v11::downgrade_fork_choice(fork_choice, log);

ops.push(updated_fork_choice.as_kv_store_op(FORK_CHOICE_DB_KEY));
}

db.store_schema_version_atomically(to, ops)?;

Ok(())
}
// Upgrade from v11 to v12 to store richer metadata in the attestation op pool.
(SchemaVersion(11), SchemaVersion(12)) => {
let ops = migration_schema_v12::upgrade_to_v12::<T>(db.clone(), log)?;
Expand Down
Loading