Handle journal_v2 file creation failure due to OOM (netdata#19965)

Author: stelfrag

src/database/engine/cache.c

@@ -2552,7 +2552,7 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
     pgc_queue_unlock(cache, &cache->hot);
 
     // callback
-    cb(section, datafile_fileno, type, JudyL_metrics, JudyL_extents_pos, count_of_unique_extents, count_of_unique_metrics, count_of_unique_pages, data);
+    bool success = cb(section, datafile_fileno, type, JudyL_metrics, JudyL_extents_pos, count_of_unique_extents, count_of_unique_metrics, count_of_unique_pages, data);
 
     {
         Pvoid_t *PValue1;
@@ -2567,12 +2567,14 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
             while ((PValue2 = JudyLFirstThenNext(mi->JudyL_pages_by_start_time, &start_time, &start_time_first))) {
                 struct jv2_page_info *pi = *PValue2;
 
-                // balance-parents: transition from hot to clean directly
                 yield_the_processor(); // do not lock too aggressively
-                page_set_clean(cache, pi->page, true, false, PGC_QUEUE_LOCK_PRIO_LOW);
-                page_transition_unlock(cache, pi->page);
-                page_release(cache, pi->page, true);
+                if (likely(success))
+                    page_set_clean(cache, pi->page, true, false, PGC_QUEUE_LOCK_PRIO_LOW);
+                else
+                    page_flag_clear(pi->page, PGC_PAGE_IS_BEING_MIGRATED_TO_V2);
 
+                page_transition_unlock(cache, pi->page);
+                page_release(cache, pi->page, success);
                 // before balance-parents:
                 // page_transition_unlock(cache, pi->page);
                 // pgc_page_hot_to_dirty_and_release(cache, pi->page, true);

src/database/engine/cache.h

@@ -227,7 +227,7 @@ int64_t pgc_get_wanted_cache_size(PGC *cache);
 void pgc_page_hot_set_end_time_s(PGC *cache, PGC_PAGE *page, time_t end_time_s, size_t additional_bytes);
 bool pgc_page_to_clean_evict_or_release(PGC *cache, PGC_PAGE *page);
 
-typedef void (*migrate_to_v2_callback)(Word_t section, unsigned datafile_fileno, uint8_t type, Pvoid_t JudyL_metrics, Pvoid_t JudyL_extents_pos, size_t count_of_unique_extents, size_t count_of_unique_metrics, size_t count_of_unique_pages, void *data);
+typedef bool (*migrate_to_v2_callback)(Word_t section, unsigned datafile_fileno, uint8_t type, Pvoid_t JudyL_metrics, Pvoid_t JudyL_extents_pos, size_t count_of_unique_extents, size_t count_of_unique_metrics, size_t count_of_unique_pages, void *data);
 void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_fileno, uint8_t type, migrate_to_v2_callback cb, void *data);
 void pgc_open_evict_clean_pages_of_datafile(PGC *cache, struct rrdengine_datafile *datafile);
 size_t pgc_count_clean_pages_having_data_ptr(PGC *cache, Word_t section, void *ptr);

src/database/engine/journalfile.c

@@ -1293,7 +1293,7 @@ static void *journalfile_v2_write_descriptors(struct journal_v2_header *j2_heade
 // startup  : if the migration is done during agent startup
 //            this will allow us to optimize certain things
 
-void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno __maybe_unused, uint8_t type __maybe_unused,
+bool journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno __maybe_unused, uint8_t type __maybe_unused,
                                         Pvoid_t JudyL_metrics, Pvoid_t JudyL_extents_pos,
                                         size_t number_of_extents, size_t number_of_metrics, size_t number_of_pages, void *user_data)
 {
@@ -1346,8 +1346,10 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno
 
     int fd_v2;
     uint8_t *data_start = nd_mmap_advanced(path, total_file_size, MAP_SHARED, 0, false, true, &fd_v2);
-    if(!data_start)
-        out_of_memory(__FUNCTION__, total_file_size, path);
+    if(!data_start) {
+        nd_log_daemon(NDLP_WARNING, "DBENGINE: Failed to allocate %"PRIu64" bytes of memory for journal file '%s'. Will retry later", total_file_size, path);
+        return false;
+    }
 
     memset(data_start, 0, extent_offset);
 
@@ -1492,7 +1494,7 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno
         internal_error(true, "DBENGINE: ACTIVATING NEW INDEX JNL %llu", (now_monotonic_usec() - start_loading) / USEC_PER_MS);
         ctx_current_disk_space_increase(ctx, total_file_size);
         freez(uuid_list);
-        return;
+        return true;
     }
     else {
         netdata_log_info("DBENGINE: failed to build index '%s', file will be skipped", path);
@@ -1506,7 +1508,7 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno
     freez(uuid_list);
 
     if (likely(resize_file_to == total_file_size))
-        return;
+        return true;
 
     int ret = truncate(path, (long) resize_file_to);
     if (ret < 0) {
@@ -1516,6 +1518,8 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno
     }
     else
         ctx_current_disk_space_increase(ctx, resize_file_to);
+
+    return true;
 }
 
 int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile,

src/database/engine/journalfile.h

@@ -264,7 +264,7 @@ int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfil
                      struct rrdengine_datafile *datafile);
 void journalfile_v2_populate_retention_to_mrg(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile);
 
-void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno __maybe_unused, uint8_t type __maybe_unused,
+bool journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno __maybe_unused, uint8_t type __maybe_unused,
                                         Pvoid_t JudyL_metrics, Pvoid_t JudyL_extents_pos,
                                         size_t number_of_extents, size_t number_of_metrics, size_t number_of_pages, void *user_data);