Merge branch 'ai-docs-updates' of github.com:netdata/netdata into ai-docs-updates

Author: shyamvalsan

CHANGELOG.md

@@ -6,6 +6,11 @@
 
 **Merged pull requests:**
 
+- Revert "ml: implement fixed time-based training windows \(\#20638\)" [\#21045](https://github.com/netdata/netdata/pull/21045) ([vkalintiris](https://github.com/vkalintiris))
+- chore\(go.d/ddsnmp\): Improve profile sorting by match specificity [\#21042](https://github.com/netdata/netdata/pull/21042) ([ilyam8](https://github.com/ilyam8))
+- Add documentation on using custom CA certificates to Learn [\#21041](https://github.com/netdata/netdata/pull/21041) ([ralphm](https://github.com/ralphm))
+- Context loading priority to vnodes [\#21040](https://github.com/netdata/netdata/pull/21040) ([stelfrag](https://github.com/stelfrag))
+- improve\(go.d/ddsnmp\): switch profile matching to `selector` [\#21039](https://github.com/netdata/netdata/pull/21039) ([ilyam8](https://github.com/ilyam8))
 - fix\(docs\): update mermaid diagrams leftovers plus syntax issues [\#21034](https://github.com/netdata/netdata/pull/21034) ([kanelatechnical](https://github.com/kanelatechnical))
 - docs: fix mdx parsing scalability.md [\#21032](https://github.com/netdata/netdata/pull/21032) ([ilyam8](https://github.com/ilyam8))
 - Revert "chore\(docs\): rename REST API sidebar to Netdata APIs" [\#21031](https://github.com/netdata/netdata/pull/21031) ([ilyam8](https://github.com/ilyam8))
@@ -442,12 +447,6 @@
 - improve\(go.d/snmp profiles\): simplify \_generic-if.yaml and add interface type tags [\#20505](https://github.com/netdata/netdata/pull/20505) ([ilyam8](https://github.com/ilyam8))
 - fix snmp prof mikrotik mem tagging [\#20504](https://github.com/netdata/netdata/pull/20504) ([ilyam8](https://github.com/ilyam8))
 - feat\(go.d/ddsnmp\): make SNMP profile collection configurable [\#20503](https://github.com/netdata/netdata/pull/20503) ([ilyam8](https://github.com/ilyam8))
-- Use ARAL for labels [\#20502](https://github.com/netdata/netdata/pull/20502) ([stelfrag](https://github.com/stelfrag))
-- SNMP: audiocodes profile [\#20501](https://github.com/netdata/netdata/pull/20501) ([Ancairon](https://github.com/Ancairon))
-- chore\(go.d/ddsnmp\): better label values sanitization [\#20500](https://github.com/netdata/netdata/pull/20500) ([ilyam8](https://github.com/ilyam8))
-- SNMP: second pass of aruba profiles [\#20499](https://github.com/netdata/netdata/pull/20499) ([Ancairon](https://github.com/Ancairon))
-- SNMP: Arista profiles [\#20498](https://github.com/netdata/netdata/pull/20498) ([Ancairon](https://github.com/Ancairon))
-- chore\(go.d/ddsnmp\): fix table metrics again [\#20497](https://github.com/netdata/netdata/pull/20497) ([ilyam8](https://github.com/ilyam8))
 
 ## [v2.5.4](https://github.com/netdata/netdata/tree/v2.5.4) (2025-06-24)
 

docs/.map/map.csv

@@ -58,6 +58,7 @@ https://github.com/netdata/netdata/edit/master/docs/netdata-agent/configuration/
 https://github.com/netdata/netdata/edit/master/docs/netdata-agent/configuration/running-the-netdata-agent-behind-a-reverse-proxy/Running-behind-h2o.md,H2O,Published,Netdata Agent/Configuration/Securing Agents/Running the Agent behind a reverse proxy,
 https://github.com/netdata/netdata/edit/master/docs/netdata-agent/configuration/optimize-the-netdata-agents-performance.md,Performance Optimization,Published,Netdata Agent/Configuration,"While the Netdata Agent is designed to monitor a system with only 1% CPU, you can optimize its performance for low-resource systems."
 https://github.com/netdata/netdata/edit/master/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts.md,"Organize systems, metrics, and alerts",Published,Netdata Agent/Configuration,
+https://github.com/netdata/netdata/edit/master/docs/netdata-agent/configuration/using-custom-ca-certificates-with-netdata.md,"Using custom CA certificates with Netdata",Published,Netdata Agent/Configuration,
 https://github.com/netdata/netdata/edit/master/src/daemon/README.md,Daemon,Published,Netdata Agent,
 https://github.com/netdata/netdata/edit/master/src/database/README.md,Database,Published,Netdata Agent,
 https://github.com/netdata/netdata/edit/master/src/libnetdata/log/README.md,Logging,Published,Netdata Agent,

docs/netdata-agent/configuration/using-custom-ca-certificates-with-netdata.md

@@ -116,6 +116,6 @@ Windows. Instead, certificates must be installed into the bundled MSYS2 environm
 using the following instructions:
 
 1. Ensure the certificate file to be installed is in PEM or DER format.
-2. Copy the certificate file to `C:\Program FIles\Netdata\etc\pki\ca-trust\source\anchors`. You may need to create
+2. Copy the certificate file to `C:\Program Files\Netdata\etc\pki\ca-trust\source\anchors`. You may need to create
    this directory.
 3. In an administrative command prompt, run `C:\Program Files\Netdata\usr\bin\update-ca-trust.exe`

packaging/version

@@ -1 +1 @@
-v2.6.0-323-nightly
+v2.6.0-330-nightly

src/database/engine/mrg-internals.h

@@ -232,16 +232,21 @@ static bool metric_release(MRG *mrg, METRIC *metric) {
 
     if (refcount_release(&metric->refcount) == 0) {
         // we are the last user
-        bool already_deleted = __atomic_load_n(&metric->deleted, __ATOMIC_ACQUIRE);
-        if (already_deleted || !acquired_metric_has_retention(mrg, metric)) {
-            if (!already_deleted) {
-                 acquired_for_deletion_metric_delete(mrg, metric);
+        if (!acquired_metric_has_retention(mrg, metric)) {
+            // This metric is eligible for deletion.
+            // Atomically check and set the 'deleted' flag.
+            // If __atomic_test_and_set returns 'true', it means the flag was already set.
+            if (!__atomic_test_and_set(&metric->deleted, __ATOMIC_ACQ_REL)) {
+                // We won the race. The flag was 'false' and we set it to 'true'.
+                // We are now responsible for deletion.
+                acquired_for_deletion_metric_delete(mrg, metric);
+                uuidmap_free(metric->uuid);
+                aral_freez(mrg->index[partition].aral, metric);
+                __atomic_sub_fetch(&mrg->index[partition].stats.entries_acquired, 1, __ATOMIC_RELAXED);
+                __atomic_sub_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED);
+                return true;
             }
-            uuidmap_free(metric->uuid);
-            aral_freez(mrg->index[partition].aral, metric);
-            __atomic_sub_fetch(&mrg->index[partition].stats.entries_acquired, 1, __ATOMIC_RELAXED);
-            __atomic_sub_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED);
-            return true;
+            // Another thread is already deleting it. nothing to do
         }
     }
 

src/database/sqlite/sqlite_aclk.c

@@ -92,9 +92,14 @@ enum {
     IDX_IS_REGISTERED,
 };
 
+struct children {
+    int vnodes;
+    int normal;
+};
+
 static int create_host_callback(void *data, int argc, char **argv, char **column)
 {
-    int *number_of_chidren = data;
+    struct children *node_data = data;
     UNUSED(argc);
     UNUSED(column);
 
@@ -175,7 +180,10 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
         pulse_host_status(host, 0, 0); // this will detect the receiver status
     }
 
-    (*number_of_chidren)++;
+    if (IS_VIRTUAL_HOST_OS(host))
+        node_data->vnodes++;
+    else
+        node_data->normal++;
 
 #ifdef NETDATA_INTERNAL_CHECKS
     char node_str[UUID_STR_LEN] = "<none>";
@@ -973,26 +981,41 @@ void create_aclk_config(RRDHOST *host, nd_uuid_t *host_uuid __maybe_unused, nd_u
     "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni "                                                     \
     "WHERE h.host_id = ni.host_id AND ni.node_id IS NOT NULL"
 
+
+uv_sem_t ctx_sem;
+
 void aclk_synchronization_init(void)
 {
     char *err_msg = NULL;
     int rc;
 
     nd_log_daemon(NDLP_INFO, "Creating archived hosts");
-    int number_of_children = 0;
-    rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_HOSTS, create_host_callback, &number_of_children, &err_msg);
+    struct children node_data = { 0, 0};
+
+    rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_HOSTS, create_host_callback, &node_data, &err_msg);
 
     if (rc != SQLITE_OK) {
         nd_log_daemon(NDLP_ERR, "SQLite error when loading archived hosts, rc = %d (%s)", rc, err_msg);
         sqlite3_free(err_msg);
     }
 
-    nd_log_daemon(NDLP_INFO, "Created %d archived hosts", number_of_children);
+    nd_log_daemon(
+        NDLP_INFO,
+        "Created %d archived hosts (%d children and %d vnodes)",
+        node_data.normal + node_data.vnodes,
+        node_data.normal,
+        node_data.vnodes);
+
+    bool sem_init = true;
+    uv_sem_init(&ctx_sem, 0);
+
     // Trigger host context load for hosts that have been created
     if (unlikely(!metadata_queue_load_host_context())) {
         nd_log_daemon(NDLP_WARNING, "Failed to queue command to load contexts for archived hosts");
         // Reset context load flag so that contexts will be loaded on demand
         reset_host_context_load_flag();
+        uv_sem_destroy(&ctx_sem);
+        sem_init = false;
     }
 
     rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_INSTANCES, aclk_config_parameters, NULL, &err_msg);
@@ -1004,9 +1027,28 @@ void aclk_synchronization_init(void)
 
     aclk_initialize_event_loop();
 
-    if (!number_of_children)
+    if (!(node_data.normal + node_data.vnodes))
         aclk_queue_node_info(localhost, true);
 
+    if (sem_init) {
+        int finished_vnodes = 0;
+        time_t deadline = now_realtime_sec() + 60;  // hard timeput to avoid infinite block
+        while (finished_vnodes < node_data.vnodes) {
+            if (uv_sem_trywait(&ctx_sem) == 0) {
+                finished_vnodes++;
+                continue;
+            }
+
+            if (now_realtime_sec() >= deadline) {
+                nd_log_daemon(NDLP_WARNING, "Vnodes context load still in progress, continue with agent start");
+                break;
+            }
+            sleep_usec(100 * USEC_PER_MS);
+        }
+        if (finished_vnodes == node_data.vnodes) {
+            uv_sem_destroy(&ctx_sem);
+        }
+    }
     nd_log_daemon(NDLP_INFO, "ACLK sync initialization completed");
 }
 

src/database/sqlite/sqlite_metadata.c

@@ -1741,6 +1741,7 @@ __thread sqlite3 *db_meta_thread = NULL;
 __thread sqlite3 *db_context_thread = NULL;
 __thread bool main_context_thread = false;
 
+extern uv_sem_t ctx_sem;
 static void restore_host_context(void *arg)
 {
     struct host_context_load_thread *hclt = arg;
@@ -1787,6 +1788,10 @@ static void restore_host_context(void *arg)
 
     aclk_queue_node_info(host, false);
 
+    if (IS_VIRTUAL_HOST_OS(host)) {
+        uv_sem_post(&ctx_sem);
+    }
+
     // Check and clear the thread local variables
     if (!main_context_thread) {
         db_meta_thread = NULL;
@@ -1847,7 +1852,7 @@ void reset_host_context_load_flag()
     RRDHOST *host;
     dfe_start_reentrant(rrdhost_root_index, host)
     {
-        rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD);
+        rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD);
     }
     dfe_done(host);
 }
@@ -1876,36 +1881,44 @@ static void ctx_hosts_load(uv_work_t *req)
     size_t host_count = 0;
     size_t sync_exec = 0;
     size_t async_exec = 0;
-    dfe_start_reentrant(rrdhost_root_index, host) {
-        if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))
-            continue;
 
-        if (unlikely(SHUTDOWN_REQUESTED(config)))
-            break;
+    for (int pass=0 ; pass < 2 ; pass++) {
+        dfe_start_reentrant(rrdhost_root_index, host) {
+            // pass 0 will do vnodes (skip the rest)
+            // pass 1 will do the rest (skip vnodes)
+            if (pass == IS_VIRTUAL_HOST_OS(host))
+                continue;
 
-        nd_log_daemon(NDLP_DEBUG, "Loading context for host %s", rrdhost_hostname(host));
-
-        int rc = 0;
-        bool thread_found = cleanup_finished_threads(hclt, max_threads, false, &thread_index);
-        if (thread_found) {
-            __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);
-            hclt[thread_index].host = host;
-            hclt[thread_index].thread = nd_thread_create("CTXLOAD", NETDATA_THREAD_OPTION_DEFAULT, restore_host_context, &hclt[thread_index]);
-            rc = (hclt[thread_index].thread == NULL);
-            async_exec += (rc == 0);
-            // if it failed, mark the thread slot as free
-            if (rc)
-                __atomic_store_n(&hclt[thread_index].busy, false, __ATOMIC_RELAXED);
-        }
-        // if single thread, thread creation failure or failure tofind slot
-        if (rc || !thread_found) {
-            sync_exec++;
-            struct host_context_load_thread hclt_sync = {.host = host};
-            restore_host_context(&hclt_sync);
+            if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))
+                continue;
+
+            if (unlikely(SHUTDOWN_REQUESTED(config)))
+                break;
+
+            nd_log_daemon(NDLP_DEBUG, "Loading context for host %s", rrdhost_hostname(host));
+
+            int rc = 0;
+            bool thread_found = cleanup_finished_threads(hclt, max_threads, false, &thread_index);
+            if (thread_found) {
+                __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);
+                hclt[thread_index].host = host;
+                hclt[thread_index].thread = nd_thread_create("CTXLOAD", NETDATA_THREAD_OPTION_DEFAULT, restore_host_context, &hclt[thread_index]);
+                rc = (hclt[thread_index].thread == NULL);
+                async_exec += (rc == 0);
+                // if it failed, mark the thread slot as free
+                if (rc)
+                    __atomic_store_n(&hclt[thread_index].busy, false, __ATOMIC_RELAXED);
+            }
+            // if single thread, thread creation failure or failure tofind slot
+            if (rc || !thread_found) {
+                sync_exec++;
+                struct host_context_load_thread hclt_sync = {.host = host};
+                restore_host_context(&hclt_sync);
+            }
+            host_count++;
         }
-        host_count++;
+        dfe_done(host);
     }
-    dfe_done(host);
 
     bool should_clean_threads = cleanup_finished_threads(hclt, max_threads, true, NULL);
 

src/ml/ml.cc

@@ -56,13 +56,13 @@ ml_dimension_calculated_numbers(ml_worker_t *worker, ml_dimension_t *dim)
     training_response.first_entry_on_response = rrddim_first_entry_s_of_tier(dim->rd, 0);
     training_response.last_entry_on_response = rrddim_last_entry_s_of_tier(dim->rd, 0);
 
-    size_t min_n = Cfg.min_training_window / dim->rd->rrdset->update_every;
-    size_t max_n = Cfg.training_window / dim->rd->rrdset->update_every;
+    size_t min_n = Cfg.min_train_samples;
+    size_t max_n = Cfg.max_train_samples;
 
     // Figure out what our time window should be.
     training_response.query_before_t = training_response.last_entry_on_response;
     training_response.query_after_t = std::max(
-        training_response.query_before_t - Cfg.training_window,  // Fixed time window
+        training_response.query_before_t - static_cast<time_t>((max_n - 1) * dim->rd->rrdset->update_every),
         training_response.first_entry_on_response
     );
 
@@ -379,7 +379,7 @@ int ml_dimension_load_models(RRDDIM *rd, sqlite3_stmt **active_stmt) {
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_int64(res, ++param, now_realtime_sec() - (Cfg.num_models_to_use * Cfg.train_every));
+    rc = sqlite3_bind_int64(res, ++param, now_realtime_sec() - (Cfg.num_models_to_use * Cfg.max_train_samples));
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
@@ -674,29 +674,13 @@ ml_dimension_train_model(ml_worker_t *worker, ml_dimension_t *dim)
         memcpy(worker->scratch_training_cns, worker->training_cns,
                training_response.total_values * sizeof(calculated_number_t));
 
-        size_t smoothing_window = (dim->rd->rrdset->update_every > nd_profile.update_every) ? 1 : Cfg.max_samples_to_smooth;
-
         ml_features_t features = {
-            Cfg.diff_n, smoothing_window, Cfg.lag_n,
+            Cfg.diff_n, Cfg.smooth_n, Cfg.lag_n,
             worker->scratch_training_cns, training_response.total_values,
             worker->training_cns, training_response.total_values,
             worker->training_samples
         };
-        
-        // Calculate dynamic sampling ratio based on expected output size
-        // After diff and smooth, we'll have approximately this many vectors
-        size_t expected_vectors = training_response.total_values;
-        if (Cfg.diff_n > 0) expected_vectors--;
-        if (smoothing_window > 1) expected_vectors = expected_vectors - smoothing_window + 1;
-        expected_vectors = expected_vectors - Cfg.lag_n;
-        
-        double sampling_ratio = 1.0;
-        if (expected_vectors > Cfg.max_training_vectors) {
-            sampling_ratio = (double)Cfg.max_training_vectors / expected_vectors;
-        }
-
-        // Apply sampling during lag feature extraction
-        ml_features_preprocess(&features, sampling_ratio);
+        ml_features_preprocess(&features);
 
         ml_kmeans_init(&dim->kmeans);
         ml_kmeans_train(&dim->kmeans, &features,  Cfg.max_kmeans_iters, training_response.query_after_t, training_response.query_before_t);
@@ -722,7 +706,7 @@ ml_dimension_predict(ml_dimension_t *dim, calculated_number_t value, bool exists
     }
 
     // Save the value and return if we don't have enough values for a sample
-    unsigned n = Cfg.diff_n + Cfg.max_samples_to_smooth + Cfg.lag_n;
+    unsigned n = Cfg.diff_n + Cfg.smooth_n + Cfg.lag_n;
     if (dim->cns.size() < n) {
         dim->cns.push_back(value);
         return false;
@@ -747,11 +731,11 @@ ml_dimension_predict(ml_dimension_t *dim, calculated_number_t value, bool exists
     memcpy(dst_cns, dim->cns.data(), n * sizeof(calculated_number_t));
 
     ml_features_t features = {
-        Cfg.diff_n, Cfg.max_samples_to_smooth, Cfg.lag_n,
+        Cfg.diff_n, Cfg.smooth_n, Cfg.lag_n,
         dst_cns, n, src_cns, n,
         dim->feature
     };
-    ml_features_preprocess(&features, 1.0);
+    ml_features_preprocess(&features);
 
     /*
      * Lock to predict