Revert "OTEL logs (netdata#21356)"

This reverts commit d0905d9.

Author: vkalintiris

.github/labeler.yml

@@ -195,8 +195,7 @@ collectors/systemd-journal:
   - any:
       - changed-files:
           - any-glob-to-any-file:
-              - src/crates/netdata-log-viewer/**
-              - docs/logs/**
+              - src/collectors/systemd-journal.plugin/**
 
 collectors/tc:
   - any:

.github/workflows/docker.yml

@@ -248,7 +248,6 @@ jobs:
           build-args: |
             OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
             EXTRA_INSTALL_OPTS=${{ needs.file-check.outputs.skip-go }}
-            BUILD_ARCH=${{ matrix.arch }}
             BUILD_VERSION=test
             BUILD_DATE=${{ github.event.repository.updated_at }}
       - name: Test Image
@@ -370,7 +369,6 @@ jobs:
           outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true
           build-args: |
             OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
-            BUILD_ARCH=${{ matrix.arch }}
             BUILD_VERSION=${{ github.event.inputs.version }}
             BUILD_DATE=${{ github.event.repository.updated_at }}
       - name: Export Digest
@@ -528,7 +526,6 @@ jobs:
           cache-from: type=local,src=/tmp/build-cache
           build-args: |
             OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
-            BUILD_ARCH=${{ matrix.arch }}
             BUILD_VERSION=${{ github.event.inputs.version }}
             BUILD_DATE=${{ github.event.repository.updated_at }}
           outputs: type=image,name=quay.io/netdata/netdata,push-by-digest=true,name-canonical=true,push=true
@@ -688,7 +685,6 @@ jobs:
           cache-from: type=local,src=/tmp/build-cache
           build-args: |
             OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
-            BUILD_ARCH=${{ matrix.arch }}
             BUILD_VERSION=${{ github.event.inputs.version }}
             BUILD_DATE=${{ github.event.repository.updated_at }}
           outputs: type=image,name=ghcr.io/netdata/netdata,push-by-digest=true,name-canonical=true,push=true

CMakeLists.txt

@@ -132,7 +132,6 @@ mark_as_advanced(ENABLE_DASHBOARD)
 # Data collection plugins
 option(ENABLE_PLUGIN_GO "Enable metric collectors written in Go" ${DEFAULT_FEATURE_STATE})
 option(ENABLE_PLUGIN_OTEL "Enable collection of OpenTelemetry metrics and logs" ${DEFAULT_FEATURE_STATE})
-option(ENABLE_NETDATA_JOURNAL_FILE_READER "Enable journal viewer plugin" ${DEFAULT_FEATURE_STATE})
 option(ENABLE_PLUGIN_PYTHON "Enable metric collectors written in Python" ${DEFAULT_FEATURE_STATE})
 
 cmake_dependent_option(ENABLE_PLUGIN_APPS "Enable per-process resource usage monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX OR OS_FREEBSD OR OS_MACOS OR OS_WINDOWS" False)
@@ -151,6 +150,7 @@ cmake_dependent_option(ENABLE_PLUGIN_NETWORK_VIEWER "Enable network viewer funct
 cmake_dependent_option(ENABLE_PLUGIN_NFACCT "Enable Linux NFACCT metric collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
 cmake_dependent_option(ENABLE_PLUGIN_PERF "Enable Linux performance counter monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
 cmake_dependent_option(ENABLE_PLUGIN_SLABINFO "Enable Linux kernel SLAB allocator monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
+cmake_dependent_option(ENABLE_PLUGIN_SYSTEMD_JOURNAL "Enable systemd journal log collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
 cmake_dependent_option(ENABLE_PLUGIN_SYSTEMD_UNITS "Enable systemd units information collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
 cmake_dependent_option(ENABLE_PLUGIN_XENSTAT "Enable Xen domain monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
 
@@ -184,6 +184,7 @@ mark_as_advanced(ENABLE_LIBUNWIND)
 cmake_dependent_option(FORCE_LEGACY_LIBBPF "Force usage of libbpf 0.0.9 instead of the latest version." False "ENABLE_PLUGIN_EBPF" False)
 mark_as_advanced(FORCE_LEGACY_LIBBPF)
 
+cmake_dependent_option(ENABLE_NETDATA_JOURNAL_FILE_READER "Enable netdata's journal file reader implementation" False "ENABLE_PLUGIN_SYSTEMD_JOURNAL" False)
 
 # Setup Rust/Corrosion for plugins that need it
 if(ENABLE_NETDATA_JOURNAL_FILE_READER OR ENABLE_PLUGIN_OTEL)
@@ -194,33 +195,8 @@ if(ENABLE_NETDATA_JOURNAL_FILE_READER OR ENABLE_PLUGIN_OTEL)
         GIT_TAG f3b91559efca32c6b54837866ef35ba98ff5b2ca # stable/v0.5
     )
     FetchContent_MakeAvailable(Corrosion)
-
-    if(ENABLE_NETDATA_JOURNAL_FILE_READER AND ENABLE_PLUGIN_OTEL)
-        corrosion_import_crate(MANIFEST_PATH src/crates/Cargo.toml
-                               CRATES journal-viewer-plugin otel-plugin)
-    elseif(ENABLE_NETDATA_JOURNAL_FILE_READER)
-        corrosion_import_crate(MANIFEST_PATH src/crates/Cargo.toml
-                               CRATES journal-viewer-plugin)
-    elseif(ENABLE_PLUGIN_OTEL)
-        corrosion_import_crate(MANIFEST_PATH src/crates/Cargo.toml
-                               CRATES otel-plugin)
-    endif()
-
-    if(ENABLE_NETDATA_JOURNAL_FILE_READER)
-      if(STATIC_BUILD)
-        corrosion_add_target_rustflags(journal-viewer-plugin --cfg=tracing_unstable "-C" "target-feature=+crt-static")
-      else()
-        corrosion_add_target_rustflags(journal-viewer-plugin --cfg=tracing_unstable)
-      endif()
-    endif()
-
-    if(ENABLE_PLUGIN_OTEL)
-      if(STATIC_BUILD)
-        corrosion_add_target_rustflags(otel-plugin --cfg=tracing_unstable "-C" "target-feature=+crt-static")
-      else()
-        corrosion_add_target_rustflags(otel-plugin --cfg=tracing_unstable)
-      endif()
-    endif()
+    corrosion_import_crate(MANIFEST_PATH src/crates/jf/Cargo.toml
+                           CRATES journal_reader_ffi otel-plugin)
 endif()
 
 option(ENABLE_MIMALLOC "Enable mimalloc allocator" OFF)
@@ -1698,6 +1674,23 @@ set(STATSD_PLUGIN_FILES
         src/collectors/statsd.plugin/statsd.c
 )
 
+set(SYSTEMD_JOURNAL_PLUGIN_FILES
+        src/collectors/systemd-journal.plugin/systemd-journal.c
+        src/collectors/systemd-journal.plugin/systemd-journal-fstat.c
+        src/collectors/systemd-journal.plugin/systemd-internals.h
+        src/collectors/systemd-journal.plugin/systemd-main.c
+        src/collectors/systemd-journal.plugin/systemd-journal.c
+        src/collectors/systemd-journal.plugin/systemd-journal-annotations.c
+        src/collectors/systemd-journal.plugin/systemd-journal-files.c
+        src/collectors/systemd-journal.plugin/systemd-journal-watcher.c
+        src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
+        src/collectors/systemd-journal.plugin/provider/netdata_provider.c
+        src/collectors/systemd-journal.plugin/provider/netdata_provider.h
+        src/collectors/systemd-journal.plugin/provider/rust_provider.h
+        src/libnetdata/os/system-maps/system-services.h
+        src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
+)
+
 set(SYSTEMD_UNITS_PLUGIN_FILES
         src/collectors/systemd-units.plugin/plugin_systemd_units.c
 )
@@ -2752,6 +2745,36 @@ if(ENABLE_PLUGIN_CGROUP_NETWORK)
             DESTINATION usr/libexec/netdata/plugins.d)
 endif()
 
+# Enable rust implementation if we don't have systemd and we want the journal plugin
+if(ENABLE_PLUGIN_SYSTEMD_JOURNAL AND NOT SYSTEMD_FOUND)
+        if (NOT ENABLE_NETDATA_JOURNAL_FILE_READER)
+                message(WARNING "Systemd journal package not found, will try netdata's journal reader which requires cargo.")
+                set(ENABLE_NETDATA_JOURNAL_FILE_READER True)
+        endif()
+endif()
+
+if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
+        add_executable(systemd-journal.plugin ${SYSTEMD_JOURNAL_PLUGIN_FILES})
+
+        if(ENABLE_NETDATA_JOURNAL_FILE_READER)
+                target_compile_definitions(systemd-journal.plugin PRIVATE HAVE_RUST_PROVIDER)
+                target_link_libraries(systemd-journal.plugin journal_reader_ffi)
+        endif()
+
+        target_link_libraries(systemd-journal.plugin libnetdata)
+
+        install(TARGETS systemd-journal.plugin
+                COMPONENT plugin-systemd-journal
+                DESTINATION usr/libexec/netdata/plugins.d)
+
+        if(BUILD_FOR_PACKAGING)
+                install(FILES
+                        ${PKG_FILES_PATH}/copyright
+                        COMPONENT plugin-systemd-journal
+                        DESTINATION usr/share/doc/netdata-plugin-systemd-journal)
+        endif()
+endif()
+
 if(ENABLE_PLUGIN_SYSTEMD_UNITS)
         if(NOT SYSTEMD_FOUND)
                 message(FATAL_ERROR "Systemd units plugin requires systemd, but systemd was not found.")
@@ -3219,28 +3242,17 @@ if(ENABLE_PLUGIN_OTEL)
                       RUNTIME DESTINATION usr/libexec/netdata/plugins.d
                       COMPONENT plugin-otel)
 
-    install(FILES src/crates/netdata-otel/otel-plugin/configs/otel.yaml
+    install(FILES src/crates/jf/otel-plugin/configs/otel.yml
             COMPONENT plugin-otel
             DESTINATION usr/lib/netdata/conf.d)
 
-    install(FILES src/crates/netdata-otel/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yaml
+    install(FILES src/crates/jf/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yml
             COMPONENT plugin-otel
             DESTINATION usr/lib/netdata/conf.d/otel.d/v1/metrics)
 
     install(DIRECTORY COMPONENT plugin-otel DESTINATION etc/netdata/otel.d/v1/metrics)
 endif()
 
-#
-# journal-viewer-plugin
-#
-# Handle journal-viewer plugin
-if(ENABLE_NETDATA_JOURNAL_FILE_READER)
-    corrosion_install(TARGETS journal-viewer-plugin
-                      PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE
-                      RUNTIME DESTINATION usr/libexec/netdata/plugins.d
-                      COMPONENT plugin-journal-viewer)
-endif()
-
 #
 # Generate config file
 #
@@ -3315,16 +3327,6 @@ set(logdir_POST "${LOG_DIR}")
 set(netdata_user_POST "${NETDATA_USER}")
 set(netdata_group_POST "${NETDATA_USER}")
 
-if(ENABLE_NETDATA_JOURNAL_FILE_READER)
-    configure_file(src/crates/netdata-log-viewer/journal-viewer-plugin/configs/journal-viewer.yaml.in
-                   src/crates/netdata-log-viewer/journal-viewer-plugin/configs/journal-viewer.yaml
-                   @ONLY)
-
-    install(FILES ${CMAKE_BINARY_DIR}/src/crates/netdata-log-viewer/journal-viewer-plugin/configs/journal-viewer.yaml
-            COMPONENT plugin-journal-viewer
-            DESTINATION usr/lib/netdata/conf.d)
-endif()
-
 if(NOT OS_WINDOWS)
         configure_file(src/claim/netdata-claim.sh.in src/claim/netdata-claim.sh @ONLY)
         install(PROGRAMS
@@ -3743,6 +3745,7 @@ endif()
 
 # confs
 install(FILES
+        src/collectors/systemd-journal.plugin/schema.d/systemd-journal%3Amonitored-directories.json
         src/health/schema.d/health%3Aalert%3Aprototype.json
         COMPONENT netdata
         DESTINATION usr/lib/netdata/conf.d/schema.d)

docs/.map/map.csv

@@ -117,7 +117,7 @@ https://github.com/netdata/netdata/edit/master/src/collectors/README.md,Collecti
 https://github.com/netdata/netdata/edit/master/src/collectors/REFERENCE.md,Collectors configuration,Published,Collecting Metrics,,
 https://github.com/netdata/agent-service-discovery/edit/master/README.md,Service discovery,Published,Collecting Metrics,,
 https://github.com/netdata/netdata/edit/master/src/collectors/statsd.plugin/README.md,StatsD,Published,Collecting Metrics,,
-https://github.com/netdata/netdata/edit/master/src/crates/netdata-otel/otel-plugin/README.md,OpenTelemetry Metrics,Published,Collecting Metrics/OpenTelemetry,,Ingesting storing and visualizing OpenTelemetry metrics
+https://github.com/netdata/netdata/edit/master/src/crates/jf/otel-plugin/README.md,OpenTelemetry Metrics,Published,Collecting Metrics/OpenTelemetry,,Ingesting storing and visualizing OpenTelemetry metrics
 https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/metrics-centralization-points/README.md,Metrics Centralization Points,Published,Netdata Parents/Metrics Centralization Points,,
 https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/metrics-centralization-points/configuration.md,Configuring Metrics Centralization Points,Published,Netdata Parents/Metrics Centralization Points,,
 https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/metrics-centralization-points/sizing-netdata-parents.md,Sizing Netdata Parents,Published,Netdata Parents/Metrics Centralization Points,,
@@ -139,15 +139,15 @@ https://github.com/netdata/netdata/edit/master/src/web/api/exporters/shell/READM
 ,,,,,
 ,,,,,
 logs_integrations,,,,,
-https://github.com/netdata/netdata/edit/master/docs/logs/README.md,Journal Viewer Plugin,Published,Logs/Systemd Journal Logs,,View and analyze logs available in systemd journal
-https://github.com/netdata/netdata/edit/master/docs/logs/forward_secure_sealing.md,Forward Secure Sealing (FSS) in Systemd-Journal,Published,Logs/Systemd Journal Logs,,
+https://github.com/netdata/netdata/edit/master/src/collectors/systemd-journal.plugin/README.md,Systemd Journal Plugin Reference,Published,Logs/Systemd Journal Logs,,View and analyze logs available in systemd journal
+https://github.com/netdata/netdata/edit/master/src/collectors/systemd-journal.plugin/forward_secure_sealing.md,Forward Secure Sealing (FSS) in Systemd-Journal,Published,Logs/Systemd Journal Logs,,
 https://github.com/netdata/netdata/edit/master/src/collectors/windows-events.plugin/README.md,Windows Events Plugin Reference,Published,Logs/Windows Event Logs,,
 https://github.com/netdata/netdata/edit/master/src/collectors/log2journal/README.md,log2journal,Published,Logs/log2journal,,
 https://github.com/netdata/netdata/edit/master/src/libnetdata/log/systemd-cat-native.md,systemd-cat-native,Published,Logs/systemd-cat-native,,
 https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/README.md,Logs Centralization Points with systemd-journald,Published,Logs/Logs Centralization Points with systemd-journald,,
 https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/passive-journal-centralization-with-encryption-using-self-signed-certificates.md,Passive journal centralization with encryption using self-signed certificates,Published,Logs/Logs Centralization Points with systemd-journald,,
 https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/passive-journal-centralization-without-encryption.md,Passive journal centralization without encryption,Published,Logs/Logs Centralization Points with systemd-journald,,
-https://github.com/netdata/netdata/edit/master/docs/logs/active_journal_centralization_guide_no_encryption.md,Active journal source without encryption,Published,Logs/Logs Centralization Points with systemd-journald,,
+https://github.com/netdata/netdata/edit/master/src/collectors/systemd-journal.plugin/active_journal_centralization_guide_no_encryption.md,Active journal source without encryption,Published,Logs/Logs Centralization Points with systemd-journald,,
 ,,,,,
 ,,,,,
 https://github.com/netdata/netdata/edit/master/docs/top-monitoring-netdata-functions.md,Top Consumers,Published,Top Consumers,,Present the Netdata Functions what these are and why they should be used.

docs/category-overview-pages/working-with-logs.md

@@ -2,7 +2,7 @@
 
 This section talks about the ways Netdata collects and visualizes logs.
 
-The [journal viewer plugin](/docs/logs/README.md) is the core Netdata component for reading systemd journal logs.
+The [systemd journal plugin](/src/collectors/systemd-journal.plugin) is the core Netdata component for reading systemd journal logs.
 
 For structured logs, Netdata provides tools like [log2journal](/src/collectors/log2journal/README.md) and [systemd-cat-native](/src/libnetdata/log/systemd-cat-native.md) to convert them into compatible systemd journal entries.
 

docs/dashboards-and-charts/logs-tab.md

@@ -1,16 +1,16 @@
 # Logs tab
 
-The Logs tab is using the [`journal-viewer` plugin](/docs/logs/README.md), to present a structured view into your infrastructure's `systemd` logs.
+The Logs tab is using the [`systemd` journal plugin](/src/collectors/systemd-journal.plugin/README.md), to present a structured view into your infrastructure's `systemd` logs.
 
 We have a thorough section explaining how you can [work with logs](https://learn.netdata.cloud/docs/logs), detailing how the plugin works, and what other utilities are used under the hood to provide you with the visualizations and the log entries.
 
-The [`journal-viewer` plugin](/docs/logs/README.md) documentation has information about:
+The [`systemd` journal plugin](/src/collectors/systemd-journal.plugin/README.md) documentation has information about:
 
-- [Key features the plugin provides](/docs/logs/README.md#key-features)
-- [Journal sources](/docs/logs/README.md#journal-sources)
-- [Journal fields](/docs/logs/README.md#journal-fields)
-- [Full-text search](/docs/logs/README.md#full-text-search)
-- [Query performance](/docs/logs/README.md#query-performance)
-- [Performance at scale](/docs/logs/README.md#performance-at-scale)
+- [Key features the plugin provides](/src/collectors/systemd-journal.plugin/README.md#key-features)
+- [Journal sources](/src/collectors/systemd-journal.plugin/README.md#journal-sources)
+- [Journal fields](/src/collectors/systemd-journal.plugin/README.md#journal-fields)
+- [Full-text search](/src/collectors/systemd-journal.plugin/README.md#full-text-search)
+- [Query performance](/src/collectors/systemd-journal.plugin/README.md#query-performance)
+- [Performance at scale](/src/collectors/systemd-journal.plugin/README.md#performance-at-scale)
 
 We recommend you to read through that document, to better understand how the plugin and the visualizations work.

docs/developer-and-contributor-corner/dyncfg.md

@@ -150,12 +150,13 @@ The health module manages alert definitions through DynCfg:
 </details>
 
 <details>
-<summary><strong>journal-viewer-plugin (External Plugin, Rust)</strong></summary><br/>
+<summary><strong>systemd-journal.plugin (External Plugin, C)</strong></summary><br/>
 
-External Rust plugin that provides systemd journal log viewing and analysis:
+External C plugin that manages journal directory configurations:
 
-- **Location**: `src/crates/netdata-log-viewer/journal-viewer-plugin/`
-- **Use Case**: Viewing and analyzing systemd journal logs
+- **File**: `src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c`
+- **Pattern**: SINGLE configuration type
+- **Use Case**: Managing journal directory paths
 
 <br/>
 </details>

docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/passive-journal-centralization-with-encryption-using-self-signed-certificates.md

@@ -20,7 +20,7 @@ This means that if both certificates are issued by the same certificate authorit
 
 ## Self-signed certificates
 
-To simplify the process of creating and managing self-signed certificates, we have created [this bash script](https://github.com/netdata/netdata/blob/master/docs/logs/systemd-journal-self-signed-certs.sh).
+To simplify the process of creating and managing self-signed certificates, we have created [this bash script](https://github.com/netdata/netdata/blob/master/src/collectors/systemd-journal.plugin/systemd-journal-self-signed-certs.sh).
 
 This helps to also automate the distribution of the certificates to your servers (it generates a new bash script for each of your servers, which includes everything required, including the certificates).
 
@@ -34,7 +34,7 @@ On the server that will issue the certificates (usually the centralization serve
 sudo apt-get install systemd-journal-remote openssl
 
 # download the script and make it executable
-curl >systemd-journal-self-signed-certs.sh "https://raw.githubusercontent.com/netdata/netdata/master/docs/logs/systemd-journal-self-signed-certs.sh"
+curl >systemd-journal-self-signed-certs.sh "https://raw.githubusercontent.com/netdata/netdata/master/src/collectors/systemd-journal.plugin/systemd-journal-self-signed-certs.sh"
 chmod 750 systemd-journal-self-signed-certs.sh
 ```