Skip to content
Build

Rework internal unit tests #6667

Sign in to view logs

Rework internal unit tests

Rework internal unit tests #6667

Workflow file for this run

.github/workflows/build.yml at a4bbfa7
name: Build
on:
push:
branches:
- dev
pull_request:
branches:
- dev
types: [opened, synchronize, reopened]
release:
types: [created]
jobs:
python-bindings:
name: Python Bindings (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v5
with:
python-version: "3.9"
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
- name: Build nDPI library
run: |
./autogen.sh && ./configure
make -j
sudo make install
- name: Generate Python bindings
run: |
pip install --upgrade pip
pip install -r python/requirements.txt
cd python
python setup.py install
cd ..
- name: Test Python Bindings
run: |
cd python
python tests.py
test-scripts:
name: Test Utils (ubuntu-latest)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install python3-netaddr git whois libxml2-utils
- name: Run Scripts
run: |
echo 'Running ./utils/bitcoinnodes.sh'
./utils/bitcoinnodes.sh >/dev/null
echo 'Running ./utils/get_routes_by_asn.sh AS714'
./utils/get_routes_by_asn.sh AS714 >/dev/null
echo 'Running ./utils/update_every_lists.sh'
./utils/update_every_lists.sh
echo 'Checking for changes in the git tree..'
git update-index --refresh || echo "::warning file=utils/update_every_lists.sh::Please re-run utils/update_every_lists.sh and commit any changes."
git diff-index --quiet HEAD -- || true
test:
name: ${{ matrix.os }} ${{ matrix.compiler }} ${{ matrix.cflags }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }}
runs-on: ${{ matrix.os }}
env:
CC: ${{ matrix.compiler }}
CFLAGS: -Werror -DNDPI_EXTENDED_SANITY_CHECKS ${{ matrix.cflags }}
strategy:
fail-fast: true
matrix:
# Only most "important" OS; the other ones are tested only via scheduled job
os: ["ubuntu-22.04", "ubuntu-24.04", "macOS-15", "macOS-26"]
compiler: ["cc"]
pcre: [""]
maxminddb: [""]
msan: [""]
cflags: ["-g -O2"] # Default value if you do not specify CFLAGS variable
include:
- compiler: "gcc-4.9" # "Oldest" gcc easily available. To simulate RHEL7
os: ubuntu-22.04
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
cflags: "-O0"
- compiler: "gcc-14" # "Newest" gcc easily available
os: ubuntu-24.04
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
cflags: "-O1"
- compiler: "clang-12" # "Oldest" clang easily available
os: ubuntu-22.04
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
cflags: "-Os"
- compiler: "clang-18" # "Newest" clang easily available. See also below...
os: ubuntu-24.04
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
cflags: "-O3"
- compiler: "cc"
os: macOS-latest
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: ""
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Install Ubuntu Prerequisites
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
sudo apt-get install rrdtool librrd-dev parallel
- name: Install Ubuntu Prerequisites [Mingw-w64] (runs only on ubuntu jobs)
if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') # Only on a few "standard" builds
run: |
sudo apt-get install gcc-mingw-w64 libc6-dev
- name: Install Ubuntu Prerequisites (libpcre2)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.pcre, '--with-pcre2')
run: |
sudo apt-get install libpcre3-dev
- name: Install Ubuntu Prerequisites (maxminddb)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.maxminddb, '--with-maxminddb')
run: |
sudo apt-get install libmaxminddb-dev
- name: Setup Ubuntu specified compiler
if: startsWith(matrix.os, 'ubuntu-22.04') && ! startsWith(matrix.compiler, 'cc')
run: |
# For gcc-4.9 (on ubuntu-22.04)
echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" | sudo tee -a /etc/apt/sources.list
echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" | sudo tee -a /etc/apt/sources.list
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
sudo apt-get update
sudo apt-get install ${{ matrix.compiler }}
- name: Installing MacOS prerequisites
if: startsWith(matrix.os, 'macOS')
run: |
# Avoid (re)installing pkg-config. See: https://github.com/actions/runner-images/issues/10984
brew install coreutils wdiff colordiff autoconf automake libtool gettext json-c rrdtool parallel
- name: Install MacOS Prerequisites (libpcre2)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.pcre, '--with-pcre2')
run: |
brew install pcre
- name: Install MacOS Prerequisites (maxminddb)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.maxminddb, '--with-maxminddb')
run: |
brew install libmaxminddb
- name: Configure nDPI
run: |
./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }}
- name: Build nDPI
run: |
make -j all
make -C example
make -C rrdtool
- name: Print nDPI long help
run: |
cd ./example && ./ndpiReader -H
- name: Install nDPI
run: |
DESTDIR=/tmp/ndpi make install
ls -alhHR /tmp/ndpi
file /tmp/ndpi/usr/include/ndpi/ndpi_api.h
test ! -r /tmp/ndpi/usr/include/ndpi/ndpi_private.h
file /tmp/ndpi/usr/lib/libndpi.a
file /tmp/ndpi/usr/lib/libndpi.so*
- name: Test nDPI [SYMBOLS]
if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') # Only on a few "standard" builds
run: |
./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; }
env:
NDPI_LIB: src/lib/libndpi.a
- name: Tests
run: |
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
./tests/do-unit.sh
./tests/do-dga.sh
- name: Generate/Verify tarball
if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') # Only on a few "standard" builds
run: |
make dist
./utils/verify_dist_tarball.sh
- name: Build Debian/Ubuntu package
if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') # Only on a few "standard" builds
run: |
cd packages/ubuntu
./configure --enable-no-sign
make
cd ../..
- name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs)
if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') # Only on a few "standard" builds
run: |
make distclean
./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --host=x86_64-w64-mingw32
make -j $(nproc) all
env:
CC:
test-windows:
name: ${{ matrix.os }} (msys2)
runs-on: ${{ matrix.os }}
env:
CFLAGS: -Werror -DNDPI_EXTENDED_SANITY_CHECKS -g -O2
strategy:
fail-fast: true
matrix:
os: ["windows-latest"]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Install Windows msys2 prerequisites
uses: msys2/setup-msys2@v2
with:
msystem: MINGW64
update: true
install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap parallel
- name: Configure nDPI on Windows msys2
run: |
msys2 -c './autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --disable-npcap'
- name: Build nDPI on Windows msys2
run: |
msys2 -c 'make -j all'
msys2 -c 'ldd ./example/ndpiReader.exe'
- name: Tests
run: |
# Don't know why but lately the script in parallel mode is stuck...
#msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh'
msys2 -c './tests/do.sh'
msys2 -c './tests/do-unit.sh'
msys2 -c './tests/do-dga.sh'
test-usdt:
name: USDT probes (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror -g -O2
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
sudo apt-get install systemtap-sdt-dev bpftrace pahole llvm
- name: Download latest bpftrace
run: |
curl -L -O https://github.com/bpftrace/bpftrace/releases/download/v0.24.2/bpftrace
chmod +x bpftrace
- name: Configure nDPI with USDT
run: |
# Workaround: C11 '_Atomic' type (from croaring code) is not fully supported for BPF CO-RE purposes
# This is needed only if you want to dereference `struct ndpi_flow_struct *`
# in the probe actions
./autogen.sh && CFLAGS="-DCROARING_ATOMIC_IMPL=1" ./configure --enable-option-checking=fatal --enable-debug-build --enable-debug-messages --enable-usdt-probes
- name: Build nDPI
run: |
make -j all
- name: Build bpftool
run: |
# Temporary workaround: some 6.14 kernels don't have bpftool (and perf) in their packages
# See: https://bugs.launchpad.net/ubuntu/+source/linux-hwe-6.14/+bug/2117147
# See: https://gist.github.com/karlivory/9111f906f4eb06370b2b237c62a6b00e
curl -L -O https://gist.githubusercontent.com/karlivory/9111f906f4eb06370b2b237c62a6b00e/raw/dbd24228dda79e05815531a1db643d0709301838/build-perf.sh
sudo bash build-perf.sh
- name: Generate header file for bpftrace
run: |
pahole -J src/lib/libndpi.so
pahole -J example/ndpiReader
bpftool btf dump file example/ndpiReader format c > ndpi_types.h
- name: Verify USDT probes are embedded (readelf)
run: |
echo "=== Probes in libndpi.so ==="
readelf -n src/lib/libndpi.so | grep -A4 stapsdt
echo "=== Probes in ndpiReader ==="
readelf -n example/ndpiReader | grep -A4 stapsdt
# Verify both probes are present
readelf -n example/ndpiReader | grep -q 'flow_classified'
readelf -n example/ndpiReader | grep -q 'hostname_set'
readelf -n example/ndpiReader | grep -q 'fragment_ipv4'
readelf -n example/ndpiReader | grep -q 'fragment_ipv6'
echo "All expected USDT probes found."
- name: List probes via bpftrace
run: |
sudo bpftrace -l "usdt:./example/ndpiReader:ndpi:*" | tee /tmp/probe_list.txt
grep -q 'flow_classified' /tmp/probe_list.txt
grep -q 'hostname_set' /tmp/probe_list.txt
grep -q 'fragment_ipv4' /tmp/probe_list.txt
grep -q 'fragment_ipv6' /tmp/probe_list.txt
echo "bpftrace can see all probes."
- name: Test flow_classified probe with ndpiReader
run: |
sudo bpftrace -e '
usdt:./example/ndpiReader:ndpi:flow_classified {
@flows = count();
@proto_master[arg0] = count();
@confidence[arg2] = count();
@category[arg3] = count();
}' -c './example/ndpiReader -q -i tests/pcap/http.pcapng' 2>&1 | tee /tmp/bpf_classified.txt
# Verify we actually traced some flows
grep -q '@flows:' /tmp/bpf_classified.txt
echo "flow_classified probe (scalar args): OK"
- name: Test flow_classified probe with flow pointer (arg4) via header
run: |
# Embedding .BTF section and reading it from bpftrace is still a bit buggy.
# Use explicit header, with latest bpftrace
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:flow_classified {
$flow = (struct ndpi_flow_struct *)arg4;
@flows = count();
if ($flow->risk != 0) { @risky[arg0] = count(); }
}' -c './example/ndpiReader -q -i tests/pcap/1kxun.pcap' 2>&1 | tee /tmp/bpf_classified_ptr.txt
grep -q '@flows:' /tmp/bpf_classified_ptr.txt
echo "flow_classified probe (struct dereference): OK"
- name: Test hostname_set probe with ndpiReader
run: |
READER=$(realpath example/ndpiReader)
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:hostname_set {
$flow = (struct ndpi_flow_struct *)arg1;
@hostnames = count();
@top[str(arg0), $flow->detected_protocol_stack[0]] = count();
}' -c './example/ndpiReader -q -i tests/pcap/tls_certificate_too_long.pcap' 2>&1 | tee /tmp/bpf_hostname.txt
# Verify we actually traced some hostnames
grep -q '@hostnames:' /tmp/bpf_hostname.txt
echo "hostname_set probe: OK"
- name: Test both hostname_set and flow_classified simultaneously
run: |
sudo bpftrace -e '
usdt:./example/ndpiReader:ndpi:hostname_set {
@hostnames = count();
}
usdt:./example/ndpiReader:ndpi:flow_classified {
@classified[arg0] = count();
}' -c './example/ndpiReader -q -i tests/pcap/dns.pcap' 2>&1 | tee /tmp/bpf_both.txt
grep -q '@classified' /tmp/bpf_both.txt
grep -q '@hostnames' /tmp/bpf_both.txt
echo "Both probes fired successfully."
- name: Test fragment_ipv4 probe with ndpiReader
run: |
READER=$(realpath example/ndpiReader)
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:fragment_ipv4 {
$iph = (struct ndpi_iphdr *)arg0;
@frags = count();
@top[ntop($iph->saddr)] = count();
}' -c './example/ndpiReader -q -i tests/pcap/ip_fragmented_garbage.pcap' 2>&1 | tee /tmp/frag_ipv4.txt
# Verify we actually traced some hostnames
grep -q '@frags:' /tmp/frag_ipv4.txt
grep -q '@top' /tmp/frag_ipv4.txt
echo "fragment_ipv4 probe: OK"
- name: Test fragment_ipv6 probe with ndpiReader
run: |
READER=$(realpath example/ndpiReader)
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:fragment_ipv6 {
$ip6h = (struct ndpi_ipv6hdr *)arg0;
@frags = count();
@top[ntop($ip6h->ip6_src.u6_addr.u6_addr8)] = count();
}' -c './example/ndpiReader -q -i tests/pcap/dns_fragmented.pcap' 2>&1 | tee /tmp/frag_ipv6.txt
# Verify we actually traced some hostnames
grep -q '@frags:' /tmp/frag_ipv6.txt
grep -q '@top' /tmp/frag_ipv6.txt
echo "fragment_ipv6 probe: OK"