Merge branch 'master' into releng#1062-add-github-action-for-detect-secrets

Author: rupeshkumarsahu

.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.194.0/containers/cpp
+{
+	"name": "Erlang/OTP",
+	"build": {
+	    "dockerfile": "../.github/dockerfiles/Dockerfile.ubuntu-base",
+            "context": "../.github",
+            "args": {
+                "BASE": "mcr.microsoft.com/vscode/devcontainers/base:focal",
+                "USER": "vscode",
+                "GROUP": "vscode"
+            }
+	},
+	"runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-vscode.cpptools",
+		"erlang-ls.erlang-ls"	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "gcc -v",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode"
+}

.dir-locals.el

@@ -2,6 +2,7 @@
 (
  (erlang-mode (indent-tabs-mode . nil))
  (autoconf-mode (indent-tabs-mode . nil))
+ (m4-mode (indent-tabs-mode . nil))
  (java-mode (indent-tabs-mode . nil))
  (perl-mode (indent-tabs-mode . nil))
  (xml-mode (indent-tabs-mode . nil))

.github/dockerfiles/Dockerfile.32-bit

@@ -1,4 +1,5 @@
-FROM docker.pkg.github.com/erlang/otp/i386-debian-base
+ARG BASE=docker.pkg.github.com/erlang/otp/i386-debian-base
+FROM $BASE
 
 ARG MAKEFLAGS=-j4
 ENV MAKEFLAGS=$MAKEFLAGS \
@@ -16,14 +17,8 @@ ENV CFLAGS="-O2 -g -Werror"
 
 ## Configure, check that no application are disabled, then make and then build doc chunks
 RUN ./configure --with-ssl --prefix=/otp && \
-        if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi
-
-RUN make && make install && \
-        make install-docs DOC_TARGETS=chunks
+        if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \
+    make && make install
 
 ## Disable -Werror as testcases do not compile with it on
 ENV CFLAGS="-O2 -g"
-
-RUN TESTSUITE_ROOT=/tests ./otp_build tests
-
-ENTRYPOINT ["bash","-c"]

.github/dockerfiles/Dockerfile.64-bit

@@ -1,32 +1,24 @@
-FROM docker.pkg.github.com/erlang/otp/ubuntu-base
-## We do a SSA lint check here
-ENV ERL_COMPILER_OPTIONS=ssalint
+ARG BASE=docker.pkg.github.com/erlang/otp/ubuntu-base
+FROM $BASE
 
 ARG MAKEFLAGS=-j4
 ENV MAKEFLAGS=$MAKEFLAGS \
-        ERLC_USE_SERVER=yes \
-        ERL_TOP=/buildroot/otp \
-        PATH=/buildroot/otp/bin:$PATH
+    ERLC_USE_SERVER=yes \
+    ERL_TOP=/buildroot/otp \
+    PATH=/otp/bin:/buildroot/otp/bin:$PATH
 
 ARG ARCHIVE=./otp.tar.gz
 COPY $ARCHIVE /buildroot/otp.tar.gz
 RUN cd /buildroot && tar -xzf ./otp.tar.gz
 
 WORKDIR /buildroot/otp/
 
-ENV CC=clang CXX=clang++ \
-        CFLAGS="-O2 -g -Werror"
+ENV CFLAGS="-O2 -g -Werror"
 
 ## Configure, check that no application are disabled and then make
-# We need --with-ssl-lib-subdir=lib/x86_64-linux-gnu since clang does not
-# give us this information such as gcc does...
-RUN  ./configure --with-ssl --with-ssl-lib-subdir=lib/x86_64-linux-gnu --prefix=/otp && \
+RUN  ./configure --prefix=/otp && \
         if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \
-        make && make install
+        make && sudo make install
 
 ## Disable -Werror as testcases do not compile with it on
 ENV CFLAGS="-O2 -g"
-
-RUN TESTSUITE_ROOT=/tests ./otp_build tests
-
-ENTRYPOINT ["bash","-c"]

.github/dockerfiles/Dockerfile.clang

@@ -0,0 +1,31 @@
+ARG BASE=docker.pkg.github.com/erlang/otp/ubuntu-base
+FROM $BASE
+## We do a SSA lint check here
+ENV ERL_COMPILER_OPTIONS=ssalint
+
+ARG MAKEFLAGS=-j4
+ENV MAKEFLAGS=$MAKEFLAGS \
+    ERLC_USE_SERVER=yes \
+    ERL_TOP=/buildroot/otp \
+    PATH=/otp/bin:/buildroot/otp/bin:$PATH
+
+RUN sudo apt-get install -y clang
+
+ARG ARCHIVE=./otp.tar.gz
+COPY $ARCHIVE /buildroot/otp.tar.gz
+RUN cd /buildroot && tar -xzf ./otp.tar.gz
+
+WORKDIR /buildroot/otp/
+
+ENV CC=clang CXX=clang++ \
+        CFLAGS="-O2 -g -Werror"
+
+## Configure, check that no application are disabled and then make
+# We need --with-ssl-lib-subdir=lib/x86_64-linux-gnu since clang does not
+# give us this information such as gcc does...
+RUN  ./configure --with-ssl --with-ssl-lib-subdir=lib/x86_64-linux-gnu --prefix=/otp && \
+        if cat lib/*/CONF_INFO || cat lib/*/SKIP || cat lib/SKIP-APPLICATIONS; then exit 1; fi && \
+        make && sudo make install
+
+## Disable -Werror as testcases do not compile with it on
+ENV CFLAGS="-O2 -g"

.github/dockerfiles/Dockerfile.cross-compile

@@ -1,7 +1,8 @@
 ##
 ## This docker file will build Erlang on 32-bit to 64-bit x86
 ##
-FROM docker.pkg.github.com/erlang/otp/i386-debian-base as build
+ARG BASE=docker.pkg.github.com/erlang/otp/i386-debian-base
+FROM $BASE as build
 
 ARG MAKEFLAGS=-j4
 ENV MAKEFLAGS=$MAKEFLAGS \
@@ -17,13 +18,6 @@ RUN tar xzf ../otp.tar.gz
 ## Build the bootstrap system
 RUN cd $ERL_TOP && CFLAGS="-Wall -O2 -g" ./configure && make && make install
 
-## Build pre-build tar ball
-RUN cd $ERL_TOP && scripts/build-otp-tar -o /buildroot/otp_clean_src.tar.gz /buildroot/otp_src.tar.gz \
-        -b /buildroot/bootstrap/otp/ /buildroot/otp.tar.gz
-
-## Prepare for a new build using pre-built tar ball
-RUN cd ../ && tar -xzf ./otp_src.tar.gz
-
 ENV HOST=$HOST_TRIP \
         CC=$HOST_TRIP-gcc \
         CPPFLAGS="--sysroot=/buildroot/sysroot" \
@@ -39,15 +33,16 @@ ENV HOST=$HOST_TRIP \
         erl_xcomp_sysroot=/buildroot/sysroot \
         ERL_TOP=/buildroot/otp
 
+RUN cd /buildroot && tar xzf otp.tar.gz
+
 WORKDIR /buildroot/otp
 
 ## Build the cross system
 # We cannot use config.guess for --build since its value clashes with the
 # canonical value of host...
-RUN ./configure --prefix=/otp/ --host=$HOST --build=x86-pc-linux-gnu
-RUN OTP_SMALL_BUILD=true V=1 make
-
-RUN make odbc && make install
+RUN ./configure --prefix=/otp/ --host=$HOST --build=x86-pc-linux-gnu && \
+    OTP_SMALL_BUILD=true V=1 make && \
+    make install
 
 ## Build the cross tests
 ENV CFLAGS="--sysroot=/buildroot/sysroot -O2 -g"
@@ -64,5 +59,3 @@ COPY --from=build /otp /otp
 COPY --from=build /buildroot/otp/release/tests /tests
 
 ENV PATH=/otp/bin:$PATH
-
-ENTRYPOINT ["bash","-c"]

.github/dockerfiles/Dockerfile.documentation

@@ -1,15 +1,74 @@
 ##
 ## This docker file will build a base image for building Erlang/OTP
 ##
-FROM ubuntu
+ARG BASE=gitpod/workspace-full
+FROM $BASE
 
-ENV INSTALL_LIBS="zlib1g-dev libncurses5-dev libssl-dev unixodbc-dev libgmp3-dev libwxbase3.0-dev libwxgtk3.0-gtk3-dev libwxgtk-webview3.0-gtk3-dev libsctp-dev lksctp-tools"
+ENV INSTALL_LIBS="zlib1g-dev libncurses5-dev libssl-dev unixodbc-dev libsctp-dev lksctp-tools libgmp3-dev libwxbase3.0-dev libwxgtk3.0-gtk3-dev libwxgtk-webview3.0-gtk3-dev"
+
+ARG EXTRA_LIBS="erlang erlang-doc"
+
+USER root
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+RUN apt-get update && apt-get -y upgrade && \
+    apt-get install -y build-essential m4 autoconf fop xsltproc \
+    default-jdk libxml2-utils flex pkg-config \
+    unixodbc odbc-postgresql postgresql \
+    tzdata ssh openssh-server groff-base sudo gdb tinyproxy bind9 nsd expect vsftpd python \
+    linux-tools-common linux-tools-generic linux-tools-`uname -r` \
+    ${INSTALL_LIBS} && \
+    for lib in ${EXTRA_LIBS}; do apt-get install -y ${lib}; done && \
+    if [ ! -f /etc/apache2/apache2.conf ]; then apt-get install -y apache2; fi && \
+    sed -i 's@# en_US.UTF-8@en_US.UTF-8@g' /etc/locale.gen && locale-gen && \
+    update-alternatives --set wx-config /usr/lib/x86_64-linux-gnu/wx/config/gtk3-unicode-3.0
+
+## EXTRA_LIBS are installed using a for loop because of bugs in the erlang-doc deb package
+## Apache2 may already be installed, if so we do not want to install it again
+
+ENV LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
+
+ARG USER=gitpod
+ARG GROUP=gitpod
+ARG gid=10
+ARG uid=421
+
+RUN echo "Europe/Stockholm" > /etc/timezone && \
+    ln -snf /usr/share/zoneinfo/$(cat /etc/timezone) /etc/localtime && \
+    if [ ! -d /home/${USER} ]; then useradd -rm -d /home/${USER} -s /bin/sh -g ${gid} -G ${gid},sudo -u ${uid} ${USER}; fi && \
+    echo "${USER} ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/${USER} && \
+    echo "/buildroot/** r," >> /etc/apparmor.d/local/usr.sbin.named && \
+    echo "/tests/** r," >> /etc/apparmor.d/local/usr.sbin.named
+
+## Setup progres so that the odbc test can run
+USER postgres
+
+RUN service postgresql start && \
+    psql -c "CREATE USER odbctest WITH SUPERUSER PASSWORD 'odbctest';" && \
+    createdb -O odbctest odbctest && \
+    service postgresql stop
+
+COPY --chown=root:${GROUP} dockerfiles/odbc.ini /etc/
+COPY --chown=root:${GROUP} dockerfiles/odbcinst.ini /etc/
+
+USER ${USER}
+
+## Need to set USER and create a keygen file for ssh tests to pass
+ENV USER=${USER}
+RUN ssh-keygen -q -t rsa -N '' -f $HOME/.ssh/id_rsa && \
+    cp $HOME/.ssh/id_rsa.pub $HOME/.ssh/authorized_keys
+
+## Java and log4j are used by fop to build documentation
+COPY --chown=${USER}:${GROUP} dockerfiles/log4j.properties /home/${USER}/
+ENV OTP_STRICT_INSTALL=yes \
+    JAVA_ARGS="-Dlog4j.configuration=file://home/${USER}/log4j.properties"
+
+RUN sudo mkdir /buildroot /tests /otp && sudo chown ${USER}:${GROUP} /buildroot /tests /otp
+
+COPY --chown=${USER}:${GROUP} dockerfiles/init.sh /buildroot/
 
-RUN apt-get update && \
-        apt-get -y upgrade && \
-        apt-get install -y build-essential m4 autoconf fop xsltproc clang clang-format \
-        default-jdk libxml2-utils $INSTALL_LIBS
+## TODO: Build Erlang versions N, N-1 and N-2 for compatability testing.
 
-RUN update-alternatives --set wx-config /usr/lib/x86_64-linux-gnu/wx/config/gtk3-unicode-3.0
+ENTRYPOINT ["/buildroot/init.sh"]

.github/dockerfiles/Dockerfile.ubuntu-base

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+sudo mkdir -p -m0755 /var/run/sshd
+
+sudo /usr/sbin/sshd
+
+sudo service postgresql start
+
+sudo -E bash -c "apt-get update && apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r`"
+
+exec /bin/bash -c "$1"

.github/dockerfiles/init.sh

@@ -0,0 +1,9 @@
+log4j.rootLogger = INFO, console
+
+log4j.appender.console = org.apache.log4j.ConsoleAppender
+
+log4j.appender.console.layout = org.apache.log4j.PatternLayour
+log4j.appender.console.layout.ConversionPattern = %d [%t] %-5p %c{1} - %m%n
+
+log4j.logger.org.apache.fop = INFO
+log4j.logger.org.apache.fop.render = INFO