[CI] Fix maintenance scripts

bin/cleanup_jenkins.sh

@@ -5,10 +5,15 @@ CLIPPER_REGISTRY=clippertesting
 
 # best effort, ignore all errors
 try_cleanup() {
-    (docker ps | grep "$@" | awk '{ print $1 }' | xargs docker kill) || true
-    (docker container ls | grep "$@" | awk '{ print $3 }' | xargs docker container rm) || true
-    (docker image ls | grep "$@" | awk '{ print $3 }' | xargs docker image rm -f) || true
+    (docker ps --all | grep "$@" | awk '{ print $1 }' | xargs docker kill) || true
+    (docker container ls --all | grep "$@" | awk '{ print $3 }' | xargs docker container rm) || true
+    (docker image ls --all | grep "$@" | awk '{ print $3 }' | xargs docker image rm -f) || true
+    docker image prune -f
 }
 
-try_cleanup CLIPPER_REGISTRY > /dev/null
+try_cleanup_docker_volume() {
+    (docker volume ls -f dangling=true | awk '{ print $2 }' | xargs docker volume rm -f) || true
+}
+try_cleanup ${CLIPPER_REGISTRY} > /dev/null
 try_cleanup shipyard > /dev/null
+try_cleanup_docker_volume > /dev/null

bin/retry_with_timeout.py

@@ -3,6 +3,14 @@
 import sys
 import time
 from random import randint
+import logging
+
+logging.basicConfig(
+    format='%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
+    datefmt='%y-%m-%d:%H:%M:%S',
+    level=logging.INFO)
+
+logger = logging.getLogger(__name__)
 
 
 def timeout_to_float(s):
@@ -49,7 +57,7 @@ def timeout_to_float(s):
 
 def run_once_with_timeout(command_to_run, timeout):
     proc = subprocess.Popen(
-        command_to_run, stdout=sys.stdout, stderr=subprocess.STDOUT
+        command_to_run, stdout=sys.stdout, stderr=sys.stderr
     )
     start = time.time()
     while True:
@@ -68,23 +76,23 @@ def run_once_with_timeout(command_to_run, timeout):
 # the ports used by each test collide. To avoid this problem, wait for a random
 # amount of time and start the test.
 sleep_time = randint(60, 600)  # 1min ~ 10min
-print("Sleep {} secs before starting a test".format(sleep_time))
+logger.info("Sleep {} secs before starting a test".format(sleep_time))
 time.sleep(sleep_time)
 
 for try_num in range(args.retry + 1):
-    print(
+    logger.info(
         "Starting Trial {try_num} with timeout {timeout} seconds".format(
             try_num=try_num, timeout=timeout
         )
     )
     return_code = run_once_with_timeout(command_to_run, timeout)
     if return_code == 0:
-        print("Success!")
+        logger.info("Success!")
         sys.exit(0)
     else:
         sleep_time = randint(60, 600)  # 1min ~ 10min
-        print("Sleep {}".format(sleep_time))
+        logger.info("Sleep {}".format(sleep_time))
         time.sleep(sleep_time)
 
-print("All retry failed.")
+logger.info("All retry failed.")
 sys.exit(1)

bin/run_ci.sh

@@ -50,5 +50,5 @@ make -j -f CI_build.Makefile all
 
 # Run all test
 make -j10 -f CI_test.Makefile unittest
-make -j10 -f CI_test.Makefile integration_py2
-make -j10 -f CI_test.Makefile integration_py3
+make -j15 -f CI_test.Makefile integration_py2
+make -j15 -f CI_test.Makefile integration_py3