Merge pull request #914 from keyboardio/f/parallel-smoke-examples

move example smoking into make for parallelization
4 years ago · 9511140e52
parent 25f0d2fc66 db93473e12
commit 9511140e52
3 changed files with 71 additions and 46 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -10,20 +10,27 @@ addons:
 env:
  global:
  - LC_ALL: C
+
 git:
  quiet: true
 before_install: 
-  - ccache --set-config=compiler_check=content -M 5G -F 0
+  - ccache --set-config=compiler_check=content -M 1G -F 0

 install:
-  - git clone --depth 1 --recurse-submodules https://github.com/keyboardio/Kaleidoscope-Bundle-Keyboardio ../hardware/keyboardio
+  - git clone --depth 1 --recurse-submodules --shallow-submodules git://github.com/keyboardio/Kaleidoscope-Bundle-Keyboardio ../hardware/keyboardio
 ## We delete the Bundle's version of Kaleidoscope, and symlink ourselves in.
 ## This makes sure we're using the current version of the library.
  - rm -rf ../hardware/keyboardio/avr/libraries/Kaleidoscope
  - ln -s $(pwd) ../hardware/keyboardio/avr/libraries/Kaleidoscope
+  - export KALEIDOSCOPE_TEMP_PATH=$TRAVIS_BUILD_DIR/.kaleidoscope-build-cache
+  - export BOARD_HARDWARE_PATH=$TRAVIS_BUILD_DIR/../hardware
+  - export ARDUINO_PATH=$TRAVIS_BUILD_DIR/arduino-1.8.13
+  - make adjust-git-timestamps
+  - make travis-install-arduino 
+  - make prepare-ccache
 jobs:
  include:
-    - env: TEST_TARGET=travis-smoke-examples
+    - env: TEST_TARGET=smoke-sketches
    - env: TEST_TARGET=travis-simulator-tests
    - env: TEST_TARGET=cpplint
    - env: TEST_TARGET=find-filename-conflicts 
@ -31,7 +38,7 @@ jobs:
    - env: TEST_TARGET=travis-check-astyle
 script:
  - unset CC
-  - make $TEST_TARGET KALEIDOSCOPE_TEMP_PATH=$(pwd)/.kaleidoscope-build-cache BOARD_HARDWARE_PATH=$(pwd)/../hardware ARDUINO_PATH="$(pwd)/arduino-1.8.13"
+  - make -j 2 $TEST_TARGET 
 notifications:
  email:
    on_success: change
--- a/bin/kaleidoscope-builder
+++ b/bin/kaleidoscope-builder
@ -26,15 +26,15 @@ absolute_filename() {


 build_version () {
-    GIT_VERSION="$(cd "$(find_sketch)"; if [ -d .git ]; then echo -n '-g' && git describe --abbrev=4 --dirty --always; fi)"
+    GIT_VERSION="$(cd "${SKETCH_DIR}"; if [ -d .git ]; then echo -n '-g' && git describe --abbrev=4 --dirty --always; fi)"
    LIB_PROPERTIES_PATH="${LIB_PROPERTIES_PATH:-"../.."}"
-    LIB_VERSION="$(cd "$(find_sketch)"; (grep version= "${LIB_PROPERTIES_PATH}/library.properties" 2>/dev/null || echo version=0.0.0) | cut -d= -f2)${GIT_VERSION}"
+    LIB_VERSION="$(cd "${SKETCH_DIR}"; (grep version= "${LIB_PROPERTIES_PATH}/library.properties" 2>/dev/null || echo version=0.0.0) | cut -d= -f2)${GIT_VERSION}"
 }

 build_paths() {
    # We need that echo because we\re piping to cksum
    # shellcheck disable=SC2005
-    SKETCH_IDENTIFIER="$(echo "$(absolute_filename "$(find_sketch)/${SKETCH}.ino")" | cksum | cut -d ' ' -f 1)-${SKETCH}.ino"
+    SKETCH_IDENTIFIER="$(echo "$(absolute_filename "${SKETCH_DIR}/${SKETCH}.ino")" | cksum | cut -d ' ' -f 1)-${SKETCH}.ino"
    KALEIDOSCOPE_TEMP_PATH="${KALEIDOSCOPE_TEMP_PATH:-${TMPDIR:-/tmp}/kaleidoscope-${USER}}"


@ -67,30 +67,30 @@ enable_ccache () {
    if [ -z "${CCACHE_NOT_SUPPORTED}" ] && [ "$(command -v ccache)" ]; then
      if ! [ -d "$CCACHE_WRAPPER_PATH" ]; then
         mkdir -p "$CCACHE_WRAPPER_PATH"
-      fi

-      if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${C_COMPILER_BASENAME}" ]; then
-         ln -s "$(command -v ccache)" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${C_COMPILER_BASENAME}"
-      fi
+          if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${C_COMPILER_BASENAME}" ]; then
+             ln -s "$(command -v ccache)" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${C_COMPILER_BASENAME}"
+          fi
    
-      if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${CXX_COMPILER_BASENAME}" ]; then
-         ln -s "$(command -v ccache)" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${CXX_COMPILER_BASENAME}"
-      fi
+          if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${CXX_COMPILER_BASENAME}" ]; then
+             ln -s "$(command -v ccache)" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${CXX_COMPILER_BASENAME}"
+          fi
    
-      if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}nm" ]; then
-         ln -s "${AVR_NM}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}nm"
-      fi
+          if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}nm" ]; then
+             ln -s "${AVR_NM}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}nm"
+          fi
    
-      if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}objcopy" ]; then
-         ln -s "${AVR_OBJCOPY}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}objcopy"
-      fi
+          if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}objcopy" ]; then
+             ln -s "${AVR_OBJCOPY}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}objcopy"
+          fi
    
-      if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}ar" ]; then
-         ln -s "${AVR_AR}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}ar"
-      fi
+          if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}ar" ]; then
+             ln -s "${AVR_AR}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}ar"
+          fi
    
-      if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}size" ]; then
-         ln -s "${AVR_SIZE}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}size"
+          if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}size" ]; then
+             ln -s "${AVR_SIZE}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}size"
+          fi
      fi

      export CCACHE_PATH=${COMPILER_PATH}/
@ -137,11 +137,16 @@ find_sketch () {
        exit 1
    fi
  
-    for path in "examples/${LIBRARY}" \
+    SKETCH_DIR="$SKETCH"
+    SKETCH_FILE=$(basename "$SKETCH")
+
+    for path in "${SKETCH_DIR}" \
+		    "examples/${LIBRARY}" \
                    "src" \
 		    "."; do
-        if [ -f "${path}/${SKETCH}.ino" ]; then
-            echo "${path}"
+        if [ -f "${path}/${SKETCH_FILE}.ino" ]; then
+      	    SKETCH_DIR="${path}"
+	    SKETCH="${SKETCH_FILE}"
            return
        fi
    done
@ -324,14 +329,13 @@ hex_with_bootloader () {
 }

 maybe_build () {
+    find_sketch
    build_version
    build_paths
    build_filenames

    if [ ! -e "${HEX_FILE_PATH}" ]; then
 	build "$@"
-    else
-      SKETCH_DIR="$(find_sketch)"
    fi

 }
@ -341,7 +345,14 @@ build () {
    size "$@"
 }

+
+prepare_ccache () {
+    build_paths
+    enable_ccache
+}
+
 compile () {
+    find_sketch
    build_version
    build_paths
    build_filenames
@ -349,8 +360,6 @@ compile () {

    install -d "${OUTPUT_PATH}"

-    SKETCH_DIR="$(find_sketch)"
-
    echo "Building ${SKETCH_DIR}/${SKETCH}"

    # This is defined in the (optional) user config.
@ -440,7 +449,7 @@ compile () {
    FQBN="${SAVED_FQBN}"
 }

-_find_all () {
+find_all_sketches () {
    for plugin in ./*.ino \
  	              $([ -d examples ] && find examples -name '*.ino') \
                  src/*.ino; do
@ -461,7 +470,7 @@ _find_all () {
 }

 build_all () {
-    plugins="$(_find_all)"
+    plugins="$(find_all_sketches)"

    for plugin in ${plugins}; do
        export SKETCH="${plugin}"
@ -472,7 +481,7 @@ build_all () {


 compile_all () {
-    plugins="$(_find_all)"
+    plugins="$(find_all_sketches)"

    for plugin in ${plugins}; do
        export SKETCH="${plugin}"
@ -737,12 +746,6 @@ done

 LIBRARY="${SKETCH}"

-case "${SKETCH}" in
-    */*)
-        SKETCH="$(basename "${SKETCH}")"
-        ;;
-esac
-
 export SKETCH
 export LIBRARY

--- a/bin/set-timestamps-from-git
+++ b/bin/set-timestamps-from-git
@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+# This script sets all of the files inside src and example to have mtimes
+# that match the times of the last git commit that touched each file
+
+# This can be useful when build tools depend on file timestamps to
+# make caching decisions
+
+find src examples -type f -exec sh -c '
+  timestamp=$(git log --pretty=format:%ad --date=format:%Y%m%d%H%M.%S -n 1 HEAD "$1" 2> /dev/null)
+  if [ "x$timestamp" != "x" ]; then
+    touch -t "$timestamp" "$1"
+  fi
+' sh {} \;
+