Merge pull request #914 from keyboardio/f/parallel-smoke-examples

move example smoking into make for parallelization

.travis.yml

@@ -10,20 +10,27 @@ addons:
 env:
   global:
   - LC_ALL: C
+
 git:
   quiet: true
 before_install: 
-  - ccache --set-config=compiler_check=content -M 5G -F 0
+  - ccache --set-config=compiler_check=content -M 1G -F 0
 
 install:
-  - git clone --depth 1 --recurse-submodules https://github.com/keyboardio/Kaleidoscope-Bundle-Keyboardio ../hardware/keyboardio
+  - git clone --depth 1 --recurse-submodules --shallow-submodules git://github.com/keyboardio/Kaleidoscope-Bundle-Keyboardio ../hardware/keyboardio
 ## We delete the Bundle's version of Kaleidoscope, and symlink ourselves in.
 ## This makes sure we're using the current version of the library.
   - rm -rf ../hardware/keyboardio/avr/libraries/Kaleidoscope
   - ln -s $(pwd) ../hardware/keyboardio/avr/libraries/Kaleidoscope
+  - export KALEIDOSCOPE_TEMP_PATH=$TRAVIS_BUILD_DIR/.kaleidoscope-build-cache
+  - export BOARD_HARDWARE_PATH=$TRAVIS_BUILD_DIR/../hardware
+  - export ARDUINO_PATH=$TRAVIS_BUILD_DIR/arduino-1.8.13
+  - make adjust-git-timestamps
+  - make travis-install-arduino 
+  - make prepare-ccache
 jobs:
   include:
-    - env: TEST_TARGET=travis-smoke-examples
+    - env: TEST_TARGET=smoke-sketches
     - env: TEST_TARGET=travis-simulator-tests
     - env: TEST_TARGET=cpplint
     - env: TEST_TARGET=find-filename-conflicts 
@@ -31,7 +38,7 @@ jobs:
     - env: TEST_TARGET=travis-check-astyle
 script:
   - unset CC
-  - make $TEST_TARGET KALEIDOSCOPE_TEMP_PATH=$(pwd)/.kaleidoscope-build-cache BOARD_HARDWARE_PATH=$(pwd)/../hardware ARDUINO_PATH="$(pwd)/arduino-1.8.13"
+  - make -j 2 $TEST_TARGET 
 notifications:
   email:
     on_success: change

bin/kaleidoscope-builder

@@ -26,15 +26,15 @@ absolute_filename() {
 
 
 build_version () {
-    GIT_VERSION="$(cd "$(find_sketch)"; if [ -d .git ]; then echo -n '-g' && git describe --abbrev=4 --dirty --always; fi)"
+    GIT_VERSION="$(cd "${SKETCH_DIR}"; if [ -d .git ]; then echo -n '-g' && git describe --abbrev=4 --dirty --always; fi)"
     LIB_PROPERTIES_PATH="${LIB_PROPERTIES_PATH:-"../.."}"
-    LIB_VERSION="$(cd "$(find_sketch)"; (grep version= "${LIB_PROPERTIES_PATH}/library.properties" 2>/dev/null || echo version=0.0.0) | cut -d= -f2)${GIT_VERSION}"
+    LIB_VERSION="$(cd "${SKETCH_DIR}"; (grep version= "${LIB_PROPERTIES_PATH}/library.properties" 2>/dev/null || echo version=0.0.0) | cut -d= -f2)${GIT_VERSION}"
 }
 
 build_paths() {
     # We need that echo because we\re piping to cksum
     # shellcheck disable=SC2005
-    SKETCH_IDENTIFIER="$(echo "$(absolute_filename "$(find_sketch)/${SKETCH}.ino")" | cksum | cut -d ' ' -f 1)-${SKETCH}.ino"
+    SKETCH_IDENTIFIER="$(echo "$(absolute_filename "${SKETCH_DIR}/${SKETCH}.ino")" | cksum | cut -d ' ' -f 1)-${SKETCH}.ino"
     KALEIDOSCOPE_TEMP_PATH="${KALEIDOSCOPE_TEMP_PATH:-${TMPDIR:-/tmp}/kaleidoscope-${USER}}"
 
 
@@ -67,7 +67,6 @@ enable_ccache () {
     if [ -z "${CCACHE_NOT_SUPPORTED}" ] && [ "$(command -v ccache)" ]; then
       if ! [ -d "$CCACHE_WRAPPER_PATH" ]; then
          mkdir -p "$CCACHE_WRAPPER_PATH"
-      fi
 
           if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${C_COMPILER_BASENAME}" ]; then
              ln -s "$(command -v ccache)" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}${C_COMPILER_BASENAME}"
@@ -92,6 +91,7 @@ enable_ccache () {
           if ! [ -h "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}size" ]; then
              ln -s "${AVR_SIZE}" "${CCACHE_WRAPPER_PATH}/${COMPILER_PREFIX}size"
           fi
+      fi
 
       export CCACHE_PATH=${COMPILER_PATH}/
       CCACHE_ENABLE="-prefs compiler.path=${CCACHE_WRAPPER_PATH}/"
@@ -137,11 +137,16 @@ find_sketch () {
         exit 1
     fi
   
-    for path in "examples/${LIBRARY}" \
+    SKETCH_DIR="$SKETCH"
+    SKETCH_FILE=$(basename "$SKETCH")
+
+    for path in "${SKETCH_DIR}" \
+		    "examples/${LIBRARY}" \
                     "src" \
 		    "."; do
-        if [ -f "${path}/${SKETCH}.ino" ]; then
-            echo "${path}"
+        if [ -f "${path}/${SKETCH_FILE}.ino" ]; then
+      	    SKETCH_DIR="${path}"
+	    SKETCH="${SKETCH_FILE}"
             return
         fi
     done
@@ -324,14 +329,13 @@ hex_with_bootloader () {
 }
 
 maybe_build () {
+    find_sketch
     build_version
     build_paths
     build_filenames
 
     if [ ! -e "${HEX_FILE_PATH}" ]; then
 	build "$@"
-    else
-      SKETCH_DIR="$(find_sketch)"
     fi
 
 }
@@ -341,7 +345,14 @@ build () {
     size "$@"
 }
 
+
+prepare_ccache () {
+    build_paths
+    enable_ccache
+}
+
 compile () {
+    find_sketch
     build_version
     build_paths
     build_filenames
@@ -349,8 +360,6 @@ compile () {
 
     install -d "${OUTPUT_PATH}"
 
-    SKETCH_DIR="$(find_sketch)"
-
     echo "Building ${SKETCH_DIR}/${SKETCH}"
 
     # This is defined in the (optional) user config.
@@ -440,7 +449,7 @@ compile () {
     FQBN="${SAVED_FQBN}"
 }
 
-_find_all () {
+find_all_sketches () {
     for plugin in ./*.ino \
   	              $([ -d examples ] && find examples -name '*.ino') \
                   src/*.ino; do
@@ -461,7 +470,7 @@ _find_all () {
 }
 
 build_all () {
-    plugins="$(_find_all)"
+    plugins="$(find_all_sketches)"
 
     for plugin in ${plugins}; do
         export SKETCH="${plugin}"
@@ -472,7 +481,7 @@ build_all () {
 
 
 compile_all () {
-    plugins="$(_find_all)"
+    plugins="$(find_all_sketches)"
 
     for plugin in ${plugins}; do
         export SKETCH="${plugin}"
@@ -737,12 +746,6 @@ done
 
 LIBRARY="${SKETCH}"
 
-case "${SKETCH}" in
-    */*)
-        SKETCH="$(basename "${SKETCH}")"
-        ;;
-esac
-
 export SKETCH
 export LIBRARY
 

bin/set-timestamps-from-git

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+# This script sets all of the files inside src and example to have mtimes
+# that match the times of the last git commit that touched each file
+
+# This can be useful when build tools depend on file timestamps to
+# make caching decisions
+
+find src examples -type f -exec sh -c '
+  timestamp=$(git log --pretty=format:%ad --date=format:%Y%m%d%H%M.%S -n 1 HEAD "$1" 2> /dev/null)
+  if [ "x$timestamp" != "x" ]; then
+    touch -t "$timestamp" "$1"
+  fi
+' sh {} \;
+