Simplify our docker execution (#1124)

* Switch docker to no longer build a bundle separate from the kaleidoscope
dir.
Stop copying over a bunch of tarballs we don't need.

* fix shellcheck

* Stop copying the core-bundled copies of Kaleidoscope over to docker

* Symlink our Kaleidoscope into the versions packaged into the arduino
cores when running in docker. We need to do this to satisfy the
sketch_header.h/sketch_footer.h system we use to be able to amend
arduino sketches on avr and virtual. (This feature is not used by the
core and may be removed in the future)

* Reimplement docker-clean to do much less work and get the same result

* Add a stub at docs for the docker test runner

Makefile

@@ -48,7 +48,7 @@ docker-simulator-tests:
 	ARDUINO_DIRECTORIES_USER="$(ARDUINO_DIRECTORIES_USER)" ./bin/run-docker "make simulator-tests $(TEST_PATH_ARG)"
 
 docker-clean:
-	ARDUINO_DIRECTORIES_USER="$(ARDUINO_DIRECTORIES_USER)" ./bin/run-docker "make clean"
+	_NO_SYNC_KALEIDOSCOPE=1 ARDUINO_DIRECTORIES_USER="$(ARDUINO_DIRECTORIES_USER)" ./bin/run-docker "rm -rf -- testing/googletest/build/* _build/* /kaleidoscope-persist/temp/*"
 
 docker-bash:
 	ARDUINO_DIRECTORIES_USER="$(ARDUINO_DIRECTORIES_USER)" ./bin/run-docker "bash"

bin/docs/example-to-doc

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-export EXAMPLE=$1
+export EXAMPLE="$1"
 EXNAME=$(echo "${EXAMPLE}" |cut -c 3-)
 cd examples ||exit; \
 install -d dirname "${EXAMPLE}"

bin/run-docker

@@ -19,13 +19,28 @@ else
     ARDUINO_LOCAL_LIB_PATH="${ARDUINO_LOCAL_LIB_PATH:-${HOME}/Arduino}"
 fi
 
-echo "Preparing Kaleidoscope and the bundle..."
+
+if [ ! -d ./.arduino/user/hardware ]; then
+	echo "Running tests in Docker requires that your Arduino environment be installed in .arduino inside the top-level Kaleidoscope directory. To set this up, run 'make setup'"
+  exit 1
+fi
+
+
 
 XFER_DIR="$(pwd)/.docker_xfer"
 mkdir -p "${XFER_DIR}"
 
+if [ -z "$_NO_SYNC_KALEIDOSCOPE" ]; then 
+
+echo "Preparing Kaleidoscope..."
+
+
+echo "The bundle is coming from ${ARDUINO_DIRECTORIES_USER}/hardware/keyboardio"
 tar  -cf "${XFER_DIR}/kaleidoscope.tar"  \
 	--exclude .arduino/data \
+	--exclude .arduino/downloads \
+	--exclude .arduino/user/hardware/keyboardio/avr/libraries/Kaleidoscope \
+	--exclude .arduino/user/hardware/keyboardio/gd32/libraries/Kaleidoscope \
 	--exclude bin/arduino-cli \
 	--exclude .docker_xfer \
 	--exclude .git \
@@ -33,9 +48,8 @@ tar  -cf "${XFER_DIR}/kaleidoscope.tar"  \
 	--exclude testing/googletest/build  \
 	.
 
-echo "The bundle is coming from ${ARDUINO_DIRECTORIES_USER}/hardware/keyboardio"
+fi
-(cd "${ARDUINO_DIRECTORIES_USER}/hardware/keyboardio" && tar  -cf "${XFER_DIR}/bundle.tar" \
+
-	--exclude .git --exclude avr/libraries/Kaleidoscope .)
 
 echo "Building the docker image..."
 docker build -q -t kaleidoscope/docker etc
@@ -49,6 +63,7 @@ docker run --rm $DOCKER_RUN_INTERACTIVE_OPTS \
        --mount type=volume,source=kaleidoscope-build,destination=/kaleidoscope/_build,consistency=delegated \
        --env ARDUINO_DIRECTORIES_DATA=/arduino-cli/data \
        --env ARDUINO_DIRECTORIES_USER=/kaleidoscope/.arduino/user/ \
+       --env _NO_SYNC_KALEIDOSCOPE=${_NO_SYNC_KALEIDOSCOPE} \
        kaleidoscope/docker "$*"
 
 

docs/codebase/docker.md

@@ -0,0 +1,25 @@
+# Docker
+
+It's possible to use Docker to run Kaleidoscope's test suite.
+
+## Running tests in Docker
+
+```
+# make docker-simulator-tests
+```
+
+## Cleaning out stale data in the Docker image:
+
+```
+# make docker-clean
+```
+
+
+## Removing the Kaleidoscope Docker image entirely:
+
+```
+# docker volume rm kaleidoscope-persist
+# docker volume rm kaleidoscope-googletest-build
+# docker volume rm kaleidoscope-build
+# docker image rm kaleidoscope/docker
+```

etc/Dockerfile

@@ -12,11 +12,13 @@ RUN curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/inst
   sh
 
 RUN ARDUINO_DIRECTORIES_DATA=/arduino-cli/data \
-    ARDUINO_BOARD_MANAGER_ADDITIONAL_URLS=https://raw.githubusercontent.com/keyboardio/boardsmanager/master/package_keyboardio_index.json \
+    ARDUINO_BOARD_MANAGER_ADDITIONAL_URLS=https://raw.githubusercontent.com/keyboardio/boardsmanager/master/devel/package_kaleidoscope_devel_index.json \
     /bin/arduino-cli config init
 RUN ARDUINO_DIRECTORIES_DATA=/arduino-cli/data /bin/arduino-cli update
 RUN ARDUINO_DIRECTORIES_DATA=/arduino-cli/data /bin/arduino-cli core update-index
 RUN ARDUINO_DIRECTORIES_DATA=/arduino-cli/data /bin/arduino-cli core install arduino:avr
+RUN ARDUINO_DIRECTORIES_DATA=/arduino-cli/data /bin/arduino-cli core install "keyboardio:avr-tools-only"
+RUN ARDUINO_DIRECTORIES_DATA=/arduino-cli/data /bin/arduino-cli core install "keyboardio:gd32-tools-only"
 
 
 COPY docker-entrypoint.sh /usr/local/bin/entrypoint

etc/docker-entrypoint.sh

@@ -6,13 +6,15 @@ install -d /kaleidoscope/ \
            /kaleidoscope-persist/temp  \
            /kaleidoscope-persist/ccache/cache
 
+if [ -z $_NO_SYNC_KALEIDOSCOPE ]; then
 
 echo "Syncing Kaleidoscope..."
 tar xf /kaleidoscope-src/kaleidoscope.tar -C /kaleidoscope/
 
-echo "Syncing the bundle..."
+ln -s  /kaleidoscope /kaleidoscope/.arduino/user/hardware/keyboardio/avr/libraries/Kaleidoscope
-tar xf /kaleidoscope-src/bundle.tar -C /kaleidoscope/.arduino/user/hardware/keyboardio
+ln -s  /kaleidoscope /kaleidoscope/.arduino/user/hardware/keyboardio/gd32/libraries/Kaleidoscope
 
+fi
 
 cd /kaleidoscope/
 export ARDUINO_DIRECTORIES_DATA=/arduino-cli/data