Merge pull request #1000 from keyboardio/f/plugins-to-libraries

Move plugins back out into their own directories

Makefile

@@ -109,3 +109,13 @@ clean:
 
 $(SMOKE_SKETCHES): force
 	$(MAKE) -C $@ -f $(KALEIDOSCOPE_ETC_DIR)/makefiles/sketch.mk compile
+
+
+build-arduino-nightly-package:
+	perl bin/build-arduino-package \
+		--kaleidoscope-tag=master \
+		--version `date +%Y.%m.%d%H%M%S` \
+		--index-filename-slug=kaleidoscope_nightly \
+		--only-one-platform-revision \
+		--push \
+		--output-repo=https://github.com/keyboardio/arduino-kaleidoscope-nightly 

README.md

@@ -12,7 +12,7 @@ If you're just getting started with the Keyboardio Model 01, the [introductory d
 
 If you want to use Kaleidoscope to customize or compile a "sketch" to power a supported keyboard, the fastest way to get started is to use the Arduino IDE. You can find [setup instructions](https://kaleidoscope.readthedocs.io/en/latest/quick_start.html) on [kaleidoscope.readthedocs.io](https://kaleidoscope.readthedocs.io)
 
-If you prefer to work from the command line or intend to work on Kaleidscope itself, please follow the instructions below
+If you prefer to work from the command line or intend to work on Kaleidscope itself, please follow the instructions below. It's important to note that the Arduino IDE needs the source code laid out in a slightly different arrangement than you'll find in this repository. If you want to use the Arduino IDE, you should follow [these instructions](https://kaleidoscope.readthedocs.io/en/latest/quick_start.html) instead.
 
 # Use git to check out a copy of Kaleidoscope
 
@@ -25,7 +25,7 @@ $ mkdir ${HOME}/git
 $ cd ${HOME}/git
 ```
 
-2. Use git to download the latest version of Kaleidoscope
+2. Download the latest version of Kaleidoscope
 
 ```sh
 $ git clone https://github.com/keyboardio/Kaleidoscope

bin/build-arduino-package

@@ -7,68 +7,118 @@
 ###################
 ###################
 
+my $executed_as = join(' ', @ARGV);
+
 use warnings;
 use strict;
 use File::Temp qw/tempdir/;
 use Cwd qw/abs_path cwd/;
 use JSON;
+      use Getopt::Long;
 
 my $sha256 = 0;
 my $size   = 0;
 
-my $tag = shift || 'master';
-
-my $dir           = tempdir( CLEANUP => 0 );
-my $checkout_dir  = "Kaleidoscope-$tag";
-my $filename      = $checkout_dir . ".tar.bz2";
-my $checkout_path = "$dir/$checkout_dir";
-chdir($dir);
-
-`git clone --depth=1 https://github.com/keyboardio/Kaleidoscope-Bundle-Keyboardio $checkout_path`;
-chdir($checkout_path);
-`git checkout $tag`;
-`make update-submodules`;
-`find $checkout_path -name .git |xargs rm -rf`;
-`rm -rf $checkout_path/toolchain`;
-`rm -rf $checkout_path/etc`;
-`rm -rf $checkout_path/doc`;
-chdir($dir);
-`rm -rf $checkout_dir/avr/build-tools`;
-`mv $checkout_dir/avr $checkout_dir/$checkout_dir`; # A hack to get consistent naming with the old setup
+my $version = '';
+my $tag              =  'master';
+my $kaleidoscope_tag =  'master';
+my $temp_dir         = tempdir( CLEANUP => 0 );
+my $index_filename_slug  ='keyboardio';
+
+my $only_latest_platform = 0;
+
+my $bundle_repo = 'https://github.com/keyboardio/Kaleidoscope-Bundle-Keyboardio';
+my $boards_repo = 'https://github.com/keyboardio/boardsmanager';
+my $push_package_repo = 0;
+
+
+
+      GetOptions ("bundle-tag=s" => \$tag,
+		  "kaleidoscope-tag=s" => \$kaleidoscope_tag,
+		  "bundle-repo=s" => \$bundle_repo,
+		  "output-repo=s" => \$boards_repo,
+		  "index-filename-slug=s" => \$index_filename_slug,
+		  "version=s" => \$version,
+		  "only-one-platform-revision" => \$only_latest_platform,
+		  "push" => \$push_package_repo)
+      or die("Error in command line arguments\n");
+
+my $index_filename = 'package_'.$index_filename_slug.'_index.json';
+
+if ( $version eq '' && $tag =~ /^v(\d.*)$/ ) {
+    $version = $1;
+}
+
+my $checkout_dir     = "Kaleidoscope-$tag-$version";
+my $filename         = $checkout_dir . ".tar.bz2";
+
+my $build_base_url = $boards_repo."/master/builds/";
+$build_base_url =~ s/github.com/raw.githubusercontent.com/;
+
+
+my @bundle_dirs_to_remove = qw|toolchain etc doc avr/build-tools|;
+
+chdir($temp_dir);
+
+`git clone --depth=1 --quiet --recurse-submodules --branch $tag $bundle_repo $checkout_dir`;
+`git clone $boards_repo boardsmanager`;
+
+
+chdir($checkout_dir);
+foreach my $dir_to_remove (@bundle_dirs_to_remove) {
+    `rm -rf $dir_to_remove`;
+}
+
+chdir("avr/libraries/Kaleidoscope");
+`git checkout --quiet $kaleidoscope_tag`;
+
+if ( -d 'plugins') { 
+chdir("plugins");
+# move the plugins to where they should live
+`mv * ../../`;
+}
+
+set_plugin_versions();
+
+
+chdir($temp_dir);
+`mv $checkout_dir/avr $checkout_dir/$checkout_dir`
+  ;    # A hack to get consistent naming with the old setup
+`find $checkout_dir -name .git |xargs rm -rf`;
 `tar cjvf $filename -C $checkout_dir/ $checkout_dir `;
+
 $sha256 = `sha256sum $filename | cut -d' ' -f 1 `;
 chomp($sha256);
-
 die "There was a problem generating the sha256" unless ($sha256);
-$size = -s $filename;
 
-`git clone https://github.com/keyboardio/boardsmanager`;
+$size = -s $filename;
 `mkdir -p boardsmanager/builds`;
+if ($only_latest_platform) {
+`rm -rf boardsmanager/builds/*`;
+}
 `cp $filename boardsmanager/builds/`;
 
 my $platform_template = {
-    'archiveFileName'   => 'Arduino-Boards-v1.14.zip',
+    'archiveFileName'   => $filename,
     'toolsDependencies' => [
-            {
-              "packager" => "arduino",
-              "name" => "avr-gcc",
-              "version" => "7.3.0-atmel3.6.1-arduino5"
-            },
-            {
-              "packager" => "arduino",
-              "name" => "avrdude",
-              "version" => "6.3.0-arduino17"
-            }
+        {
+            "packager" => "arduino",
+            "name"     => "avr-gcc",
+            "version"  => "7.3.0-atmel3.6.1-arduino7"
+        },
+        {
+            "packager" => "arduino",
+            "name"     => "avrdude",
+            "version"  => "6.3.0-arduino17"
+        }
     ],
-    'url'  => 'https://github.com/keyboardio/Arduino-Boards/archive/v1.14.zip',
-    'name' => 'keyboardio',
-    'version' => '1.1.4',
-    'checksum' =>
-'SHA-256:fd0017ea2950f6fb3afb46a503f0193c75111d68dfc95fa2775fbd63f0cf4528',
-    'size'   => '1008405',
-    'boards' => [
-        { 'name' => 'Keyboardio Model 01' },
-	{ 'name' => 'Keyboardio Atreus' }
+    'url'      => $build_base_url . '/' . $filename,
+    'name'     => "Kaleidoscope keyboards - nightly builds",
+    'version'  => $version,
+    'checksum' => 'SHA-256:' . $sha256,
+    'size'     => $size,
+    'boards'   => [
+        { 'name' => 'Keyboardio Model 01' }, { 'name' => 'Keyboardio Atreus' }
     ],
     'architecture' => 'avr',
     'category'     => 'Contributed',
@@ -77,35 +127,34 @@ my $platform_template = {
     }
 };
 
-my $version = $tag;
-if ($version =~ /^v(\d.*)$/) {
-	$version = $1
-};
-
-$platform_template->{archiveFileName} = $filename;
-$platform_template->{version} = $version;
-$platform_template->{url} =
-  'https://raw.githubusercontent.com/keyboardio/boardsmanager/master/builds/'
-  . $filename;
-$platform_template->{checksum} = 'SHA-256:' . $sha256;
-$platform_template->{size}     = $size;
-
 my $json = JSON->new->allow_nonref;
-
 local $/;
-open( my $fh, '<', 'boardsmanager/package_keyboardio_index.json' );
+open( my $fh, '<', 'boardsmanager/'.$index_filename) || die "Could not open boardsmanager/$index_filename $!";
 my $json_text = <$fh>;
 my $data      = from_json($json_text);
-
+if ($only_latest_platform) {
+@{ $data->{'packages'}->[0]->{'platforms'} } = ( $platform_template);
+} else {
 push @{ $data->{'packages'}->[0]->{'platforms'} }, $platform_template;
-
+}
 my $json_out = $json->canonical->pretty->encode($data);
-
-open( my $out_fh, '>', 'boardsmanager/package_keyboardio_index.json' );
+open( my $out_fh, '>',  'boardsmanager/'.$index_filename);
 print $out_fh $json_out;
 close($out_fh);
-
-#	rm -rf /tmp/boardsmanager
+chdir('boardsmanager');
+`git add $index_filename`;
+`git add builds`;
+`git commit -a -m 'Built by $executed_as'`;
+if ($push_package_repo) {
+	`git push`;
+} else {
 
 print
-  "Now, you need to cd to $dir/boardsmanager check the content and commit it\n";
+"Now, you need to cd to $temp_dir/boardsmanager check the content and push it\n";
+}
+
+
+sub set_plugin_versions {
+	chdir("$temp_dir/$checkout_dir/avr/libraries");
+	`perl -pi -e's/version=0.0.0/version=$version/' */library.properties`
+}

docs/Makefile

@@ -18,6 +18,11 @@ clean:
 	rm -rf doxyoutput/ api/
 	@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
+copy-plugin-readmes:
+	install -d plugins
+	(cd ../plugins; \
+		ls -d * |xargs -n 1 -I % cp %/README.md ../docs/plugins/%.md )
+
 
 examples:
 	install -d examples

docs/UPGRADING.md

@@ -13,6 +13,7 @@ If any of this does not make sense to you, or you have trouble updating your .in
     - [Bidirectional communication for plugins](#bidirectional-communication-for-plugins)
     - [Consistent timing](#consistent-timing)
   + [Breaking changes](#breaking-changes)
+    - [git checkouts aren't compatible with Arduino IDE (GUI)]([#repository-rearchitecture)
     - [Layer system switched to activation-order](#layer-system-switched-to-activation-order)
     - [The `RxCy` macros and peeking into the keyswitch state](#the-rxcy-macros-and-peeking-into-the-keyswitch-state)
     - [HostOS](#hostos)
@@ -323,6 +324,14 @@ As a developer, one can continue using `millis()`, but migrating to `Kaleidoscop
 
 ## Breaking changes
 
+### Repository rearchitecture
+
+To improve build times and to better highlight Kaleidoscope's many plugins, plugins have been move into directories inside the Kaleidoscope directory. 
+
+The "breaking change" part of this is that git checkouts of Kaleidoscope are no longer directly compatible with the Arduino IDE, since plugins aren't in a directory the IDE looks in. They are, of course, visible to tools using our commandline build infrastructure / Makefiles.
+
+When we build releases, those plugins are moved into directories inside the arduino platform packages for each architecture to make them visible to the Arduino IDE.
+
 ### Layer system switched to activation order
 
 The layer system used to be index-ordered, meaning that we'd look keys up on

docs/_sphinx/copy-plugin-readmes.py

@@ -0,0 +1,11 @@
+"""Crude Sphinx extension to run a makefile step
+"""
+
+import os
+
+
+def config_inited(app, config):
+    os.system('make copy-plugin-readmes')
+
+def setup(app):
+    app.connect('config-inited', config_inited)

docs/conf.py

@@ -37,6 +37,7 @@ extensions = [
 ]
 
 extensions.append('copy-examples')
+extensions.append('copy-plugin-readmes')
 
 
 # Setup the breathe extension

docs/setup_toolchain.md

@@ -1,9 +1,12 @@
 # Setting up your development environment
 
-If you'd like to customize your keyboard's layout or functionality, the most robust and flexible option is to use the Arduino IDE.
 
 Arduino is one of the world's most widely used (and user friendly) platforms for programming "embedded" devices like the chip inside your keyboard.
 
+To customize your keyboard's layout or functionality, the most robust and flexible option is to use the Arduino IDE.
+
+If you're planning to modify Kaleidoscope itself or plan on developing Kaleidoscope plugins, you should be checking out the source code from our git repository instead. You can find instructions for that at https://github.com/keyboardio/Kaleidoscope
+
 
 # Set up the Arduino IDE
 
@@ -22,8 +25,6 @@ The right way to install Arduino is a little bit different depending on what ope
 
 ## <a name="Arduino-macOS"></a>Install Arduino on macOS
 
-
-
 1. Download the Arduino IDE install package from https://www.arduino.cc/en/Main/Software
 
 As of this writing, the latest version is v1.8.13, which you can download from https://www.arduino.cc/download_handler.php?f=/arduino-1.8.13-macosx.zip

etc/makefiles/sketch.mk

@@ -177,6 +177,7 @@ compile:
 	$(QUIET) install -d "${OUTPUT_PATH}"
 	$(QUIET) $(ARDUINO_CLI) compile --fqbn "${FQBN}" ${ARDUINO_VERBOSE} --warnings all ${ccache_wrapper_property} ${local_cflags_property} \
 	  --libraries "${KALEIDOSCOPE_PLATFORM_LIB_DIR}" \
+	  --libraries "${KALEIDOSCOPE_DIR}/plugins/" \
 	  --build-path "${BUILD_PATH}" \
 	  --output-dir "${OUTPUT_PATH}" \
 	  --build-cache-path "${CORE_CACHE_PATH}" \

examples/Features/HostOS/Makefile

@@ -0,0 +1,48 @@
+# This makefile for a Kaleidoscope sketch pulls in all the targets
+# required to build the example
+
+
+
+
+ifneq ($(KALEIDOSCOPE_DIR),)
+search_path += $(KALEIDOSCOPE_DIR)
+endif
+
+ifneq ($(ARDUINO_DIRECTORIES_USER),)
+search_path += $(ARDUINO_DIRECTORIES_USER)/hardware/keyboardio/avr/libraries/Kaleidoscope
+endif
+
+ifeq ($(shell uname -s),Darwin)
+search_path += $(HOME)/Documents/Arduino/hardware/keyboardio/avr/libraries/Kaleidoscope
+else
+search_path += $(HOME)/Arduino/hardware/keyboardio/avr/libraries/Kaleidoscope
+endif
+
+sketch_makefile := etc/makefiles/sketch.mk
+
+$(foreach candidate, $(search_path), $(if $(wildcard $(candidate)/$(sketch_makefile)), $(eval ks_dir ?= $(candidate))))
+
+ifneq ($(ks_dir),)
+
+$(info Using Kaleidoscope from $(ks_dir))
+
+export KALEIDOSCOPE_DIR := $(ks_dir)
+include $(ks_dir)/$(sketch_makefile)
+
+else
+
+$(info I can't find your Kaleidoscope installation.)
+$(info )
+$(info I tried looking in:)
+$(info )
+$(foreach candidate, $(search_path), $(info $(candidate)))
+$(info )
+$(info The easiest way to fix this is to set the 'KALEIDOSCOPE_DIR' environment)
+$(info variable to the location of your Kaleidoscope directory.)
+
+endif
+
+
+null-target:
+	$(info You should never see this message)
+	@:

library.properties

@@ -1,9 +1,9 @@
 name=Kaleidoscope
-version=0.0.1
+version=0.0.0
 author=Jesse Vincent
 maintainer=Jesse Vincent <jesse@keyboard.io>
 sentence=Firmware for the Keyboardio Model 01, and other Arduino-powered keyboards.
-paragraph=...
+paragraph=
 category=Communication
 url=https://github.com/keyboardio/Kaleidoscope
 architectures=*

plugins/Kaleidoscope-Colormap/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-Colormap
+version=0.0.0
+sentence=Per-layer colormap effect
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-Cycle/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-Cycle
+version=0.0.0
+sentence=Key sequence cycling dead key for Kaleidoscope.
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-CycleTimeReport/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-CycleTimeReport
+version=0.0.0
+sentence=Scan cycle time reporting
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-DynamicMacros/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-DynamicMacros
+version=0.0.0
+sentence=Dynamic macro support for Kaleidoscope.
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-DynamicMacros/src/Kaleidoscope-DynamicMacros.h

@@ -1,4 +1,4 @@
-/* DynamicMacros - Dynamic macro support for Kaleidoscope.
+/* Kaleidoscope-DynamicMacros -- Dynamic macro support for Kaleidoscope.
  * Copyright (C) 2019  Keyboard.io, Inc.
  *
  * This program is free software: you can redistribute it and/or modify it under
@@ -16,4 +16,5 @@
 
 #pragma once
 
+#include "Kaleidoscope-Macros.h"
 #include "kaleidoscope/plugin/DynamicMacros.h"

plugins/Kaleidoscope-DynamicTapDance/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-DynamicTapDance
+version=0.0.0
+sentence=Dynamic TapDance support for Kaleidoscope
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-DynamicTapDance/src/Kaleidoscope-DynamicTapDance.h

@@ -1,4 +1,4 @@
-/* DynamicTapDance -- Dynamic TapDance support for Kaleidoscope
+/* Kaleidoscope-DynamicTapDance -- Dynamic TapDance support for Kaleidoscope
  * Copyright (C) 2019  Keyboard.io, Inc
  * Copyright (C) 2019  Dygma Lab S.L.
  *
@@ -17,4 +17,5 @@
 
 #pragma once
 
+#include "Kaleidoscope-TapDance.h"
 #include "kaleidoscope/plugin/DynamicTapDance.h"

plugins/Kaleidoscope-EEPROM-Keymap-Programmer/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-EEPROM-Keymap-Programmer
+version=0.0.0
+sentence=On-the-fly reprogrammable keymap.
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-EEPROM-Keymap/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-EEPROM-Keymap
+version=0.0.0
+sentence=EEPROM-based keymap support.
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-EEPROM-Settings/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-EEPROM-Settings
+version=0.0.0
+sentence=Basic EEPROM settings plugin for Kaleidoscope.
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-Escape-OneShot/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-Escape-OneShot
+version=0.0.0
+sentence=Turn ESC into a key that cancels OneShots, if active.
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-FingerPainter/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-FingerPainter
+version=0.0.0
+sentence=On-the-fly keyboard painting.
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-FirmwareDump/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-FirmwareDump
+version=0.0.0
+sentence=Firmware dumper for Kaleidoscope
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-FocusSerial/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-FocusSerial
+version=0.0.0
+sentence=Bidirectional communication plugin
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-GhostInTheFirmware/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-GhostInTheFirmware
+version=0.0.0
+sentence=Let the keyboard write for you!
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-Hardware-Dygma-Raise/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-Hardware-Dygma-Raise
+version=0.0.0
+sentence=Kaleidoscope device plugin for Dygma Raise
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=DygmaLab S.L.
+paragraph=

plugins/Kaleidoscope-Hardware-Dygma-Raise/src/Kaleidoscope-Hardware-Dygma-Raise.h

@@ -1,5 +1,5 @@
 /* -*- mode: c++ -*-
- * kaleidoscope::device::dygma::Raise -- Kaleidoscope device plugin for Dygma Raise
+ * Kaleidoscope-Hardware-Dygma-Raise -- Kaleidoscope device plugin for Dygma Raise
  * Copyright (C) 2017-2019  Keyboard.io, Inc
  * Copyright (C) 2017-2019  Dygma Lab S.L.
  *

plugins/Kaleidoscope-Hardware-EZ-ErgoDox/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-Hardware-EZ-ErgoDox
+version=0.0.0
+sentence=ErgoDox hardware support for Kaleidoscope
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=

plugins/Kaleidoscope-Hardware-KBDFans-KBD4x/library.properties

@@ -0,0 +1,7 @@
+name=Kaleidoscope-Hardware-KBDFans-KBD4x
+version=0.0.0
+sentence=KBD4x hardware support for Kaleidoscope
+maintainer=Kaleidoscope's Developers <jesse@keyboard.io>
+url=https://github.com/keyboardio/Kaleidoscope
+author=Keyboardio
+paragraph=