diff --git a/classes/external-toolchain.bbclass b/classes/external-toolchain.bbclass
index fa8721b..0ff818c 100644
--- a/classes/external-toolchain.bbclass
+++ b/classes/external-toolchain.bbclass
@@ -60,6 +60,14 @@ EXTERNAL_AUTO_PROVIDE ?= "0"
 EXTERNAL_AUTO_PROVIDE[type] = "boolean"
 EXTERNAL_AUTO_PROVIDE_class-target ?= "1"
 
+# We don't care if this path references other variables
+EXTERNAL_TOOLCHAIN[vardepvalue] = "${EXTERNAL_TOOLCHAIN}"
+
+# We don't want to rebuild if the path to the toolchain changes, only if the
+# toolchain changes
+external_toolchain_do_install[vardepsexclude] += "EXTERNAL_TOOLCHAIN"
+EXTERNAL_INSTALL_SOURCE_PATHS[vardepsexclude] += "EXTERNAL_TOOLCHAIN"
+
 python () {
     # Skipping only matters up front
     if d.getVar('BB_WORKERCONTEXT', True) == '1':
diff --git a/conf/distro/include/external-run.inc b/conf/distro/include/external-run.inc
index 7931e38..c9dd175 100644
--- a/conf/distro/include/external-run.inc
+++ b/conf/distro/include/external-run.inc
@@ -20,3 +20,5 @@ def external_run(d, cmd, *args):
             return output
 
     return 'UNKNOWN'
+
+external_run[vardepsexclude] += "EXTERNAL_TOOLCHAIN TMPDIR"
diff --git a/conf/distro/include/tcmode-external-sourcery.inc b/conf/distro/include/tcmode-external-sourcery.inc
index 717080d..a1f1520 100644
--- a/conf/distro/include/tcmode-external-sourcery.inc
+++ b/conf/distro/include/tcmode-external-sourcery.inc
@@ -82,7 +82,14 @@ def external_target_sys(d):
             return triplet
     return '${TARGET_SYS}'
 
+# We need our -cross recipes to rebuild when the external toolchain changes
+BB_HASHBASE_WHITELIST_remove = "EXTERNAL_TOOLCHAIN"
+
+# All we care about for the signatures is the result, not how we got there, so
+# we don't want EXTERNAL_TARGET_SYSTEMS included.
 EXTERNAL_TARGET_SYS ?= "${@external_target_sys(d)}"
+EXTERNAL_TARGET_SYS[vardepvalue] = "${EXTERNAL_TARGET_SYS}"
+EXTERNAL_TARGET_SYS[vardepsexclude] += "EXTERNAL_TARGET_SYSTEMS EXTERNAL_TOOLCHAIN"
 TARGET_PREFIX = "${EXTERNAL_TARGET_SYS}-"
 
 TOOLCHAIN_OPTIONS = " --sysroot=${STAGING_DIR_HOST}"
@@ -175,6 +182,7 @@ def sourcery_version(d):
 
 SOURCERY_VERSION = "${@sourcery_version(d)}"
 SOURCERY_VERSION_allarch = ""
+SOURCERY_VERSION[vardepvalue] = "${SOURCERY_VERSION}"
 
 # Add sourcery toolchain version to external recipe versions
 EXTERNAL_PV_SUFFIX ?= "-${SOURCERY_VERSION}"
diff --git a/conf/layer.conf b/conf/layer.conf
index 104dcea..ddcf39e 100644
--- a/conf/layer.conf
+++ b/conf/layer.conf
@@ -11,3 +11,12 @@ BBFILES += "${@' '.join('${LAYERDIR}/%s/recipes*/*/*.%s' % (layer, ext) \
                for layer in '${BBFILE_COLLECTIONS}'.split() for ext in ['bb', 'bbappend'])}"
 
 TCMODE = "external-sourcery"
+
+# These will rebuild when EXTERNAL_TOOLCHAIN changes, but we don't want
+# everything to rebuild in such a case.
+SIGGEN_EXCLUDERECIPES_ABISAFE += "\
+    gcc-external-cross \
+    gcc-external-cross-${TARGET_ARCH} \
+    binutils-external-cross  \
+    binutils-external-cross-${TARGET_ARCH} \
+"