diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml
index 3cab4e0fb3..dee18e49cb 100644
--- a/.buildkite/pipeline.yaml
+++ b/.buildkite/pipeline.yaml
@@ -44,3 +44,12 @@ steps:
     plugins:
       - docker#v3.0.1:
           image: "node:10"
+
+  - label: "🌐 i18n"
+    command:
+      # Do the things needed to actually run the i18n stuff
+      - "yarn install"
+      - "yarn diff-i18n"
+    plugins:
+      - docker#v3.0.1:
+          image: "node:10"
\ No newline at end of file
diff --git a/package.json b/package.json
index aa2290b1ad..c1fe79e14c 100644
--- a/package.json
+++ b/package.json
@@ -30,6 +30,7 @@
     "reskindex:watch": "reskindex -h src/header -w",
     "i18n": "matrix-gen-i18n",
     "prunei18n": "matrix-prune-i18n",
+    "diff-i18n": "cp src/i18n/strings/en_EN.json src/i18n/strings/en_EN_orig.json && yarn i18n && node scripts/compare-file.js src/i18n/strings/en_EN_orig.json src/i18n/strings/en_EN.json",
     "build:res": "node scripts/copy-res.js",
     "build:modernizr": "modernizr -c .modernizr.json -d src/vector/modernizr.js",
     "build:compile": "yarn reskindex && babel --source-maps -d lib src",
diff --git a/scripts/compare-file.js b/scripts/compare-file.js
new file mode 100644
index 0000000000..f53275ebfa
--- /dev/null
+++ b/scripts/compare-file.js
@@ -0,0 +1,10 @@
+const fs = require("fs");
+
+if (process.argv.length < 4) throw new Error("Missing source and target file arguments");
+
+const sourceFile = fs.readFileSync(process.argv[2], 'utf8');
+const targetFile = fs.readFileSync(process.argv[3], 'utf8');
+
+if (sourceFile !== targetFile) {
+    throw new Error("Files do not match");
+}