From 4b82c598d5bbec72c1a723ad58fd61a3c4e14fa0 Mon Sep 17 00:00:00 2001 From: Jeremy Wentworth Date: Fri, 22 Dec 2017 14:34:40 -0500 Subject: [PATCH] improved tests --- package-lock.json | 160 +++++++++++++++++++++++++++++++++++++++ package.json | 2 + spec/README.md | 14 ++-- spec/basic.tests.spec.js | 2 +- spec/manifest.json | 17 ++++- spec/zip.tests.spec.js | 110 +++++++++++++++++++++++++++ 6 files changed, 295 insertions(+), 10 deletions(-) create mode 100644 spec/zip.tests.spec.js diff --git a/package-lock.json b/package-lock.json index e390742a..cbadd441 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,12 @@ "integrity": "sha1-WYc/Nej89sc2HBAjkmHXbhU0i7I=", "dev": true }, + "adm-zip": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.7.tgz", + "integrity": "sha1-hgbCy/HEJs6MjsABdER/1Jtur8E=", + "dev": true + }, "ajv": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", @@ -28,6 +34,15 @@ "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", "dev": true }, + "argparse": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz", + "integrity": "sha1-c9g7wmP4bpf4zE9rrhsOkKfSLIY=", + "dev": true, + "requires": { + "sprintf-js": "1.0.3" + } + }, "asn1": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", @@ -40,6 +55,12 @@ "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", "dev": true }, + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", + "dev": true + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -245,6 +266,12 @@ "iconv-lite": "0.4.19" } }, + "esprima": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", + "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==", + "dev": true + }, "exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", @@ -333,6 +360,16 @@ "path-is-absolute": "1.0.1" } }, + "glob-all": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/glob-all/-/glob-all-3.1.0.tgz", + "integrity": "sha1-iRPd+17hrHgSZWJBsD1SF8ZLAqs=", + "dev": true, + "requires": { + "glob": "7.1.2", + "yargs": "1.2.6" + } + }, "graceful-fs": { "version": "4.1.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", @@ -361,6 +398,19 @@ "har-schema": "2.0.0" } }, + "hash-files": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/hash-files/-/hash-files-1.1.1.tgz", + "integrity": "sha1-X4nGTvIezmnIJhJUt2Wb1A6R2N4=", + "dev": true, + "requires": { + "async": "1.5.2", + "glob-all": "3.1.0", + "opter": "1.1.0", + "read-files": "0.1.0", + "underscore": "1.8.3" + } + }, "hawk": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", @@ -463,6 +513,16 @@ "integrity": "sha1-vMl5rh+f0FcB5F5S5l06XWPxok4=", "dev": true }, + "js-yaml": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.10.0.tgz", + "integrity": "sha512-O2v52ffjLa9VeM43J4XocZE//WT9N0IiwDa3KSHH7Tu8CtH+1qM8SIZvnsTh6v+4yFy5KUY3BHUVwjpfAWsjIA==", + "dev": true, + "requires": { + "argparse": "1.0.9", + "esprima": "4.0.0" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", @@ -500,6 +560,18 @@ "verror": "1.10.0" } }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", + "dev": true + }, + "lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=", + "dev": true + }, "mail-notifier": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/mail-notifier/-/mail-notifier-0.3.0.tgz", @@ -638,6 +710,12 @@ "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=", "dev": true }, + "object-path": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/object-path/-/object-path-0.11.4.tgz", + "integrity": "sha1-NwrnUvvzfePqcKhhwju6iRVpGUk=", + "dev": true + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -647,6 +725,20 @@ "wrappy": "1.0.2" } }, + "opter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/opter/-/opter-1.1.0.tgz", + "integrity": "sha1-JZiuu2Cz8acyKvEJcIb055jISnY=", + "dev": true, + "requires": { + "commander": "2.8.1", + "js-yaml": "3.10.0", + "object-path": "0.11.4", + "underscore": "1.8.3", + "z-schema": "3.19.0", + "z-schema-errors": "0.0.1" + } + }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -677,6 +769,12 @@ "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==", "dev": true }, + "read-files": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/read-files/-/read-files-0.1.0.tgz", + "integrity": "sha1-YGu3oBd5Utai+YPVmAlglDFNMh4=", + "dev": true + }, "readable-stream": { "version": "1.1.14", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", @@ -784,6 +882,12 @@ "integrity": "sha1-1PVSp712PK1ywY4tnwjclB8AK+I=", "dev": true }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, "sshpk": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", @@ -854,6 +958,12 @@ "dev": true, "optional": true }, + "underscore": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", + "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=", + "dev": true + }, "utf7": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/utf7/-/utf7-1.0.2.tgz", @@ -875,6 +985,12 @@ "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==", "dev": true }, + "validator": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-9.2.0.tgz", + "integrity": "sha512-6Ij4Eo0KM4LkR0d0IegOwluG5453uqT5QyF5SV5Ezvm8/zmkKI/L4eoraafZGlZPC9guLkwKzgypcw8VGWWnGA==", + "dev": true + }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", @@ -891,6 +1007,50 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", + "dev": true + }, + "yargs": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-1.2.6.tgz", + "integrity": "sha1-nHtKgv1dWVsr8Xq23MQxNUMv40s=", + "dev": true, + "requires": { + "minimist": "0.1.0" + }, + "dependencies": { + "minimist": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.1.0.tgz", + "integrity": "sha1-md9lelJXTCHJBXSX33QnkLK0wN4=", + "dev": true + } + } + }, + "z-schema": { + "version": "3.19.0", + "resolved": "https://registry.npmjs.org/z-schema/-/z-schema-3.19.0.tgz", + "integrity": "sha512-V94f3ODuluBS4kQLLjNhwoMek0dyIXCsvNu/A17dAyJ6sMhT5KkJQwSn07R0naByLIXJWMDk+ruMfI/3G3hS4Q==", + "dev": true, + "requires": { + "commander": "2.8.1", + "lodash.get": "4.4.2", + "lodash.isequal": "4.5.0", + "validator": "9.2.0" + } + }, + "z-schema-errors": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/z-schema-errors/-/z-schema-errors-0.0.1.tgz", + "integrity": "sha1-4GJwpMpDklcp8ldkeJyp9Gv/D30=", + "dev": true, + "requires": { + "xtend": "4.0.1" + } } } } diff --git a/package.json b/package.json index fd2e3b5c..c4b3fde1 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,9 @@ }, "homepage": "https://github.com/VCVRack/community#readme", "devDependencies": { + "adm-zip": "^0.4.7", "ajv": "^5.5.2", + "hash-files": "^1.1.1", "jasmine": "^2.8.0", "node-virustotal": "^2.4.2", "request": "^2.83.0" diff --git a/spec/README.md b/spec/README.md index 98302cc6..ed22a3ac 100644 --- a/spec/README.md +++ b/spec/README.md @@ -5,15 +5,18 @@ [Jasmine Intro](https://jasmine.github.io/2.8/introduction.html) To run all tests: - ``` npm test ``` -To run only the zip tests for example: +To run one set of tests: +``` +npm test spec/basic.tests.spec.js +``` +To force the zip tests to run for certain manifest files: ``` -./node_modules/.bin/jasmine --filter=zip +TEST_MANIFEST_ZIPS=plugins/JW-Modules.json npm test ``` ## Virus Total @@ -24,9 +27,4 @@ To run only the zip tests for example: [Public API](https://www.virustotal.com/en/documentation/public-api/v2/) -[Node Module](https://github.com/natewatson999/node-virustotal) - - -## Schema Validation -[AJV](https://github.com/epoberezkin/ajv) diff --git a/spec/basic.tests.spec.js b/spec/basic.tests.spec.js index 83fdf6ee..ca754a71 100644 --- a/spec/basic.tests.spec.js +++ b/spec/basic.tests.spec.js @@ -2,7 +2,7 @@ const CUR_VER_PREFIX = '0.5'; const MF_DIR = 'plugins'; const fs = require("fs"); -const Ajv = require('ajv'); +const Ajv = require('ajv'); //https://github.com/epoberezkin/ajv const ajv = new Ajv({allErrors:true}); const validate = ajv.compile(require('./manifest.json')); diff --git a/spec/manifest.json b/spec/manifest.json index c1ef1786..e60088ac 100644 --- a/spec/manifest.json +++ b/spec/manifest.json @@ -1,12 +1,12 @@ { "id": "manifest.json", "type": "object", + "additionalProperties": false, "required": [ "slug", "version" ], "description": "A plugin manifest file for VCV Rack.", - "additionalProperties": false, "properties": { "slug": { "type": "string", @@ -58,6 +58,11 @@ "properties": { "win": { "type": "object", + "additionalProperties": false, + "required": [ + "download", + "sha256" + ], "properties": { "download": { "type": "string", @@ -71,6 +76,11 @@ }, "lin": { "type": "object", + "additionalProperties": false, + "required": [ + "download", + "sha256" + ], "properties": { "download": { "type": "string", @@ -84,6 +94,11 @@ }, "mac": { "type": "object", + "additionalProperties": false, + "required": [ + "download", + "sha256" + ], "properties": { "download": { "type": "string", diff --git a/spec/zip.tests.spec.js b/spec/zip.tests.spec.js new file mode 100644 index 00000000..c613c92e --- /dev/null +++ b/spec/zip.tests.spec.js @@ -0,0 +1,110 @@ +const TEMP_DIR = ".tmp-zip-downloads/"; +const fs = require("fs"); +const request = require("request"); +const { execSync } = require('child_process'); +const AdmZip = require('adm-zip'); //https://github.com/cthackers/adm-zip +const hashFiles = require('hash-files'); //https://github.com/mac-/hash-files +const manifestsToTest = getManifestsThatApply(); + +if(manifestsToTest.length === 0){ + console.log("No manifest zip files to test."); + process.exit(0); +} + +//virus total stuff +const VIRUS_TOTAL_ENABLED = false; +const vt = require("node-virustotal"); //https://github.com/natewatson999/node-virustotal +const con = vt.MakePublicConnection(); +if(VIRUS_TOTAL_ENABLED){ + con.setKey(process.env.VT_API_KEY); + con.setDelay(15000); + jasmine.DEFAULT_TIMEOUT_INTERVAL = manifestsToTest.length * 4 * con.getDelay(); +} + +describe("zips", function() { + + beforeEach(()=>{ + execSync(`mkdir -p ${TEMP_DIR}`) + }); + + afterEach(()=>{ + execSync(`rm ${TEMP_DIR}/*.zip`) + fs.rmdirSync(TEMP_DIR); + }); + + const testZipsInMF = (filePath) => { + it("valid zips in manifest", function(done) { + const mfObj = JSON.parse(fs.readFileSync(filePath, 'utf8')); + if(mfObj.downloads){ + const urlsChecked = []; + let lastSha256; + ['win', 'lin', 'mac'].map(os => { + const osObj = mfObj.downloads[os]; + if(osObj && osObj.download && osObj.sha256){ + const zipUrl = osObj.download; + if(urlsChecked.includes(zipUrl)){ + if(lastSha256 !== osObj.sha256){ + fail("SHA256 should be the same if the download URL is the same"); + } + } else { + urlsChecked.push(zipUrl); + lastSha256 = osObj.sha256; + testOneZip(mfObj.slug, osObj, done); + } + } + }); + } + }); + }; + + manifestsToTest.map(testZipsInMF); +}); + +function testOneZip(expectedRootDir, osObj, done) { + const urlParts = osObj.download.split('/'); + const zipName = urlParts[urlParts.length - 1].split('\?')[0]; + request(osObj.download).pipe(fs.createWriteStream(TEMP_DIR+zipName)).on('finish', ()=>{ + + console.log(`Downloaded ${TEMP_DIR+zipName}`); + const zip = new AdmZip(TEMP_DIR+zipName); + const zipEntries = zip.getEntries(); + const invalidEntry = zipEntries.find(ze => !ze.entryName.startsWith(expectedRootDir)); + if(invalidEntry){ + fail(`Zip entries should all be under a dir named ${expectedRootDir} but this entry was found: ${invalidEntry.entryName}`); + } + + if(VIRUS_TOTAL_ENABLED){ + con.FileEvaluation(zipName, "application/zip", fs.readFileSync(TEMP_DIR+zipName), function(data) { + console.log(data); + if(osObj.sha256 !== data.sha256){ + throw new Error(`Invalid sha256 value. manifest:${osObj.sha256} virustotal:${data.sha256}`); + } + if(data.positives > 2){ + throw new Error(`Too many positives from virustotal.`); + } + done(); + }, function(err) { + if (err){ throw err; } + done(); + }); + } else { + hashFiles({files:[TEMP_DIR+zipName], algorithm:'sha256'}, function(error, hash) { + if(osObj.sha256 !== hash){ + throw new Error(`Invalid sha256 value. manifest:${osObj.sha256} hash:${hash}`); + } + done(); + }); + } + }); +} + +function getManifestsThatApply(){ + let paths = ""; + if(process.env.TEST_MANIFEST_ZIPS){ + paths = process.env.TEST_MANIFEST_ZIPS; + } else { + paths = execSync('git diff -w --stat --name-only origin/master -- plugins/', {encoding:'utf8'}); + } + return paths.trim().split('\n').filter(s=>s.trim() !== ''); +} +