improved tests

package-lock.json

@@ -10,6 +10,12 @@
       "integrity": "sha1-WYc/Nej89sc2HBAjkmHXbhU0i7I=",
       "dev": true
     },
+    "adm-zip": {
+      "version": "0.4.7",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.7.tgz",
+      "integrity": "sha1-hgbCy/HEJs6MjsABdER/1Jtur8E=",
+      "dev": true
+    },
     "ajv": {
       "version": "5.5.2",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
@@ -28,6 +34,15 @@
       "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=",
       "dev": true
     },
+    "argparse": {
+      "version": "1.0.9",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz",
+      "integrity": "sha1-c9g7wmP4bpf4zE9rrhsOkKfSLIY=",
+      "dev": true,
+      "requires": {
+        "sprintf-js": "1.0.3"
+      }
+    },
     "asn1": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
@@ -40,6 +55,12 @@
       "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
       "dev": true
     },
+    "async": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
+      "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
+      "dev": true
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -245,6 +266,12 @@
         "iconv-lite": "0.4.19"
       }
     },
+    "esprima": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
+      "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==",
+      "dev": true
+    },
     "exit": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz",
@@ -333,6 +360,16 @@
         "path-is-absolute": "1.0.1"
       }
     },
+    "glob-all": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/glob-all/-/glob-all-3.1.0.tgz",
+      "integrity": "sha1-iRPd+17hrHgSZWJBsD1SF8ZLAqs=",
+      "dev": true,
+      "requires": {
+        "glob": "7.1.2",
+        "yargs": "1.2.6"
+      }
+    },
     "graceful-fs": {
       "version": "4.1.11",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz",
@@ -361,6 +398,19 @@
         "har-schema": "2.0.0"
       }
     },
+    "hash-files": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/hash-files/-/hash-files-1.1.1.tgz",
+      "integrity": "sha1-X4nGTvIezmnIJhJUt2Wb1A6R2N4=",
+      "dev": true,
+      "requires": {
+        "async": "1.5.2",
+        "glob-all": "3.1.0",
+        "opter": "1.1.0",
+        "read-files": "0.1.0",
+        "underscore": "1.8.3"
+      }
+    },
     "hawk": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz",
@@ -463,6 +513,16 @@
       "integrity": "sha1-vMl5rh+f0FcB5F5S5l06XWPxok4=",
       "dev": true
     },
+    "js-yaml": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.10.0.tgz",
+      "integrity": "sha512-O2v52ffjLa9VeM43J4XocZE//WT9N0IiwDa3KSHH7Tu8CtH+1qM8SIZvnsTh6v+4yFy5KUY3BHUVwjpfAWsjIA==",
+      "dev": true,
+      "requires": {
+        "argparse": "1.0.9",
+        "esprima": "4.0.0"
+      }
+    },
     "jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
@@ -500,6 +560,18 @@
         "verror": "1.10.0"
       }
     },
+    "lodash.get": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
+      "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=",
+      "dev": true
+    },
+    "lodash.isequal": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
+      "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=",
+      "dev": true
+    },
     "mail-notifier": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/mail-notifier/-/mail-notifier-0.3.0.tgz",
@@ -638,6 +710,12 @@
       "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=",
       "dev": true
     },
+    "object-path": {
+      "version": "0.11.4",
+      "resolved": "https://registry.npmjs.org/object-path/-/object-path-0.11.4.tgz",
+      "integrity": "sha1-NwrnUvvzfePqcKhhwju6iRVpGUk=",
+      "dev": true
+    },
     "once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -647,6 +725,20 @@
         "wrappy": "1.0.2"
       }
     },
+    "opter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/opter/-/opter-1.1.0.tgz",
+      "integrity": "sha1-JZiuu2Cz8acyKvEJcIb055jISnY=",
+      "dev": true,
+      "requires": {
+        "commander": "2.8.1",
+        "js-yaml": "3.10.0",
+        "object-path": "0.11.4",
+        "underscore": "1.8.3",
+        "z-schema": "3.19.0",
+        "z-schema-errors": "0.0.1"
+      }
+    },
     "path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
@@ -677,6 +769,12 @@
       "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==",
       "dev": true
     },
+    "read-files": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/read-files/-/read-files-0.1.0.tgz",
+      "integrity": "sha1-YGu3oBd5Utai+YPVmAlglDFNMh4=",
+      "dev": true
+    },
     "readable-stream": {
       "version": "1.1.14",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
@@ -784,6 +882,12 @@
       "integrity": "sha1-1PVSp712PK1ywY4tnwjclB8AK+I=",
       "dev": true
     },
+    "sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+      "dev": true
+    },
     "sshpk": {
       "version": "1.13.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz",
@@ -854,6 +958,12 @@
       "dev": true,
       "optional": true
     },
+    "underscore": {
+      "version": "1.8.3",
+      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz",
+      "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=",
+      "dev": true
+    },
     "utf7": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/utf7/-/utf7-1.0.2.tgz",
@@ -875,6 +985,12 @@
       "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==",
       "dev": true
     },
+    "validator": {
+      "version": "9.2.0",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-9.2.0.tgz",
+      "integrity": "sha512-6Ij4Eo0KM4LkR0d0IegOwluG5453uqT5QyF5SV5Ezvm8/zmkKI/L4eoraafZGlZPC9guLkwKzgypcw8VGWWnGA==",
+      "dev": true
+    },
     "verror": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
@@ -891,6 +1007,50 @@
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
       "dev": true
+    },
+    "xtend": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
+      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=",
+      "dev": true
+    },
+    "yargs": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-1.2.6.tgz",
+      "integrity": "sha1-nHtKgv1dWVsr8Xq23MQxNUMv40s=",
+      "dev": true,
+      "requires": {
+        "minimist": "0.1.0"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "0.1.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.1.0.tgz",
+          "integrity": "sha1-md9lelJXTCHJBXSX33QnkLK0wN4=",
+          "dev": true
+        }
+      }
+    },
+    "z-schema": {
+      "version": "3.19.0",
+      "resolved": "https://registry.npmjs.org/z-schema/-/z-schema-3.19.0.tgz",
+      "integrity": "sha512-V94f3ODuluBS4kQLLjNhwoMek0dyIXCsvNu/A17dAyJ6sMhT5KkJQwSn07R0naByLIXJWMDk+ruMfI/3G3hS4Q==",
+      "dev": true,
+      "requires": {
+        "commander": "2.8.1",
+        "lodash.get": "4.4.2",
+        "lodash.isequal": "4.5.0",
+        "validator": "9.2.0"
+      }
+    },
+    "z-schema-errors": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/z-schema-errors/-/z-schema-errors-0.0.1.tgz",
+      "integrity": "sha1-4GJwpMpDklcp8ldkeJyp9Gv/D30=",
+      "dev": true,
+      "requires": {
+        "xtend": "4.0.1"
+      }
     }
   }
 }

package.json

@@ -14,7 +14,9 @@
   },
   "homepage": "https://github.com/VCVRack/community#readme",
   "devDependencies": {
+    "adm-zip": "^0.4.7",
     "ajv": "^5.5.2",
+    "hash-files": "^1.1.1",
     "jasmine": "^2.8.0",
     "node-virustotal": "^2.4.2",
     "request": "^2.83.0"

spec/README.md

@@ -5,15 +5,18 @@
 [Jasmine Intro](https://jasmine.github.io/2.8/introduction.html)
 
 To run all tests:
-
 ```
 npm test
 ```
 
-To run only the zip tests for example:
+To run one set of tests:
+```
+npm test spec/basic.tests.spec.js
+```
 
+To force the zip tests to run for certain manifest files:
 ```
-./node_modules/.bin/jasmine --filter=zip
+TEST_MANIFEST_ZIPS=plugins/JW-Modules.json npm test
 ```
 
 ## Virus Total
@@ -24,9 +27,4 @@ To run only the zip tests for example:
 
 [Public API](https://www.virustotal.com/en/documentation/public-api/v2/)
 
-[Node Module](https://github.com/natewatson999/node-virustotal)
-
-
-## Schema Validation 
 
-[AJV](https://github.com/epoberezkin/ajv)

spec/basic.tests.spec.js

@@ -2,7 +2,7 @@ const CUR_VER_PREFIX = '0.5';
 const MF_DIR = 'plugins';
 const fs = require("fs");
 
-const Ajv = require('ajv');
+const Ajv = require('ajv'); //https://github.com/epoberezkin/ajv
 const ajv = new Ajv({allErrors:true});
 const validate = ajv.compile(require('./manifest.json'));
 

spec/manifest.json

@@ -1,12 +1,12 @@
 {
   "id": "manifest.json",
   "type": "object",
+  "additionalProperties": false,
   "required": [
     "slug",
     "version"
   ],
   "description": "A plugin manifest file for VCV Rack.",
-  "additionalProperties": false,
   "properties": {
     "slug": {
       "type": "string",
@@ -58,6 +58,11 @@
       "properties": {
         "win": {
           "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "download",
+            "sha256"
+          ],
           "properties": {
             "download": {
               "type": "string",
@@ -71,6 +76,11 @@
         },
         "lin": {
           "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "download",
+            "sha256"
+          ],
           "properties": {
             "download": {
               "type": "string",
@@ -84,6 +94,11 @@
         },
         "mac": {
           "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "download",
+            "sha256"
+          ],
           "properties": {
             "download": {
               "type": "string",

spec/zip.tests.spec.js

@@ -0,0 +1,110 @@
+const TEMP_DIR = ".tmp-zip-downloads/";
+const fs = require("fs");
+const request = require("request");
+const { execSync } = require('child_process');
+const AdmZip = require('adm-zip'); //https://github.com/cthackers/adm-zip
+const hashFiles = require('hash-files'); //https://github.com/mac-/hash-files
+const manifestsToTest = getManifestsThatApply();
+
+if(manifestsToTest.length === 0){
+    console.log("No manifest zip files to test.");
+    process.exit(0);
+}
+
+//virus total stuff
+const VIRUS_TOTAL_ENABLED = false;
+const vt = require("node-virustotal"); //https://github.com/natewatson999/node-virustotal
+const con = vt.MakePublicConnection();
+if(VIRUS_TOTAL_ENABLED){
+    con.setKey(process.env.VT_API_KEY);
+    con.setDelay(15000);
+    jasmine.DEFAULT_TIMEOUT_INTERVAL = manifestsToTest.length * 4 * con.getDelay();
+}
+
+describe("zips", function() {
+    
+    beforeEach(()=>{
+        execSync(`mkdir -p ${TEMP_DIR}`)
+    });
+    
+    afterEach(()=>{
+        execSync(`rm ${TEMP_DIR}/*.zip`)
+        fs.rmdirSync(TEMP_DIR);
+    });
+
+    const testZipsInMF = (filePath) => {
+        it("valid zips in manifest", function(done) {
+            const mfObj = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+            if(mfObj.downloads){
+                const urlsChecked = [];
+                let lastSha256;
+                ['win', 'lin', 'mac'].map(os => {
+                    const osObj = mfObj.downloads[os];
+                    if(osObj && osObj.download && osObj.sha256){
+                        const zipUrl = osObj.download;
+                        if(urlsChecked.includes(zipUrl)){
+                            if(lastSha256 !== osObj.sha256){
+                                fail("SHA256 should be the same if the download URL is the same");
+                            }
+                        } else {
+                            urlsChecked.push(zipUrl);
+                            lastSha256 = osObj.sha256;
+                            testOneZip(mfObj.slug, osObj, done);
+                        }
+                    }
+                });
+            }
+        });
+    };
+
+    manifestsToTest.map(testZipsInMF);
+});
+
+function testOneZip(expectedRootDir, osObj, done) {
+    const urlParts = osObj.download.split('/');
+    const zipName = urlParts[urlParts.length - 1].split('\?')[0];
+    request(osObj.download).pipe(fs.createWriteStream(TEMP_DIR+zipName)).on('finish', ()=>{
+
+        console.log(`Downloaded ${TEMP_DIR+zipName}`);
+        const zip = new AdmZip(TEMP_DIR+zipName);
+        const zipEntries = zip.getEntries();
+        const invalidEntry = zipEntries.find(ze => !ze.entryName.startsWith(expectedRootDir));
+        if(invalidEntry){
+            fail(`Zip entries should all be under a dir named ${expectedRootDir} but this entry was found: ${invalidEntry.entryName}`);
+        }
+
+        if(VIRUS_TOTAL_ENABLED){
+            con.FileEvaluation(zipName, "application/zip", fs.readFileSync(TEMP_DIR+zipName), function(data) {
+                console.log(data);
+                if(osObj.sha256 !== data.sha256){
+                    throw new Error(`Invalid sha256 value.  manifest:${osObj.sha256} virustotal:${data.sha256}`);
+                }
+                if(data.positives > 2){
+                    throw new Error(`Too many positives from virustotal.`);   
+                }
+                done();
+            }, function(err) {
+                if (err){ throw err; }
+                done();
+            });
+        } else {
+            hashFiles({files:[TEMP_DIR+zipName], algorithm:'sha256'}, function(error, hash) {
+                if(osObj.sha256 !== hash){
+                    throw new Error(`Invalid sha256 value.  manifest:${osObj.sha256} hash:${hash}`);
+                }
+                done();
+            });
+        }
+    });
+}
+
+function getManifestsThatApply(){
+    let paths = "";
+    if(process.env.TEST_MANIFEST_ZIPS){
+         paths = process.env.TEST_MANIFEST_ZIPS;
+    } else {
+        paths = execSync('git diff -w --stat --name-only origin/master -- plugins/', {encoding:'utf8'});
+    }
+    return paths.trim().split('\n').filter(s=>s.trim() !== '');
+}
+