Add manifest-cache.json which currently caches plugin (and soon module) creation timestamps.

scripts/build.py

@@ -1,37 +1,32 @@
 import os
 import sys
+import common
 
 
 RACK_SDK = os.path.abspath("Rack-SDK")
 STAGE_DIR = "stage"
 
 
-def system(cmd):
-	print(cmd)
-	err = os.system(cmd)
-	if err:
-		raise Exception(f"Command failed with error {err}: {cmd}")
-
 
 def stage_package(plugin_dir):
-	system(f'mkdir -p {STAGE_DIR}')
-	system(f'mv {plugin_dir}/dist/*.zip {STAGE_DIR}/')
+	common.system(f'mkdir -p {STAGE_DIR}')
+	common.system(f'mv {plugin_dir}/dist/*.zip {STAGE_DIR}/')
 
 
 def delete_stage():
-	system(f'rm -rf {STAGE_DIR}')
+	common.system(f'rm -rf {STAGE_DIR}')
 
 
 def build_mac(plugin_dir):
 	print(f"Building {plugin_dir} for mac")
 	env = f'CC=x86_64-apple-darwin17-clang CXX=x86_64-apple-darwin17-clang++-libc++ STRIP=x86_64-apple-darwin17-strip RACK_DIR={RACK_SDK}'
 	make = f'{env} make -j$(nproc) -C {plugin_dir}'
-	system(f'{make} clean')
-	system(f'{make} cleandep')
-	system(f'{make} dep')
-	system(f'{make} dist')
+	common.system(f'{make} clean')
+	common.system(f'{make} cleandep')
+	common.system(f'{make} dep')
+	common.system(f'{make} dist')
 	stage_package(plugin_dir)
-	system(f'{make} clean')
+	common.system(f'{make} clean')
 	print(f"Built {plugin_dir} for mac")
 
 
@@ -39,12 +34,12 @@ def build_win(plugin_dir):
 	print(f"Building {plugin_dir} for win")
 	env = f'CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ STRIP=x86_64-w64-mingw32-strip RACK_DIR={RACK_SDK}'
 	make = f'{env} make -j$(nproc) -C {plugin_dir}'
-	system(f'{make} clean')
-	system(f'{make} cleandep')
-	system(f'{make} dep')
-	system(f'{make} dist')
+	common.system(f'{make} clean')
+	common.system(f'{make} cleandep')
+	common.system(f'{make} dep')
+	common.system(f'{make} dist')
 	stage_package(plugin_dir)
-	system(f'{make} clean')
+	common.system(f'{make} clean')
 	print(f"Built {plugin_dir} for win")
 
 
@@ -55,12 +50,12 @@ def build_lin(plugin_dir):
 	# TODO Make this Docker image publicly available
 	# It's essentially just Ubuntu 16.04 with plugin build dependencies installed, the workdir, and a user account set up so it matches my own machine's UID to solve file permissions issues.
 	docker = f'docker run --rm -v {RACK_SDK}:/Rack-SDK -v {plugin_abs}:/workdir -w /workdir -u vortico -e RACK_DIR=/Rack-SDK rackplugin:1'
-	system(f'{docker} {make} clean')
-	system(f'{docker} {make} cleandep')
-	system(f'{docker} {make} dep')
-	system(f'{docker} {make} dist')
+	common.system(f'{docker} {make} clean')
+	common.system(f'{docker} {make} cleandep')
+	common.system(f'{docker} {make} dep')
+	common.system(f'{docker} {make} dist')
 	stage_package(plugin_dir)
-	system(f'{docker} {make} clean')
+	common.system(f'{docker} {make} clean')
 	print(f"Built {plugin_dir} for lin")
 
 

scripts/common.py

@@ -0,0 +1,9 @@
+import subprocess
+
+
+def system(cmd):
+	print(cmd)
+	result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE)
+	if result.returncode != 0:
+		raise Exception(f"Command failed with error {result.returncode}: {cmd}")
+	return result.stdout.decode('utf-8')

scripts/update.py

@@ -6,8 +6,10 @@ import time
 import zipfile
 import re
 
+import common
 import build
 import update_modulargrid
+import update_cache
 
 
 PACKAGES_DIR = "../packages"
@@ -17,9 +19,9 @@ RACK_USER_DIR = "$HOME/.Rack"
 RACK_USER_PLUGIN_DIR = os.path.join(RACK_USER_DIR, "plugins-v1")
 
 # Update git before continuing
-build.system("git pull")
-build.system("git submodule sync --quiet")
-build.system("git submodule update --init --recursive")
+common.system("git pull")
+common.system("git submodule sync --quiet")
+common.system("git submodule update --init --recursive")
 
 plugin_filenames = sys.argv[1:]
 
@@ -80,8 +82,8 @@ for plugin_filename in plugin_filenames:
 		try:
 			build.delete_stage()
 			build.build(plugin_filename)
-			build.system(f'cp -vi stage/* "{PACKAGES_DIR}"')
-			build.system(f'cp -vi stage/*-lin.zip "{RACK_USER_PLUGIN_DIR}"')
+			common.system(f'cp -vi stage/* "{PACKAGES_DIR}"')
+			common.system(f'cp -vi stage/*-lin.zip "{RACK_USER_PLUGIN_DIR}"')
 		except Exception as e:
 			print(e)
 			print(f"{slug} build failed")
@@ -101,10 +103,10 @@ for plugin_filename in plugin_filenames:
 
 		# Copy package
 		package_dest = os.path.join(PACKAGES_DIR, os.path.basename(plugin_filename))
-		build.system(f'cp "{plugin_filename}" "{package_dest}"')
-		build.system(f'touch "{package_dest}"')
+		common.system(f'cp "{plugin_filename}" "{package_dest}"')
+		common.system(f'touch "{package_dest}"')
 		if arch == 'lin':
-			build.system(f'cp "{plugin_filename}" "{RACK_USER_PLUGIN_DIR}"')
+			common.system(f'cp "{plugin_filename}" "{RACK_USER_PLUGIN_DIR}"')
 
 	# Copy manifest
 	with open(library_manifest_filename, "w") as f:
@@ -112,7 +114,7 @@ for plugin_filename in plugin_filenames:
 
 	# Delete screenshot cache
 	screenshots_dir = os.path.join(SCREENSHOTS_DIR, slug)
-	build.system(f'rm -rf "{screenshots_dir}"')
+	common.system(f'rm -rf "{screenshots_dir}"')
 
 	updated_slugs.add(slug)
 
@@ -121,6 +123,7 @@ if not updated_slugs:
 	print("Nothing to build")
 	exit(0)
 
+update_cache.update()
 update_modulargrid.update()
 
 # Upload data
@@ -133,15 +136,16 @@ print(f"Press enter to upload the following packages and push the library repo:
 input()
 
 # Upload packages
-build.system("cd ../packages && make upload")
+common.system("cd ../packages && make upload")
 
 # Upload screenshots
-build.system("cd ../screenshots && make upload")
+common.system("cd ../screenshots && make upload")
 
 # Commit repository
-build.system("git add manifests")
-build.system(f"git commit -m 'Update manifest for {built_slugs_str}'")
-build.system("git push")
+common.system("git add manifests")
+common.system("git add manifest-cache.json ModularGrid-VCVLibrary.json")
+common.system(f"git commit -m 'Update manifest for {built_slugs_str}'")
+common.system("git push")
 
 print()
 print(f"Updated {built_slugs_str}")

scripts/update_cache.py

@@ -0,0 +1,66 @@
+import sys
+import os
+import glob
+import json
+import re
+import requests
+
+import common
+
+
+# Get the timestamp of the earliest commit touching the manifest file
+def get_plugin_creation(manifest_filename):
+	stdout = common.system(f"git log --format=%ct -- {manifest_filename} | tail -1")
+	return float(stdout.strip())
+
+
+# Get the timestamp of the earliest commit having the module slug
+def get_module_creation(manifest_filename, module_slug):
+	stdout = common.system(f"git log --format='%H %ct' -- {manifest_filename}")
+	for line in stdout.strip().split("\n"):
+		hash, time = line.split(" ")
+		stdout = common.system(f"git show {hash}:{manifest_filename}")
+		print(stdout)
+
+
+def update():
+	# Load existing dataset
+	cache_filename = "manifest-cache.json"
+	with open(cache_filename) as f:
+		cache = json.load(f)
+
+	# Iterate plugins
+	for manifest_filename in glob.glob('manifests/*.json'):
+		with open(manifest_filename) as f:
+			plugin = json.load(f)
+
+		plugin_slug = plugin['slug']
+		cache_plugin = cache.get(plugin_slug, {})
+
+		# Get plugin creation
+		if 'creationTimestamp' not in cache_plugin:
+			cache_plugin['creationTimestamp'] = get_plugin_creation(manifest_filename)
+
+		# Iterate modules in plugin
+		cache_modules = cache.get('modules', {})
+		for module in plugin.get('modules', []):
+			module_slug = module['slug']
+			cache_module = cache_modules.get(module_slug, {})
+
+			# TODO
+			# Get module creation
+			# get_module_creation()
+			# exit()
+
+			cache_modules[module_slug] = cache_module
+		cache_plugin['modules'] = cache_modules
+
+		cache[plugin_slug] = cache_plugin
+
+
+	with open(cache_filename, 'w') as f:
+		json.dump(cache, f, indent=2)
+
+
+if __name__ == "__main__":
+	update()

scripts/update_modulargrid.py

@@ -8,6 +8,7 @@ import requests
 
 def update():
 	# Load existing dataset
+	# This dataset is standardized by ModularGrid. Do not change the schema or filename.
 	mg_filename = "ModularGrid-VCVLibrary.json"
 	with open(mg_filename) as f:
 		mg = json.load(f)