From de0a75863b43ab17f34930deb36201a34ba4166d Mon Sep 17 00:00:00 2001 From: Andrew Belt Date: Fri, 5 Apr 2024 00:05:20 -0400 Subject: [PATCH] Tweak update scripts. Add BUILD_MAC_ARM64_ONLY and several other adjustments. --- scripts/update.py | 192 +++++++++++++++++++++++++--------- scripts/update_cache.py | 2 +- scripts/update_modulargrid.py | 3 + 3 files changed, 146 insertions(+), 51 deletions(-) diff --git a/scripts/update.py b/scripts/update.py index d43cb092..0811ed1f 100644 --- a/scripts/update.py +++ b/scripts/update.py @@ -5,6 +5,7 @@ import json import time import re import tempfile +import random import common import update_modulargrid @@ -18,11 +19,49 @@ RACK_SYSTEM_DIR = "../Rack2" RACK_USER_DIR = "$HOME/.Rack2" SCREENSHOTS_DIR = os.path.join(RACK_USER_DIR, "screenshots") PLUGIN_DIR = os.path.join(RACK_USER_DIR, "plugins-lin-x64") +FILES_DIR = "../files/packages" + +BUILD_MAC_ARM64_ONLY = False + + +def package_exists(slug, version, os_, cpu): + if os.path.isfile(os.path.join(PACKAGES_DIR, f"{slug}-{version}-{os_}-{cpu}.vcvplugin")): + return True + if cpu == 'x64': + if os.path.isfile(os.path.join(PACKAGES_DIR, f"{slug}-{version}-{os_}.vcvplugin")): + return True + return False + + +# Get missing Mac ARM64 slugs +def print_missing_mac_arm64(): + emails = set() + + for manifest_path in glob.glob("manifests/*.json"): + with open(manifest_path, "r") as f: + manifest = json.load(f) + slug = manifest['slug'] + version = manifest['version'] + author = manifest['author'] + license = manifest.get('license', "") + email = manifest.get('authorEmail', "") + + if not package_exists(slug, version, 'mac', 'x64'): + continue + if package_exists(slug, version, 'mac', 'arm64'): + continue + + print(f"{slug} {version}\t\t{email}\t\t{license}") + +# print_missing_mac_arm64() +# exit(0) + # Update git before continuing -common.system("git pull") -common.system("git submodule sync --quiet") -common.system("git submodule update --init --recursive") +if not BUILD_MAC_ARM64_ONLY: + common.system("git pull") + common.system("git submodule sync --recursive --quiet") + common.system("git submodule update --init --recursive") plugin_paths = sys.argv[1:] @@ -30,12 +69,17 @@ plugin_paths = sys.argv[1:] if not plugin_paths: plugin_paths = glob.glob("repos/*") +# Randomize repo order +plugin_paths = random.sample(plugin_paths, len(plugin_paths)) + manifest_versions = {} for plugin_path in plugin_paths: plugin_path = os.path.abspath(plugin_path) (plugin_basename, plugin_ext) = os.path.splitext(os.path.basename(plugin_path)) - # Extract manifest from plugin dir or package + + # Get manifest + # Source dir if os.path.isdir(plugin_path): manifest_filename = os.path.join(plugin_path, "plugin.json") try: @@ -47,58 +91,90 @@ for plugin_path in plugin_paths: continue slug = manifest['slug'] version = manifest['version'] - # Extract manifest from .vcvplugin + + # Plugin package elif plugin_ext == ".vcvplugin": - m = re.match(r'^(.*)-(2\..*?)-(.*)$', plugin_basename) + m = re.match(r'^(.*)-(\d\..*?)-(.*?)-(.*?)$', plugin_basename) if not m: raise Exception(f"Filename {plugin_path} invalid format") slug = m[1] version = m[2] - arch = m[3] - # Open ZIP - tempdir = tempfile.mkdtemp() - common.system(f'zstd -d < "{plugin_path}" | tar -x -C "{tempdir}"') - # Unzip manifest - manifest_filename = os.path.join(tempdir, slug, "plugin.json") - with open(manifest_filename) as f: - manifest = json.load(f) - if manifest['slug'] != slug: - raise Exception(f"Manifest slug does not match filename slug {slug}") - if manifest['version'] != version: - raise Exception(f"Manifest version does not match filename version {version}") - common.system(f'rm -rf "{tempdir}"') + os_ = m[3] + cpu = m[4] + # Extract ZIP to temp dir + try: + tempdir = tempfile.mkdtemp() + common.system(f'zstd -d < "{plugin_path}" | tar -x -C "{tempdir}"') + # Read manifest + manifest_filename = os.path.join(tempdir, slug, "plugin.json") + with open(manifest_filename) as f: + manifest = json.load(f) + if manifest['slug'] != slug: + raise Exception(f"Manifest slug {manifest['slug']} does not match filename slug {slug}") + if manifest['version'] != version: + raise Exception(f"Manifest version {manifest['version']} does not match filename version {version}") + finally: + common.system(f'rm -rf "{tempdir}"') + else: raise Exception(f"Plugin {plugin_path} is not a valid format") + # Slug blacklist + # if slug == 'questionablemodules': + # continue + # Get library manifest library_manifest_filename = os.path.join(MANIFESTS_DIR, f"{slug}.json") - # Warn if manifest is new - # if not os.path.exists(library_manifest_filename): - # print(f"Manifest {slug} is new, press enter to approve.") - # input() + library_manifest = None + try: + with open(library_manifest_filename, "r") as f: + library_manifest = json.load(f) + except IOError: + # Warn if manifest is new + print(f"Manifest {slug} is new, press enter to approve.") + if input() != '': + continue + pass + # Source dir if os.path.isdir(plugin_path): - # Check if the library manifest is up to date - try: - with open(library_manifest_filename, "r") as f: - library_manifest = json.load(f) - if library_manifest and version == library_manifest['version']: + # Check library manifest version + if library_manifest and version == library_manifest['version']: + # Build only if library manifest is out of date + if not BUILD_MAC_ARM64_ONLY: + continue + else: + # Build Mac ARM64 only if library manifest is up to date + if BUILD_MAC_ARM64_ONLY: continue - except IOError: - pass # Build repo + if BUILD_MAC_ARM64_ONLY: + if package_exists(slug, version, 'mac', 'arm64'): + print(f"{slug} {version} Mac ARM64 package already exists") + continue + if not package_exists(slug, version, 'mac', 'x64'): + print(f"{slug} {version} Mac x64 doesn't exist, not building") + continue + print() print(f"Building {slug}") try: + # Clean repo common.system(f'cd "{TOOLCHAIN_DIR}" && make plugin-build-clean') + # Build repo for each arch common.system(f'cd "{TOOLCHAIN_DIR}" && make -j2 plugin-build-mac-arm64 PLUGIN_DIR="{plugin_path}"') - common.system(f'cd "{TOOLCHAIN_DIR}" && make -j2 plugin-build-mac-x64 PLUGIN_DIR="{plugin_path}"') - common.system(f'cd "{TOOLCHAIN_DIR}" && make -j2 plugin-build-win-x64 PLUGIN_DIR="{plugin_path}"') - common.system(f'cd "{TOOLCHAIN_DIR}" && make -j2 plugin-build-lin-x64 PLUGIN_DIR="{plugin_path}"') + if not BUILD_MAC_ARM64_ONLY: + common.system(f'cd "{TOOLCHAIN_DIR}" && make -j2 plugin-build-mac-x64 PLUGIN_DIR="{plugin_path}"') + common.system(f'cd "{TOOLCHAIN_DIR}" && make -j2 plugin-build-win-x64 PLUGIN_DIR="{plugin_path}"') + common.system(f'cd "{TOOLCHAIN_DIR}" && make -j2 plugin-build-lin-x64 PLUGIN_DIR="{plugin_path}"') + # Copy package to packages dir common.system(f'cp -v "{TOOLCHAIN_DIR}"/plugin-build/* "{PACKAGES_DIR}"/') + # Copy Mac ARM64 package to files dir for testing + common.system(f'cp -v "{TOOLCHAIN_DIR}"/plugin-build/*-mac-arm64.vcvplugin "{FILES_DIR}"/') # Install Linux package for testing - common.system(f'cp -v "{TOOLCHAIN_DIR}"/plugin-build/*-lin-x64.vcvplugin "{PLUGIN_DIR}"/') + if not BUILD_MAC_ARM64_ONLY: + common.system(f'cp -v "{TOOLCHAIN_DIR}"/plugin-build/*-lin-x64.vcvplugin "{PLUGIN_DIR}"/') except Exception as e: print(e) print(f"{slug} build failed") @@ -108,8 +184,10 @@ for plugin_path in plugin_paths: common.system(f'cd "{TOOLCHAIN_DIR}" && make plugin-build-clean') # Open plugin issue thread - os.system(f"xdg-open 'https://github.com/VCVRack/library/issues?utf8=%E2%9C%93&q=is%3Aissue+is%3Aopen+in%3Atitle+{slug}' &") + if not BUILD_MAC_ARM64_ONLY: + os.system(f"xdg-open 'https://github.com/VCVRack/library/issues?q=is%3Aissue+in%3Atitle+{slug}' &") + # Plugin package elif plugin_ext == ".vcvplugin": # Review manifest for errors print(json.dumps(manifest, indent=" ")) @@ -117,24 +195,38 @@ for plugin_path in plugin_paths: input() # Copy package - package_filename = os.path.basename(plugin_path) - common.system(f'cp "{plugin_path}" "{PACKAGES_DIR}/{package_filename}"') + common.system(f'cp "{plugin_path}" "{PACKAGES_DIR}/"') # Update file timestamp + package_filename = os.path.basename(plugin_path) common.system(f'touch "{PACKAGES_DIR}/{package_filename}"') + # Copy Mac ARM64 package to files dir for testing + if os_ == 'mac' and cpu == 'arm64': + common.system(f'cp -v "{plugin_path}" "{FILES_DIR}"/') # Install Linux package for testing - if arch == 'lin' or arch == 'lin-x64': - common.system(f'cp "{plugin_path}" "{PLUGIN_DIR}/{package_filename}"') + if os_ == 'lin': + common.system(f'cp "{plugin_path}" "{PLUGIN_DIR}/"') # Copy manifest - with open(library_manifest_filename, "w") as f: - json.dump(manifest, f, indent=" ") + if not BUILD_MAC_ARM64_ONLY: + with open(library_manifest_filename, "w") as f: + json.dump(manifest, f, indent=" ") - # Delete screenshot cache - screenshots_dir = os.path.join(SCREENSHOTS_DIR, slug) - common.system(f'rm -rf "{screenshots_dir}"') + if not BUILD_MAC_ARM64_ONLY: + # Delete screenshot cache + screenshots_dir = os.path.join(SCREENSHOTS_DIR, slug) + common.system(f'rm -rf "{screenshots_dir}"') manifest_versions[slug] = version +manifest_versions_str = ", ".join(map(lambda pair: pair[0] + " to " + pair[1], manifest_versions.items())) + + +if BUILD_MAC_ARM64_ONLY: + print() + print(f"Test plugins on Mac ARM64 and press enter if approved: {manifest_versions_str}") + input() + common.system("cd ../packages && make upload") + exit(0) if not manifest_versions: print("Nothing to build") @@ -145,8 +237,6 @@ update_modulargrid.update() # Upload data -manifest_versions_str = ", ".join(map(lambda pair: pair[0] + " to " + pair[1], manifest_versions.items())) - print() print(f"Press enter to launch Rack and test the following packages: {manifest_versions_str}") input() @@ -166,16 +256,18 @@ except: print(f"Rack failed! Enter to continue if desired") common.system("cd ../screenshots && make -j$(nproc)") +# Commit git repo +common.system("git add manifests") +common.system("git add manifests-cache.json ModularGrid-VCVLibrary.json") +common.system(f"git commit -m 'Update manifest {manifest_versions_str}'") + # Upload packages common.system("cd ../packages && make upload") # Upload screenshots common.system("cd ../screenshots && make upload") -# Commit repository -common.system("git add manifests") -common.system("git add manifests-cache.json ModularGrid-VCVLibrary.json") -common.system(f"git commit -m 'Update manifest {manifest_versions_str}'") +# Push git repo common.system("git push") print() diff --git a/scripts/update_cache.py b/scripts/update_cache.py index 0400c8d3..481ab15d 100644 --- a/scripts/update_cache.py +++ b/scripts/update_cache.py @@ -82,7 +82,7 @@ def update(): cache_plugin = cache.get(plugin_slug, {}) # Get plugin build - print(f"Getting buildTimestamp for plugin {plugin_slug}") + # print(f"Getting buildTimestamp for plugin {plugin_slug}") buildTimestamp = get_plugin_build(plugin) if buildTimestamp: cache_plugin['buildTimestamp'] = buildTimestamp diff --git a/scripts/update_modulargrid.py b/scripts/update_modulargrid.py index 4bdcb6e7..efb6363b 100644 --- a/scripts/update_modulargrid.py +++ b/scripts/update_modulargrid.py @@ -46,6 +46,9 @@ def update(): mg_url = mg_data['mgUrl'] r = requests.get(mg_url) m = re.search(r'data-module-id\w*=\w*"(\d+)"', r.text) + if not m: + print(f"No ModularGrid ID found for {plugin_slug} {module_slug}") + continue mg_id = m.group(1) if not mg_id: print(f"No ModularGrid ID found for {plugin_slug} {module_slug}")