Make Float/BoolArray metaclasses for LuaJIT. Switch to Python 3.8.

Makefile

@@ -15,8 +15,8 @@ include $(RACK_DIR)/arch.mk
 
 DUKTAPE ?= 0
 QUICKJS ?= 1
+LUAJIT ?= 1
 PYTHON ?= 1
-LUA ?= 1
 
 # Entropia File System Watcher
 efsw := dep/lib/libefsw-static-release.a
@@ -60,36 +60,52 @@ $(quickjs):
 	cd QuickJS && $(MAKE) $(QUICKJS_MAKE_FLAGS) install
 endif
 
+# LuaJIT
+ifeq ($(LUAJIT), 1)
+SOURCES += src/LuaJITEngine.cpp
+luajit := dep/lib/libluajit-5.1.a
+OBJECTS += $(luajit)
+DEPS += $(luajit)
+$(luajit):
+	$(WGET) "http://luajit.org/download/LuaJIT-2.0.5.tar.gz"
+	$(SHA256) LuaJIT-2.0.5.tar.gz 874b1f8297c697821f561f9b73b57ffd419ed8f4278c82e05b48806d30c1e979
+	cd dep && $(UNTAR) ../LuaJIT-2.0.5.tar.gz
+	cd dep/LuaJIT-2.0.5 && $(MAKE)
+	cd dep/LuaJIT-2.0.5 && $(MAKE) PREFIX="$(DEP_PATH)" install
+endif
+
 # Python
 ifeq ($(PYTHON), 1)
 SOURCES += src/PythonEngine.cpp
 # Note this is a dynamic library, not static.
-python := dep/lib/libpython3.7m.so.1.0
+python := dep/lib/libpython3.8.so.1.0
 DEPS += $(python) $(numpy)
-FLAGS += -Idep/include/python3.7m
+FLAGS += -Idep/include/python3.8
 # TODO Test these flags on all platforms
 # Make dynamic linker look in the plugin folder for libpython.
 LDFLAGS += -Wl,-rpath,'$$ORIGIN'/dep/lib
-LDFLAGS += -Ldep/lib -lpython3.7m
+LDFLAGS += -Ldep/lib -lpython3.8
 LDFLAGS += -lcrypt -lpthread -ldl -lutil -lm
 DISTRIBUTABLES += $(python)
-DISTRIBUTABLES += dep/lib/python3.7
+DISTRIBUTABLES += dep/lib/python3.8
 $(python):
-	$(WGET) "https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tar.xz"
-	$(SHA256) Python-3.7.4.tar.xz fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f
-	cd dep && $(UNTAR) ../Python-3.7.4.tar.xz
-	cd dep/Python-3.7.4 && $(CONFIGURE) --build=$(MACHINE) --enable-shared --enable-optimizations
-	cd dep/Python-3.7.4 && $(MAKE) build_all
-	cd dep/Python-3.7.4 && $(MAKE) install
-
-numpy := dep/lib/python3.7/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg
-FLAGS += -Idep/lib/python3.7/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg/numpy/core/include
+	$(WGET) "https://www.python.org/ftp/python/3.8.0/Python-3.8.0rc1.tar.xz"
+	$(SHA256) Python-3.8.0rc1.tar.xz ae44df6ccf5d70059dd4d04c97156f5fcace74384a6f3cfb2fdf9baddb90a821
+	cd dep && $(UNTAR) ../Python-3.8.0rc1.tar.xz
+	cd dep/Python-3.8.0rc1 && $(CONFIGURE) --build=$(MACHINE) --enable-shared --enable-optimizations
+	cd dep/Python-3.8.0rc1 && $(MAKE) build_all
+	cd dep/Python-3.8.0rc1 && $(MAKE) install
+
+numpy := dep/lib/python3.8/site-packages/numpy
+FLAGS += -Idep/lib/python3.8/site-packages/numpy/core/include
 $(numpy): $(python)
 	$(WGET) "https://github.com/numpy/numpy/releases/download/v1.17.2/numpy-1.17.2.tar.gz"
 	$(SHA256) numpy-1.17.2.tar.gz 81a4f748dcfa80a7071ad8f3d9f8edb9f8bc1f0a9bdd19bfd44fd42c02bd286c
 	cd dep && $(UNTAR) ../numpy-1.17.2.tar.gz
 	# Don't try to find an external BLAS and LAPACK library.
-	cd dep/numpy-1.17.2 && LD_LIBRARY_PATH=../lib NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install
+	# Don't install to an egg folder.
+	# Make sure to use our built Python.
+	cd dep/numpy-1.17.2 && LD_LIBRARY_PATH=../lib NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.8 setup.py build -j4 install --single-version-externally-managed --root=/
 
 # scipy: $(numpy)
 # 	$(WGET) "https://github.com/scipy/scipy/releases/download/v1.3.1/scipy-1.3.1.tar.xz"
@@ -98,20 +114,6 @@ $(numpy): $(python)
 # 	cd dep/scipy-1.3.1 && "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install
 endif
 
-# LuaJIT
-ifeq ($(LUA), 1)
-SOURCES += src/LuaJITEngine.cpp
-luajit := dep/lib/libluajit-5.1.a
-OBJECTS += $(luajit)
-DEPS += $(luajit)
-$(luajit):
-	$(WGET) "http://luajit.org/download/LuaJIT-2.0.5.tar.gz"
-	$(SHA256) LuaJIT-2.0.5.tar.gz 874b1f8297c697821f561f9b73b57ffd419ed8f4278c82e05b48806d30c1e979
-	cd dep && $(UNTAR) ../LuaJIT-2.0.5.tar.gz
-	cd dep/LuaJIT-2.0.5 && $(MAKE)
-	cd dep/LuaJIT-2.0.5 && $(MAKE) PREFIX="$(DEP_PATH)" install
-endif
-
 # # Julia
 # julia := dep/lib/libjulia.a
 # DEPS += $(julia)

src/LuaJITEngine.cpp

@@ -5,6 +5,11 @@
 struct LuaJITEngine : ScriptEngine {
 	lua_State* L = NULL;
 
+	struct SafeArray {
+		void* p;
+		size_t len;
+	};
+
 	~LuaJITEngine() {
 		if (L)
 			lua_close(L);
@@ -34,8 +39,8 @@ struct LuaJITEngine : ScriptEngine {
 		lua_setglobal(L, "_engine");
 
 		// Set global functions
-		lua_pushcfunction(L, native_print);
-		lua_setglobal(L, "print");
+		// lua_pushcfunction(L, native_print);
+		// lua_setglobal(L, "print");
 
 		lua_pushcfunction(L, native_display);
 		lua_setglobal(L, "display");
@@ -93,64 +98,99 @@ struct LuaJITEngine : ScriptEngine {
 			return -1;
 		}
 
+		// FloatArray metatable
+		lua_newtable(L);
+		{
+			// __index
+			lua_pushcfunction(L, native_FloatArray_index);
+			lua_setfield(L, -2, "__index");
+			// __newindex
+			lua_pushcfunction(L, native_FloatArray_newindex);
+			lua_setfield(L, -2, "__newindex");
+			// __len
+			lua_pushcfunction(L, native_FloatArray_len);
+			lua_setfield(L, -2, "__len");
+		}
+		lua_setglobal(L, "FloatArray");
+
+		// BoolArray metatable
+		lua_newtable(L);
+		{
+			// __index
+			lua_pushcfunction(L, native_BoolArray_index);
+			lua_setfield(L, -2, "__index");
+			// __newindex
+			lua_pushcfunction(L, native_BoolArray_newindex);
+			lua_setfield(L, -2, "__newindex");
+			// __len
+			lua_pushcfunction(L, native_BoolArray_len);
+			lua_setfield(L, -2, "__len");
+		}
+		lua_setglobal(L, "BoolArray");
+
 		// Create block object
 		lua_newtable(L);
 		{
 			// inputs
 			lua_newtable(L);
 			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_newtable(L);
-				for (int j = 0; j < block->bufferSize; j++) {
-					lua_pushnumber(L, 0.0);
-					lua_rawseti(L, -2, j + 1);
-				}
+				SafeArray* input = (SafeArray*) lua_newuserdata(L, sizeof(SafeArray));
+				input->p = &block->inputs[i];
+				input->len = block->bufferSize;
+				lua_getglobal(L, "FloatArray");
+				lua_setmetatable(L, -2);
 				lua_rawseti(L, -2, i + 1);
 			}
 			lua_setfield(L, -2, "inputs");
+
 			// outputs
 			lua_newtable(L);
 			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_newtable(L);
-				for (int j = 0; j < block->bufferSize; j++) {
-					lua_pushnumber(L, 0.0);
-					lua_rawseti(L, -2, j + 1);
-				}
+				SafeArray* output = (SafeArray*) lua_newuserdata(L, sizeof(SafeArray));
+				output->p = &block->outputs[i];
+				output->len = block->bufferSize;
+				lua_getglobal(L, "FloatArray");
+				lua_setmetatable(L, -2);
 				lua_rawseti(L, -2, i + 1);
 			}
 			lua_setfield(L, -2, "outputs");
+
 			// knobs
-			lua_newtable(L);
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_pushnumber(L, 0.0);
-				lua_rawseti(L, -2, i + 1);
-			}
+			SafeArray* knobs = (SafeArray*) lua_newuserdata(L, sizeof(SafeArray));
+			knobs->p = &block->knobs;
+			knobs->len = 6;
+			lua_getglobal(L, "FloatArray");
+			lua_setmetatable(L, -2);
 			lua_setfield(L, -2, "knobs");
+
 			// switches
-			lua_newtable(L);
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_pushboolean(L, false);
-				lua_rawseti(L, -2, i + 1);
-			}
+			SafeArray* switches = (SafeArray*) lua_newuserdata(L, sizeof(SafeArray));
+			switches->p = &block->switches;
+			switches->len = 6;
+			lua_getglobal(L, "BoolArray");
+			lua_setmetatable(L, -2);
 			lua_setfield(L, -2, "switches");
+
 			// lights
 			lua_newtable(L);
 			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_newtable(L);
-				for (int c = 0; c < 3; c++) {
-					lua_pushnumber(L, 0.0);
-					lua_rawseti(L, -2, c + 1);
-				}
+				SafeArray* light = (SafeArray*) lua_newuserdata(L, sizeof(SafeArray));
+				light->p = &block->lights[i];
+				light->len = 3;
+				lua_getglobal(L, "FloatArray");
+				lua_setmetatable(L, -2);
 				lua_rawseti(L, -2, i + 1);
 			}
 			lua_setfield(L, -2, "lights");
+
 			// switchLights
 			lua_newtable(L);
 			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_newtable(L);
-				for (int c = 0; c < 3; c++) {
-					lua_pushnumber(L, 0.0);
-					lua_rawseti(L, -2, c + 1);
-				}
+				SafeArray* switchLight = (SafeArray*) lua_newuserdata(L, sizeof(SafeArray));
+				switchLight->p = &block->switchLights[i];
+				switchLight->len = 3;
+				lua_getglobal(L, "FloatArray");
+				lua_setmetatable(L, -2);
 				lua_rawseti(L, -2, i + 1);
 			}
 			lua_setfield(L, -2, "switchLights");
@@ -175,32 +215,6 @@ struct LuaJITEngine : ScriptEngine {
 			// bufferSize
 			lua_pushinteger(L, block->bufferSize);
 			lua_setfield(L, -2, "bufferSize");
-
-			// inputs
-			lua_getfield(L, -1, "inputs");
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_rawgeti(L, -1, i + 1);
-				for (int j = 0; j < block->bufferSize; j++) {
-					lua_pushnumber(L, block->inputs[i][j]);
-					lua_rawseti(L, -2, j + 1);
-				}
-				lua_pop(L, 1);
-			}
-			lua_pop(L, 1);
-			// knobs
-			lua_getfield(L, -1, "knobs");
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_pushnumber(L, block->knobs[i]);
-				lua_rawseti(L, -2, i + 1);
-			}
-			lua_pop(L, 1);
-			// switches
-			lua_getfield(L, -1, "switches");
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_pushboolean(L, block->switches[i]);
-				lua_rawseti(L, -2, i + 1);
-			}
-			lua_pop(L, 1);
 		}
 
 		// Duplicate process function
@@ -215,54 +229,6 @@ struct LuaJITEngine : ScriptEngine {
 			return -1;
 		}
 
-		// Get block
-		{
-			// outputs
-			lua_getfield(L, -1, "outputs");
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_rawgeti(L, -1, i + 1);
-				for (int j = 0; j < block->bufferSize; j++) {
-					lua_rawgeti(L, -1, j + 1);
-					block->outputs[i][j] = lua_tonumber(L, -1);
-					lua_pop(L, 1);
-				}
-				lua_pop(L, 1);
-			}
-			lua_pop(L, 1);
-			// knobs
-			lua_getfield(L, -1, "knobs");
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_rawgeti(L, -1, i + 1);
-				block->knobs[i] = lua_tonumber(L, -1);
-				lua_pop(L, 1);
-			}
-			lua_pop(L, 1);
-			// lights
-			lua_getfield(L, -1, "lights");
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_rawgeti(L, -1, i + 1);
-				for (int c = 0; c < 3; c++) {
-					lua_rawgeti(L, -1, c + 1);
-					block->lights[i][c] = lua_tonumber(L, -1);
-					lua_pop(L, 1);
-				}
-				lua_pop(L, 1);
-			}
-			lua_pop(L, 1);
-			// switchLights
-			lua_getfield(L, -1, "switchLights");
-			for (int i = 0; i < NUM_ROWS; i++) {
-				lua_rawgeti(L, -1, i + 1);
-				for (int c = 0; c < 3; c++) {
-					lua_rawgeti(L, -1, c + 1);
-					block->switchLights[i][c] = lua_tonumber(L, -1);
-					lua_pop(L, 1);
-				}
-				lua_pop(L, 1);
-			}
-			lua_pop(L, 1);
-		}
-
 		return 0;
 	}
 
@@ -273,14 +239,14 @@ struct LuaJITEngine : ScriptEngine {
 		return engine;
 	}
 
-	static int native_print(lua_State* L) {
-		lua_getglobal(L, "tostring");
-		lua_pushvalue(L, 1);
-		lua_call(L, 1, 1);
-		const char* s = lua_tostring(L, 1);
-		INFO("LuaJIT: %s", s);
-		return 0;
-	}
+	// static int native_print(lua_State* L) {
+	// 	lua_getglobal(L, "tostring");
+	// 	lua_pushvalue(L, 1);
+	// 	lua_call(L, 1, 1);
+	// 	const char* s = lua_tostring(L, 1);
+	// 	INFO("LuaJIT: %s", s);
+	// 	return 0;
+	// }
 
 	static int native_display(lua_State* L) {
 		lua_getglobal(L, "tostring");
@@ -292,6 +258,62 @@ struct LuaJITEngine : ScriptEngine {
 		getEngine(L)->display(s);
 		return 0;
 	}
+
+	static int native_FloatArray_index(lua_State* L) {
+		SafeArray* a = (SafeArray*) lua_touserdata(L, 1);
+		float* data = (float*) a->p;
+		size_t index = lua_tointeger(L, 2) - 1;
+		if (index >= a->len) {
+			lua_pushstring(L, "Array out of bounds");
+			lua_error(L);
+		}
+		lua_pushnumber(L, data[index]);
+		return 1;
+	}
+	static int native_FloatArray_newindex(lua_State* L) {
+		SafeArray* a = (SafeArray*) lua_touserdata(L, 1);
+		float* data = (float*) a->p;
+		size_t index = lua_tointeger(L, 2) - 1;
+		if (index >= a->len) {
+			lua_pushstring(L, "Array out of bounds");
+			lua_error(L);
+		}
+		data[index] = lua_tonumber(L, 3);
+		return 0;
+	}
+	static int native_FloatArray_len(lua_State* L) {
+		SafeArray* a = (SafeArray*) lua_touserdata(L, 1);
+		lua_pushinteger(L, a->len);
+		return 1;
+	}
+
+	static int native_BoolArray_index(lua_State* L) {
+		SafeArray* a = (SafeArray*) lua_touserdata(L, 1);
+		bool* data = (bool*) a->p;
+		size_t index = lua_tointeger(L, 2) - 1;
+		if (index >= a->len) {
+			lua_pushstring(L, "Array out of bounds");
+			lua_error(L);
+		}
+		lua_pushboolean(L, data[index]);
+		return 1;
+	}
+	static int native_BoolArray_newindex(lua_State* L) {
+		SafeArray* a = (SafeArray*) lua_touserdata(L, 1);
+		bool* data = (bool*) a->p;
+		size_t index = lua_tointeger(L, 2) - 1;
+		if (index >= a->len) {
+			lua_pushstring(L, "Array out of bounds");
+			lua_error(L);
+		}
+		data[index] = lua_toboolean(L, 3);
+		return 0;
+	}
+	static int native_BoolArray_len(lua_State* L) {
+		SafeArray* a = (SafeArray*) lua_touserdata(L, 1);
+		lua_pushinteger(L, a->len);
+		return 1;
+	}
 };
 
 

src/Prototype.cpp

@@ -298,11 +298,10 @@ void ScriptEngine::display(const std::string& message) {
 	module->message = message;
 }
 void ScriptEngine::setFrameDivider(int frameDivider) {
-	module->frameDivider = frameDivider;
+	module->frameDivider = std::max(frameDivider, 1);
 }
 void ScriptEngine::setBufferSize(int bufferSize) {
-	bufferSize = clamp(bufferSize, 1, MAX_BUFFER_SIZE);
-	module->block->bufferSize = bufferSize;
+	module->block->bufferSize = clamp(bufferSize, 1, MAX_BUFFER_SIZE);
 }
 ProcessBlock* ScriptEngine::getProcessBlock() {
 	return module->block;

src/PythonEngine.cpp

@@ -9,6 +9,14 @@ extern "C" {
 #include <thread>
 
 
+/*
+TODO:
+- Allow multiple instances with GIL.
+- Fix destructors.
+- Add config object.
+*/
+
+
 extern rack::Plugin* pluginInstance;
 
 
@@ -16,13 +24,12 @@ static void initPython() {
 	if (Py_IsInitialized())
 		return;
 
-	std::string pythonDir = rack::asset::plugin(pluginInstance, "dep/lib/python3.7");
+	std::string pythonDir = rack::asset::plugin(pluginInstance, "dep/lib/python3.8");
 	// Set python path
 	std::string sep = ":";
 	std::string pythonPath = pythonDir;
 	pythonPath += sep + pythonDir + "/lib-dynload";
-	// TODO Don't install to egg
-	pythonPath += sep + pythonDir + "/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg";
+	pythonPath += sep + pythonDir + "/site-packages";
 	wchar_t* pythonPathW = Py_DecodeLocale(pythonPath.c_str(), NULL);
 	Py_SetPath(pythonPathW);
 	PyMem_RawFree(pythonPathW);
@@ -30,10 +37,13 @@ static void initPython() {
 	Py_InitializeEx(0);
 	assert(Py_IsInitialized());
 
-	PyEval_InitThreads();
+	// PyEval_InitThreads();
 
 	// Import numpy
-	assert(_import_array() == 0);
+	if (_import_array()) {
+		PyErr_Print();
+		abort();
+	}
 }
 
 
@@ -59,6 +69,7 @@ struct PythonEngine : ScriptEngine {
 	}
 
 	int run(const std::string& path, const std::string& script) override {
+		ProcessBlock* block = getProcessBlock();
 		initPython();
 
 		// PyThreadState* tstate = PyThreadState_Get();
@@ -73,8 +84,8 @@ struct PythonEngine : ScriptEngine {
 		assert(mainDict);
 
 		// Set context pointer
-		PyObject* thisO = PyCapsule_New(this, NULL, NULL);
-		PyDict_SetItemString(mainDict, "this", thisO);
+		PyObject* engineObj = PyCapsule_New(this, NULL, NULL);
+		PyDict_SetItemString(mainDict, "_engine", engineObj);
 
 		// Add functions to globals
 		static PyMethodDef native_functions[] = {
@@ -86,6 +97,25 @@ struct PythonEngine : ScriptEngine {
 			return -1;
 		}
 
+		// Set config
+		static PyStructSequence_Field configFields[] = {
+			{"frameDivider", ""},
+			{"bufferSize", ""},
+			{NULL, NULL},
+		};
+		static PyStructSequence_Desc configDesc = {"Config", "", configFields, LENGTHOF(configFields) - 1};
+		PyTypeObject* configType = PyStructSequence_NewType(&configDesc);
+		assert(configType);
+
+		PyObject* configObj = PyStructSequence_New(configType);
+		assert(configObj);
+		PyDict_SetItemString(mainDict, "config", configObj);
+
+		// frameDivider
+		PyStructSequence_SetItem(configObj, 0, PyLong_FromLong(32));
+		// bufferSize
+		PyStructSequence_SetItem(configObj, 1, PyLong_FromLong(1));
+
 		// Compile string
 		PyObject* code = Py_CompileString(script.c_str(), path.c_str(), Py_file_input);
 		if (!code) {
@@ -103,7 +133,6 @@ struct PythonEngine : ScriptEngine {
 		DEFER({Py_DECREF(result);});
 
 		// Create block
-		ProcessBlock* block = getProcessBlock();
 		static PyStructSequence_Field blockFields[] = {
 			{"inputs", ""},
 			{"outputs", ""},
@@ -197,10 +226,10 @@ struct PythonEngine : ScriptEngine {
 	static PyObject* nativeDisplay(PyObject* self, PyObject* args) {
 		PyObject* mainDict = PyEval_GetGlobals();
 		assert(mainDict);
-		PyObject* thatO = PyDict_GetItemString(mainDict, "this");
-		assert(thatO);
-		PythonEngine* that = (PythonEngine*) PyCapsule_GetPointer(thatO, NULL);
-		assert(that);
+		PyObject* engineObj = PyDict_GetItemString(mainDict, "_engine");
+		assert(engineObj);
+		PythonEngine* engine = (PythonEngine*) PyCapsule_GetPointer(engineObj, NULL);
+		assert(engine);
 
 		PyObject* msgO = PyTuple_GetItem(args, 0);
 		if (!msgO)
@@ -210,7 +239,7 @@ struct PythonEngine : ScriptEngine {
 		DEFER({Py_DECREF(msgS);});
 
 		const char* msg = PyUnicode_AsUTF8(msgS);
-		that->display(msg);
+		engine->display(msg);
 
 		Py_INCREF(Py_None);
 		return Py_None;

tests/hello.lua

@@ -3,8 +3,10 @@ config.frameDivider = 1
 config.bufferSize = 16
 
 function process(block)
-	for j=1,block.bufferSize do
-		block.outputs[1][j] = math.random()
+	for i=1,6 do
+		for j=1,block.bufferSize do
+			block.outputs[i][j] = block.inputs[i][j]
+		end
 	end
 end
 

tests/hello.py

@@ -1,9 +1,13 @@
 frame = 0
 
+print(config)
+print(config.frameDivider)
+print(config.bufferSize)
+
 def process(block):
 	global frame
 	frame += 1
-	print(frame)
+	display(frame)
 	# print(block)
 	# block.switch_lights[:, 2] = 1
 	print(block.inputs)
@@ -12,6 +16,6 @@ def process(block):
 	print(block.switches)
 	print(block.lights)
 	print(block.switch_lights)
-	print("===")
+	# print("===")
 
 print("Hello, world!")