VCVRack
/
VCV-Prototype
mirror of https://github.com/VCVRack/VCV-Prototype.git
1
0
Fork 0
Code Issues 0 Releases 3 Wiki Activity
Browse Source

Add WIP Python engine.

tags/v1.3.0
Andrew Belt 6 years ago
parent
730409458d
commit
d77713dba2
5 changed files with 481 additions and 16 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +56
    -15
      Makefile
  2. +238
    -0
      src/DuktapeEngine.cpp
  3. +1
    -1
      src/Prototype.cpp
  4. +182
    -0
      src/PythonEngine.cpp
  5. +4
    -0
      src/ScriptEngine.hpp

+ 56
- 15
Makefile View File

@@ -6,13 +6,17 @@ CXXFLAGS +=

LDFLAGS +=

SOURCES += $(wildcard src/*.cpp)
SOURCES += src/Prototype.cpp

DISTRIBUTABLES += res examples
DISTRIBUTABLES += $(wildcard LICENSE*)

include $(RACK_DIR)/arch.mk

DUKTAPE ?= 0
QUICKJS ?= 1
PYTHON ?= 0

# Entropia File System Watcher
efsw := dep/lib/libefsw-static-release.a
DEPS += $(efsw)
@@ -23,10 +27,26 @@ $(efsw):
cd dep && $(UNZIP) e6afbec564e2.zip
cd dep/SpartanJ-efsw-e6afbec564e2 && premake4 gmake
cd dep/SpartanJ-efsw-e6afbec564e2 && $(MAKE) -C make/* config=release efsw-static-lib
mkdir -p dep/lib dep/include
cd dep/SpartanJ-efsw-e6afbec564e2 && cp lib/libefsw-static-release.a $(DEP_PATH)/lib/
cd dep/SpartanJ-efsw-e6afbec564e2 && cp -R include/efsw $(DEP_PATH)/include/

# Duktape
ifeq ($(DUKTAPE), 1)
SOURCES += src/DuktapeEngine.cpp
duktape := dep/duktape-2.4.0/src/duktape.c
DEPS += $(duktape)
SOURCES += $(duktape)
FLAGS += -Idep/duktape-2.4.0/src
$(duktape):
$(WGET) "https://duktape.org/duktape-2.4.0.tar.xz"
$(SHA256) duktape-2.4.0.tar.xz 86a89307d1633b5cedb2c6e56dc86e92679fc34b05be551722d8cc69ab0771fc
cd dep && $(UNTAR) ../duktape-2.4.0.tar.xz
endif

# QuickJS
ifeq ($(QUICKJS), 1)
SOURCES += src/QuickJSEngine.cpp
quickjs := dep/lib/quickjs/libquickjs.a
DEPS += $(quickjs)
OBJECTS += $(quickjs)
@@ -37,6 +57,41 @@ endif
$(quickjs):
cd QuickJS && $(MAKE) $(QUICKJS_MAKE_FLAGS)
cd QuickJS && $(MAKE) $(QUICKJS_MAKE_FLAGS) install
endif

# Python
ifeq ($(PYTHON), 1)
SOURCES += src/PythonEngine.cpp
python := dep/lib/libpython3.7m.so
DEPS += $(python) $(numpy)
# OBJECTS += $(python)
FLAGS += -Idep/include/python3.7m
# TODO Test these flags on all platforms
LDFLAGS += -Ldep/lib -lpython3.7m
LDFLAGS += -lcrypt -lpthread -ldl -lutil -lm
$(python):
$(WGET) "https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tar.xz"
$(SHA256) Python-3.7.4.tar.xz fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f
cd dep && $(UNTAR) ../Python-3.7.4.tar.xz
cd dep/Python-3.7.4 && $(CONFIGURE) --build=$(MACHINE) --enable-shared --enable-optimizations
cd dep/Python-3.7.4 && $(MAKE) build_all
cd dep/Python-3.7.4 && $(MAKE) install

numpy := dep/lib/python3.7/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg
FLAGS += -Idep/lib/python3.7/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg/numpy/core/include
$(numpy): $(python)
$(WGET) "https://github.com/numpy/numpy/releases/download/v1.17.2/numpy-1.17.2.tar.gz"
$(SHA256) numpy-1.17.2.tar.gz 81a4f748dcfa80a7071ad8f3d9f8edb9f8bc1f0a9bdd19bfd44fd42c02bd286c
cd dep && $(UNTAR) ../numpy-1.17.2.tar.gz
# Don't try to find an external BLAS and LAPACK library.
cd dep/numpy-1.17.2 && NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install install_headers

# scipy: $(numpy)
# $(WGET) "https://github.com/scipy/scipy/releases/download/v1.3.1/scipy-1.3.1.tar.xz"
# $(SHA256) scipy-1.3.1.tar.xz 326ffdad79f113659ed0bca80f5d0ed5e28b2e967b438bb1f647d0738073a92e
# cd dep && $(UNTAR) ../scipy-1.3.1.tar.xz
# cd dep/scipy-1.3.1 && "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install
endif

# # LuaJIT
# luajit := dep/lib/luajit.a
@@ -55,20 +110,6 @@ $(quickjs):
# $(SHA256) julia-1.2.0-full.tar.gz 2419b268fc5c3666dd9aeb554815fe7cf9e0e7265bc9b94a43957c31a68d9184
# cd dep && $(UNTAR) ../julia-1.2.0-full.tar.gz

# # Python
# python := dep/lib/libpython3.7m.a
# DEPS += $(python)
# OBJECTS += $(python)
# FLAGS += -Idep/include/python3.7m
# LDFLAGS += -lcrypt -lpthread -ldl -lutil -lm
# $(python):
# $(WGET) "https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tar.xz"
# $(SHA256) Python-3.7.4.tar.xz fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f
# cd dep && $(UNTAR) ../Python-3.7.4.tar.xz
# cd dep/Python-3.7.4 && $(CONFIGURE) --build=$(MACHINE) --enable-optimizations
# cd dep/Python-3.7.4 && $(MAKE) build_all
# cd dep/Python-3.7.4 && $(MAKE) install

# # Csound
# csound := dep/lib/libcsound.a
# DEPS += $(csound)


+ 238
- 0
src/DuktapeEngine.cpp View File

@@ -0,0 +1,238 @@
#include "ScriptEngine.hpp"
#include <duktape.h>


struct DuktapeEngine : ScriptEngine {
duk_context* ctx = NULL;

~DuktapeEngine() {
if (ctx)
duk_destroy_heap(ctx);
}

std::string getEngineName() override {
return "JavaScript";
}

int run(const std::string& path, const std::string& script) override {
assert(!ctx);
// Create duktape context
ctx = duk_create_heap_default();
if (!ctx) {
setMessage("Could not create duktape context");
return -1;
}

// Initialize globals
// user pointer
duk_push_pointer(ctx, this);
duk_put_global_string(ctx, DUK_HIDDEN_SYMBOL("p"));

// console
duk_idx_t consoleIdx = duk_push_object(ctx);
{
// log
duk_push_c_function(ctx, native_console_log, 1);
duk_put_prop_string(ctx, consoleIdx, "log");
// info (alias for log)
duk_push_c_function(ctx, native_console_log, 1);
duk_put_prop_string(ctx, consoleIdx, "info");
// debug
duk_push_c_function(ctx, native_console_debug, 1);
duk_put_prop_string(ctx, consoleIdx, "debug");
// warn
duk_push_c_function(ctx, native_console_warn, 1);
duk_put_prop_string(ctx, consoleIdx, "warn");
}
duk_put_global_string(ctx, "console");

// display
duk_push_c_function(ctx, native_display, 1);
duk_put_global_string(ctx, "display");

// config: Set defaults
duk_idx_t configIdx = duk_push_object(ctx);
{
// frameDivider
duk_push_int(ctx, 32);
duk_put_prop_string(ctx, configIdx, "frameDivider");
// bufferSize
duk_push_int(ctx, 1);
duk_put_prop_string(ctx, configIdx, "bufferSize");
}
duk_put_global_string(ctx, "config");

// Compile string
duk_push_string(ctx, path.c_str());
if (duk_pcompile_lstring_filename(ctx, 0, script.c_str(), script.size()) != 0) {
const char* s = duk_safe_to_string(ctx, -1);
rack::WARN("duktape: %s", s);
setMessage(s);
duk_pop(ctx);
return -1;
}
// Execute function
if (duk_pcall(ctx, 0)) {
const char* s = duk_safe_to_string(ctx, -1);
rack::WARN("duktape: %s", s);
setMessage(s);
duk_pop(ctx);
return -1;
}
// Ignore return value
duk_pop(ctx);

// config: Read values
duk_get_global_string(ctx, "config");
{
// frameDivider
duk_get_prop_string(ctx, -1, "frameDivider");
setFrameDivider(duk_get_int(ctx, -1));
duk_pop(ctx);
// bufferSize
duk_get_prop_string(ctx, -1, "bufferSize");
block->bufferSize = duk_get_int(ctx, -1);
duk_pop(ctx);
}
duk_pop(ctx);

// Keep process function on stack for faster calling
duk_get_global_string(ctx, "process");
if (!duk_is_function(ctx, -1)) {
setMessage("No process() function");
return -1;
}

// block (keep on stack)
duk_idx_t blockIdx = duk_push_object(ctx);
{
// inputs
duk_idx_t inputsIdx = duk_push_array(ctx);
for (int i = 0; i < NUM_ROWS; i++) {
duk_push_external_buffer(ctx);
duk_config_buffer(ctx, -1, block->inputs[i], sizeof(float) * block->bufferSize);
duk_push_buffer_object(ctx, -1, 0, sizeof(float) * block->bufferSize, DUK_BUFOBJ_FLOAT32ARRAY);
duk_put_prop_index(ctx, inputsIdx, i);
duk_pop(ctx);
}
duk_put_prop_string(ctx, blockIdx, "inputs");

// outputs
duk_idx_t outputsIdx = duk_push_array(ctx);
for (int i = 0; i < NUM_ROWS; i++) {
duk_push_external_buffer(ctx);
duk_config_buffer(ctx, -1, block->outputs[i], sizeof(float) * block->bufferSize);
duk_push_buffer_object(ctx, -1, 0, sizeof(float) * block->bufferSize, DUK_BUFOBJ_FLOAT32ARRAY);
duk_put_prop_index(ctx, outputsIdx, i);
duk_pop(ctx);
}
duk_put_prop_string(ctx, blockIdx, "outputs");

// knobs
duk_push_external_buffer(ctx);
duk_config_buffer(ctx, -1, block->knobs, sizeof(float) * NUM_ROWS);
duk_push_buffer_object(ctx, -1, 0, sizeof(float) * NUM_ROWS, DUK_BUFOBJ_FLOAT32ARRAY);
duk_put_prop_string(ctx, blockIdx, "knobs");
duk_pop(ctx);

// switches
duk_push_external_buffer(ctx);
duk_config_buffer(ctx, -1, block->switches, sizeof(bool) * NUM_ROWS);
duk_push_buffer_object(ctx, -1, 0, sizeof(bool) * NUM_ROWS, DUK_BUFOBJ_UINT8ARRAY);
duk_put_prop_string(ctx, blockIdx, "switches");
duk_pop(ctx);

// lights
duk_idx_t lightsIdx = duk_push_array(ctx);
for (int i = 0; i < NUM_ROWS; i++) {
duk_push_external_buffer(ctx);
duk_config_buffer(ctx, -1, block->lights[i], sizeof(float) * 3);
duk_push_buffer_object(ctx, -1, 0, sizeof(float) * 3, DUK_BUFOBJ_FLOAT32ARRAY);
duk_put_prop_index(ctx, lightsIdx, i);
duk_pop(ctx);
}
duk_put_prop_string(ctx, blockIdx, "lights");

// switchLights
duk_idx_t switchLightsIdx = duk_push_array(ctx);
for (int i = 0; i < NUM_ROWS; i++) {
duk_push_external_buffer(ctx);
duk_config_buffer(ctx, -1, block->switchLights[i], sizeof(float) * 3);
duk_push_buffer_object(ctx, -1, 0, sizeof(float) * 3, DUK_BUFOBJ_FLOAT32ARRAY);
duk_put_prop_index(ctx, switchLightsIdx, i);
duk_pop(ctx);
}
duk_put_prop_string(ctx, blockIdx, "switchLights");
}

return 0;
}

int process() override {
// block
duk_idx_t blockIdx = duk_get_top(ctx) - 1;
{
// sampleRate
duk_push_number(ctx, block->sampleRate);
duk_put_prop_string(ctx, blockIdx, "sampleRate");

// sampleTime
duk_push_number(ctx, block->sampleTime);
duk_put_prop_string(ctx, blockIdx, "sampleTime");

// bufferSize
duk_push_int(ctx, block->bufferSize);
duk_put_prop_string(ctx, blockIdx, "bufferSize");
}

// Duplicate process function
duk_dup(ctx, -2);
// Duplicate block object
duk_dup(ctx, -2);
// Call process function
if (duk_pcall(ctx, 1)) {
const char* s = duk_safe_to_string(ctx, -1);
rack::WARN("duktape: %s", s);
setMessage(s);
duk_pop(ctx);
return -1;
}
// return value
duk_pop(ctx);

return 0;
}

static DuktapeEngine* getDuktapeEngine(duk_context* ctx) {
duk_get_global_string(ctx, DUK_HIDDEN_SYMBOL("p"));
DuktapeEngine* engine = (DuktapeEngine*) duk_get_pointer(ctx, -1);
duk_pop(ctx);
return engine;
}

static duk_ret_t native_console_log(duk_context* ctx) {
const char* s = duk_safe_to_string(ctx, -1);
rack::INFO("VCV Prototype: %s", s);
return 0;
}
static duk_ret_t native_console_debug(duk_context* ctx) {
const char* s = duk_safe_to_string(ctx, -1);
rack::DEBUG("VCV Prototype: %s", s);
return 0;
}
static duk_ret_t native_console_warn(duk_context* ctx) {
const char* s = duk_safe_to_string(ctx, -1);
rack::WARN("VCV Prototype: %s", s);
return 0;
}
static duk_ret_t native_display(duk_context* ctx) {
const char* s = duk_safe_to_string(ctx, -1);
getDuktapeEngine(ctx)->setMessage(s);
return 0;
}
};


ScriptEngine* createDuktapeEngine() {
return new DuktapeEngine;
}

+ 1
- 1
src/Prototype.cpp View File

@@ -193,6 +193,7 @@ struct Prototype : Module {

if (script == "")
return;
this->script = script;

// Create script engine from path extension
std::string ext = string::filenameExtension(string::filename(path));
@@ -212,7 +213,6 @@ struct Prototype : Module {
return;
}
block->bufferSize = clamp(block->bufferSize, 1, MAX_BUFFER_SIZE);
this->script = script;
this->engineName = scriptEngine->getEngineName();
}



+ 182
- 0
src/PythonEngine.cpp View File

@@ -0,0 +1,182 @@
extern "C" {
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <numpy/arrayobject.h>
}
#include <dlfcn.h>
#include "ScriptEngine.hpp"
#include <thread>


extern rack::Plugin* pluginInstance;


static void initPython() {
if (Py_IsInitialized())
return;

// Okay, this is an IQ 200 solution for fixing the following issue.
// - Rack (the "application") `dlopen()`s this plugin with RTLD_LOCAL.
// - This plugin links with libpython, either statically or dynamically. In either case, symbols are hidden to "outside" libraries.
// - A Python script runs `import math` for example, which loads math.cpython*.so.
// - Since that's an "outside" library, it can't access libpython symbols, because it doesn't link to libpython itself.
// The best solution I have is to dlopen() with RTLD_GLOBAL within the plugin, which will make all libpython symbols available to the *entire* Rack application.
// The plugin still needs to -lpython because otherwise Rack will complain that there are unresolved symbols in the plugin, so after the following lines, libpython will be in memory *twice*, unless dlopen() is doing some optimization I'm not aware of.
std::string libDir = rack::asset::plugin(pluginInstance, "dep/lib");
std::string pythonLib = libDir + "/libpython3.7m.so";
void* handle = dlopen(pythonLib.c_str(), RTLD_NOW | RTLD_GLOBAL);
assert(handle);
// Set python path
std::string sep = ":";
std::string pythonPath = libDir + "/python3.7";
pythonPath += sep + libDir + "/python3.7/lib-dynload";
wchar_t* pythonPathW = Py_DecodeLocale(pythonPath.c_str(), NULL);
Py_SetPath(pythonPathW);
PyMem_RawFree(pythonPathW);
// Initialize but don't register signal handlers
Py_InitializeEx(0);

// PyEval_InitThreads();
}


struct PythonEngine : ScriptEngine {
PyObject* mainDict = NULL;
PyObject* processFunc = NULL;
PyObject* blockObj = NULL;
PyInterpreterState* interp = NULL;

~PythonEngine() {
if (interp)
PyInterpreterState_Delete(interp);
}

std::string getEngineName() override {
return "Python";
}

int run(const std::string& path, const std::string& script) override {
initPython();

// PyThreadState* tstate = PyThreadState_Get();
// interp = PyInterpreterState_New();
// PyThreadState_Swap(tstate);

// Get globals dictionary
PyObject* mainModule = PyImport_AddModule("__main__");
assert(mainModule);
mainDict = PyModule_GetDict(mainModule);
assert(mainDict);

// Add functions to globals
DEFER({Py_DECREF(mainDict);});
static PyMethodDef native_functions[] = {
{"display", native_display, METH_VARARGS, ""},
{NULL, NULL, 0, NULL},
};
if (PyModule_AddFunctions(mainModule, native_functions)) {
WARN("Could not add global functions");
return -1;
}

// Compile string
PyObject* code = Py_CompileString(script.c_str(), path.c_str(), Py_file_input);
if (!code) {
PyErr_Print();
return -1;
}
DEFER({Py_DECREF(code);});

// Evaluate string
PyObject* result = PyEval_EvalCode(code, mainDict, mainDict);
if (!result) {
PyErr_Print();
return -1;
}
DEFER({Py_DECREF(result);});

// Get process function from globals
processFunc = PyDict_GetItemString(mainDict, "process");
if (!processFunc) {
setMessage("No process() function");
return -1;
}
if (!PyCallable_Check(processFunc)) {
setMessage("process() is not callable");
return -1;
}

// Create block
static PyStructSequence_Field blockFields[] = {
{"inputs", ""},
{"outputs", ""},
{"knobs", ""},
{"switches", ""},
{"lights", ""},
{"switch_lights", ""},
{NULL, NULL},
};
static PyStructSequence_Desc blockDesc = {"block", "", blockFields, LENGTHOF(blockFields) - 1};
PyTypeObject* blockType = PyStructSequence_NewType(&blockDesc);
assert(blockType);
blockObj = PyStructSequence_New(blockType);
assert(blockObj);
PyStructSequence_SetItem(blockObj, 1, PyFloat_FromDouble(42.f));
PyStructSequence_SetItem(blockObj, 2, PyFloat_FromDouble(42.f));
PyStructSequence_SetItem(blockObj, 3, PyFloat_FromDouble(42.f));
PyStructSequence_SetItem(blockObj, 4, PyFloat_FromDouble(42.f));
PyStructSequence_SetItem(blockObj, 5, PyFloat_FromDouble(42.f));

// inputs
// npy_intp dims[] = {NUM_ROWS, MAX_BUFFER_SIZE};
// PyObject* inputs = PyArray_SimpleNewFromData(2, dims, NPY_FLOAT32, block->inputs);
// PyStructSequence_SetItem(blockObj, 0, inputs);

return 0;
}

int process() override {
// Call process()
PyObject* args = PyTuple_Pack(1, blockObj);
assert(args);
DEFER({Py_DECREF(args);});
PyObject* processResult = PyObject_CallObject(processFunc, args);
if (!processResult) {
PyErr_Print();

// PyObject *ptype, *pvalue, *ptraceback;
// PyErr_Fetch(&ptype, &pvalue, &ptraceback);
// const char* str = PyUnicode_AsUTF8(pvalue);
// if (!str)
// return -1;

// setMessage(str);
return -1;
}
DEFER({Py_DECREF(processResult);});

// PyThreadState* tstate = PyThreadState_New(interp);
// PyEval_RestoreThread(tstate);
// PyThreadState_Clear(tstate);
// PyThreadState_DeleteCurrent();
return 0;
}

static PyObject* native_display(PyObject* self, PyObject* args) {
PyObject* msg = PyTuple_GetItem(args, 0);
if (!msg)
return NULL;

const char* msgS = PyUnicode_AsUTF8(msg);
DEBUG("%s", msgS);

Py_INCREF(Py_None);
return Py_None;
}
};


ScriptEngine* createPythonEngine() {
return new PythonEngine;
}

+ 4
- 0
src/ScriptEngine.hpp View File

@@ -50,12 +50,16 @@ struct ScriptEngine {
// List of ScriptEngines

// Add your createMyEngine() function here.
ScriptEngine* createDuktapeEngine();
ScriptEngine* createQuickJSEngine();
ScriptEngine* createPythonEngine();

inline ScriptEngine* createScriptEngine(std::string ext) {
ext = rack::string::lowercase(ext);
if (ext == "js")
return createQuickJSEngine();
// else if (ext == "py")
// return createPythonEngine();
// Add your file extension check here.
return NULL;
}

Write Preview
Loading…
Cancel
Save