From ed1972a0a7d66deac76449f9cc15318281d88989 Mon Sep 17 00:00:00 2001 From: clfreville2 Date: Mon, 13 May 2024 15:34:43 +0200 Subject: [PATCH 1/2] Allow multiple runner backends --- config.toml | 13 +++++++ src/config.cpp | 32 +++++++++++++++++ src/config.hpp | 14 ++++++++ src/main.cpp | 18 +++++----- src/runner.cpp | 94 ++++++++++++++++++++++++++++++++++++++++++-------- src/runner.hpp | 53 +++++++++++++++++++++++++--- 6 files changed, 195 insertions(+), 29 deletions(-) diff --git a/config.toml b/config.toml index dca0aa4..bbf0fdb 100644 --- a/config.toml +++ b/config.toml @@ -5,3 +5,16 @@ threads = 1 [runner] timeout = 2 + +[[runners]] +[javascript] +type = "bubblewrap" +binary = ["node", "-e", "{}"] + +[bash] +type = "bubblewrap" +binary = ["bash", "-c", "{}"] + +[moshell] +type = "docker" +image = "ghcr.io/moshell-lang/moshell:master" diff --git a/src/config.cpp b/src/config.cpp index 89c4686..0a0607b 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -4,6 +4,37 @@ #include static sk::config read(const toml::table &config) { + std::unordered_map strategies; + auto *table = config["runners"].as_table(); + if (table != nullptr) { + for (auto [name, backend_config] : *table) { + auto *backend_table = backend_config.as_table(); + if (backend_table == nullptr) { + std::cerr << "Invalid runner config for " << name << ": expected table\n"; + continue; + } + auto *backend_type = backend_table->get_as("type"); + if (backend_type == nullptr) { + std::cerr << "Invalid runner config for " << name << ": missing type field\n"; + continue; + } + auto *args = backend_table->get_as("args"); + if (args == nullptr) { + std::cerr << "Invalid runner config for " << name << ": missing args field\n"; + continue; + } + std::vector args_vector; + for (const auto &arg : *args) { + auto *arg_string = arg.as_string(); + if (arg_string == nullptr) { + std::cerr << "Invalid runner config for " << name << ": args must be strings\n"; + continue; + } + args_vector.emplace_back(*arg_string); + } + strategies.emplace(name, sk::bubblewrap_config{std::move(args_vector)}); + } + } return sk::config{ sk::queue_config{ config["queue"]["pull"].value_or("tcp://localhost:5557"), @@ -12,6 +43,7 @@ static sk::config read(const toml::table &config) { }, sk::runner_config{ config["runner"]["timeout"].value_or(2u), + std::move(strategies), }, }; } diff --git a/src/config.hpp b/src/config.hpp index efb39ab..cd8f865 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -2,6 +2,9 @@ #include #include +#include +#include +#include namespace sk { struct queue_config { @@ -10,8 +13,19 @@ struct queue_config { unsigned int nb_threads; }; +struct bubblewrap_config { + std::vector args; +}; + +struct docker_config : bubblewrap_config { + std::string image; +}; + +using runner_strategy_config = std::variant; + struct runner_config { unsigned int timeout; + std::unordered_map strategies; }; struct config { diff --git a/src/main.cpp b/src/main.cpp index 04bc138..00bb0c5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -36,7 +36,7 @@ sk::runner_backend detect_backend() { return sk::runner_backend::BubbleWrap; } -sk::runner *global_runner = nullptr; +sk::runner_list *global_runners = nullptr; int main(int argc, char **argv) { int opt; @@ -62,12 +62,12 @@ int main(int argc, char **argv) { } sk::config config = sk::config::read_or_default(config_path, !may_use_default_config); - sk::runner runner(selected_backend.has_value() ? selected_backend.value() : detect_backend(), config.runner); - global_runner = &runner; + sk::runner_list runners(selected_backend.has_value() ? selected_backend.value() : detect_backend(), config.runner); + global_runners = &runners; struct sigaction action {}; action.sa_handler = [](int) { - if (global_runner) { - global_runner->exit_active_jobs(); + if (global_runners) { + global_runners->exit_active_jobs(); } }; sigaction(SIGINT, &action, nullptr); @@ -82,8 +82,8 @@ int main(int argc, char **argv) { std::stringstream buffer; buffer << t.rdbuf(); std::string code = buffer.str(); - sk::program program{"sample-code", code, "ghcr.io/moshell-lang/moshell:master"}; - sk::run_result result = runner.run_blocking(program); + sk::program program{"sample-code", code, "moshell"}; + sk::run_result result = runners.run_blocking(program); std::cout << "exited with: " << result.exit_code << "\n"; std::cout << "out: " << result.out << "\n"; std::cout << "err: " << result.err << "\n"; @@ -132,7 +132,7 @@ int main(int argc, char **argv) { std::cout << "Executing " << codeLen << " bytes code.\n"; #endif sk::program program{std::move(jobId), std::move(requestString), std::move(imageString)}; - sk::run_result result = runner.run_blocking(program); + sk::run_result result = runners.run_blocking(program); if (!result.out.empty()) { send(STDOUT_CLIENT_BOUND, program.name, result.out); @@ -151,7 +151,7 @@ int main(int argc, char **argv) { continue; } std::string jobId(message, sk::JOB_ID_LEN); - runner.kill_active(jobId); + runners.kill_active(jobId); break; } default: diff --git a/src/runner.cpp b/src/runner.cpp index b855335..06a52fd 100644 --- a/src/runner.cpp +++ b/src/runner.cpp @@ -18,8 +18,67 @@ static auto ensure = [](int res) -> void { } }; +template inline constexpr bool always_false_v = false; + namespace sk { -runner::runner(runner_backend backend, const runner_config &config) : backend{backend}, timeout{static_cast(config.timeout)} {} +execution_strategy::execution_strategy(std::vector patterns) : patterns(std::move(patterns)) {} + +std::unique_ptr execution_strategy::create(const runner_strategy_config &config) { + return std::visit( + [](auto &&strategy) -> std::unique_ptr { + using T = std::decay_t; + std::vector patterns; + if constexpr (std::is_same_v) { + return std::make_unique(std::move(patterns)); + } else if constexpr (std::is_same_v) { + return std::make_unique(std::move(patterns), strategy.image); + } else { + static_assert(always_false_v, "non-exhaustive visitor!"); + } + }, + config); +} + +bubblewrap_execution_strategy::bubblewrap_execution_strategy(std::vector patterns) : execution_strategy{std::move(patterns)} {} + +std::vector bubblewrap_execution_strategy::start(const program &program) { return {"bwrap", "--ro-bind", "/usr", "/usr", "--dir", "/tmp", "--dir", "/var", "--proc", "/proc", "--dev", "/dev", "--symlink", "usr/lib", "/lib", "--symlink", "usr/lib64", "/lib64", "--symlink", "usr/bin", "/bin", "--symlink", "usr/sbin", "/sbin", "--unshare-all", "--die-with-parent", "/bin/sh", nullptr}; } + +void bubblewrap_execution_strategy::stop(const active_job &job) { ensure(kill(job.pid, SIGINT)); } + +docker_execution_strategy::docker_execution_strategy(std::vector patterns, std::string image) : execution_strategy{std::move(patterns)}, image{std::move(image)} {} + +std::vector docker_execution_strategy::start(const program &program) { return {"docker", "run", "--rm", "-i", "--name", program.name.c_str(), "--pull=never", "--cap-drop=ALL", "--network=none", "--memory=64m", "--memory-swap=64m", "--pids-limit=128", image.c_str(), nullptr}; } + +void docker_execution_strategy::stop(const active_job &job) { + const char *const kill_args[] = {"docker", "kill", job.job_id.c_str(), nullptr}; + pid_t kill_pid; + ensure(posix_spawnp(&kill_pid, kill_args[0], nullptr, nullptr, const_cast(kill_args), nullptr)); +} + +runner::runner(const runner_strategy_config &config, unsigned int timeout) : backend{execution_strategy::create(config)}, timeout{static_cast(timeout)} {} + +runner::runner(const runner &other) : backend{&*other.backend}, timeout{other.timeout} {} + +runner::runner(runner &&other) noexcept : backend{std::move(other.backend)}, timeout{other.timeout} {} + +runner &runner::operator=(const runner &other) { + if (this == &other) { + return *this; + } + std::unique_ptr new_backend(&*other.backend); + backend = std::move(new_backend); + timeout = other.timeout; + return *this; +} + +runner &runner::operator=(runner &&other) noexcept { + if (this == &other) { + return *this; + } + backend = std::move(other.backend); + timeout = other.timeout; + return *this; +} run_result runner::run_blocking(const program &program) { // Open file descriptors ahead of time @@ -51,16 +110,11 @@ run_result runner::run_blocking(const program &program) { posix_spawn_file_actions_adddup2(&actions, in_pipe[0], STDIN_FILENO); posix_spawn_file_actions_adddup2(&actions, out_pipe[1], STDOUT_FILENO); posix_spawn_file_actions_adddup2(&actions, err_pipe[1], STDERR_FILENO); - const char *const docker_args[] = {"docker", "run", "--rm", "-i", "--name", program.name.c_str(), "--pull=never", "--cap-drop=ALL", "--network=none", "--memory=64m", "--memory-swap=64m", "--pids-limit=128", program.image.c_str(), nullptr}; - const char *const bwrap_args[] = {"bwrap", "--ro-bind", "/usr", "/usr", "--dir", "/tmp", "--dir", "/var", "--proc", "/proc", "--dev", "/dev", "--symlink", "usr/lib", "/lib", "--symlink", "usr/lib64", "/lib64", "--symlink", "usr/bin", "/bin", "--symlink", "usr/sbin", "/sbin", "--unshare-all", "--die-with-parent", "/bin/sh", nullptr}; - const char *const *args = docker_args; - if (backend == runner_backend::BubbleWrap) { - args = bwrap_args; - } + std::vector args = backend->start(program); pid_t pid; bool killed = false; int exit_code; - if (posix_spawnp(&pid, args[0], &actions, nullptr, const_cast(args), nullptr) != 0) { + if (posix_spawnp(&pid, args[0], &actions, nullptr, const_cast(args.data()), nullptr) != 0) { throw std::system_error{errno, std::generic_category()}; } @@ -184,13 +238,23 @@ void runner::exit_active_jobs() { active_jobs.clear(); } -void runner::exit(const active_job &job) { - if (backend == runner_backend::Docker) { - const char *const kill_args[] = {"docker", "kill", job.job_id.c_str(), nullptr}; - pid_t kill_pid; - ensure(posix_spawnp(&kill_pid, kill_args[0], nullptr, nullptr, const_cast(kill_args), nullptr)); - } else { - ensure(kill(job.pid, SIGINT)); +void runner::exit(const active_job &job) { backend->stop(job); } + +runner_list::runner_list(sk::runner_backend preferred_backend, const runner_config &config) { + for (const auto &strategy : config.strategies) { + runners.emplace(std::string{strategy.first}, runner{strategy.second, config.timeout}); + } +} + +run_result runner_list::run_blocking(const program &program) { return runners.at(program.name).run_blocking(program); } + +bool runner_list::kill_active(const std::string &jobId) { + return std::any_of(runners.begin(), runners.end(), [&jobId](auto &runner) { return runner.second.kill_active(jobId); }); +} + +void runner_list::exit_active_jobs() { + for (auto &[name, runner] : runners) { + runner.exit_active_jobs(); } } } diff --git a/src/runner.hpp b/src/runner.hpp index 9fe7505..657602a 100644 --- a/src/runner.hpp +++ b/src/runner.hpp @@ -2,8 +2,10 @@ #include "config.hpp" #include "program.hpp" +#include #include #include +#include #include namespace sk { @@ -19,20 +21,61 @@ struct [[nodiscard]] active_job { }; enum class runner_backend { BubbleWrap, Docker }; +using pattern = std::variant; + +class execution_strategy { + std::vector patterns; + + public: + explicit execution_strategy(std::vector patterns); + virtual std::vector start(const program &program) = 0; + virtual void stop(const active_job &job) = 0; + virtual ~execution_strategy() = default; + static std::unique_ptr create(const runner_strategy_config &config); +}; + +class bubblewrap_execution_strategy : public execution_strategy { + public: + bubblewrap_execution_strategy(std::vector patterns); + std::vector start(const program &program) override; + void stop(const active_job &job) override; +}; + +class docker_execution_strategy : public execution_strategy { + private: + std::string image; + + public: + docker_execution_strategy(std::vector patterns, std::string image); + std::vector start(const program &program) override; + void stop(const active_job &job) override; +}; class runner { - runner_backend backend; - int timeout; std::vector active_jobs; std::mutex active_jobs_mutex; + std::unique_ptr backend; + int timeout; public: - runner(runner_backend backend, const runner_config &config); + explicit runner(const runner_strategy_config &config, unsigned int timeout = 2u); + runner(const runner &other); + runner(runner &&other) noexcept; + runner &operator=(const runner &other); + runner &operator=(runner &&other) noexcept; run_result run_blocking(const program &program); bool kill_active(const std::string &jobId); void exit_active_jobs(); - - private: void exit(const active_job &job); }; + +class runner_list { + std::unordered_map runners; + + public: + runner_list(sk::runner_backend preferred_backend, const runner_config &config); + run_result run_blocking(const program &program); + bool kill_active(const std::string &jobId); + void exit_active_jobs(); +}; } From eeccee7763aaf0cd6322c81b7b591aa4a673c19b Mon Sep 17 00:00:00 2001 From: clfreville2 Date: Thu, 16 May 2024 20:36:37 +0200 Subject: [PATCH 2/2] Support patterns in config --- config.toml | 16 +++++++++------ src/config.cpp | 13 +++++++++++- src/main.cpp | 18 +++++++++++++++-- src/runner.cpp | 54 ++++++++++++++++++++++++++++++++++++++++++++------ src/runner.hpp | 10 ++++++++-- 5 files changed, 94 insertions(+), 17 deletions(-) diff --git a/config.toml b/config.toml index bbf0fdb..cc9e2c2 100644 --- a/config.toml +++ b/config.toml @@ -6,15 +6,19 @@ threads = 1 [runner] timeout = 2 -[[runners]] -[javascript] +[runners.javascript] type = "bubblewrap" -binary = ["node", "-e", "{}"] +args = ["node", "-e", "{}"] -[bash] +[runners.bun] type = "bubblewrap" -binary = ["bash", "-c", "{}"] +args = ["bun", "run", "/dev/stdin"] -[moshell] +[runners.bash] +type = "bubblewrap" +args = ["bash", "-c", "{}"] + +[runners.moshell] type = "docker" image = "ghcr.io/moshell-lang/moshell:master" +args = ["moshell", "-c", "{}"] diff --git a/src/config.cpp b/src/config.cpp index 0a0607b..226f2d7 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -32,7 +32,18 @@ static sk::config read(const toml::table &config) { } args_vector.emplace_back(*arg_string); } - strategies.emplace(name, sk::bubblewrap_config{std::move(args_vector)}); + if (*backend_type == "docker") { + auto *image = backend_table->get_as("image"); + if (image == nullptr) { + std::cerr << "Invalid runner config for " << name << ": missing image field\n"; + continue; + } + strategies.emplace(name, sk::docker_config{{std::move(args_vector)}, std::string{*image}}); + } else if (*backend_type == "bubblewrap") { + strategies.emplace(name, sk::bubblewrap_config{std::move(args_vector)}); + } else { + std::cerr << "Invalid runner config for " << name << ": unknown type " << *backend_type << "\n"; + } } } return sk::config{ diff --git a/src/main.cpp b/src/main.cpp index 00bb0c5..d864360 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -83,7 +83,12 @@ int main(int argc, char **argv) { buffer << t.rdbuf(); std::string code = buffer.str(); sk::program program{"sample-code", code, "moshell"}; - sk::run_result result = runners.run_blocking(program); + sk::runner *runner = runners.find_runner_for(program); + if (runner == nullptr) { + std::cerr << "No runner found for " << program.image << std::endl; + return 1; + } + sk::run_result result = runner->run_blocking(program); std::cout << "exited with: " << result.exit_code << "\n"; std::cout << "out: " << result.out << "\n"; std::cout << "err: " << result.err << "\n"; @@ -132,7 +137,16 @@ int main(int argc, char **argv) { std::cout << "Executing " << codeLen << " bytes code.\n"; #endif sk::program program{std::move(jobId), std::move(requestString), std::move(imageString)}; - sk::run_result result = runners.run_blocking(program); + sk::runner *runner = runners.find_runner_for(program); + if (runner == nullptr) { + send(STDERR_CLIENT_BOUND, program.name, "No runner found for " + program.image); + sk::prepare_headers(sizeof(uint32_t), EXIT_CLIENT_BOUND, program.name); + auto [reply, reply_bytes] = sk::prepare_headers(sizeof(uint32_t), EXIT_CLIENT_BOUND, program.name); + sk::write_uint32(reply_bytes, -1); + sender.send(reply, zmq::send_flags::none); + continue; + } + sk::run_result result = runner->run_blocking(program); if (!result.out.empty()) { send(STDOUT_CLIENT_BOUND, program.name, result.out); diff --git a/src/runner.cpp b/src/runner.cpp index 06a52fd..5c1819c 100644 --- a/src/runner.cpp +++ b/src/runner.cpp @@ -21,17 +21,26 @@ static auto ensure = [](int res) -> void { template inline constexpr bool always_false_v = false; namespace sk { +execution_strategy::execution_strategy(const std::vector &patterns) { + for (const auto &pattern : patterns) { + if (pattern == "{}") { + this->patterns.emplace_back(nullptr); + } else { + this->patterns.emplace_back(pattern); + } + } +} + execution_strategy::execution_strategy(std::vector patterns) : patterns(std::move(patterns)) {} std::unique_ptr execution_strategy::create(const runner_strategy_config &config) { return std::visit( [](auto &&strategy) -> std::unique_ptr { using T = std::decay_t; - std::vector patterns; if constexpr (std::is_same_v) { - return std::make_unique(std::move(patterns)); + return std::make_unique(strategy.args); } else if constexpr (std::is_same_v) { - return std::make_unique(std::move(patterns), strategy.image); + return std::make_unique(strategy.args, strategy.image); } else { static_assert(always_false_v, "non-exhaustive visitor!"); } @@ -39,15 +48,41 @@ std::unique_ptr execution_strategy::create(const runner_stra config); } +void execution_strategy::concat_patterns(std::vector &args, const program &program) const { + for (const auto &pattern : patterns) { + std::visit( + [&args, &program](const auto &arg) { + if constexpr (std::is_same_v>) { + args.push_back(arg.c_str()); + } else { + args.push_back(program.code.c_str()); + } + }, + pattern); + } +} + +bubblewrap_execution_strategy::bubblewrap_execution_strategy(const std::vector &patterns) : execution_strategy{patterns} {} + bubblewrap_execution_strategy::bubblewrap_execution_strategy(std::vector patterns) : execution_strategy{std::move(patterns)} {} -std::vector bubblewrap_execution_strategy::start(const program &program) { return {"bwrap", "--ro-bind", "/usr", "/usr", "--dir", "/tmp", "--dir", "/var", "--proc", "/proc", "--dev", "/dev", "--symlink", "usr/lib", "/lib", "--symlink", "usr/lib64", "/lib64", "--symlink", "usr/bin", "/bin", "--symlink", "usr/sbin", "/sbin", "--unshare-all", "--die-with-parent", "/bin/sh", nullptr}; } +std::vector bubblewrap_execution_strategy::start(const program &program) { + std::vector args{"bwrap", "--ro-bind", "/usr", "/usr", "--dir", "/tmp", "--dir", "/var", "--proc", "/proc", "--dev", "/dev", "--symlink", "usr/lib", "/lib", "--symlink", "usr/lib64", "/lib64", "--symlink", "usr/bin", "/bin", "--symlink", "usr/sbin", "/sbin", "--unshare-all", "--die-with-parent"}; + concat_patterns(args, program); + return args; +} void bubblewrap_execution_strategy::stop(const active_job &job) { ensure(kill(job.pid, SIGINT)); } +docker_execution_strategy::docker_execution_strategy(const std::vector &patterns, std::string image) : execution_strategy{patterns}, image{std::move(image)} {} + docker_execution_strategy::docker_execution_strategy(std::vector patterns, std::string image) : execution_strategy{std::move(patterns)}, image{std::move(image)} {} -std::vector docker_execution_strategy::start(const program &program) { return {"docker", "run", "--rm", "-i", "--name", program.name.c_str(), "--pull=never", "--cap-drop=ALL", "--network=none", "--memory=64m", "--memory-swap=64m", "--pids-limit=128", image.c_str(), nullptr}; } +std::vector docker_execution_strategy::start(const program &program) { + std::vector args{"docker", "run", "--rm", "-i", "--name", program.name.c_str(), "--pull=never", "--cap-drop=ALL", "--network=none", "--memory=64m", "--memory-swap=64m", "--pids-limit=128", image.c_str()}; + concat_patterns(args, program); + return args; +} void docker_execution_strategy::stop(const active_job &job) { const char *const kill_args[] = {"docker", "kill", job.job_id.c_str(), nullptr}; @@ -111,6 +146,7 @@ run_result runner::run_blocking(const program &program) { posix_spawn_file_actions_adddup2(&actions, out_pipe[1], STDOUT_FILENO); posix_spawn_file_actions_adddup2(&actions, err_pipe[1], STDERR_FILENO); std::vector args = backend->start(program); + args.push_back(nullptr); pid_t pid; bool killed = false; int exit_code; @@ -246,7 +282,13 @@ runner_list::runner_list(sk::runner_backend preferred_backend, const runner_conf } } -run_result runner_list::run_blocking(const program &program) { return runners.at(program.name).run_blocking(program); } +runner *runner_list::find_runner_for(const program &program) { + auto it = runners.find(program.image); + if (it != runners.end()) { + return &it->second; + } + return nullptr; +} bool runner_list::kill_active(const std::string &jobId) { return std::any_of(runners.begin(), runners.end(), [&jobId](auto &runner) { return runner.second.kill_active(jobId); }); diff --git a/src/runner.hpp b/src/runner.hpp index 657602a..ba03d7d 100644 --- a/src/runner.hpp +++ b/src/runner.hpp @@ -27,16 +27,21 @@ class execution_strategy { std::vector patterns; public: + explicit execution_strategy(const std::vector &patterns); explicit execution_strategy(std::vector patterns); virtual std::vector start(const program &program) = 0; virtual void stop(const active_job &job) = 0; virtual ~execution_strategy() = default; static std::unique_ptr create(const runner_strategy_config &config); + + protected: + void concat_patterns(std::vector &args, const program &program) const; }; class bubblewrap_execution_strategy : public execution_strategy { public: - bubblewrap_execution_strategy(std::vector patterns); + explicit bubblewrap_execution_strategy(const std::vector &patterns); + explicit bubblewrap_execution_strategy(std::vector patterns); std::vector start(const program &program) override; void stop(const active_job &job) override; }; @@ -46,6 +51,7 @@ class docker_execution_strategy : public execution_strategy { std::string image; public: + docker_execution_strategy(const std::vector &patterns, std::string image); docker_execution_strategy(std::vector patterns, std::string image); std::vector start(const program &program) override; void stop(const active_job &job) override; @@ -74,7 +80,7 @@ class runner_list { public: runner_list(sk::runner_backend preferred_backend, const runner_config &config); - run_result run_blocking(const program &program); + runner *find_runner_for(const program &program); bool kill_active(const std::string &jobId); void exit_active_jobs(); };