From 65c2033120ae76120cae5957bee0a43d5ed9f37d Mon Sep 17 00:00:00 2001 From: virusdefender <1670873886@qq.com> Date: Mon, 1 Feb 2016 22:17:33 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E4=BD=BF=E7=94=A8=20judger=20=E8=BF=90?= =?UTF-8?q?=E8=A1=8C=E7=BC=96=E8=AF=91=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- judge/compiler.py | 57 ++++++++++++++++++++++------------------------- judge/settings.py | 8 ------- 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/judge/compiler.py b/judge/compiler.py index 127750be..3643bd0e 100644 --- a/judge/compiler.py +++ b/judge/compiler.py @@ -1,42 +1,39 @@ # coding=utf-8 +import time +import os +import judger import commands - -from settings import lrun_uid, lrun_gid from judge_exceptions import CompileError, JudgeClientError -from utils import parse_lrun_output from logger import logger +from settings import judger_workspace def compile_(language_item, src_path, exe_path): - compile_command = language_item["compile_command"].format(src_path=src_path, exe_path=exe_path) + compile_command = language_item["compile_command"].format(src_path=src_path, exe_path=exe_path).split(" ") + compiler = compile_command[0] + compile_args = compile_command[1:] + compiler_output_file = os.path.join(judger_workspace, str(time.time()) + ".out") - # 防止编译器卡死 或者 include 等 - execute_command = "lrun" + \ - " --max-real-time 5" + \ - " --uid " + str(lrun_uid) + \ - " --gid " + str(lrun_gid) + \ - " " + \ - compile_command + \ - " 3>&2" - status, output = commands.getstatusoutput(execute_command) + compile_result = judger.run(exe_path=compiler, + in_file="/dev/null", + out_file=compiler_output_file, + max_cpu_time=2000, + max_memory=200000000, + args=compile_args, + env=["PATH=" + os.environ["PATH"]], + use_sandbox=False) - output_start = output.rfind("MEMORY") + compile_output_handler = open(compiler_output_file) + compile_output = compile_output_handler.read() + compile_output_handler.close() + os.remove(compiler_output_file) - if output_start == -1: + if compile_result["flag"] != 0: logger.error("Compiler error") - logger.error(output) - raise JudgeClientError("Error running compiler in lrun") - - # 返回值不为 0 或者 stderr 中 lrun 的输出之前有 erro r字符串 - # 判断 error 字符串的原因是链接的时候可能会有一些不推荐使用的函数的的警告， - # 但是 -w 参数并不能关闭链接时的警告 - if status or "error" in output[0:output_start]: - raise CompileError(output[0:output_start]) - - parse_result = parse_lrun_output(output[output_start:]) - - if parse_result["exit_code"] or parse_result["term_sig"] or parse_result["siginaled"] or parse_result["exceed"]: - logger.error("Compiler error") - logger.error(output) + logger.error(compile_output) + logger.error(str(compile_result)) raise CompileError("Compile error") - return exe_path + else: + if "error" in compile_output: + raise CompileError(compile_output) + return exe_path diff --git a/judge/settings.py b/judge/settings.py index 224cf5dd..5d9fa432 100644 --- a/judge/settings.py +++ b/judge/settings.py @@ -14,11 +14,3 @@ lrun_gid = 1002 # judger工作目录 judger_workspace = "/var/judger/" - -submission_db = { - "host": os.environ.get("MYSQL_PORT_3306_TCP_ADDR", "127.0.0.1"), - "port": 3306, - "db": "oj_submission", - "user": "root", - "password": os.environ.get("MYSQL_ENV_MYSQL_ROOT_PASSWORD", "root") -} From ee35981fcbb5b3201b6bb81438814720ae34cc75 Mon Sep 17 00:00:00 2001 From: virusdefender <1670873886@qq.com> Date: Mon, 1 Feb 2016 22:45:13 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E7=BC=96=E8=AF=91=E5=99=A8=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E7=BB=9D=E5=AF=B9=E8=B7=AF=E5=BE=84=EF=BC=8C=E5=90=A6?= =?UTF-8?q?=E5=88=99=E6=8A=A5=E6=89=BE=E4=B8=8D=E5=88=B0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9A=84=E5=BC=82=E5=B8=B8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- judge/compiler.py | 5 ++--- judge/language.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/judge/compiler.py b/judge/compiler.py index 3643bd0e..8e741ecb 100644 --- a/judge/compiler.py +++ b/judge/compiler.py @@ -2,7 +2,6 @@ import time import os import judger -import commands from judge_exceptions import CompileError, JudgeClientError from logger import logger from settings import judger_workspace @@ -14,11 +13,11 @@ def compile_(language_item, src_path, exe_path): compile_args = compile_command[1:] compiler_output_file = os.path.join(judger_workspace, str(time.time()) + ".out") - compile_result = judger.run(exe_path=compiler, + compile_result = judger.run(path=compiler, in_file="/dev/null", out_file=compiler_output_file, max_cpu_time=2000, - max_memory=200000000, + max_memory=2000000000, args=compile_args, env=["PATH=" + os.environ["PATH"]], use_sandbox=False) diff --git a/judge/language.py b/judge/language.py index e6406566..4f5658bb 100644 --- a/judge/language.py +++ b/judge/language.py @@ -7,7 +7,7 @@ languages = { "src_name": "main.c", "code": 1, "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", - "compile_command": "gcc -DONLINE_JUDGE -O2 -w -std=c99 {src_path} -lm -o {exe_path}/main", + "compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -w -std=c99 {src_path} -lm -o {exe_path}/main", "execute_command": "{exe_path}/main" }, 2: { @@ -15,7 +15,7 @@ languages = { "src_name": "main.cpp", "code": 2, "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", - "compile_command": "g++ -DONLINE_JUDGE -O2 -w -std=c++11 {src_path} -lm -o {exe_path}/main", + "compile_command": "/usr/bin/g++ -DONLINE_JUDGE -O2 -w -std=c++11 {src_path} -lm -o {exe_path}/main", "execute_command": "{exe_path}/main" }, 3: { @@ -23,7 +23,7 @@ languages = { "src_name": "Main.java", "code": 3, "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone[a&268435456==268435456]:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", - "compile_command": "javac {src_path} -d {exe_path}", + "compile_command": "/usr/bin/javac {src_path} -d {exe_path}", "execute_command": "java -cp {exe_path} -Djava.security.manager -Djava.security.policy==policy Main" } } From 8dcd36b3d91b5932ca699f8166922e073b8f6bc5 Mon Sep 17 00:00:00 2001 From: virusdefender <1670873886@qq.com> Date: Tue, 2 Feb 2016 10:14:18 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E6=9B=B4=E6=8D=A2=E5=88=A4=E9=A2=98?= =?UTF-8?q?=E4=B8=BA=E6=96=B0=E7=9A=84=20judger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- judge/client.py | 109 +++++++++++----------------------------------- judge/compiler.py | 7 ++- judge/language.py | 5 +-- judge/result.py | 1 + judge/runner.py | 10 ++--- judge/utils.py | 40 ----------------- 6 files changed, 36 insertions(+), 136 deletions(-) delete mode 100644 judge/utils.py diff --git a/judge/client.py b/judge/client.py index f5b12966..c39c840d 100644 --- a/judge/client.py +++ b/judge/client.py @@ -3,13 +3,13 @@ import os import json import commands import hashlib +import judger from multiprocessing import Pool from settings import max_running_number, lrun_gid, lrun_uid, judger_workspace from language import languages from result import result from judge_exceptions import JudgeClientError -from utils import parse_lrun_output from logger import logger @@ -20,13 +20,11 @@ def _run(instance, test_case_id): class JudgeClient(object): - def __init__(self, language_code, exe_path, max_cpu_time, - max_real_time, max_memory, test_case_dir): + def __init__(self, language_code, exe_path, max_cpu_time, max_memory, test_case_dir, judge_base_path): """ :param language_code: 语言编号 :param exe_path: 可执行文件路径 :param max_cpu_time: 最大cpu时间，单位ms - :param max_real_time: 最大执行时间，单位ms :param max_memory: 最大内存，单位MB :param test_case_dir: 测试用例文件夹路径 :return:返回结果list @@ -34,13 +32,13 @@ class JudgeClient(object): self._language = languages[language_code] self._exe_path = exe_path self._max_cpu_time = max_cpu_time - self._max_real_time = max_real_time self._max_memory = max_memory self._test_case_dir = test_case_dir # 进程池 self._pool = Pool(processes=max_running_number) # 测试用例配置项 self._test_case_info = self._load_test_case_info() + self._judge_base_path = judge_base_path def _load_test_case_info(self): # 读取测试用例信息 转换为dict @@ -52,55 +50,9 @@ class JudgeClient(object): except ValueError: raise JudgeClientError("Test case config file format error") - def _generate_command(self, test_case_id): - """ - 设置相关运行限制 进制访问网络 如果启用tmpfs 就把代码输出写入tmpfs，否则写入硬盘 - """ - # todo 系统调用白名单 chroot等参数 - command = "lrun" + \ - " --max-cpu-time " + str(self._max_cpu_time / 1000.0) + \ - " --max-real-time " + str(self._max_real_time / 1000.0 * 2) + \ - " --max-memory " + str(self._max_memory * 1000 * 1000) + \ - " --network false" + \ - " --remount-dev true " + \ - " --reset-env true " + \ - " --syscalls '" + self._language["syscalls"] + "'" + \ - " --max-nprocess 20" + \ - " --uid " + str(lrun_uid) + \ - " --gid " + str(lrun_gid) - - execute_command = self._language["execute_command"].format(exe_path=self._exe_path) - - command += (" " + - execute_command + - # 0就是stdin - " 0<" + self._test_case_dir + str(test_case_id) + ".in" + - # 1就是stdout - " 1>" + judger_workspace + str(test_case_id) + ".out" + - # 3是stderr，包含lrun的输出和程序的异常输出 - " 3>&2") - return command - - def _parse_lrun_output(self, output): - # 要注意的是 lrun把结果输出到了stderr，所以有些情况下lrun的输出可能与程序的一些错误输出的混合的，要先分离一下 - error = None - # 倒序找到MEMORY的位置,lrun的 MEMORY 输出后面有3个空格，而 EXCEEDED 也有可能是MEMORY，所以需要判断空格 - output_start = output.rfind("MEMORY ") - if output_start == -1: - logger.error("Lrun result parse error") - logger.error(output) - raise JudgeClientError("Lrun result parse error") - # 如果不是0，说明lrun输出前面有输出，也就是程序的stderr有内容 - if output_start != 0: - error = output[0:output_start] - # 分离出lrun的输出 - output = output[output_start:] - - return error, parse_lrun_output(output) - def _compare_output(self, test_case_id): test_case_config = self._test_case_info["test_cases"][str(test_case_id)] - output_path = judger_workspace + str(test_case_id) + ".out" + output_path = os.path.join(self._judge_base_path, str(test_case_id) + ".out") try: f = open(output_path, "rb") @@ -130,40 +82,31 @@ class JudgeClient(object): return output_md5, output_md5 == test_case_config["striped_output_md5"] def _judge_one(self, test_case_id): - # 运行lrun程序 接收返回值 - command = self._generate_command(test_case_id) - status_code, output = commands.getstatusoutput(command) - if status_code: - raise JudgeClientError(output) - error, run_result = self._parse_lrun_output(output) + execute_command = self._language["execute_command"].format(exe_path=self._exe_path).split(" ") - run_result["test_case_id"] = test_case_id - - # 代表内存或者时间超过限制了 程序被终止掉 要在runtime error 之前判断 - if run_result["exceed"]: - if run_result["exceed"] == "memory": - run_result["result"] = result["memory_limit_exceeded"] - elif run_result["exceed"] in ["cpu_time", "real_time"]: - run_result["result"] = result["time_limit_exceeded"] + run_result = judger.run(path=execute_command[0], + max_cpu_time=self._max_cpu_time, + max_memory=self._max_memory, + in_file=os.path.join(self._test_case_dir, str(test_case_id) + ".in"), + out_file=os.path.join(self._judge_base_path, str(test_case_id) + ".out"), + args=execute_command[1:], + env=["PATH=" + os.environ["PATH"]], + use_sandbox=True) + if run_result["flag"] == 0: + output_md5, r = self._compare_output(test_case_id) + if r: + run_result["result"] = result["accepted"] else: - logger.error("Error exceeded type: " + run_result["exceed"]) - logger.error(output) - raise JudgeClientError("Error exceeded type: " + run_result["exceed"]) - return run_result - - # 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束 - if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error: + run_result["result"] = result["wrong_answer"] + run_result["output_md5"] = output_md5 + elif run_result["flag"] in [1, 2]: + run_result["result"] = result["time_limit_exceeded"] + elif run_result["flag"] == 3: + run_result["result"] = result["memory_limit_exceeded"] + elif run_result["flag"] == 4: run_result["result"] = result["runtime_error"] - return run_result - - # 下面就是代码正常运行了 需要判断代码的输出是否正确 - output_md5, r = self._compare_output(test_case_id) - if r: - run_result["result"] = result["accepted"] - else: - run_result["result"] = result["wrong_answer"] - run_result["output_md5"] = output_md5 - + elif run_result["flag"] == 5: + run_result["result"] = result["system_error"] return run_result def run(self): diff --git a/judge/compiler.py b/judge/compiler.py index 8e741ecb..42b575ab 100644 --- a/judge/compiler.py +++ b/judge/compiler.py @@ -7,11 +7,11 @@ from logger import logger from settings import judger_workspace -def compile_(language_item, src_path, exe_path): +def compile_(language_item, src_path, exe_path, judge_base_path): compile_command = language_item["compile_command"].format(src_path=src_path, exe_path=exe_path).split(" ") compiler = compile_command[0] compile_args = compile_command[1:] - compiler_output_file = os.path.join(judger_workspace, str(time.time()) + ".out") + compiler_output_file = os.path.join(judge_base_path, "compiler.out") compile_result = judger.run(path=compiler, in_file="/dev/null", @@ -25,13 +25,12 @@ def compile_(language_item, src_path, exe_path): compile_output_handler = open(compiler_output_file) compile_output = compile_output_handler.read() compile_output_handler.close() - os.remove(compiler_output_file) if compile_result["flag"] != 0: logger.error("Compiler error") logger.error(compile_output) logger.error(str(compile_result)) - raise CompileError("Compile error") + raise CompileError("Compile error, info: " + str(compile_result)) else: if "error" in compile_output: raise CompileError(compile_output) diff --git a/judge/language.py b/judge/language.py index 4f5658bb..d0a08770 100644 --- a/judge/language.py +++ b/judge/language.py @@ -6,7 +6,6 @@ languages = { "name": "c", "src_name": "main.c", "code": 1, - "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", "compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -w -std=c99 {src_path} -lm -o {exe_path}/main", "execute_command": "{exe_path}/main" }, @@ -14,7 +13,6 @@ languages = { "name": "cpp", "src_name": "main.cpp", "code": 2, - "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", "compile_command": "/usr/bin/g++ -DONLINE_JUDGE -O2 -w -std=c++11 {src_path} -lm -o {exe_path}/main", "execute_command": "{exe_path}/main" }, @@ -22,9 +20,8 @@ languages = { "name": "java", "src_name": "Main.java", "code": 3, - "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone[a&268435456==268435456]:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", "compile_command": "/usr/bin/javac {src_path} -d {exe_path}", - "execute_command": "java -cp {exe_path} -Djava.security.manager -Djava.security.policy==policy Main" + "execute_command": "/usr/bin/java -cp {exe_path} -Djava.security.manager -Djava.security.policy==policy Main" } } diff --git a/judge/result.py b/judge/result.py index ee0d4798..f42e63fd 100644 --- a/judge/result.py +++ b/judge/result.py @@ -1,6 +1,7 @@ # coding=utf-8 +# 这个映射关系是前后端通用的,判题服务器提供接口,也应该遵守这个,可能需要一些转换 result = { "accepted": 0, "runtime_error": 1, diff --git a/judge/runner.py b/judge/runner.py index 5ca5c205..e023e87a 100644 --- a/judge/runner.py +++ b/judge/runner.py @@ -35,7 +35,7 @@ class JudgeInstanceRunner(object): # 编译 try: - exe_path = compile_(language, src_path, judge_base_path) + exe_path = compile_(language, src_path, judge_base_path, judge_base_path) except Exception as e: shutil.rmtree(judge_base_path, ignore_errors=True) return {"code": 1, "data": {"error": str(e), "server": host_name}} @@ -45,14 +45,14 @@ class JudgeInstanceRunner(object): client = JudgeClient(language_code=language_code, exe_path=exe_path, max_cpu_time=int(time_limit), - max_real_time=int(time_limit) * 2, - max_memory=int(memory_limit), - test_case_dir=judger_workspace + "test_case/" + test_case_id + "/") + max_memory=int(memory_limit) * 1024 * 1024, + test_case_dir=judger_workspace + "test_case/" + test_case_id + "/", + judge_base_path=judge_base_path) judge_result = {"result": result["accepted"], "info": client.run(), "accepted_answer_time": None, "server": host_name} for item in judge_result["info"]: - if item["result"]: + if item["result"] != 0: judge_result["result"] = item["result"] break else: diff --git a/judge/utils.py b/judge/utils.py deleted file mode 100644 index aed1057c..00000000 --- a/judge/utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# coding=utf-8 -from judge_exceptions import JudgeClientError - - -def parse_lrun_output(output): - lines = output.split("\n") - if len(lines) != 7: - raise JudgeClientError("Lrun result parse error") - result = {} - # 将lrun输出的各种带下划线 不带下划线的字符串统一处理 - translate = {"MEMORY": "memory", - "CPUTIME": "cpu_time", - "CPU_TIME": "cpu_time", - "REALTIME": "real_time", - "REAL_TIME": "real_time", - "TERMSIG": "term_sig", - "SIGNALED": "siginaled", - "EXITCODE": "exit_code", - "EXCEED": "exceed"} - for line in lines: - name = line[:9].strip(" ") - value = line[9:] - if name == "MEMORY": - result[translate[name]] = int(value) - elif name == "CPUTIME": - result[translate[name]] = int(float(value) * 1000) - elif name == "REALTIME": - result[translate[name]] = int(float(value) * 1000) - elif name == "EXITCODE": - result[translate[name]] = int(value) - elif name == "TERMSIG": - result[translate[name]] = int(value) - elif name == "SIGNALED": - result[translate[name]] = int(value) - elif name == "EXCEED": - if value == "none": - result[translate[name]] = None - else: - result[translate[name]] = translate[value] - return result From 6963b2879b9b2f2c601b78686e791fd564754d65 Mon Sep 17 00:00:00 2001 From: virusdefender <1670873886@qq.com> Date: Tue, 2 Feb 2016 10:23:23 +0800 Subject: [PATCH 4/4] =?UTF-8?q?java=20=E4=B8=8D=E5=86=8D=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E6=B2=99=E7=AE=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- judge/client.py | 2 +- judge/language.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/judge/client.py b/judge/client.py index c39c840d..da871c20 100644 --- a/judge/client.py +++ b/judge/client.py @@ -91,7 +91,7 @@ class JudgeClient(object): out_file=os.path.join(self._judge_base_path, str(test_case_id) + ".out"), args=execute_command[1:], env=["PATH=" + os.environ["PATH"]], - use_sandbox=True) + use_sandbox=self._language["use_sandbox"]) if run_result["flag"] == 0: output_md5, r = self._compare_output(test_case_id) if r: diff --git a/judge/language.py b/judge/language.py index d0a08770..7a74f00a 100644 --- a/judge/language.py +++ b/judge/language.py @@ -7,21 +7,24 @@ languages = { "src_name": "main.c", "code": 1, "compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -w -std=c99 {src_path} -lm -o {exe_path}/main", - "execute_command": "{exe_path}/main" + "execute_command": "{exe_path}/main", + "use_sandbox": True }, 2: { "name": "cpp", "src_name": "main.cpp", "code": 2, "compile_command": "/usr/bin/g++ -DONLINE_JUDGE -O2 -w -std=c++11 {src_path} -lm -o {exe_path}/main", - "execute_command": "{exe_path}/main" + "execute_command": "{exe_path}/main", + "use_sandbox": True }, 3: { "name": "java", "src_name": "Main.java", "code": 3, "compile_command": "/usr/bin/javac {src_path} -d {exe_path}", - "execute_command": "/usr/bin/java -cp {exe_path} -Djava.security.manager -Djava.security.policy==policy Main" + "execute_command": "/usr/bin/java -cp {exe_path} -Djava.security.manager -Djava.security.policy==policy Main", + "use_sandbox": False } }