mirror of
https://github.com/QingdaoU/OnlineJudge.git
synced 2024-09-21 08:23:20 +00:00
增加判题中的日志记录,没有测试
This commit is contained in:
parent
ee9951abf1
commit
1dc0e8b742
@ -7,9 +7,9 @@ from multiprocessing import Pool
|
||||
from settings import max_running_number, lrun_gid, lrun_uid, judger_workspace
|
||||
from language import languages
|
||||
from result import result
|
||||
from compiler import compile_
|
||||
from judge_exceptions import JudgeClientError, CompileError
|
||||
from judge_exceptions import JudgeClientError
|
||||
from utils import parse_lrun_output
|
||||
from logger import logger
|
||||
|
||||
|
||||
# 下面这个函数作为代理访问实例变量,否则Python2会报错,是Python2的已知问题
|
||||
@ -82,6 +82,8 @@ class JudgeClient(object):
|
||||
# 倒序找到MEMORY的位置
|
||||
output_start = output.rfind("MEMORY")
|
||||
if output_start == -1:
|
||||
logger.error("Lrun result parse error")
|
||||
logger.error(output)
|
||||
raise JudgeClientError("Lrun result parse error")
|
||||
# 如果不是0,说明lrun输出前面有输出,也就是程序的stderr有内容
|
||||
if output_start != 0:
|
||||
@ -123,21 +125,23 @@ class JudgeClient(object):
|
||||
|
||||
run_result["test_case_id"] = test_case_id
|
||||
|
||||
# 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
|
||||
if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
|
||||
run_result["result"] = result["runtime_error"]
|
||||
return run_result
|
||||
|
||||
# 代表内存或者时间超过限制了
|
||||
# 代表内存或者时间超过限制了 程序被终止掉 要在runtime error 之前判断
|
||||
if run_result["exceed"]:
|
||||
if run_result["exceed"] == "memory":
|
||||
run_result["result"] = result["memory_limit_exceeded"]
|
||||
elif run_result["exceed"] in ["cpu_time", "real_time"]:
|
||||
run_result["result"] = result["time_limit_exceeded"]
|
||||
else:
|
||||
logger.error("Error exceeded type: " + run_result["exceed"])
|
||||
logger.error(output)
|
||||
raise JudgeClientError("Error exceeded type: " + run_result["exceed"])
|
||||
return run_result
|
||||
|
||||
# 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
|
||||
if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
|
||||
run_result["result"] = result["runtime_error"]
|
||||
return run_result
|
||||
|
||||
# 下面就是代码正常运行了 需要判断代码的输出是否正确
|
||||
if self._compare_output(test_case_id):
|
||||
run_result["result"] = result["accepted"]
|
||||
@ -160,8 +164,8 @@ class JudgeClient(object):
|
||||
try:
|
||||
results.append(item.get())
|
||||
except Exception as e:
|
||||
# todo logging
|
||||
print e
|
||||
logger.error("system error")
|
||||
logger.error(e)
|
||||
results.append({"result": result["system_error"]})
|
||||
return results
|
||||
|
||||
|
@ -4,6 +4,7 @@ import commands
|
||||
from settings import lrun_uid, lrun_gid
|
||||
from judge_exceptions import CompileError, JudgeClientError
|
||||
from utils import parse_lrun_output
|
||||
from logger import logger
|
||||
|
||||
|
||||
def compile_(language_item, src_path, exe_path):
|
||||
@ -22,6 +23,8 @@ def compile_(language_item, src_path, exe_path):
|
||||
output_start = output.rfind("MEMORY")
|
||||
|
||||
if output_start == -1:
|
||||
logger.error("Compiler error")
|
||||
logger.error(output)
|
||||
raise JudgeClientError("Error running compiler in lrun")
|
||||
|
||||
# 返回值不为 0 或者 stderr 中 lrun 的输出之前有 erro r字符串
|
||||
@ -33,5 +36,7 @@ def compile_(language_item, src_path, exe_path):
|
||||
parse_result = parse_lrun_output(output[output_start:])
|
||||
|
||||
if parse_result["exit_code"] or parse_result["term_sig"] or parse_result["siginaled"] or parse_result["exceed"]:
|
||||
logger.error("Compiler error")
|
||||
logger.error(output)
|
||||
raise CompileError("Compile error")
|
||||
return exe_path
|
||||
|
8
judge/judger/logger.py
Normal file
8
judge/judger/logger.py
Normal file
@ -0,0 +1,8 @@
|
||||
# coding=utf-8
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG,
|
||||
format='%(asctime)s [%(threadName)s:%(thread)d] [%(name)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s',
|
||||
filename='log/judge.log')
|
||||
|
||||
logger = logging
|
@ -1,163 +0,0 @@
|
||||
# coding=utf-8
|
||||
import json
|
||||
import commands
|
||||
import hashlib
|
||||
from multiprocessing import Pool
|
||||
|
||||
from settings import max_running_number, lrun_gid, lrun_uid, judger_workspace
|
||||
from language import languages
|
||||
from result import result
|
||||
from compiler import compile_
|
||||
from judge_exceptions import JudgeClientError, CompileError
|
||||
from utils import parse_lrun_output
|
||||
|
||||
|
||||
# 下面这个函数作为代理访问实例变量,否则Python2会报错,是Python2的已知问题
|
||||
# http://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-pythons-multiprocessing-pool-ma/7309686
|
||||
def _run(instance, test_case_id):
|
||||
return instance._judge_one(test_case_id)
|
||||
|
||||
|
||||
class JudgeClient(object):
|
||||
def __init__(self, language_code, exe_path, max_cpu_time,
|
||||
max_real_time, max_memory, test_case_dir):
|
||||
"""
|
||||
:param language_code: 语言编号
|
||||
:param exe_path: 可执行文件路径
|
||||
:param max_cpu_time: 最大cpu时间,单位ms
|
||||
:param max_real_time: 最大执行时间,单位ms
|
||||
:param max_memory: 最大内存,单位MB
|
||||
:param test_case_dir: 测试用例文件夹路径
|
||||
:return:返回结果list
|
||||
"""
|
||||
self._language = languages[language_code]
|
||||
self._exe_path = exe_path
|
||||
self._max_cpu_time = max_cpu_time
|
||||
self._max_real_time = max_real_time
|
||||
self._max_memory = max_memory
|
||||
self._test_case_dir = test_case_dir
|
||||
# 进程池
|
||||
self._pool = Pool(processes=max_running_number)
|
||||
|
||||
def _generate_command(self, test_case_id):
|
||||
"""
|
||||
设置相关运行限制 进制访问网络 如果启用tmpfs 就把代码输出写入tmpfs,否则写入硬盘
|
||||
"""
|
||||
# todo 系统调用白名单 chroot等参数
|
||||
command = "lrun" + \
|
||||
" --max-cpu-time " + str(self._max_cpu_time / 1000.0) + \
|
||||
" --max-real-time " + str(self._max_real_time / 1000.0 * 2) + \
|
||||
" --max-memory " + str(self._max_memory * 1000 * 1000) + \
|
||||
" --network false" + \
|
||||
" --uid " + str(lrun_uid) + \
|
||||
" --gid " + str(lrun_gid)
|
||||
|
||||
execute_command = self._language["execute_command"].format(exe_path=self._exe_path)
|
||||
|
||||
command += (" " +
|
||||
execute_command +
|
||||
# 0就是stdin
|
||||
" 0<" + self._test_case_dir + str(test_case_id) + ".in" +
|
||||
# 1就是stdout
|
||||
" 1>" + judger_workspace + str(test_case_id) + ".out" +
|
||||
# 3是stderr,包含lrun的输出和程序的异常输出
|
||||
" 3>&2")
|
||||
return command
|
||||
|
||||
def _parse_lrun_output(self, output):
|
||||
# 要注意的是 lrun把结果输出到了stderr,所以有些情况下lrun的输出可能与程序的一些错误输出的混合的,要先分离一下
|
||||
error = None
|
||||
# 倒序找到MEMORY的位置
|
||||
output_start = output.rfind("MEMORY")
|
||||
if output_start == -1:
|
||||
raise JudgeClientError("Lrun result parse error")
|
||||
# 如果不是0,说明lrun输出前面有输出,也就是程序的stderr有内容
|
||||
if output_start != 0:
|
||||
error = output[0:output_start]
|
||||
# 分离出lrun的输出
|
||||
output = output[output_start:]
|
||||
|
||||
return error, parse_lrun_output(output)
|
||||
|
||||
def _compare_output(self, test_case_id):
|
||||
|
||||
output_path = judger_workspace + str(test_case_id) + ".out"
|
||||
|
||||
try:
|
||||
f = open(output_path, "r")
|
||||
except IOError:
|
||||
# 文件不存在等引发的异常 返回结果错误
|
||||
return False
|
||||
try:
|
||||
std = open(self._test_case_dir+test_case_id+".out", "r")
|
||||
except IOError:
|
||||
# 文件不存在等引发的异常 返回结果错误
|
||||
return False
|
||||
lines=std.readline()
|
||||
line_conut = len(lines)
|
||||
for i in range(0, line_conut-2):
|
||||
if lines[i]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def _judge_one(self, test_case_id):
|
||||
# 运行lrun程序 接收返回值
|
||||
command = self._generate_command(test_case_id)
|
||||
status_code, output = commands.getstatusoutput(command)
|
||||
if status_code:
|
||||
raise JudgeClientError(output)
|
||||
error, run_result = self._parse_lrun_output(output)
|
||||
|
||||
run_result["test_case_id"] = test_case_id
|
||||
|
||||
# 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
|
||||
if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
|
||||
run_result["result"] = result["runtime_error"]
|
||||
return run_result
|
||||
|
||||
# 代表内存或者时间超过限制了
|
||||
if run_result["exceed"]:
|
||||
if run_result["exceed"] == "memory":
|
||||
run_result["result"] = result["memory_limit_exceeded"]
|
||||
elif run_result["exceed"] in ["cpu_time", "real_time"]:
|
||||
run_result["result"] = result["time_limit_exceeded"]
|
||||
else:
|
||||
raise JudgeClientError("Error exceeded type: " + run_result["exceed"])
|
||||
return run_result
|
||||
|
||||
# 下面就是代码正常运行了 需要判断代码的输出是否正确
|
||||
if self._compare_output(test_case_id):
|
||||
run_result["result"] = result["accepted"]
|
||||
else:
|
||||
run_result["result"] = result["wrong_answer"]
|
||||
|
||||
return run_result
|
||||
|
||||
def run(self):
|
||||
# 添加到任务队列
|
||||
_results = []
|
||||
results = []
|
||||
for i in range(self._test_case_info["test_case_number"]):
|
||||
_results.append(self._pool.apply_async(_run, (self, i + 1)))
|
||||
self._pool.close()
|
||||
self._pool.join()
|
||||
for item in _results:
|
||||
# 注意多进程中的异常只有在get()的时候才会被引发
|
||||
# http://stackoverflow.com/questions/22094852/how-to-catch-exceptions-in-workers-in-multiprocessing
|
||||
try:
|
||||
results.append(item.get())
|
||||
except Exception as e:
|
||||
# todo logging
|
||||
print e
|
||||
results.append({"result": result["system_error"]})
|
||||
return results
|
||||
|
||||
def __getstate__(self):
|
||||
# 不同的pool之间进行pickle的时候要排除自己,否则报错
|
||||
# http://stackoverflow.com/questions/25382455/python-notimplementederror-pool-objects-cannot-be-passed-between-processes
|
||||
self_dict = self.__dict__.copy()
|
||||
del self_dict['_pool']
|
||||
return self_dict
|
@ -2,21 +2,13 @@
|
||||
import sys
|
||||
import json
|
||||
import MySQLdb
|
||||
import os
|
||||
|
||||
# 判断判题模式
|
||||
judge_model = os.environ.get("judge_model", "default")
|
||||
if judge_model == "default":
|
||||
from client import JudgeClient
|
||||
elif judge_model == "loose":
|
||||
from loose_client import JudgeClient
|
||||
|
||||
from client import JudgeClient
|
||||
from language import languages
|
||||
from compiler import compile_
|
||||
from result import result
|
||||
from settings import judger_workspace
|
||||
|
||||
from settings import submission_db
|
||||
from settings import judger_workspace, submission_db
|
||||
from logger import logger
|
||||
|
||||
|
||||
# 简单的解析命令行参数
|
||||
@ -67,7 +59,6 @@ except Exception as e:
|
||||
conn.commit()
|
||||
exit()
|
||||
|
||||
print "Compile successfully"
|
||||
# 运行
|
||||
try:
|
||||
client = JudgeClient(language_code=language_code,
|
||||
@ -87,16 +78,13 @@ try:
|
||||
judge_result["accepted_answer_time"] = l[-1]["cpu_time"]
|
||||
|
||||
except Exception as e:
|
||||
print e
|
||||
logger.error(e)
|
||||
conn = db_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute("update submission set result=%s, info=%s where id=%s", (result["system_error"], str(e), submission_id))
|
||||
conn.commit()
|
||||
exit()
|
||||
|
||||
print "Run successfully"
|
||||
print judge_result
|
||||
|
||||
conn = db_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute("update submission set result=%s, info=%s, accepted_answer_time=%s where id=%s",
|
||||
|
9
judge/judger_controller/logger.py
Normal file
9
judge/judger_controller/logger.py
Normal file
@ -0,0 +1,9 @@
|
||||
# coding=utf-8
|
||||
import logging
|
||||
|
||||
# 此处的 celery 代码如果在 docker 中运行,需要将filename 修改为映射路径
|
||||
logging.basicConfig(level=logging.DEBUG,
|
||||
format='%(asctime)s [%(threadName)s:%(thread)d] [%(name)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s',
|
||||
filename='/var/log/judge.log')
|
||||
|
||||
logger = logging
|
@ -19,6 +19,8 @@ docker_config = {
|
||||
test_case_dir = "/var/mnt/source/test_case/"
|
||||
# 源代码路径,也就是 manage.py 所在的实际路径
|
||||
source_code_dir = "/var/mnt/source/OnlineJudge/"
|
||||
# 日志文件夹路径
|
||||
log_dir = "/var/log/"
|
||||
|
||||
|
||||
# 存储提交信息的数据库,是 celery 使用的,与 oj.settings/local_settings 等区分,那是 web 服务器访问的地址
|
||||
|
@ -5,7 +5,8 @@ import MySQLdb
|
||||
import subprocess
|
||||
from ..judger.result import result
|
||||
from ..judger_controller.celery import app
|
||||
from settings import docker_config, source_code_dir, test_case_dir, submission_db, redis_config
|
||||
from logger import logger
|
||||
from settings import docker_config, source_code_dir, test_case_dir, log_dir, submission_db, redis_config
|
||||
|
||||
|
||||
@app.task
|
||||
@ -14,17 +15,19 @@ def judge(submission_id, time_limit, memory_limit, test_case_id):
|
||||
command = "%s run -t -i --privileged --rm=true " \
|
||||
"-v %s:/var/judger/test_case/ " \
|
||||
"-v %s:/var/judger/code/ " \
|
||||
"-v %s:/var/judger/code/log/ " \
|
||||
"%s " \
|
||||
"python judge/judger/run.py " \
|
||||
"--solution_id %s --time_limit %s --memory_limit %s --test_case_id %s" % \
|
||||
(docker_config["docker_path"],
|
||||
test_case_dir,
|
||||
source_code_dir,
|
||||
log_dir,
|
||||
docker_config["image_name"],
|
||||
submission_id, str(time_limit), str(memory_limit), test_case_id)
|
||||
subprocess.call(command, shell=docker_config["shell"])
|
||||
except Exception as e:
|
||||
print e
|
||||
logger.error(e)
|
||||
conn = MySQLdb.connect(db=submission_db["db"],
|
||||
user=submission_db["user"],
|
||||
passwd=submission_db["password"],
|
||||
|
@ -1,12 +0,0 @@
|
||||
#include <stdio.h>
|
||||
int main()
|
||||
{
|
||||
int a = 0;
|
||||
int i = 0;
|
||||
for(i = 0; i < 9999999999;i++)
|
||||
{
|
||||
a += i;
|
||||
}
|
||||
printf("%d", a);
|
||||
return 0;
|
||||
}
|
@ -1,6 +0,0 @@
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
int main()
|
||||
{
|
||||
|
||||
}
|
@ -1,8 +0,0 @@
|
||||
# include <stdio.h>
|
||||
int main()
|
||||
{
|
||||
int a, b;
|
||||
scanf("%d %d", &a, &b);
|
||||
printf("%d", a + b);
|
||||
return 0;
|
||||
}
|
Loading…
Reference in New Issue
Block a user