Accept Merge Request #228 更新判题时细节 : (virusdefender-dev -> dev)

Merge Request: 更新判题时细节
Created By: @virusdefender
Accepted By: @virusdefender
URL: https://coding.net/u/virusdefender/p/qduoj/git/merge/228

judge/judger/client.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+import os
 import json
 import commands
 import hashlib
@@ -7,9 +8,9 @@ from multiprocessing import Pool
 from settings import max_running_number, lrun_gid, lrun_uid, judger_workspace
 from language import languages
 from result import result
-from compiler import compile_
+from judge_exceptions import JudgeClientError
-from judge_exceptions import JudgeClientError, CompileError
 from utils import parse_lrun_output
+from logger import logger
 
 
 # 下面这个函数作为代理访问实例变量，否则Python2会报错，是Python2的已知问题
@@ -82,6 +83,8 @@ class JudgeClient(object):
         # 倒序找到MEMORY的位置
         output_start = output.rfind("MEMORY")
         if output_start == -1:
+            logger.error("Lrun result parse error")
+            logger.error(output)
             raise JudgeClientError("Lrun result parse error")
         # 如果不是0，说明lrun输出前面有输出，也就是程序的stderr有内容
         if output_start != 0:
@@ -92,7 +95,8 @@ class JudgeClient(object):
         return error, parse_lrun_output(output)
 
     def _compare_output(self, test_case_id):
-        test_case_md5 = self._test_case_info["test_cases"][str(test_case_id)]["output_md5"]
+        test_case_config = self._test_case_info["test_cases"][str(test_case_id)]
+        test_case_md5 = test_case_config["output_md5"]
         output_path = judger_workspace + str(test_case_id) + ".out"
 
         try:
@@ -102,6 +106,7 @@ class JudgeClient(object):
             return False
 
         # 计算输出文件的md5 和之前测试用例文件的md5进行比较
+        # 现在比较的是完整的文件
         md5 = hashlib.md5()
         while True:
             data = f.read(2 ** 8)
@@ -109,9 +114,18 @@ class JudgeClient(object):
                 break
             md5.update(data)
 
-        # 对比文件是否一致
+        if md5.hexdigest() == test_case_md5:
-        # todo 去除最后的空行
+            return True
-        return md5.hexdigest() == test_case_md5
+        else:
+            # 这时候需要去除用户输出最后的空格和换行 再去比较md5
+            # 兼容之前没有striped_output_md5的测试用例
+            if "striped_output_md5" not in test_case_config:
+                return False
+            f.seek(0)
+            striped_md5 = hashlib.md5()
+            # 比较和返回去除空格后的md5比较结果
+            striped_md5.update(f.read().rstrip())
+            return striped_md5.hexdigest() == test_case_config["striped_output_md5"]
 
     def _judge_one(self, test_case_id):
         # 运行lrun程序 接收返回值
@@ -123,21 +137,23 @@ class JudgeClient(object):
 
         run_result["test_case_id"] = test_case_id
 
-        # 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
+        # 代表内存或者时间超过限制了 程序被终止掉 要在runtime error 之前判断
-        if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
-            run_result["result"] = result["runtime_error"]
-            return run_result
-
-        # 代表内存或者时间超过限制了
         if run_result["exceed"]:
             if run_result["exceed"] == "memory":
                 run_result["result"] = result["memory_limit_exceeded"]
             elif run_result["exceed"] in ["cpu_time", "real_time"]:
                 run_result["result"] = result["time_limit_exceeded"]
             else:
+                logger.error("Error exceeded type: " + run_result["exceed"])
+                logger.error(output)
                 raise JudgeClientError("Error exceeded type: " + run_result["exceed"])
             return run_result
 
+        # 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
+        if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
+            run_result["result"] = result["runtime_error"]
+            return run_result
+
         # 下面就是代码正常运行了 需要判断代码的输出是否正确
         if self._compare_output(test_case_id):
             run_result["result"] = result["accepted"]
@@ -160,8 +176,8 @@ class JudgeClient(object):
             try:
                 results.append(item.get())
             except Exception as e:
-                # todo logging
+                logger.error("system error")
-                print e
+                logger.error(e)
                 results.append({"result": result["system_error"]})
         return results
 

judge/judger/compiler.py

@@ -4,6 +4,7 @@ import commands
 from settings import lrun_uid, lrun_gid
 from judge_exceptions import CompileError, JudgeClientError
 from utils import parse_lrun_output
+from logger import logger
 
 
 def compile_(language_item, src_path, exe_path):
@@ -22,14 +23,20 @@ def compile_(language_item, src_path, exe_path):
     output_start = output.rfind("MEMORY")
 
     if output_start == -1:
+        logger.error("Compiler error")
+        logger.error(output)
         raise JudgeClientError("Error running compiler in lrun")
 
-    # 返回值不为0 或者 stderr中lrun的输出之前有东西
+    # 返回值不为 0 或者 stderr 中 lrun 的输出之前有 erro r字符串
-    if status or output_start:
+    # 判断 error 字符串的原因是链接的时候可能会有一些不推荐使用的函数的的警告，
+    # 但是 -w 参数并不能关闭链接时的警告
+    if status or "error" in output[0:output_start]:
         raise CompileError(output[0:output_start])
 
-    parse_result = parse_lrun_output(output)
+    parse_result = parse_lrun_output(output[output_start:])
 
     if parse_result["exit_code"] or parse_result["term_sig"] or parse_result["siginaled"] or parse_result["exceed"]:
+        logger.error("Compiler error")
+        logger.error(output)
         raise CompileError("Compile error")
     return exe_path

judge/judger/logger.py

@@ -0,0 +1,8 @@
+# coding=utf-8
+import logging
+
+logging.basicConfig(level=logging.DEBUG,
+                    format='%(asctime)s [%(threadName)s:%(thread)d] [%(name)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s',
+                    filename='log/judge.log')
+
+logger = logging

judge/judger/loose_client.py

@@ -1,184 +0,0 @@
-# coding=utf-8
-import json
-import commands
-import hashlib
-from multiprocessing import Pool
-
-from settings import max_running_number, lrun_gid, lrun_uid, judger_workspace
-from language import languages
-from result import result
-from compiler import compile_
-from judge_exceptions import JudgeClientError, CompileError
-from utils import parse_lrun_output
-
-
-# 下面这个函数作为代理访问实例变量，否则Python2会报错，是Python2的已知问题
-# http://stackoverflow.com/questions/1816958/cant-pickle-type-instancemethod-when-using-pythons-multiprocessing-pool-ma/7309686
-def _run(instance, test_case_id):
-    return instance._judge_one(test_case_id)
-
-
-class JudgeClient(object):
-    def __init__(self, language_code, exe_path, max_cpu_time,
-                 max_real_time, max_memory, test_case_dir):
-        """
-        :param language_code: 语言编号
-        :param exe_path: 可执行文件路径
-        :param max_cpu_time: 最大cpu时间，单位ms
-        :param max_real_time: 最大执行时间，单位ms
-        :param max_memory: 最大内存，单位MB
-        :param test_case_dir: 测试用例文件夹路径
-        :return:返回结果list
-        """
-        self._language = languages[language_code]
-        self._exe_path = exe_path
-        self._max_cpu_time = max_cpu_time
-        self._max_real_time = max_real_time
-        self._max_memory = max_memory
-        self._test_case_dir = test_case_dir
-        # 进程池
-        self._pool = Pool(processes=max_running_number)
-        self._test_case_info = self._load_test_case_info()
-    def _load_test_case_info(self):
-        # 读取测试用例信息 转换为dict
-        try:
-            f = open(self._test_case_dir + "info")
-            return json.loads(f.read())
-        except IOError:
-            raise JudgeClientError("Test case config file not found")
-        except ValueError:
-            raise JudgeClientError("Test case config file format error")
-
-    def _generate_command(self, test_case_id):
-        """
-        设置相关运行限制 进制访问网络 如果启用tmpfs 就把代码输出写入tmpfs，否则写入硬盘
-        """
-        # todo 系统调用白名单 chroot等参数
-        command = "lrun" + \
-                  " --max-cpu-time " + str(self._max_cpu_time / 1000.0) + \
-                  " --max-real-time " + str(self._max_real_time / 1000.0 * 2) + \
-                  " --max-memory " + str(self._max_memory * 1000 * 1000) + \
-                  " --network false" + \
-                  " --uid " + str(lrun_uid) + \
-                  " --gid " + str(lrun_gid)
-
-        execute_command = self._language["execute_command"].format(exe_path=self._exe_path)
-
-        command += (" " +
-                    execute_command +
-                    # 0就是stdin
-                    " 0<" + self._test_case_dir + str(test_case_id) + ".in" +
-                    # 1就是stdout
-                    " 1>" + judger_workspace + str(test_case_id) + ".out" +
-                    # 3是stderr，包含lrun的输出和程序的异常输出
-                    " 3>&2")
-        return command
-
-    def _parse_lrun_output(self, output):
-        # 要注意的是 lrun把结果输出到了stderr，所以有些情况下lrun的输出可能与程序的一些错误输出的混合的，要先分离一下
-        error = None
-        # 倒序找到MEMORY的位置
-        output_start = output.rfind("MEMORY")
-        if output_start == -1:
-            raise JudgeClientError("Lrun result parse error")
-        # 如果不是0，说明lrun输出前面有输出，也就是程序的stderr有内容
-        if output_start != 0:
-            error = output[0:output_start]
-        # 分离出lrun的输出
-        output = output[output_start:]
-
-        return error, parse_lrun_output(output)
-
-    def _compare_output(self, test_case_id):
-
-        output_path = judger_workspace + str(test_case_id) + ".out"
-
-        try:
-            f = open(output_path, "r")
-        except IOError:
-            # 文件不存在等引发的异常 返回结果错误
-            return False
-        try:
-            std = open(self._test_case_dir+str(test_case_id)+".out", "r")
-        except IOError:
-            # 文件不存在等引发的异常 返回结果错误
-            return False
-        sline = std.readline().strip('\n')
-        line = f.readline().strip('\n')
-        while sline:
-            try:
-                while line[-1]==' ':
-                    line = line[:-1]
-            except IndexError:
-                return False
-            if sline != line: 
-                return False
-            sline = std.readline().strip('\n')
-            line = f.readline().strip('\n')
-        if line:
-            return False
-        line = f.readline()
-        while line and line == '\n':
-            line = f.readline()
-        if line:
-            return False
-        return True
-
-    def _judge_one(self, test_case_id):
-        # 运行lrun程序 接收返回值
-        command = self._generate_command(test_case_id)
-        status_code, output = commands.getstatusoutput(command)
-        if status_code:
-            raise JudgeClientError(output)
-        error, run_result = self._parse_lrun_output(output)
-
-        run_result["test_case_id"] = test_case_id
-
-        # 代表内存或者时间超过限制了
-        if run_result["exceed"]:
-            if run_result["exceed"] == "memory":
-                run_result["result"] = result["memory_limit_exceeded"]
-            elif run_result["exceed"] in ["cpu_time", "real_time"]:
-                run_result["result"] = result["time_limit_exceeded"]
-            else:
-                raise JudgeClientError("Error exceeded type: " + run_result["exceed"])
-            return run_result
-
-        # 如果返回值非0 或者信号量不是0 或者程序的stderr有输出 代表非正常结束
-        if run_result["exit_code"] or run_result["term_sig"] or run_result["siginaled"] or error:
-            run_result["result"] = result["runtime_error"]
-            return run_result
-
-        # 下面就是代码正常运行了 需要判断代码的输出是否正确
-        if self._compare_output(test_case_id):
-            run_result["result"] = result["accepted"]
-        else:
-            run_result["result"] = result["wrong_answer"]
-
-        return run_result
-
-    def run(self):
-        # 添加到任务队列
-        _results = []
-        results = []
-        for i in range(self._test_case_info["test_case_number"]):
-            _results.append(self._pool.apply_async(_run, (self, i + 1)))
-        self._pool.close()
-        self._pool.join()
-        for item in _results:
-            # 注意多进程中的异常只有在get()的时候才会被引发
-            # http://stackoverflow.com/questions/22094852/how-to-catch-exceptions-in-workers-in-multiprocessing
-            try:
-                results.append(item.get())
-            except Exception as e:
-                # todo logging
-                print e
-                results.append({"result": result["system_error"]})
-        return results
-
-    def __getstate__(self):
-        # 不同的pool之间进行pickle的时候要排除自己，否则报错
-        # http://stackoverflow.com/questions/25382455/python-notimplementederror-pool-objects-cannot-be-passed-between-processes
-        self_dict = self.__dict__.copy()
-        del self_dict['_pool']
-        return self_dict

judge/judger/run.py

@@ -2,21 +2,13 @@
 import sys
 import json
 import MySQLdb
-import os
-
-# 判断判题模式
-judge_model = os.environ.get("judge_model", "default")
-if judge_model == "default":
-    from client import JudgeClient
-elif judge_model == "loose":
-    from loose_client import JudgeClient
 
+from client import JudgeClient
 from language import languages
 from compiler import compile_
 from result import result
-from settings import judger_workspace
+from settings import judger_workspace, submission_db
-
+from logger import logger
-from settings import submission_db
 
 
 # 简单的解析命令行参数
@@ -67,7 +59,6 @@ except Exception as e:
     conn.commit()
     exit()
 
-print "Compile successfully"
 # 运行
 try:
     client = JudgeClient(language_code=language_code,
@@ -87,16 +78,13 @@ try:
         judge_result["accepted_answer_time"] = l[-1]["cpu_time"]
 
 except Exception as e:
-    print e
+    logger.error(e)
     conn = db_conn()
     cur = conn.cursor()
     cur.execute("update submission set result=%s, info=%s where id=%s", (result["system_error"], str(e), submission_id))
     conn.commit()
     exit()
 
-print "Run successfully"
-print judge_result
-
 conn = db_conn()
 cur = conn.cursor()
 cur.execute("update submission set result=%s, info=%s, accepted_answer_time=%s where id=%s",

judge/judger_controller/settings.py

@@ -19,6 +19,8 @@ docker_config = {
 test_case_dir = "/var/mnt/source/test_case/"
 # 源代码路径，也就是 manage.py 所在的实际路径
 source_code_dir = "/var/mnt/source/OnlineJudge/"
+# 日志文件夹路径
+log_dir = "/var/log/"
 
 
 # 存储提交信息的数据库，是 celery 使用的，与 oj.settings/local_settings 等区分，那是 web 服务器访问的地址

judge/judger_controller/tasks.py

@@ -5,7 +5,7 @@ import MySQLdb
 import subprocess
 from ..judger.result import result
 from ..judger_controller.celery import app
-from settings import docker_config, source_code_dir, test_case_dir, submission_db, redis_config
+from settings import docker_config, source_code_dir, test_case_dir, log_dir, submission_db, redis_config
 
 
 @app.task
@@ -14,17 +14,18 @@ def judge(submission_id, time_limit, memory_limit, test_case_id):
         command = "%s run -t -i --privileged --rm=true " \
                   "-v %s:/var/judger/test_case/ " \
                   "-v %s:/var/judger/code/ " \
+                  "-v %s:/var/judger/code/log/ " \
                   "%s " \
                   "python judge/judger/run.py " \
                   "--solution_id %s --time_limit %s --memory_limit %s --test_case_id %s" % \
                   (docker_config["docker_path"],
                    test_case_dir,
                    source_code_dir,
+                   log_dir,
                    docker_config["image_name"],
                    submission_id, str(time_limit), str(memory_limit), test_case_id)
         subprocess.call(command, shell=docker_config["shell"])
     except Exception as e:
-        print e
         conn = MySQLdb.connect(db=submission_db["db"],
                                user=submission_db["user"],
                                passwd=submission_db["password"],

judge/tests/c/cpu_time_timeout.c

@@ -1,12 +0,0 @@
-#include <stdio.h>
-int main()
-{
-    int a = 0;
-    int i = 0;
-    for(i = 0; i < 9999999999;i++)
-    {
-        a += i;
-    }
-    printf("%d", a);
-    return 0;
-}

judge/tests/c/real_time_timeout.c

@@ -1,6 +0,0 @@
-#include <stdio.h>
-#include <unistd.h>
-int main()
-{
-
-}

judge/tests/c/success.c

@@ -1,8 +0,0 @@
-# include <stdio.h>
-int main()
-{
-    int a, b;
-    scanf("%d %d", &a, &b);
-    printf("%d", a + b);
-    return 0;
-}

problem/views.py

@@ -201,16 +201,24 @@ class TestCaseUploadAPIView(APIView):
             # 计算输出文件的md5
             for i in range(len(l) / 2):
                 md5 = hashlib.md5()
+                striped_md5 = hashlib.md5()
                 f = open(test_case_dir + str(i + 1) + ".out", "r")
+                # 完整文件的md5
                 while True:
                     data = f.read(2 ** 8)
                     if not data:
                         break
                     md5.update(data)
 
+                # 删除标准输出最后的空格和换行
+                # 这时只能一次全部读入了，分块读的话，没办法确定文件结尾
+                f.seek(0)
+                striped_md5.update(f.read().rstrip())
+
                 file_info["test_cases"][str(i + 1)] = {"input_name": str(i + 1) + ".in",
                                                        "output_name": str(i + 1) + ".out",
                                                        "output_md5": md5.hexdigest(),
+                                                       "striped_output_md5": striped_md5.hexdigest(),
                                                        "output_size": os.path.getsize(test_case_dir + str(i + 1) + ".out")}
                 # 写入配置文件
                 open(test_case_dir + "info", "w").write(json.dumps(file_info))
@@ -242,8 +250,6 @@ def problem_list_page(request, page=1):
     else:
         difficulty_order = "difficulty"
 
-
-
     # 按照标签筛选
     tag_text = request.GET.get("tag", None)
     if tag_text:

static/src/js/app/admin/problem/addProblem.js

@@ -13,6 +13,10 @@ require(["jquery", "avalon", "editor", "uploader", "bsAlert", "csrfToken", "tagE
                             bsAlert("题目描述不能为空!");
                             return false;
                         }
+                        if (vm.timeLimit < 100 || vm.timeLimit > 5000) {
+                            bsAlert("保证时间限制是一个100-5000的合法整数");
+                            return false;
+                        }
                         if (vm.samples.length == 0) {
                             bsAlert("请至少添加一组样例!");
                             return false;

static/src/js/app/admin/problem/editProblem.js

@@ -14,8 +14,8 @@ require(["jquery", "avalon", "editor", "uploader", "bsAlert", "csrfToken", "tagE
                             bsAlert("题目描述不能为空!");
                             return false;
                         }
-                        if (vm.timeLimit < 1000 || vm.timeLimit > 5000) {
+                        if (vm.timeLimit < 100 || vm.timeLimit > 5000) {
-                            bsAlert("保证时间限制是一个1000-5000的合法整数");
+                            bsAlert("保证时间限制是一个100-5000的合法整数");
                             return false;
                         }
                         if (vm.samples.length == 0) {

template/src/admin/problem/add_problem.html

@@ -18,7 +18,7 @@
         <div class="col-md-3">
             <div class="form-group"><label>时间限制(ms)</label>
                 <input type="number" name="timeLimit" class="form-control" ms-duplex="timeLimit"
-                       data-error="请输入时间限制(保证是一个1000-5000的合法整数)" required>
+                       data-error="请输入时间限制(保证是一个100-5000的合法整数)" required>
                 <div class="help-block with-errors"></div>
             </div>
         </div>
@@ -31,8 +31,12 @@
         </div>
         <div class="col-md-3">
             <div class="form-group"><label>难度</label>
-                <input type="number" name="difficulty" class="form-control" ms-duplex="difficulty"
+                <select name="difficulty" class="form-control" ms-duplex="difficulty"
-                       data-error="请输入难度(保证是一个合法整数)" required>
+                       data-error="请选择难度" required>
+                    <option value="1" selected="selected">简单</option>
+                    <option value="2">中等</option>
+                    <option value="3">难</option>
+                </select>
                 <div class="help-block with-errors"></div>
             </div>
         </div>

template/src/admin/problem/edit_problem.html

@@ -24,7 +24,7 @@
         <div class="col-md-3">
             <div class="form-group"><label>时间限制(ms)</label>
                 <input type="number" name="timeLimit" class="form-control" ms-duplex="timeLimit"
-                       data-error="请输入时间限制(保证是一个1000-5000的合法整数)" required>
+                       data-error="请输入时间限制(保证是一个100-5000的合法整数)" required>
                 <div class="help-block with-errors"></div>
             </div>
         </div>
@@ -37,8 +37,12 @@
         </div>
         <div class="col-md-3">
             <div class="form-group"><label>难度</label>
-                <input type="number" name="difficulty" class="form-control" ms-duplex="difficulty"
+                <select name="difficulty" class="form-control" ms-duplex="difficulty"
-                       data-error="请输入难度(保证是一个合法整数)" required>
+                       data-error="请选择难度" required>
+                    <option value="1">简单</option>
+                    <option value="2">中等</option>
+                    <option value="3">难</option>
+                </select>
                 <div class="help-block with-errors"></div>
             </div>
         </div>

template/src/oj/problem/problem_list.html

@@ -31,7 +31,16 @@
                                 <th><span class="glyphicon glyphicon-ok ac-flag"></span></th>
                                 <th scope="row"><a href="/problem/{{ item.id }}/">{{ item.id }}</a></th>
                                 <td><a href="/problem/{{ item.id }}/">{{ item.title }}</a></td>
-                                <td>{{ item.difficulty }}</td>
+                                <td>
+                                    {% ifequal item.difficulty 1 %}
+                                        简单
+                                    {% else %}
+                                        {% ifequal item.difficulty 2 %}
+                                            中等
+                                        {% else %}
+                                            难
+                                        {% endifequal %}
+                                    {% endifequal %}</td>
                                 <td>{{ item|accepted_radio }}</td>
                             </tr>
                         {% endfor %}
@@ -65,10 +74,11 @@
                     </div>
                     <ul class="list-group">
                         {% for item in tags %}
-                        <li class="list-group-item problem-tag" onclick="location.href='/problems/?tag={{ item.name }}'">
+                            <li class="list-group-item problem-tag"
-                            <span class="badge">{{ item.problem_number }}</span>
+                                onclick="location.href='/problems/?tag={{ item.name }}'">
-                            {{ item.name }}
+                                <span class="badge">{{ item.problem_number }}</span>
-                        </li>
+                                {{ item.name }}
+                            </li>
                         {% endfor %}
                     </ul>
                 </div>