From 3311a4c8990faeab67241de2a8ab01d5a8f29a58 Mon Sep 17 00:00:00 2001 From: virusdefender <1670873886@qq.com> Date: Sun, 29 Nov 2015 21:29:26 +0800 Subject: [PATCH] =?UTF-8?q?rpc=20=E9=80=9A=E4=BF=A1=E5=92=8C=E5=88=A4?= =?UTF-8?q?=E9=A2=98=E5=88=9D=E6=AD=A5=E6=B5=8B=E8=AF=95=E9=80=9A=E8=BF=87?= =?UTF-8?q?=EF=BC=9B=E5=88=A4=E9=A2=98=E6=9C=8D=E5=8A=A1=E5=99=A8=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E4=BE=9D=E8=B5=96=20redis=20=E5=92=8C=20mysql?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contest/models.py | 2 +- judge/language.py | 8 ++++---- judge/runner.py | 1 + judge_dispatcher/judge.py | 16 ---------------- judge_dispatcher/models.py | 2 +- judge_dispatcher/settings.py | 9 --------- judge_dispatcher/tasks.py | 34 ++++++++++++++++++++++++++++++++++ monitor/views.py | 6 +++--- oj/__init__.py | 7 ++++++- oj/celery.py | 17 +++++++++++++++++ oj/local_settings.py | 5 +++++ oj/server_settings.py | 5 +++++ oj/settings.py | 12 ++++++++++++ submission/models.py | 2 +- submission/views.py | 12 ++++++------ 15 files changed, 96 insertions(+), 42 deletions(-) delete mode 100644 judge_dispatcher/judge.py delete mode 100644 judge_dispatcher/settings.py create mode 100644 judge_dispatcher/tasks.py create mode 100644 oj/celery.py diff --git a/contest/models.py b/contest/models.py index 2213f43e..4e3631ba 100644 --- a/contest/models.py +++ b/contest/models.py @@ -7,7 +7,7 @@ from problem.models import AbstractProblem from group.models import Group from utils.models import RichTextField from jsonfield import JSONField -from judge.judger.result import result +from judge.result import result GROUP_CONTEST = 0 diff --git a/judge/language.py b/judge/language.py index c7a14eb4..e6406566 100644 --- a/judge/language.py +++ b/judge/language.py @@ -7,16 +7,16 @@ languages = { "src_name": "main.c", "code": 1, "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", - "compile_command": "gcc -DONLINE_JUDGE -O2 -w -std=c99 {src_path} -lm -o {exe_path}main", - "execute_command": "{exe_path}main" + "compile_command": "gcc -DONLINE_JUDGE -O2 -w -std=c99 {src_path} -lm -o {exe_path}/main", + "execute_command": "{exe_path}/main" }, 2: { "name": "cpp", "src_name": "main.cpp", "code": 2, "syscalls": "!execve:k,flock:k,ptrace:k,sync:k,fdatasync:k,fsync:k,msync,sync_file_range:k,syncfs:k,unshare:k,setns:k,clone:k,query_module:k,sysinfo:k,syslog:k,sysfs:k", - "compile_command": "g++ -DONLINE_JUDGE -O2 -w -std=c++11 {src_path} -lm -o {exe_path}main", - "execute_command": "{exe_path}main" + "compile_command": "g++ -DONLINE_JUDGE -O2 -w -std=c++11 {src_path} -lm -o {exe_path}/main", + "execute_command": "{exe_path}/main" }, 3: { "name": "java", diff --git a/judge/runner.py b/judge/runner.py index 4b5349e4..9eb78b0f 100644 --- a/judge/runner.py +++ b/judge/runner.py @@ -21,6 +21,7 @@ class JudgeInstanceRunner(object): try: os.mkdir(judge_base_path) + os.chmod(judge_base_path, 0777) # 将代码写入文件 src_path = os.path.join(judge_base_path, language["src_name"]) diff --git a/judge_dispatcher/judge.py b/judge_dispatcher/judge.py deleted file mode 100644 index 8e4e1dd8..00000000 --- a/judge_dispatcher/judge.py +++ /dev/null @@ -1,16 +0,0 @@ -# coding=utf-8 -import socket -import redis - -from .rpc_client import TimeoutServerProxy -from .settings import redis_config -from .models import JudgeServer - - -class JudgeDispatcher(object): - def __init__(self): - self.redis = redis.StrictRedis(host=redis_config["host"], port=redis_config["port"], db=redis_config["db"]) - - def judge(self): - pass - diff --git a/judge_dispatcher/models.py b/judge_dispatcher/models.py index 29f52267..67401c2b 100644 --- a/judge_dispatcher/models.py +++ b/judge_dispatcher/models.py @@ -3,7 +3,7 @@ from django.db import models class JudgeServer(models.Model): - ip = models.IPAddressField() + ip = models.GenericIPAddressField() port = models.IntegerField() # 这个服务器最大可能运行的判题实例数量 max_instance_number = models.IntegerField() diff --git a/judge_dispatcher/settings.py b/judge_dispatcher/settings.py deleted file mode 100644 index 332cefe8..00000000 --- a/judge_dispatcher/settings.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -import os - - -redis_config = { - "host": os.environ.get("REDIS_PORT_6379_TCP_ADDR"), - "port": 6379, - "db": 0 -} \ No newline at end of file diff --git a/judge_dispatcher/tasks.py b/judge_dispatcher/tasks.py new file mode 100644 index 00000000..b5bfcd4a --- /dev/null +++ b/judge_dispatcher/tasks.py @@ -0,0 +1,34 @@ +# coding=utf-8 +import json + +from celery import shared_task +from rpc_client import TimeoutServerProxy + +from judge.result import result +from submission.models import Submission + + +@shared_task +def create_judge_task(submission_id, code, language_code, time_limit, memory_limit, test_case_id): + submission = Submission.objects.get(id=submission_id) + try: + s = TimeoutServerProxy('http://121.42.198.156:8080', timeout=20) + data = s.run(submission_id, language_code, code, time_limit, memory_limit, test_case_id) + print data + # 编译错误 + if data["code"] == 1: + submission.result = result["compile_error"] + submission.info = data["data"]["error"] + # system error + elif data["code"] == 2: + submission.result = result["system_error"] + submission.info = data["data"]["error"] + elif data["code"] == 0: + submission.result = data["data"]["result"] + submission.info = json.dumps(data["data"]["info"]) + submission.accepted_answer_time = data["data"]["accepted_answer_time"] + except Exception as e: + submission.result = result["system_error"] + submission.info = str(e) + finally: + submission.save() diff --git a/monitor/views.py b/monitor/views.py index 15c62b4c..f502579f 100644 --- a/monitor/views.py +++ b/monitor/views.py @@ -2,15 +2,15 @@ import redis import datetime from rest_framework.views import APIView -from judge.judger.result import result -from judge.judger_controller.settings import redis_config +from judge.result import result +from django.conf import settings from utils.shortcuts import success_response from submission.models import Submission class QueueLengthMonitorAPIView(APIView): def get(self, request): - r = redis.Redis(host=redis_config["host"], port=redis_config["port"], db=redis_config["db"]) + r = redis.Redis(host=settings.redis_config["host"], port=settings.redis_config["port"], db=settings.redis_config["db"]) waiting_number = r.get("judge_queue_length") if waiting_number is None: waiting_number = 0 diff --git a/oj/__init__.py b/oj/__init__.py index c19c0794..d22a8460 100644 --- a/oj/__init__.py +++ b/oj/__init__.py @@ -6,4 +6,9 @@ \___/ |_| |_||_||_||_| |_| \___| \___/ \__,_| \__,_| \__, | \___| |_.__/ \__, | \__, | \__,_| \__,_| |___/ |___/ |_| https://github.com/QingdaoU/OnlineJudge -""" \ No newline at end of file +""" +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app diff --git a/oj/celery.py b/oj/celery.py new file mode 100644 index 00000000..2f53c0ca --- /dev/null +++ b/oj/celery.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings') + +from django.conf import settings + +app = Celery('oj') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) \ No newline at end of file diff --git a/oj/local_settings.py b/oj/local_settings.py index a08ae7e2..965fd6b8 100644 --- a/oj/local_settings.py +++ b/oj/local_settings.py @@ -22,6 +22,11 @@ REDIS_CACHE = { "db": 1 } +# for celery +REDIS_HOST = "localhost" +REDIS_PORT = 6379 +REDIS_DB = 0 + DEBUG = True ALLOWED_HOSTS = [] diff --git a/oj/server_settings.py b/oj/server_settings.py index bba27c6f..f8942a92 100644 --- a/oj/server_settings.py +++ b/oj/server_settings.py @@ -31,6 +31,11 @@ REDIS_CACHE = { "db": 1 } +# for celery +REDIS_HOST = os.environ.get("REDIS_PORT_6379_TCP_ADDR", "127.0.0.1") +REDIS_PORT = 6379 +REDIS_DB = 0 + DEBUG = False ALLOWED_HOSTS = ['*'] diff --git a/oj/settings.py b/oj/settings.py index 83d49ef7..13fb8f9a 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -22,6 +22,15 @@ if ENV == "local": elif ENV == "server": from .server_settings import * +import djcelery +djcelery.setup_loader() + +BROKER_BACKEND = "redis" +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -51,9 +60,12 @@ INSTALLED_APPS = ( 'mq', 'contest', 'mail', + 'judge', + 'judge_dispatcher', 'django_extensions', 'rest_framework', + 'djcelery', ) if DEBUG: diff --git a/submission/models.py b/submission/models.py index b28bd5ab..660b6d83 100644 --- a/submission/models.py +++ b/submission/models.py @@ -1,7 +1,7 @@ # coding=utf-8 from django.db import models from utils.shortcuts import rand_str -from judge.judger.result import result +from judge.result import result class Submission(models.Model): diff --git a/submission/views.py b/submission/views.py index 1f2acb16..94043cf4 100644 --- a/submission/views.py +++ b/submission/views.py @@ -7,7 +7,7 @@ from django.shortcuts import render from django.core.paginator import Paginator from rest_framework.views import APIView -from judge.judger_controller.tasks import judge +from judge_dispatcher.tasks import create_judge_task from account.decorators import login_required, super_admin_required from account.models import SUPER_ADMIN, User from problem.models import Problem @@ -23,8 +23,8 @@ from .serializers import (CreateSubmissionSerializer, SubmissionSerializer, logger = logging.getLogger("app_info") -def _judge(submission_id, time_limit, memory_limit, test_case_id): - judge.delay(submission_id, time_limit, memory_limit, test_case_id) +def _judge(submission_id, code, language_code, time_limit, memory_limit, test_case_id): + create_judge_task.delay(submission_id, code, language_code, time_limit, memory_limit, test_case_id) get_cache_redis().incr("judge_queue_length") @@ -49,7 +49,7 @@ class SubmissionAPIView(APIView): problem_id=problem.id) try: - _judge(submission.id, problem.time_limit, problem.memory_limit, problem.test_case_id) + _judge(submission.id, submission.code, submission.language, problem.time_limit, problem.memory_limit, problem.test_case_id) except Exception as e: logger.error(e) return error_response(u"提交判题任务失败") @@ -94,7 +94,7 @@ class ContestSubmissionAPIView(APIView): code=data["code"], problem_id=problem.id) try: - _judge(submission.id, problem.time_limit, problem.memory_limit, problem.test_case_id) + _judge(submission.id, submission.code, submission.language, problem.time_limit, problem.memory_limit, problem.test_case_id) except Exception as e: logger.error(e) return error_response(u"提交判题任务失败") @@ -279,7 +279,7 @@ class SubmissionRejudgeAdminAPIView(APIView): except Problem.DoesNotExist: return error_response(u"题目不存在") try: - _judge(submission.id, problem.time_limit, problem.memory_limit, problem.test_case_id) + _judge(submission.id, submission.code, submission.language, problem.time_limit, problem.memory_limit, problem.test_case_id) except Exception as e: logger.error(e) return error_response(u"提交判题任务失败")