QingdaoU/OnlineJudge
1
0
Fork 0
mirror of https://github.com/QingdaoU/OnlineJudge.git synced 2024-09-21 00:13:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

废弃 huey，多数据库连接的时候存在 connection 无法释放的问题，回到 celery

Browse Source
This commit is contained in:
virusdefender 2015-12-12 15:41:57 +08:00
parent 84d390362b
commit b687d2067b
14 changed files with 85 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -52,7 +52,7 @@ db.db
#redis dump
*.rdb
#*.out
db.sqlite3
*.sqlite3
.DS_Store
log/
static/release/css

8
account/tasks.py Normal file
View File

@ -0,0 +1,8 @@
# coding=utf-8
from celery import shared_task
from utils.mail import send_email
@shared_task
def _send_email(from_name, to_email, to_name, subject, content):
send_email(from_name, to_email, to_name, subject, content)

37
account/views.py
View File

@ -14,7 +14,7 @@ from rest_framework.response import Response
from utils.shortcuts import (serializer_invalid_response, error_response,
success_response, error_page, paginate, rand_str)
from utils.captcha import Captcha
from utils.mail import send_email
from .tasks import _send_email
from .decorators import login_required
from .models import User, UserProfile
@ -63,7 +63,7 @@ def index_page(request):
return render(request, "oj/index.html")
if request.META.get('HTTP_REFERER') or request.GET.get("index"):
return render(request, "oj/index.html")
return render(request, "oj/index.html")
else:
return http.HttpResponseRedirect('/problems/')
@ -151,9 +151,9 @@ class EmailCheckAPIView(APIView):
检测邮箱是否存在，用状态码标识结果
---
"""
#这里是为了适应前端表单验证空间的要求
# 这里是为了适应前端表单验证空间的要求
reset = request.GET.get("reset", None)
#如果reset为true说明该请求是重置密码页面发出的，要返回的状态码应正好相反
# 如果reset为true说明该请求是重置密码页面发出的，要返回的状态码应正好相反
if reset:
existed = 200
does_not_existed = 400
@ -287,22 +287,25 @@ class ApplyResetPasswordAPIView(APIView):
user = User.objects.get(email=data["email"])
except User.DoesNotExist:
return error_response(u"用户不存在")
if user.reset_password_token_create_time and (now() - user.reset_password_token_create_time).total_seconds() < 20 * 60:
if user.reset_password_token_create_time and (
now() - user.reset_password_token_create_time).total_seconds() < 20 * 60:
return error_response(u"20分钟内只能找回一次密码")
user.reset_password_token = rand_str()
user.reset_password_token_create_time = now()
user.save()
email_template = codecs.open(settings.TEMPLATES[0]["DIRS"][0] + "utils/reset_password_email.html", "r", "utf-8").read()
email_template = codecs.open(settings.TEMPLATES[0]["DIRS"][0] + "utils/reset_password_email.html", "r",
"utf-8").read()
email_template = email_template.replace("{{ username }}", user.username).\
replace("{{ website_name }}", settings.WEBSITE_INFO["website_name"]).\
replace("{{ link }}", request.scheme + "://" + request.META['HTTP_HOST'] + "/reset_password/t/" + user.reset_password_token)
email_template = email_template.replace("{{ username }}", user.username). \
replace("{{ website_name }}", settings.WEBSITE_INFO["website_name"]). \
replace("{{ link }}", request.scheme + "://" + request.META[
'HTTP_HOST'] + "/reset_password/t/" + user.reset_password_token)
send_email(settings.WEBSITE_INFO["website_name"],
user.email,
user.username,
settings.WEBSITE_INFO["website_name"] + u" 登录信息找回邮件",
email_template)
_send_email.delay(settings.WEBSITE_INFO["website_name"],
user.email,
user.username,
settings.WEBSITE_INFO["website_name"] + u" 登录信息找回邮件",
email_template)
return success_response(u"邮件发送成功,请前往您的邮箱查收")
else:
return serializer_invalid_response(serializer)
@ -352,7 +355,8 @@ class SSOAPIView(APIView):
user = User.objects.get(auth_token=serializer.data["token"])
user.auth_token = None
user.save()
return success_response({"username": user.username, "admin_type": user.admin_type, "avatar": user.userprofile.avatar})
return success_response(
{"username": user.username, "admin_type": user.admin_type, "avatar": user.userprofile.avatar})
except User.DoesNotExist:
return error_response(u"用户不存在")
else:
@ -366,7 +370,8 @@ class SSOAPIView(APIView):
token = rand_str()
request.user.auth_token = token
request.user.save()
return render(request, "oj/account/sso.html", {"redirect_url": callback + "?token=" + token, "callback": callback})
return render(request, "oj/account/sso.html",
{"redirect_url": callback + "?token=" + token, "callback": callback})
def reset_password_page(request, token):

0
db1.sqlite3
View File

3
dockerfiles/oj_web_server/requirements.txt
View File

@ -11,4 +11,5 @@ supervisor
pillow
jsonfield
Envelopes
huey
celery
djcelery

4
dockerfiles/oj_web_server/task_queue.conf
View File

@ -1,6 +1,6 @@
[program:mq]
[program:task_queue]
command=python manage.py run_huey
command=python manage.py celeryd -B -l DEBUG
directory=/code/
user=root

4
judge_dispatcher/tasks.py
View File

@ -32,7 +32,7 @@ class JudgeDispatcher(object):
if servers.exists():
return servers.first()
def judge(self, is_waiting_task=False):
def judge(self):
self.submission.judge_start_time = int(time.time() * 1000)
with transaction.atomic():
@ -89,7 +89,7 @@ class JudgeDispatcher(object):
submission = Submission.objects.get(id=waiting_submission.submission_id)
waiting_submission.delete()
_judge(submission, time_limit=waiting_submission.time_limit,
_judge.delay(submission, time_limit=waiting_submission.time_limit,
memory_limit=waiting_submission.memory_limit, test_case_id=waiting_submission.test_case_id,
is_waiting_task=True)

5
oj/__init__.py
View File

@ -7,3 +7,8 @@
|___/ |___/ |_|
https://github.com/QingdaoU/OnlineJudge
"""
from __future__ import absolute_import
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app

19
oj/celery.py Normal file
View File

@ -0,0 +1,19 @@
from __future__ import absolute_import
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings')
from django.conf import settings
app = Celery('oj')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
# load task modules from all registered Django app configs.
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

6
oj/local_settings.py
View File

@ -28,6 +28,12 @@ REDIS_QUEUE = {
"db": 2
}
# for celery
BROKER_URL = 'redis://%s:%s/%s' % (REDIS_QUEUE["host"], str(REDIS_QUEUE["port"]), str(REDIS_QUEUE["db"]))
ACCEPT_CONTENT = ['json']
DEBUG = True
ALLOWED_HOSTS = []

6
oj/server_settings.py
View File

@ -37,6 +37,12 @@ REDIS_QUEUE = {
"db": 2
}
# for celery
BROKER_URL = 'redis://%s:%s/%s' % (REDIS_QUEUE["host"], str(REDIS_QUEUE["port"]), str(REDIS_QUEUE["db"]))
ACCEPT_CONTENT = ['json']
DEBUG = False
ALLOWED_HOSTS = ['*']

15
oj/settings.py
View File

@ -10,7 +10,7 @@ https://docs.djangoproject.com/en/1.8/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""
from __future__ import absolute_import
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
@ -22,6 +22,9 @@ if ENV == "local":
elif ENV == "server":
from .server_settings import *
import djcelery
djcelery.setup_loader()
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@ -53,7 +56,7 @@ INSTALLED_APPS = (
'judge_dispatcher',
'rest_framework',
'huey.djhuey',
'djcelery',
)
if DEBUG:
@ -186,14 +189,6 @@ WEBSITE_INFO = {"website_name": "qduoj",
"website_footer": u"青岛大学信息工程学院 创新实验室 <a href=\"http://www.miibeian.gov.cn/\">京ICP备15062075号-1</a>",
"url": "https://qduoj.com"}
HUEY = {
'backend': 'huey.backends.redis_backend',
'name': 'task_queue',
'connection': {'host': REDIS_QUEUE["host"], 'port': REDIS_QUEUE["port"], 'db': REDIS_QUEUE["db"]},
'always_eager': False, # Defaults to False when running via manage.py run_huey
# Options to pass into the consumer when running ``manage.py run_huey``
'consumer_options': {'workers': 50},
}
SMTP_CONFIG = {"smtp_server": "smtp.mxhichina.com",
"email": "noreply@qduoj.com",

10
submission/tasks.py
View File

@ -1,9 +1,9 @@
# coding=utf-8
from huey.djhuey import db_task
from __future__ import absolute_import
from celery import shared_task
from judge_dispatcher.tasks import JudgeDispatcher
@db_task()
def _judge(submission, time_limit, memory_limit, test_case_id, is_waiting_task=False):
JudgeDispatcher(submission, time_limit, memory_limit, test_case_id).judge(is_waiting_task)
@shared_task
def _judge(submission, time_limit, memory_limit, test_case_id):
JudgeDispatcher(submission, time_limit, memory_limit, test_case_id).judge()

6
submission/views.py
View File

@ -43,7 +43,7 @@ class SubmissionAPIView(APIView):
problem_id=problem.id)
try:
_judge(submission, problem.time_limit, problem.memory_limit, problem.test_case_id)
_judge.delay(submission, problem.time_limit, problem.memory_limit, problem.test_case_id)
except Exception as e:
logger.error(e)
return error_response(u"提交判题任务失败")
@ -88,7 +88,7 @@ class ContestSubmissionAPIView(APIView):
code=data["code"],
problem_id=problem.id)
try:
_judge(submission, problem.time_limit, problem.memory_limit, problem.test_case_id)
_judge.delay(submission, problem.time_limit, problem.memory_limit, problem.test_case_id)
except Exception as e:
logger.error(e)
return error_response(u"提交判题任务失败")
@ -273,7 +273,7 @@ class SubmissionRejudgeAdminAPIView(APIView):
except Problem.DoesNotExist:
return error_response(u"题目不存在")
try:
_judge(submission, problem.time_limit, problem.memory_limit, problem.test_case_id)
_judge.delay(submission, problem.time_limit, problem.memory_limit, problem.test_case_id)
except Exception as e:
logger.error(e)
return error_response(u"提交判题任务失败")