QingdaoU
/
OnlineJudge
mirror of https://github.com/QingdaoU/OnlineJudge.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fixes

This commit is contained in:
virusdefender 2017-10-06 17:46:14 +08:00 committed by zema1
parent a324d55364
commit 93bd77d8d8
16 changed files with 91 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
account/middleware.py
View File

@ -1,10 +1,5 @@
import time
import pytz
from django.contrib import auth
from django.utils import timezone
from django.utils.translation import ugettext as _
from django.db import connection
from django.utils.timezone import now
from django.utils.deprecation import MiddlewareMixin
from utils.api import JSONResponse
@ -14,14 +9,11 @@ class SessionRecordMiddleware(MiddlewareMixin):
def process_request(self, request):
if request.user.is_authenticated():
session = request.session
ip = request.META.get("HTTP_X_REAL_IP", "UNKNOWN IP")
user_agent = request.META.get("HTTP_USER_AGENT", "")
_ip = session.setdefault("ip", ip)
_user_agent = session.setdefault("user_agent", user_agent)
if ip != _ip or user_agent != _user_agent:
session.modified = True
session["user_agent"] = request.META.get("HTTP_USER_AGENT", "")
session["ip"] = request.META.get("HTTP_X_REAL_IP", "UNKNOWN IP")
session["last_activity"] = now()
user_sessions = request.user.session_keys
if request.session.session_key not in user_sessions:
if session.session_key not in user_sessions:
user_sessions.append(session.session_key)
request.user.save()

2
account/migrations/0001_initial.py
View File

@ -50,7 +50,7 @@ class Migration(migrations.Migration):
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('problems_status', jsonfield.fields.JSONField(default={})),
('avatar', models.CharField(default=account.models._default_avatar, max_length=50)),
('avatar', models.CharField(default="default.png", max_length=50)),
('blog', models.URLField(blank=True, null=True)),
('mood', models.CharField(blank=True, max_length=200, null=True)),
('accepted_problem_number', models.IntegerField(default=0)),

2
account/models.py
View File

@ -72,7 +72,7 @@ class UserProfile(models.Model):
oi_problems_status = JSONField(default={})
real_name = models.CharField(max_length=32, blank=True, null=True)
avatar = models.CharField(max_length=256, default=f"{settings.IMAGE_UPLOAD_DIR}/default.png")
avatar = models.CharField(max_length=256, default=f"/{settings.IMAGE_UPLOAD_DIR}/default.png")
blog = models.URLField(blank=True, null=True)
mood = models.CharField(max_length=256, blank=True, null=True)
github = models.CharField(max_length=64, blank=True, null=True)

2
account/serializers.py
View File

@ -26,7 +26,6 @@ class UserRegisterSerializer(serializers.Serializer):
class UserChangePasswordSerializer(serializers.Serializer):
old_password = serializers.CharField()
new_password = serializers.CharField(min_length=6)
captcha = serializers.CharField()
class UserSerializer(serializers.ModelSerializer):
@ -46,6 +45,7 @@ class UserProfileSerializer(serializers.ModelSerializer):
class Meta:
model = UserProfile
fields = "__all__"
class UserInfoSerializer(serializers.ModelSerializer):

12
account/tests.py
View File

@ -8,11 +8,9 @@ from otpauth import OtpAuth
from utils.api.tests import APIClient, APITestCase
from utils.shortcuts import rand_str
from utils.cache import default_cache
from utils.constants import CacheKey
from options.options import SysOptions
from .models import AdminType, ProblemPermission, User
from conf.models import WebsiteConfig
class PermissionDecoratorTest(APITestCase):
@ -157,13 +155,9 @@ class UserRegisterAPITest(CaptchaTest):
self.data = {"username": "test_user", "password": "testuserpassword",
"real_name": "real_name", "email": "test@qduoj.com",
"captcha": self._set_captcha(self.client.session)}
# clea cache in redis
default_cache.delete(CacheKey.website_config)
def test_website_config_limit(self):
website = WebsiteConfig.objects.create()
website.allow_register = False
website.save()
SysOptions.allow_register = False
resp = self.client.post(self.register_url, data=self.data)
self.assertDictEqual(resp.data, {"error": "error", "data": "Register have been disabled by admin"})
@ -247,7 +241,6 @@ class TwoFactorAuthAPITest(APITestCase):
def setUp(self):
self.url = self.reverse("two_factor_auth_api")
self.create_user("test", "test123")
self.create_website_config()
def _get_tfa_code(self):
user = User.objects.first()
@ -295,7 +288,6 @@ class ApplyResetPasswordAPITest(CaptchaTest):
user.email = "test@oj.com"
user.save()
self.url = self.reverse("apply_reset_password_api")
self.create_website_config()
self.data = {"email": "test@oj.com", "captcha": self._set_captcha(self.client.session)}
def _refresh_captcha(self):

2
account/urls/oj.py
View File

@ -3,7 +3,7 @@ from django.conf.urls import url
from ..views.oj import (ApplyResetPasswordAPI, ResetPasswordAPI,
UserChangePasswordAPI, UserRegisterAPI,
UserLoginAPI, UserLogoutAPI, UsernameOrEmailCheck,
SSOAPI, AvatarUploadAPI, TwoFactorAuthAPI, UserProfileAPI,
AvatarUploadAPI, TwoFactorAuthAPI, UserProfileAPI,
UserRankAPI, CheckTFARequiredAPI, SessionManagementAPI)
from utils.captcha.views import CaptchaAPIView

10
account/views/oj.py
View File

@ -1,5 +1,4 @@
import os
import pickle
from datetime import timedelta
from importlib import import_module
@ -16,15 +15,14 @@ from utils.constants import ContestRuleType
from options.options import SysOptions
from utils.api import APIView, validate_serializer
from utils.captcha import Captcha
from utils.shortcuts import rand_str, img2base64, timestamp2utcstr
from utils.shortcuts import rand_str, img2base64, datetime2str
from ..decorators import login_required
from ..models import User, UserProfile
from ..serializers import (ApplyResetPasswordSerializer, ResetPasswordSerializer,
UserChangePasswordSerializer, UserLoginSerializer,
UserRegisterSerializer, UsernameOrEmailCheckSerializer,
RankInfoSerializer)
from ..serializers import (SSOSerializer, TwoFactorAuthCodeSerializer,
UserProfileSerializer,
from ..serializers import (TwoFactorAuthCodeSerializer, UserProfileSerializer,
EditUserProfileSerializer, AvatarUploadForm)
from ..tasks import send_email_async
@ -81,7 +79,7 @@ class AvatarUploadAPI(APIView):
img.write(chunk)
user_profile = request.user.userprofile
user_profile.avatar = f"{settings.IMAGE_UPLOAD_DIR}/{name}"
user_profile.avatar = f"/{settings.IMAGE_UPLOAD_DIR}/{name}"
user_profile.save()
return self.success("Succeeded")
@ -327,7 +325,7 @@ class SessionManagementAPI(APIView):
s["current_session"] = True
s["ip"] = session["ip"]
s["user_agent"] = session["user_agent"]
s["last_activity"] = timestamp2utcstr(session["last_activity"])
s["last_activity"] = datetime2str(session["last_activity"])
s["session_key"] = key
result.append(s)
if modified:

3
contest/models.py
View File

@ -1,8 +1,9 @@
from utils.constants import ContestRuleType # noqa
from django.db import models
from django.utils.timezone import now
from jsonfield import JSONField
from utils.constants import ContestStatus, ContestRuleType, ContestType
from utils.constants import ContestStatus, ContestType
from account.models import User, AdminType
from utils.models import RichTextField

2
contest/views/oj.py
View File

@ -4,7 +4,7 @@ from utils.api import APIView, validate_serializer
from utils.constants import CacheKey
from account.decorators import login_required, check_contest_permission
from utils.constants import ContestRuleType, ContestType, ContestStatus
from utils.constants import ContestRuleType, ContestStatus
from ..models import ContestAnnouncement, Contest, OIContestRank, ACMContestRank
from ..serializers import ContestAnnouncementSerializer
from ..serializers import ContestSerializer, ContestPasswordVerifySerializer

28
judge/dispatcher.py
View File

@ -14,7 +14,7 @@ from judge.languages import languages
from options.options import SysOptions
from problem.models import Problem, ProblemRuleType
from submission.models import JudgeStatus, Submission
from utils.cache import judge_cache, default_cache
from utils.cache import cache
from utils.constants import CacheKey
logger = logging.getLogger(__name__)
@ -22,31 +22,28 @@ logger = logging.getLogger(__name__)
# 继续处理在队列中的问题
def process_pending_task():
if judge_cache.llen(CacheKey.waiting_queue):
if cache.llen(CacheKey.waiting_queue):
# 防止循环引入
from judge.tasks import judge_task
data = json.loads(judge_cache.rpop(CacheKey.waiting_queue).decode("utf-8"))
data = json.loads(cache.rpop(CacheKey.waiting_queue).decode("utf-8"))
judge_task.delay(**data)
class JudgeDispatcher(object):
def __init__(self, submission_id, problem_id):
self.token = hashlib.sha256(SysOptions.judge_server_token.encode("utf-8")).hexdigest()
self.redis_conn = judge_cache
self.submission = Submission.objects.get(pk=submission_id)
self.submission = Submission.objects.get(id=submission_id)
self.contest_id = self.submission.contest_id
if self.contest_id:
self.problem = Problem.objects.select_related("contest") \
.get(id=problem_id, contest_id=self.contest_id)
self.problem = Problem.objects.select_related("contest").get(id=problem_id, contest_id=self.contest_id)
self.contest = self.problem.contest
else:
self.problem = Problem.objects.get(id=problem_id)
def _request(self, url, data=None):
kwargs = {"headers": {"X-Judge-Server-Token": self.token,
"Content-Type": "application/json"}}
kwargs = {"headers": {"X-Judge-Server-Token": self.token}}
if data:
kwargs["data"] = json.dumps(data)
kwargs["json"] = data
try:
return requests.post(url, **kwargs).json()
except Exception as e:
@ -55,7 +52,6 @@ class JudgeDispatcher(object):
@staticmethod
def choose_judge_server():
with transaction.atomic():
# TODO: use more reasonable way
servers = JudgeServer.objects.select_for_update().all().order_by("task_number")
servers = [s for s in servers if s.status == "normal"]
if servers:
@ -65,10 +61,10 @@ class JudgeDispatcher(object):
return server
@staticmethod
def release_judge_res(judge_server_id):
def release_judge_server(judge_server_id):
with transaction.atomic():
# 使用原子操作, 同时因为use和release中间间隔了判题过程,需要重新查询一下
server = JudgeServer.objects.select_for_update().get(id=judge_server_id)
server = JudgeServer.objects.get(id=judge_server_id)
server.used_instance_number = F("task_number") - 1
server.save()
@ -94,7 +90,7 @@ class JudgeDispatcher(object):
server = self.choose_judge_server()
if not server:
data = {"submission_id": self.submission.id, "problem_id": self.problem.id}
self.redis_conn.lpush(CacheKey.waiting_queue, json.dumps(data))
cache.lpush(CacheKey.waiting_queue, json.dumps(data))
return
sub_config = list(filter(lambda item: self.submission.language == item["name"], languages))[0]
@ -138,7 +134,7 @@ class JudgeDispatcher(object):
else:
self.submission.result = JudgeStatus.PARTIALLY_ACCEPTED
self.submission.save()
self.release_judge_res(server.id)
self.release_judge_server(server.id)
self.update_problem_status()
if self.contest_id:
@ -223,7 +219,7 @@ class JudgeDispatcher(object):
if self.contest_id and self.contest.status != ContestStatus.CONTEST_UNDERWAY:
return
if self.contest.real_time_rank:
default_cache.delete(CacheKey.contest_rank_cache + str(self.contest_id))
cache.delete(CacheKey.contest_rank_cache + str(self.contest_id))
with transaction.atomic():
if self.contest.rule_type == ContestRuleType.ACM:
acm_rank, _ = ACMContestRank.objects.select_for_update(). \

8
oj/local_settings.py
View File

@ -5,8 +5,12 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'HOST': '127.0.0.1',
'PORT': 5433,
'NAME': "onlinejudge",
'USER': "onlinejudge",
'PASSWORD': 'onlinejudge'
}
}

51
oj/settings.py
View File

@ -61,7 +61,6 @@ MIDDLEWARE_CLASSES = (
'account.middleware.SessionRecordMiddleware',
# 'account.middleware.LogSqlMiddleware',
)
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
ROOT_URLCONF = 'oj.urls'
TEMPLATES = [
@ -166,41 +165,33 @@ LOGGING = {
}
REST_FRAMEWORK = {
'TEST_REQUEST_DEFAULT_FORMAT': 'json',
'DEFAULT_RENDERER_CLASSES': (
'rest_framework.renderers.JSONRenderer',
)
}
REDIS_URL = "redis://127.0.0.1:6379"
CACHE_JUDGE_QUEUE = "judge_queue"
CACHE_THROTTLING = "throttling"
def redis_config(db):
def make_key(key, key_prefix, version):
return key
return {
"BACKEND": "utils.cache.MyRedisCache",
"LOCATION": f"{REDIS_URL}/{db}",
"TIMEOUT": None,
"KEY_PREFIX": "",
"KEY_FUNCTION": make_key
}
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379/1",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
}
},
CACHE_JUDGE_QUEUE: {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379/2",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
}
},
CACHE_THROTTLING: {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379/3",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
}
}
"default": redis_config(db=1)
}
CELERY_RESULT_BACKEND = CELERY_BROKER_URL = f"{REDIS_URL}/2"
CELERY_TASK_SOFT_TIME_LIMIT = CELERY_TASK_TIME_LIMIT = 180
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"
# For celery
REDIS_QUEUE = {
"host": "127.0.0.1",

8
options/options.py
View File

@ -113,15 +113,15 @@ class _SysOptionsMeta(type):
@property
def website_base_url(cls):
return cls._get_option(OptionKeys.website_base_url)
@website_base_url.setter
def website_base_url(cls, value):
cls._set_option(OptionKeys.website_base_url, value)
@property
def website_name(cls):
return cls._get_option(OptionKeys.website_name)
@website_name.setter
def website_name(cls, value):
cls._set_option(OptionKeys.website_name, value)
@ -173,7 +173,7 @@ class _SysOptionsMeta(type):
@judge_server_token.setter
def judge_server_token(cls, value):
cls._set_option(OptionKeys.judge_server_token, value)
class SysOptions(metaclass=_SysOptionsMeta):
pass

2
problem/serializers.py
View File

@ -71,6 +71,7 @@ class CreateContestProblemSerializer(CreateOrEditProblemSerializer):
class TagSerializer(serializers.ModelSerializer):
class Meta:
model = ProblemTag
fields = "__all__"
class BaseProblemSerializer(serializers.ModelSerializer):
@ -88,6 +89,7 @@ class BaseProblemSerializer(serializers.ModelSerializer):
class ProblemAdminSerializer(BaseProblemSerializer):
class Meta:
model = Problem
fields = "__all__"
class ContestProblemAdminSerializer(BaseProblemSerializer):

4
submission/views/oj.py
View File

@ -5,16 +5,16 @@ from problem.models import Problem, ProblemRuleType
from contest.models import Contest, ContestStatus, ContestRuleType
from utils.api import APIView, validate_serializer
from utils.throttling import TokenBucket, BucketController
from utils.cache import cache
from ..models import Submission
from ..serializers import CreateSubmissionSerializer, SubmissionModelSerializer
from ..serializers import SubmissionSafeSerializer, SubmissionListSerializer
from utils.cache import throttling_cache
def _submit(response, user, problem_id, language, code, contest_id):
# TODO: 预设默认值，需修改
controller = BucketController(user_id=user.id,
redis_conn=throttling_cache,
redis_conn=cache,
default_capacity=30)
bucket = TokenBucket(fill_rate=10, capacity=20,
last_capacity=controller.last_capacity,

31
utils/cache.py
View File

@ -1,6 +1,27 @@
from django.conf import settings
from django_redis import get_redis_connection
from django.core.cache import cache, caches # noqa
from django.conf import settings # noqa
judge_cache = get_redis_connection(settings.CACHE_JUDGE_QUEUE)
throttling_cache = get_redis_connection(settings.CACHE_THROTTLING)
default_cache = get_redis_connection("default")
from django_redis.cache import RedisCache
from django_redis.client.default import DefaultClient
class MyRedisClient(DefaultClient):
def __getattr__(self, item):
client = self.get_client(write=True)
return getattr(client, item)
def redis_incr(self, key, count=1):
"""
django 默认的 incr 在 key 不存在时候会抛异常
"""
client = self.get_client(write=True)
return client.incr(key, count)
class MyRedisCache(RedisCache):
def __init__(self, server, params):
super().__init__(server, params)
self._client_cls = MyRedisClient
def __getattr__(self, item):
return getattr(self.client, item)