From 65f9c7f52b501bee4580c65b4c947560fd053bcb Mon Sep 17 00:00:00 2001 From: Chiaki Date: Mon, 8 May 2017 17:29:01 +0800 Subject: [PATCH 1/2] Add submission module --- account/urls/user.py | 3 +- account/views/user.py | 18 +++ oj/settings.py | 1 + oj/urls.py | 3 +- requirements.txt | 3 +- submission/__init__.py | 0 submission/migrations/0001_initial.py | 39 ++++++ submission/migrations/__init__.py | 0 submission/models.py | 44 +++++++ submission/serializers.py | 10 ++ submission/tasks.py | 6 + submission/test.py | 0 submission/urls/__init__.py | 0 submission/urls/admin.py | 0 submission/urls/oj.py | 9 ++ submission/views/__init__.py | 0 submission/views/admin.py | 0 submission/views/oj.py | 172 ++++++++++++++++++++++++++ utils/shortcuts.py | 14 +++ utils/throttling.py | 91 ++++++++++++++ 20 files changed, 410 insertions(+), 3 deletions(-) create mode 100644 submission/__init__.py create mode 100644 submission/migrations/0001_initial.py create mode 100644 submission/migrations/__init__.py create mode 100644 submission/models.py create mode 100644 submission/serializers.py create mode 100644 submission/tasks.py create mode 100644 submission/test.py create mode 100644 submission/urls/__init__.py create mode 100644 submission/urls/admin.py create mode 100644 submission/urls/oj.py create mode 100644 submission/views/__init__.py create mode 100644 submission/views/admin.py create mode 100644 submission/views/oj.py create mode 100644 utils/throttling.py diff --git a/account/urls/user.py b/account/urls/user.py index 1c3ad368..921faf69 100644 --- a/account/urls/user.py +++ b/account/urls/user.py @@ -1,9 +1,10 @@ from django.conf.urls import url from ..views.user import (SSOAPI, AvatarUploadAPI, TwoFactorAuthAPI, - UserInfoAPI, UserProfileAPI) + UserNameAPI, UserInfoAPI, UserProfileAPI) urlpatterns = [ + url(r"^username/?$", UserNameAPI.as_view(), name="user_name_api"), url(r"^user/(?P\w+)/?$", UserInfoAPI.as_view(), name="user_info_api"), url(r"^profile/?$", UserProfileAPI.as_view(), name="user_profile_api"), url(r"^avatar/upload/?$", AvatarUploadAPI.as_view(), name="avatar_upload_api"), diff --git a/account/views/user.py b/account/views/user.py index d0b83930..0c0ea111 100644 --- a/account/views/user.py +++ b/account/views/user.py @@ -19,6 +19,24 @@ from ..serializers import (SSOSerializer, TwoFactorAuthCodeSerializer, EditUserProfileSerializer, AvatarUploadForm) +class UserNameAPI(APIView): + def get(self, request): + """ + Return Username to valid login status + """ + try: + user = User.objects.get(id=request.user.id) + except User.DoesNotExist: + return self.success({ + "username": "User does not exist", + "isLogin": False + }) + return self.success({ + "username": user.username, + "isLogin": True + }) + + class UserInfoAPI(APIView): # @login_required @method_decorator(ensure_csrf_cookie) diff --git a/oj/settings.py b/oj/settings.py index e7199d92..c33efe61 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -45,6 +45,7 @@ INSTALLED_APPS = ( 'problem', 'contest', 'utils', + 'submission', 'rest_framework', ) diff --git a/oj/urls.py b/oj/urls.py index 79e6b038..1e79aab5 100644 --- a/oj/urls.py +++ b/oj/urls.py @@ -10,5 +10,6 @@ urlpatterns = [ url(r"^api/", include("problem.urls.oj")), url(r"^api/admin/", include("problem.urls.admin")), url(r"^api/admin/", include("contest.urls.admin")), - url(r"^api/", include("contest.urls.oj")) + url(r"^api/", include("contest.urls.oj")), + url(r"^api/", include("submission.urls.oj")), ] diff --git a/requirements.txt b/requirements.txt index 9b0d8052..eab93e01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ celery Envelopes pytz jsonfield -qrcode \ No newline at end of file +qrcode +redis \ No newline at end of file diff --git a/submission/__init__.py b/submission/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/submission/migrations/0001_initial.py b/submission/migrations/0001_initial.py new file mode 100644 index 00000000..74a0559d --- /dev/null +++ b/submission/migrations/0001_initial.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2017-05-08 09:27 +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields +import utils.models +import utils.shortcuts + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Submission', + fields=[ + ('id', models.CharField(db_index=True, default=utils.shortcuts.rand_str, max_length=32, primary_key=True, serialize=False)), + ('contest_id', models.IntegerField(db_index=True)), + ('problem_id', models.IntegerField(db_index=True)), + ('created_time', models.DateTimeField(auto_now_add=True)), + ('user_id', models.IntegerField(db_index=True)), + ('code', utils.models.RichTextField()), + ('result', models.IntegerField(default=6)), + ('info', jsonfield.fields.JSONField()), + ('language', models.CharField(max_length=20)), + ('shared', models.BooleanField(default=False)), + ('accepted_time', models.IntegerField(blank=True, null=True)), + ('accepted_info', jsonfield.fields.JSONField()), + ], + options={ + 'db_table': 'submission', + }, + ), + ] diff --git a/submission/migrations/__init__.py b/submission/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/submission/models.py b/submission/models.py new file mode 100644 index 00000000..7cb6cc74 --- /dev/null +++ b/submission/models.py @@ -0,0 +1,44 @@ +from django.db import models +from jsonfield import JSONField + +# from judge.languages import language_names +from utils.models import RichTextField +from utils.shortcuts import rand_str + + +class JudgeStatus: + COMPILE_ERROR = -2 + WRONG_ANSWER = -1 + ACCEPTED = 0 + CPU_TIME_LIMIT_EXCEEDED = 1 + REAL_TIME_LIMIT_EXCEEDED = 2 + MEMORY_LIMIT_EXCEEDED = 3 + RUNTIME_ERROR = 4 + SYSTEM_ERROR = 5 + PENDING = 6 + JUDGING = 7 + # TODO: 部分正确 + + +class Submission(models.Model): + id = models.CharField(max_length=32, default=rand_str, primary_key=True, db_index=True) + contest_id = models.IntegerField(db_index=True) + problem_id = models.IntegerField(db_index=True) + created_time = models.DateTimeField(auto_now_add=True) + user_id = models.IntegerField(db_index=True) + code = RichTextField() + result = models.IntegerField(default=JudgeStatus.PENDING) + # 判题结果的详细信息 + info = JSONField() + # TODO: choice + language = models.CharField(max_length=20) + shared = models.BooleanField(default=False) + # 题目状态为 Accepted 时才会存储相关info + accepted_time = models.IntegerField(blank=True, null=True) + accepted_info = JSONField() + + class Meta: + db_table = "submission" + + def __str__(self): + return self.id diff --git a/submission/serializers.py b/submission/serializers.py new file mode 100644 index 00000000..f5cd1a03 --- /dev/null +++ b/submission/serializers.py @@ -0,0 +1,10 @@ +from utils.api import serializers + +# from account.models import User +# from .models import Submission + + +class CreateSubmissionSerializer(serializers.Serializer): + problem_id = serializers.IntegerField() + language = serializers.CharField(max_length=20) + code = serializers.CharField(max_length=20000) diff --git a/submission/tasks.py b/submission/tasks.py new file mode 100644 index 00000000..f6802ae3 --- /dev/null +++ b/submission/tasks.py @@ -0,0 +1,6 @@ +from celery import shared_task + + +@shared_task +def _judge(submission_obj, problem_obj): + pass diff --git a/submission/test.py b/submission/test.py new file mode 100644 index 00000000..e69de29b diff --git a/submission/urls/__init__.py b/submission/urls/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/submission/urls/admin.py b/submission/urls/admin.py new file mode 100644 index 00000000..e69de29b diff --git a/submission/urls/oj.py b/submission/urls/oj.py new file mode 100644 index 00000000..9f54c055 --- /dev/null +++ b/submission/urls/oj.py @@ -0,0 +1,9 @@ +from django.conf.urls import url + +from ..views.oj import (SubmissionAPI, SubmissionListAPI) + +urlpatterns = [ + url(r"^submission/?$", SubmissionAPI.as_view(), name="submissiob_api"), + url(r"^submissions/?$", SubmissionListAPI.as_view(), name="submission_list_api"), + url(r"^submissions/(?P\d+)/?$", SubmissionListAPI.as_view(), name="submission_list_page_api"), +] diff --git a/submission/views/__init__.py b/submission/views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/submission/views/admin.py b/submission/views/admin.py new file mode 100644 index 00000000..e69de29b diff --git a/submission/views/oj.py b/submission/views/oj.py new file mode 100644 index 00000000..ddc21485 --- /dev/null +++ b/submission/views/oj.py @@ -0,0 +1,172 @@ +import redis + +from django.core.paginator import Paginator + +from account.decorators import login_required +from account.models import AdminType, User +from problem.models import Problem + +from utils.api import APIView, validate_serializer +from utils.shortcuts import build_query_string +from utils.throttling import TokenBucket, BucketController + +from ..models import Submission +from ..serializers import CreateSubmissionSerializer +from ..tasks import _judge + + +def _submit_code(response, user, problem_id, language, code): + controller = BucketController(user_id=user.id, + redis_conn=redis.Redis(), + default_capacity=30) + bucket = TokenBucket(fill_rate=10, + capacity=20, + last_capacity=controller.last_capacity, + last_timestamp=controller.last_timestamp) + if bucket.consume(): + controller.last_capacity -= 1 + else: + return response.error("Please wait %d seconds" % int(bucket.expected_time() + 1)) + + try: + problem = Problem.objects.get(id=problem_id) + except Problem.DoesNotExist: + return response.error("Problem not exist") + + submission = Submission.objects.create(user_id=user.id, + language=language, + code=code, + problem_id=problem.id) + + try: + # TODO 参数 + _judge.delay(submission, problem) + except Exception as e: + return response.error("Failed") + + return response.success({"submission_id": submission.id}) + + +class SubmissionAPI(APIView): + @validate_serializer(CreateSubmissionSerializer) + @login_required + def post(self, request): + data = request.data + return _submit_code(self, request.user, data["problem_id"], data["language"], data["code"]) + + @login_required + def get(self, request): + submission_id = request.GET.get("submission_id") + if not submission_id: + return self.error("Parameter error") + try: + submission = Submission.objects.get(id=submission_id, user_id=request.user.id) + except Submission.DoesNotExist: + return self.error("Submission not exist") + + response_data = {"result": submission.result} + if submission.result == 0: + response_data["accepted_answer_time"] = submission.accepted_answer_time + return self.success(response_data) + + +class MyProblemSubmissionListAPI(APIView): + """ + 用户单个题目的全部提交列表 + """ + + def get(self, request): + problem_id = request.GET.get("problem_id") + try: + problem = Problem.objects.get(id=problem_id, visible=True) + except Problem.DoesNotExist: + return self.error("Problem not exist") + + submissions = Submission.objects.filter(user_id=request.user.id, problem_id=problem.id, + contest_id__isnull=True). \ + order_by("-created_time"). \ + values("id", "result", "created_time", "accepted_time", "language") + + return self.success({"submissions": submissions, "problem": problem}) + + +class SubmissionListAPI(APIView): + """ + 所有提交的列表 + """ + + def get(self, request, **kwargs): + submission_filter = {"my": None, "user_id": None} + show_all = False + page = kwargs.get("page", 1) + + user_id = request.GET.get("user_id") + if user_id and request.user.admin_type == AdminType.SUPER_ADMIN: + submission_filter["user_id"] = user_id + submissions = Submission.objects.filter(user_id=user_id, contest_id__isnull=True) + else: + show_all = True + if request.GET.get("my") == "true": + submission_filter["my"] = "true" + show_all = False + if show_all: + submissions = Submission.objects.filter(contest_id__isnull=True) + else: + submissions = Submission.objects.filter(user_id=request.user.id, contest_id__isnull=True) + + submissions = submissions.values("id", "user_id", "problem_id", "result", "created_time", + "accepted_time", "language").order_by("-created_time") + + language = request.GET.get("language") + if language: + submissions = submissions.filter(language=language) + submission_filter["language"] = language + + result = request.GET.get("result") + if result: + # TODO： 转换为数字结果 + submissions = submissions.filter(result=int(result)) + submission_filter["result"] = result + + paginator = Paginator(submissions, 20) + try: + submissions = paginator.page(int(page)) + except Exception: + return self.error("Page not exist") + + # Cache + cache_result = {"problem": {}, "user": {}} + for item in submissions: + problem_id = item["problem_id"] + if problem_id not in cache_result["problem"]: + problem = Problem.objects.get(id=problem_id) + cache_result["problem"][problem_id] = problem.title + item["title"] = cache_result["problem"][problem_id] + + user_id = item["user_id"] + if user_id not in cache_result["user"]: + user = User.objects.get(id=user_id) + cache_result["user"][user_id] = user + item["user"] = cache_result["user"][user_id] + + if item["user_id"] == request.user.id or request.user.admin_type == AdminType.SUPER_ADMIN: + item["show_link"] = True + else: + item["show_link"] = False + + previous_page = next_page = None + try: + previous_page = submissions.previous_page_number() + except Exception: + pass + try: + next_page = submissions.next_page_number() + except Exception: + pass + + return self.success({"submissions": submissions, "page": int(page), + "previous_page": previous_page, "next_page": next_page, + "start_id": int(page) * 20 - 20, + "query": build_query_string(submission_filter), + "submission_filter": submission_filter, + "show_all": show_all}) diff --git a/utils/shortcuts.py b/utils/shortcuts.py index 9962ade6..a7fcd116 100644 --- a/utils/shortcuts.py +++ b/utils/shortcuts.py @@ -44,3 +44,17 @@ def rand_str(length=32, type="lower_hex"): return random.choice("123456789abcdef") + get_random_string(length - 1, allowed_chars="0123456789abcdef") else: return random.choice("123456789") + get_random_string(length - 1, allowed_chars="0123456789") + + +def build_query_string(kv_data, ignore_none=True): + # {"a": 1, "b": "test"} -> "?a=1&b=test" + query_string = "" + for k, v in kv_data.iteritems(): + if ignore_none is True and kv_data[k] is None: + continue + if query_string != "": + query_string += "&" + else: + query_string = "?" + query_string += (k + "=" + str(v)) + return query_string diff --git a/utils/throttling.py b/utils/throttling.py new file mode 100644 index 00000000..c1fe8dce --- /dev/null +++ b/utils/throttling.py @@ -0,0 +1,91 @@ +from __future__ import print_function +import time + + +class TokenBucket: + def __init__(self, fill_rate, capacity, last_capacity, last_timestamp): + self.capacity = float(capacity) + self._left_tokens = last_capacity + self.fill_rate = float(fill_rate) + self.timestamp = last_timestamp + + def consume(self, tokens=1): + if tokens <= self.tokens: + self._left_tokens -= tokens + return True + return False + + def expected_time(self, tokens=1): + _tokens = self.tokens + tokens = max(tokens, _tokens) + return (tokens - _tokens) / self.fill_rate * 60 + + @property + def tokens(self): + if self._left_tokens < self.capacity: + now = time.time() + delta = self.fill_rate * ((now - self.timestamp) / 60) + self._left_tokens = min(self.capacity, self._left_tokens + delta) + self.timestamp = now + return self._left_tokens + + +class BucketController: + def __init__(self, user_id, redis_conn, default_capacity): + self.user_id = user_id + self.default_capacity = default_capacity + self.redis = redis_conn + self.key = "bucket_" + str(self.user_id) + + @property + def last_capacity(self): + value = self.redis.hget(self.key, "last_capacity") + if value is None: + self.last_capacity = self.default_capacity + return self.default_capacity + return int(value) + + @last_capacity.setter + def last_capacity(self, value): + self.redis.hset(self.key, "last_capacity", value) + + @property + def last_timestamp(self): + value = self.redis.hget(self.key, "last_timestamp") + if value is None: + timestamp = int(time.time()) + self.last_timestamp = timestamp + return timestamp + return int(value) + + @last_timestamp.setter + def last_timestamp(self, value): + self.redis.hset(self.key, "last_timestamp", value) + + +""" +# # Token bucket, to limit submission rate +# # Demo + +success = failure = 0 +current_user_id = 1 +token_bucket_default_capacity = 50 +token_bucket_fill_rate = 10 +for i in range(5000): + controller = BucketController(user_id=current_user_id, + redis_conn=redis.Redis(), + default_capacity=token_bucket_default_capacity) + bucket = TokenBucket(fill_rate=token_bucket_fill_rate, + capacity=token_bucket_default_capacity, + last_capacity=controller.last_capacity, + last_timestamp=controller.last_timestamp) + time.sleep(0.05) + if bucket.consume(): + success += 1 + print(i, ": Accepted") + controller.last_capacity -= 1 + else: + failure += 1 + print(i, "Dropped, time left ", bucket.expected_time()) +print(success, failure) +""" From 4dd52f6727fe98354246f8fe045ab7317a48b070 Mon Sep 17 00:00:00 2001 From: Chiaki Date: Mon, 8 May 2017 17:31:16 +0800 Subject: [PATCH 2/2] fix ci --- deploy/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/requirements.txt b/deploy/requirements.txt index 3d8bf5ad..f1868900 100644 --- a/deploy/requirements.txt +++ b/deploy/requirements.txt @@ -11,3 +11,4 @@ celery Envelopes qrcode flake8-coding +redis