From 080ecf1bcf81dc9f206ada138e130c0ebc595337 Mon Sep 17 00:00:00 2001 From: zema1 Date: Wed, 11 Oct 2017 21:43:29 +0800 Subject: [PATCH] migrate to postgres json field --- account/migrations/0008_auto_20171011_1214.py | 105 ++++++++++++++++++ account/models.py | 23 ++-- .../migrations/0002_auto_20171011_1214.py | 20 ++++ conf/migrations/0002_auto_20171011_1214.py | 39 +++++++ contest/migrations/0006_auto_20171011_1214.py | 26 +++++ contest/models.py | 6 +- judge/dispatcher.py | 18 +-- options/migrations/0002_auto_20171011_1214.py | 21 ++++ options/models.py | 2 +- problem/migrations/0009_auto_20171011_1214.py | 41 +++++++ problem/models.py | 4 +- problem/views/oj.py | 4 +- .../migrations/0008_auto_20171011_1214.py | 26 +++++ submission/models.py | 6 +- utils/models.py | 1 + 15 files changed, 315 insertions(+), 27 deletions(-) create mode 100644 account/migrations/0008_auto_20171011_1214.py create mode 100644 announcement/migrations/0002_auto_20171011_1214.py create mode 100644 conf/migrations/0002_auto_20171011_1214.py create mode 100644 contest/migrations/0006_auto_20171011_1214.py create mode 100644 options/migrations/0002_auto_20171011_1214.py create mode 100644 problem/migrations/0009_auto_20171011_1214.py create mode 100644 submission/migrations/0008_auto_20171011_1214.py diff --git a/account/migrations/0008_auto_20171011_1214.py b/account/migrations/0008_auto_20171011_1214.py new file mode 100644 index 00000000..7426a1f1 --- /dev/null +++ b/account/migrations/0008_auto_20171011_1214.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-10-11 12:14 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0007_auto_20170920_0254'), + ] + + operations = [ + migrations.RemoveField( + model_name='userprofile', + name='language', + ), + migrations.AlterField( + model_name='user', + name='admin_type', + field=models.CharField(default='Regular User', max_length=32), + ), + migrations.AlterField( + model_name='user', + name='auth_token', + field=models.CharField(max_length=32, null=True), + ), + migrations.AlterField( + model_name='user', + name='email', + field=models.EmailField(max_length=64, null=True), + ), + migrations.AlterField( + model_name='user', + name='open_api_appkey', + field=models.CharField(max_length=32, null=True), + ), + migrations.AlterField( + model_name='user', + name='problem_permission', + field=models.CharField(default='None', max_length=32), + ), + migrations.AlterField( + model_name='user', + name='reset_password_token', + field=models.CharField(max_length=32, null=True), + ), + migrations.AlterField( + model_name='user', + name='session_keys', + field=django.contrib.postgres.fields.jsonb.JSONField(default=list), + ), + migrations.AlterField( + model_name='user', + name='tfa_token', + field=models.CharField(max_length=32, null=True), + ), + migrations.AlterField( + model_name='user', + name='username', + field=models.CharField(max_length=32, unique=True), + ), + migrations.AlterField( + model_name='userprofile', + name='acm_problems_status', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='userprofile', + name='avatar', + field=models.CharField(default='/static/avatar/default.png', max_length=256), + ), + migrations.AlterField( + model_name='userprofile', + name='github', + field=models.CharField(blank=True, max_length=64, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='major', + field=models.CharField(blank=True, max_length=64, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='mood', + field=models.CharField(blank=True, max_length=256, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='oi_problems_status', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='userprofile', + name='real_name', + field=models.CharField(blank=True, max_length=32, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='school', + field=models.CharField(blank=True, max_length=64, null=True), + ), + ] diff --git a/account/models.py b/account/models.py index 3ba07d9a..0b0e6ca5 100644 --- a/account/models.py +++ b/account/models.py @@ -1,7 +1,7 @@ from django.contrib.auth.models import AbstractBaseUser from django.conf import settings from django.db import models -from jsonfield import JSONField +from utils.models import JSONField class AdminType(object): @@ -36,7 +36,7 @@ class User(AbstractBaseUser): auth_token = models.CharField(max_length=32, null=True) two_factor_auth = models.BooleanField(default=False) tfa_token = models.CharField(max_length=32, null=True) - session_keys = JSONField(default=[]) + session_keys = JSONField(default=list) # open api key open_api = models.BooleanField(default=False) open_api_appkey = models.CharField(max_length=32, null=True) @@ -65,11 +65,20 @@ class User(AbstractBaseUser): class UserProfile(models.Model): user = models.OneToOneField(User) - # Store user problem solution status with json string format - # {problems: {1: JudgeStatus.ACCEPTED}, contest_problems: {1: JudgeStatus.ACCEPTED}}, record problem_id and status - acm_problems_status = JSONField(default={}) - # {problems: {1: 33}, contest_problems: {1: 44}, record problem_id and score - oi_problems_status = JSONField(default={}) + # acm_problems_status examples: + # { + # "problems": { + # "1": { + # "status": JudgeStatus.ACCEPTED, + # "_id": "1000" + # } + # }, + # "contest_problems": { + # } + # } + acm_problems_status = JSONField(default=dict) + # like acm_problems_status, merely add "score" field + oi_problems_status = JSONField(default=dict) real_name = models.CharField(max_length=32, blank=True, null=True) avatar = models.CharField(max_length=256, default=f"/{settings.IMAGE_UPLOAD_DIR}/default.png") diff --git a/announcement/migrations/0002_auto_20171011_1214.py b/announcement/migrations/0002_auto_20171011_1214.py new file mode 100644 index 00000000..ffe4c969 --- /dev/null +++ b/announcement/migrations/0002_auto_20171011_1214.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-10-11 12:14 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('announcement', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='announcement', + name='title', + field=models.CharField(max_length=64), + ), + ] diff --git a/conf/migrations/0002_auto_20171011_1214.py b/conf/migrations/0002_auto_20171011_1214.py new file mode 100644 index 00000000..ef355b50 --- /dev/null +++ b/conf/migrations/0002_auto_20171011_1214.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-10-11 12:14 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('conf', '0001_initial'), + ] + + operations = [ + migrations.DeleteModel( + name='JudgeServerToken', + ), + migrations.DeleteModel( + name='SMTPConfig', + ), + migrations.DeleteModel( + name='WebsiteConfig', + ), + migrations.AlterField( + model_name='judgeserver', + name='hostname', + field=models.CharField(max_length=128), + ), + migrations.AlterField( + model_name='judgeserver', + name='judger_version', + field=models.CharField(max_length=32), + ), + migrations.AlterField( + model_name='judgeserver', + name='service_url', + field=models.CharField(blank=True, max_length=256, null=True), + ), + ] diff --git a/contest/migrations/0006_auto_20171011_1214.py b/contest/migrations/0006_auto_20171011_1214.py new file mode 100644 index 00000000..d429742f --- /dev/null +++ b/contest/migrations/0006_auto_20171011_1214.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-10-11 12:14 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('contest', '0005_auto_20170823_0918'), + ] + + operations = [ + migrations.AlterField( + model_name='acmcontestrank', + name='submission_info', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='oicontestrank', + name='submission_info', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ] diff --git a/contest/models.py b/contest/models.py index 10581ccb..42519988 100644 --- a/contest/models.py +++ b/contest/models.py @@ -1,7 +1,7 @@ from utils.constants import ContestRuleType # noqa from django.db import models from django.utils.timezone import now -from jsonfield import JSONField +from utils.models import JSONField from utils.constants import ContestStatus, ContestType from account.models import User, AdminType @@ -65,7 +65,7 @@ class ACMContestRank(AbstractContestRank): total_time = models.IntegerField(default=0) # {23: {"is_ac": True, "ac_time": 8999, "error_number": 2, "is_first_ac": True}} # key is problem id - submission_info = JSONField(default={}) + submission_info = JSONField(default=dict) class Meta: db_table = "acm_contest_rank" @@ -75,7 +75,7 @@ class OIContestRank(AbstractContestRank): total_score = models.IntegerField(default=0) # {23: 333}} # key is problem id, value is current score - submission_info = JSONField(default={}) + submission_info = JSONField(default=dict) class Meta: db_table = "oi_contest_rank" diff --git a/judge/dispatcher.py b/judge/dispatcher.py index 29b4475f..e457b29c 100644 --- a/judge/dispatcher.py +++ b/judge/dispatcher.py @@ -186,13 +186,10 @@ class JudgeDispatcher(object): # update user_profile if problem_id not in acm_problems_status: - acm_problems_status[problem_id] = self.submission.result + acm_problems_status[problem_id] = {"status": self.submission.result, "_id": self.problem._id} # skip if the problem has been accepted - elif acm_problems_status[problem_id] != JudgeStatus.ACCEPTED: - if self.submission.result == JudgeStatus.ACCEPTED: - acm_problems_status[problem_id] = JudgeStatus.ACCEPTED - else: - acm_problems_status[problem_id] = self.submission.result + elif acm_problems_status[problem_id]["status"] != JudgeStatus.ACCEPTED: + acm_problems_status[problem_id]["status"] = self.submission.result user_profile.acm_problems_status[key] = acm_problems_status else: @@ -204,11 +201,14 @@ class JudgeDispatcher(object): # update user_profile if problem_id not in oi_problems_status: user_profile.add_score(score) - oi_problems_status[problem_id] = score + oi_problems_status[problem_id] = {"status": self.submission.result, + "_id": self.problem._id, + "score": score} else: # minus last time score, add this time score - user_profile.add_score(this_time_score=score, last_time_score=oi_problems_status[problem_id]) - oi_problems_status[problem_id] = score + user_profile.add_score(this_time_score=score, last_time_score=oi_problems_status[problem_id]["score"]) + oi_problems_status[problem_id]["score"] = score + oi_problems_status[problem_id]["status"] = self.submission.result user_profile.oi_problems_status[key] = oi_problems_status problem.save(update_fields=["submission_number", "accepted_number", "statistic_info"]) diff --git a/options/migrations/0002_auto_20171011_1214.py b/options/migrations/0002_auto_20171011_1214.py new file mode 100644 index 00000000..ee52ffa4 --- /dev/null +++ b/options/migrations/0002_auto_20171011_1214.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-10-11 12:14 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('options', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='sysoptions', + name='value', + field=django.contrib.postgres.fields.jsonb.JSONField(), + ), + ] diff --git a/options/models.py b/options/models.py index 6d9ac75c..04dee5e2 100644 --- a/options/models.py +++ b/options/models.py @@ -1,5 +1,5 @@ from django.db import models -from jsonfield import JSONField +from utils.models import JSONField class SysOptions(models.Model): diff --git a/problem/migrations/0009_auto_20171011_1214.py b/problem/migrations/0009_auto_20171011_1214.py new file mode 100644 index 00000000..7073b8f9 --- /dev/null +++ b/problem/migrations/0009_auto_20171011_1214.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-10-11 12:14 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('problem', '0008_auto_20170923_1318'), + ] + + operations = [ + migrations.AlterField( + model_name='problem', + name='languages', + field=django.contrib.postgres.fields.jsonb.JSONField(), + ), + migrations.AlterField( + model_name='problem', + name='samples', + field=django.contrib.postgres.fields.jsonb.JSONField(), + ), + migrations.AlterField( + model_name='problem', + name='statistic_info', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='problem', + name='template', + field=django.contrib.postgres.fields.jsonb.JSONField(), + ), + migrations.AlterField( + model_name='problem', + name='test_case_score', + field=django.contrib.postgres.fields.jsonb.JSONField(), + ), + ] diff --git a/problem/models.py b/problem/models.py index 0e9c5e2f..72f3fc9d 100644 --- a/problem/models.py +++ b/problem/models.py @@ -1,5 +1,5 @@ from django.db import models -from jsonfield import JSONField +from utils.models import JSONField from account.models import User from contest.models import Contest @@ -66,7 +66,7 @@ class Problem(models.Model): submission_number = models.BigIntegerField(default=0) accepted_number = models.BigIntegerField(default=0) # ACM rule_type: {JudgeStatus.ACCEPTED: 3, JudgeStaus.WRONG_ANSWER: 11}, the number means count - statistic_info = JSONField(default={}) + statistic_info = JSONField(default=dict) class Meta: db_table = "problem" diff --git a/problem/views/oj.py b/problem/views/oj.py index 0a6b066b..6764249d 100644 --- a/problem/views/oj.py +++ b/problem/views/oj.py @@ -55,9 +55,9 @@ class ProblemAPI(APIView): oi_problems_status = profile.oi_problems_status.get("problems", {}) for problem in data["results"]: if problem["rule_type"] == ProblemRuleType.ACM: - problem["my_status"] = acm_problems_status.get(str(problem["id"]), None) + problem["my_status"] = acm_problems_status.get(str(problem["id"]), {}).get("status") else: - problem["my_status"] = oi_problems_status.get(str(problem["id"]), None) + problem["my_status"] = oi_problems_status.get(str(problem["id"]), {}).get("status") return self.success(data) diff --git a/submission/migrations/0008_auto_20171011_1214.py b/submission/migrations/0008_auto_20171011_1214.py new file mode 100644 index 00000000..1c585d8a --- /dev/null +++ b/submission/migrations/0008_auto_20171011_1214.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-10-11 12:14 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('submission', '0007_auto_20170923_1318'), + ] + + operations = [ + migrations.AlterField( + model_name='submission', + name='info', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + migrations.AlterField( + model_name='submission', + name='statistic_info', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ] diff --git a/submission/models.py b/submission/models.py index f4cb2f2e..ef408935 100644 --- a/submission/models.py +++ b/submission/models.py @@ -1,5 +1,5 @@ from django.db import models -from jsonfield import JSONField +from utils.models import JSONField from account.models import AdminType from problem.models import Problem from contest.models import Contest @@ -31,12 +31,12 @@ class Submission(models.Model): code = models.TextField() result = models.IntegerField(db_index=True, default=JudgeStatus.PENDING) # 判题结果的详细信息 - info = JSONField(default={}) + info = JSONField(default=dict) language = models.CharField(max_length=20) shared = models.BooleanField(default=False) # 存储该提交所用时间和内存值，方便提交列表显示 # {time_cost: "", memory_cost: "", err_info: "", score: 0} - statistic_info = JSONField(default={}) + statistic_info = JSONField(default=dict) def check_user_permission(self, user): return self.user_id == user.id or \ diff --git a/utils/models.py b/utils/models.py index b651aa2d..3c114522 100644 --- a/utils/models.py +++ b/utils/models.py @@ -1,3 +1,4 @@ +from django.contrib.postgres.fields import JSONField # NOQA from django.db import models from utils.xss_filter import XssHtml