From 7cc33d07011b4f8400b418f4ffbb295ec342db47 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Fri, 24 Nov 2017 22:26:56 +0800 Subject: [PATCH 01/14] use bulk_create and transcation for importing user --- account/tests.py | 17 +++++++++++------ account/views/admin.py | 38 +++++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/account/tests.py b/account/tests.py index 4279c96c..bf537273 100644 --- a/account/tests.py +++ b/account/tests.py @@ -557,20 +557,25 @@ class AdminUserTest(APITestCase): def test_import_users(self): data = {"users": [["user1", "pass1", "eami1@e.com"], - ["user1", "pass1", "eami1@e.com"], - ["user2", "pass2"], ["user3", "pass3", "eamil3@e.com"]] + ["user2", "pass3", "eamil3@e.com"]] } resp = self.client.post(self.url, data) self.assertSuccess(resp) - self.assertDictEqual(resp.data["data"], {"omitted_count": 1, - "created_count": 2, - "get_count": 1}) # successfully created 2 users self.assertEqual(User.objects.all().count(), 4) + def test_import_duplicate_user(self): + data = {"users": [["user1", "pass1", "eami1@e.com"], + ["user1", "pass1", "eami1@e.com"]] + } + resp = self.client.post(self.url, data) + self.assertFailed(resp, "DETAIL: Key (username)=(user1) already exists.") + # no user is created + self.assertEqual(User.objects.all().count(), 2) + def test_delete_users(self): self.test_import_users() - user_ids = User.objects.filter(username__in=["user1", "user3"]).values_list("id", flat=True) + user_ids = User.objects.filter(username__in=["user1", "user2"]).values_list("id", flat=True) user_ids = ",".join([str(id) for id in user_ids]) resp = self.client.delete(self.url + "?id=" + user_ids) self.assertSuccess(resp) diff --git a/account/views/admin.py b/account/views/admin.py index 062fdcf1..f0b93c65 100644 --- a/account/views/admin.py +++ b/account/views/admin.py @@ -1,8 +1,11 @@ import os import re import xlsxwriter + +from django.db import transaction, IntegrityError from django.db.models import Q from django.http import HttpResponse +from django.contrib.auth.hashers import make_password from submission.models import Submission from utils.api import APIView, validate_serializer @@ -18,26 +21,27 @@ class UserAdminAPI(APIView): @validate_serializer(ImportUserSeralizer) @super_admin_required def post(self, request): + """ + Generate user + """ data = request.data["users"] - omitted_count = created_count = get_count = 0 + + user_list = [] for user_data in data: if len(user_data) != 3 or len(user_data[0]) > 32: - omitted_count += 1 - continue - user, created = User.objects.get_or_create(username=user_data[0]) - user.set_password(user_data[1]) - user.email = user_data[2] - user.save() - if created: - UserProfile.objects.create(user=user) - created_count += 1 - else: - get_count += 1 - return self.success({ - "omitted_count": omitted_count, - "created_count": created_count, - "get_count": get_count - }) + return self.error(f"Error occurred while processing data '{user_data}'") + user_list.append(User(username=user_data[0], password=make_password(user_data[1]), email=user_data[2])) + + try: + with transaction.atomic(): + ret = User.objects.bulk_create(user_list) + UserProfile.objects.bulk_create([UserProfile(user=user) for user in ret]) + return self.success() + except IntegrityError as e: + # Extract detail from exception message + # duplicate key value violates unique constraint "user_username_key" + # DETAIL: Key (username)=(root11) already exists. + return self.error(str(e).split("\n")[1]) @validate_serializer(EditUserSerializer) @super_admin_required From 9889ac5b4adc8ab65b099e85db13afa20feda7ac Mon Sep 17 00:00:00 2001 From: virusdefender Date: Fri, 24 Nov 2017 23:29:40 +0800 Subject: [PATCH 02/14] fix directory traversal --- account/views/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/account/views/admin.py b/account/views/admin.py index f0b93c65..3e8fb471 100644 --- a/account/views/admin.py +++ b/account/views/admin.py @@ -150,7 +150,7 @@ class GenerateUserAPI(APIView): file_id = request.GET.get("file_id") if not file_id: return self.error("Invalid Parameter, file_id is required") - if not re.match(r"[a-zA-Z0-9]+", file_id): + if not re.match(r"^[a-zA-Z0-9]+$", file_id): return self.error("Illegal file_id") file_path = f"/tmp/{file_id}.xlsx" if not os.path.isfile(file_path): From 2d038c7bcc2161a9b39c4cba3f3f29aae52bee65 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Fri, 24 Nov 2017 23:30:17 +0800 Subject: [PATCH 03/14] use bulk_create and transaction for user generator --- account/serializers.py | 3 +-- account/tests.py | 4 +--- account/views/admin.py | 48 ++++++++++++++++++++---------------------- oj/dev_settings.py | 2 +- 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/account/serializers.py b/account/serializers.py index 945e0967..d3466776 100644 --- a/account/serializers.py +++ b/account/serializers.py @@ -40,8 +40,7 @@ class GenerateUserSerializer(serializers.Serializer): suffix = serializers.CharField(max_length=16, allow_blank=True) number_from = serializers.IntegerField() number_to = serializers.IntegerField() - default_email = serializers.CharField(max_length=64) - password_length = serializers.IntegerField(required=False, max_value=16) + password_length = serializers.IntegerField(max_value=16, default=8) class ImportUserSeralizer(serializers.Serializer): diff --git a/account/tests.py b/account/tests.py index bf537273..43a09fe2 100644 --- a/account/tests.py +++ b/account/tests.py @@ -1,4 +1,5 @@ import time + from unittest import mock from datetime import timedelta from copy import deepcopy @@ -610,6 +611,3 @@ class GenerateUserAPITest(APITestCase): resp = self.client.post(self.url, data=self.data) self.assertSuccess(resp) mock_workbook.assert_called() - data = resp.data["data"] - self.assertEqual(data["created_count"], 6) - self.assertEqual(data["get_count"], 0) diff --git a/account/views/admin.py b/account/views/admin.py index 3e8fb471..b1ef6889 100644 --- a/account/views/admin.py +++ b/account/views/admin.py @@ -173,9 +173,6 @@ class GenerateUserAPI(APIView): if data["number_from"] > data["number_to"]: return self.error("Start number must be lower than end number") - password_length = data.get("password_length", 8) - default_email = data.get("default_email") - file_id = rand_str(8) filename = f"/tmp/{file_id}.xlsx" workbook = xlsxwriter.Workbook(filename) @@ -184,26 +181,27 @@ class GenerateUserAPI(APIView): worksheet.write("A1", "Username") worksheet.write("B1", "Password") i = 1 - created_count = 0 - get_count = 0 + + user_list = [] for number in range(data["number_from"], data["number_to"] + 1): - username = f"{data['prefix']}{number}{data['suffix']}" - password = rand_str(password_length) - user, created = User.objects.get_or_create(username=username) - user.email = default_email - user.set_password(password) - user.save() - if created: - UserProfile.objects.create(user=user) - created_count += 1 - else: - get_count += 1 - worksheet.write_string(i, 0, username) - worksheet.write_string(i, 1, password) - i += 1 - workbook.close() - return self.success({ - "file_id": file_id, - "created_count": created_count, - "get_count": get_count - }) + raw_password = rand_str(data["password_length"]) + user = User(username=f"{data['prefix']}{number}{data['suffix']}", password=make_password(raw_password)) + user.raw_password = raw_password + user_list.append(user) + + try: + with transaction.atomic(): + + ret = User.objects.bulk_create(user_list) + UserProfile.objects.bulk_create([UserProfile(user=user) for user in ret]) + for item in user_list: + worksheet.write_string(i, 0, item.username) + worksheet.write_string(i, 1, item.raw_password) + i += 1 + workbook.close() + return self.success({"file_id": file_id}) + except IntegrityError as e: + # Extract detail from exception message + # duplicate key value violates unique constraint "user_username_key" + # DETAIL: Key (username)=(root11) already exists. + return self.error(str(e).split("\n")[1]) diff --git a/oj/dev_settings.py b/oj/dev_settings.py index 724a5dc4..5d9cb9e5 100644 --- a/oj/dev_settings.py +++ b/oj/dev_settings.py @@ -7,7 +7,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'HOST': '127.0.0.1', - 'PORT': 5433, + 'PORT': 5432, 'NAME': "onlinejudge", 'USER': "onlinejudge", 'PASSWORD': 'onlinejudge' From 6d08011e2d864ee71dacc55762b04751c8a9ad47 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 25 Nov 2017 03:37:40 +0800 Subject: [PATCH 04/14] deploy script --- .gitignore | 23 ++++++------ data/log/.gitkeep | 0 data/public/avatar/default.png | Bin 0 -> 16219 bytes data/public/upload/.gitkeep | 0 data/ssl/.gitkeep | 0 data/testcase/.gitkeep | 0 deploy/Dockerfile | 7 ++-- deploy/oj.conf | 62 +++++++++++++++++++++++++++++++++ deploy/run.sh | 37 +++++++------------- deploy/supervisor.conf | 28 ++++++++++----- oj/dev_settings.py | 16 ++------- oj/production_settings.py | 16 +++------ oj/settings.py | 14 +++++++- 13 files changed, 127 insertions(+), 76 deletions(-) create mode 100644 data/log/.gitkeep create mode 100644 data/public/avatar/default.png create mode 100644 data/public/upload/.gitkeep create mode 100644 data/ssl/.gitkeep create mode 100644 data/testcase/.gitkeep create mode 100644 deploy/oj.conf diff --git a/.gitignore b/.gitignore index e9a0324f..822575d7 100644 --- a/.gitignore +++ b/.gitignore @@ -54,21 +54,18 @@ db.db #*.out *.sqlite3 .DS_Store -log/ -static/release/css -static/release/js -static/release/img -static/src/upload_image/* build.txt tmp/ -test_case/ -release/ -upload/ custom_settings.py -docker-compose.yml *.zip -rsyncd.passwd -node_modules/ -update.sh -ssh.sh +data/log/* +!data/log/.gitkeep +data/testcase/* +!data/testcase/.gitkeep +data/ssl/* +!data/ssl/.gitkeep +data/static/upload/* +!data/static/upload/.gitkeep +data/static/avatar/* +!data/static/avatar/default.png diff --git a/data/log/.gitkeep b/data/log/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/data/public/avatar/default.png b/data/public/avatar/default.png new file mode 100644 index 0000000000000000000000000000000000000000..97f349564f0eefb2b07297ec77309597175c2b49 GIT binary patch literal 16219 zcmeHucRbba`~T}44mxIK6DrEc%E)$- zTa63&1#{6-xD6G5Wn2ay@GWK3WFV+4hWN+=0Y0B|R5S#Gh|lAGU;z)zogrFdS=O&Do}O&P1keUpVF=eDCM;f)*Yv4x zy~XduL`L#ym;{D!4hAWbzd8?rKi4sdOb~{4N@cbmFP#wg0ECZ$C~)JZvj`!UPrKjt zDo(7AXJHcHfGkiOo+m`^xeSBCHW~$`iUN#I!Dk2s5kVWcrA)mc=$Y8 z^~kp@6}!>*4>?Y`O&do1^aJj5*>xN!`VB;%`3G0YC-mOWMP-`_N@wof@;ZzXr%Teqn9-G6_#*7+lo zw7+eMS#y=^v&EjY`n8{H!)5`M$A`P)3*E2nx|4-=mUGg^;yJW!|19+hn`5D0Lq)HH z9gYvSz4RJNJXvJPAuPqmt_qhSMOnn}b45<=z#GqA!V zh>K)`LD0%vPEKy9_^vpiBy8but-1R3Er&-WCAKLjw1n5@yI&zTd%BVXRd(Y><&U}x zO&h)OZ%;myh|LDGBnO3@_Z3y}$L9S|3(Rnu{WO%L!F+hM)w4Q~r@ixMx!;zbt6*`^ zjal0NyVU;N#b3Q|gBj`ukC&x34E!ZRe{X!d1WcbdD9hWIjg8ITfAxE<%g$2YRiB-e zq(<-U&dCO^I9>F^UvqW#q}|9T%@r_}qH;q6LxVFZpVUxe-|*r*AMK%eE~Z$H zCGMJ6pw=v%Z}vhY!|}Kt^#o;$P`>!?0#+8e6$(a2$XN9 zqv!gG0xZyRe{+7Q+QBSQMezWirHk;%=KgDN>Q6|FQAgi9rHc&0wy!RUIAXJII^yf2 zNC~&!=*CsJ{Tj0Or=Wm;@V(8E*h~t+%xxj|Jt2q5?`)-9DVK})eOjCn8XauD{uwE| zKhzu$kcdwE!qVmd`|j%nL2#@AzBs9u4Nd69>*_V}p{|h{mgJW{iL+Vma#Y?~X0h{iQblcUh=V=L zQ)9+Xa?G0j^-N9e^-D~=u-7kYZ#fl^CQ;flVENMpTch%-8q|<#2$e_2tVqg z9j~%`J<;UHAT}=p+a?%XN8`!51do!*zhR!9AH%7DJWr{$(#hQCxYg;F;2rQ=;)%ci z`+Rt^%S74zr=-kM-;1yh6WG%`g>v6-``{iRz#t3l#wwf=ly!QOY(bcizf-d|oTt0Ar`!1A@o@2ISCYVle1lHx+v~45 zFq_AIAT(}~p|+3WpA;pej!iwgI)=>M!@gisVZ*y`bM^jsj6nPi1bm2I;E?JAPW>gWNb4_CqN$ixZ#TRB z($iO=jNK!BQ6m+ATavn=;?{&^n%|>a_TM2ceq9vB08*2M2%065!F>%!wP=W+kDX@H z-qfwNm2o`WS=BQ#>U_hp!OFYA%I*te3GJ@6ASNcBcr%s2r9Z=bT#J{3b!a|bh@k%S zd*bz_#c&SM7!HV{|KkT&wb#OK#%piHJ;+>HRbkxs#u~iJf1P`uiGL?U=573!wLU!j zH==kso1b(hRoAb6mbCa&;4J=YxTG5Rt>@PjZ9f~@oanc+fO(_ICq=(_6yV_@k2T z$a}$JhUDn{wYk%8f^`wT7 zln{AaAl&2kcc+i#z$1SRt!sNz*!G4cnhXk~yL0P)FahcJ?5gV;sjQos7;V&{c>cn# zr_BCVK4GFNGD+s7>I9ny1&p+;Bap?X0gzM*&<*V~ z-!8IbHE(HcoyXSL&48UfH&#~8ftRk=b75~+^nsyIOEQjOn z@4H;Qyw^ojJt;+)vhbQZd*CP%nh{3;464D;#SfY+_YoDE?{~;n`g6$LZig!c-q)(5 zz{kAejvY-Fw7$2s&|RLRk@kr3NCbvd417w?JXYuSYYA~74*v0->yMlsJX9BThb^a7 zVbCxmJH<=5k(}@T=EGlonS|Eo+pZ%2vs)dj+}YchEksDG zN5Pe!F^f2|xCFN4RLp4Cky1ioE#Gb$0JC3dXAWS!4d$Xkzg-cw8+$k3IU|^Sq#4iMy7=wscJg#AZ}ZVTU=_+=(v}HOQsfXnErseR9zbOeTa8EJFCt8v ze60ag6P|wq=WLcqC|cAYr`XG4G||bL1wP-_^L5;MPmCgLKACe-&=?lGbwH{I4zEqd zLri_6l;M^=MUp0qC9=a|%nl#DPnvHliSwc0>nXq8O9j#{)Cc=-I~%a4M~l(`P1VQy zB^Q~0_KG940HYq0-Wc0aJUq!W;D0>{&o<#ozdPIh?2>VT225uYN?5hTDST0&9F=G& zDZ?XR^q<{UW_Qup-xKwgUlMt9ZjzMc4;duz@WEKJ^9OZ|LUInGSex%;`f`-wTuATR zJ4L2yx1de0W}j%_LWbydj4G@wEPGOgxK$QWfxhgHGi~P%r-R9DwmDH-3|7l${{Jxv z=l0LRa)6HTntf&c{I@g^OcI{vMkI0enAt6;>>5tJ4eq0H>Z|sOFYvpxdAvTtwAp`E zdEHtNo(roi#U*^^(J(oxeGrgIp)LB8eJ74|i@oPp72v3QdRD*yUggOPSnq8Cceu0A z&5*%|oYM`g_gFLX^7683;Vm@9UkrDMp#;i7kEaZ<_v*>YT>mAu=*Bp)!?so9OVjX$u;}l}yaH%fKByRPK38s6fthX8C(xQ8yi@QQ~etbshvU z(*(h_ra#AB$DyrInee-Gx4@RKdFO;r2Aul&oE(X2f@!ia5~7x086OW78h(QD^)um5 z#dz@I4sR12wqQ_ko`Vo6s|L%u<9ob4oGoVru}?ysyIXh|sC0}a5l z*lTRZDma{5Vu)`CDkCbC$|Hmzz5v+E<#IyY8qhZY=EfU+ZuL-+FsH`R~r$^g6*!%z%rh4e#HDW8nZ+K44?#zu|b52)dVQ)7>hp2wfvXTCrJaYHCgc z+9H+-tQQmj;%?NPy+wNcLIVrVF{tB+z_9ns==<=VTrJZz2VS)JifS*^v3o>-TM6Q; z`jJLF1(4@B{khnE()Z~-%;0L85#)pVM{{u+Va4neUWj;N5zlpVNRg2kP$XeaCyy9c z%d3EQ?I4uqp+Ep*vdol^Mp7OH?a=R;I(y^L_6fJA)$1hSpI$6Qz@O_+AMI}md;hWe zuQ}d)_;4GSI9+8vszFiKxczzw4DK>|J@b0vn^cotOW^LNr-Bp!f*2v8scO3o(+y6N z@)LttpCJHzxIgLvmg%;PPSKra2_5uo}o7N;ArK$$;z z>az}r1@KAIhq(4PjA z2tVkc`1j3ETECf@xo2GKoV9rQ<#^0{c)k1T`zF6b0;Ipi?llp!Bky={_9g*O!N)^g zfW4rf-)pb*D#qTL^1UzVvxARmD?{P9xy6)EdHN-(r1XLvAVZ(`dNse3nUDQBSWu*$ zqaGNJ$wk_B*@5jID&brSmKAot47AhjTr|GG%X2m^kC-_wc- zO>6W$0Y5AbN^Q~|nzyqezcV~w10RUJD@XSpT zdKscUU!|vHmu}AIe~Zh!PBVZ@(1z$y5fKq90D$gn&a0%&a_=(nBjVwjT3T8K0l@?c zO^LpxK>rLLLw$Tp`ML!wASsVDAewXb)ieM6f^8^U?X}S{amNdCV9~TdlZ^1f^hWb0 zX8=p!_pJK6Pa7qzw0}dM6Sd&ph9nlJ-bFCkf|X=pI~$Ov_vMDfWzk&O-uY#7;_Z3JXNl++`Upx zN5^!O6lL!W4fiK4drr3A8NxTYLKkq%1ui)x}GiABn$$bqJ8`%C}P`yxs%VEAx_to4K3D8!#T+g2540ia=eGC4Kk!uFHVgv|-#;V<1FUTJ>Ei z)eqG2O^^R#i2y>7m&x+k{axj;W-5JEz$4`%BHkrGMbVj#cAr}sgG06QfpU%5a~H3n z>8@R8Utd#*KPvua$>~oTe}vKENfQ98(Gk7`{Np%KHr0>_Q8bGxqGGE9#|qd4T*$fk zOnWep3c6Pq6+MXKcclF###uw?_C*u5%}-B`ao)$>pGEy4V-7$koe|+WblEskkw)>k zZ3k609JtXTU_pg@!ijh6j`RedIxNHBAJIXP`CA3b&7Hd7QfQHHBz=bDqv=0>C2z|` z-=ma7Ma2tUwRZV7RAf93oR_M&yGuGM8+pU5>APUF>KaFKV-BkXjW-6xKL7~;v{e0& zlo0tK1BLSiOcEnHPVDCbU;2byYlDHHB!s1;rXB42BMB;`ATgQQ-xi#z?gD6fG08v}ra^bTmt?!DhXUW&lHy5SlJ zfL(`RfC~U5uCvT(yC$%1<^TY(Rt``mNJ*YDA_DFCHl3SBN=o4j$az$t4pnbh;KGTR z$8TGJkiR79Z;9f!J1lw_ykufQcPXc98a<(IGpTXBJh<1sD4~zKNjB;IZ<&t7 zVVXzWBNf*DpH_d!J2xyoQ2DWA26B(F&0IiD8g6Cs%A_2sHg4dc^AP+QXWO6E@!_V! z-}RKYS(rBr0|DS4Ul}c5K0Y7wo;v0gUYm)iP$Ef=*xO8nm@2?e?lDf)?$T3w{#kks zIH=EKm5&o$0=^k5H&(~X7A=9>DkQvQ_dpB~z}o3oh2YeBx+s4U=ee-P)00EPU5g8N zBD?W)SRm>y@992tuF>w;@G8Kq*(SVXl(Fz?hp8$ym6xw}5?EhZQg`+7`W*wW(WNxq z_uie`QN;g9B=Vklnl>sjvuS^ULAaicV4jtLpD?U%^O0bZ2BI9KHII$fQM?q$lg%%@ z+4(YmOuntvrW=YR1BNF;loZ8jf3&}tHeO_0&4Ou1TIK9o_h%c*$jGQ46N-F*DQLOk z0|>APQ+&N>|GlPHjJ$SNAILehD(daNbCL}h4{@)Nk$6vdB^MAxO@n`KS>uI zd@r|*S>7)nJl|O6yUU6@9x+tO{%EAsf`>LANu;nz)$1bVcThn}ce9QDmi-%=sd@3~ z&BMN+Gh&TcR{4eg$uLMhR(GcYG^Ah>O?M>C?BXTve);cc)vZ-j`VROzg>JJXAdD&Oa6=Zw9*^R(k)6rJqg*5RV`sZx@_jOyGhi6lyu z-Y||DSZj^btnE~j-`u%RP>{0gRHIMzuYJwgIIk*^u1YvCRYOScH@mdW0J}c_09!*HbdM1xm9Kt5F57d$-&$#I=xz6yttrmZYX*F*>{fgS_8J4 zf>_azqT}B(9~%kP$SEj3Q9nthu^lc>0hC`i8Ka0f2<$zV5E6%Mq+X`=?%mY9rnVro zu(qVsoFA`gVPsiQd*a0vT`0l#&n0;%yDsi{7y z_4h6=gd?7WpL>u@bAyMcYP8()HRp3HYpHtW2Lv)~Rb~yIeVt#v+4la#U}zz>1LkjR-3zB3Ne= z%f-1(`DcvQ0AN?*+!Ek+sUgIP-+W6jp*x@yhB4?na04ik)d$1G!lLRLt1a1wO9Fuj zIRF-;3@{M;3YA8`4P3H~P^^9xF7=&BnqMxiv(fqffna7CExS8k3NXbXH{owDEG01? zo$ADs{k*g`)jS1wkWW9=XM*&d0-0DscR#EfHN4%IDWdfQ;Kk&w=G0R;Fv?-d|F8&G zB1sl{ydYfs@Hw%hG_$uqm^zJS+UM}09%k6)69|>{Og#})Tr0q-ny!lG9^M7;Lgi5+ z8L-)%Z^Tjv@86XNp`?;1)RNu$MIv2uW0qIa0ZrC4F5It_#@>obaocNpcQklynJ5SA zx#zbln7?aZEC*=-#7cBBj7LwuI$m9(_C};n>TpG%`g9shd(YVM5G2CC4}_|7;4obM zXk1G^LqK9fCLzaS5=sAOukK? zWt&o5{RW(ppk({S<3RH?&T{^)pQ-7iNdZ5Ry=i{nl~1Wvb#)Gc$)!r?za-YDhT7QV zD`enrR-fv~$huvn%B8tsU_MdjCXjG>>&JYacMP&OAplu*pnX4mQ$sXt`AGNJetw^` zXYF-WompUEMa6*B@gHsdOr6Gi+<)@E#%ZKUxxHos2x44F{!>lfpH2{uyKAX6SR9e{7h0bkN z3>0{$zmHEb&z4h6cG1l&$hLS$dihKF<~u1~-Y@+M<+b?6t6+bE~`54iU`j5lf<_AhOKw&^_^W-hQtt zxr}COuh>+N^r&ae<+coO60lQx%`YMIkS&vwKJ!GM=Y6e_9NbtK`+$Ri(}#U zWOt8Ovw4I>Dy36A?Mr4N;mz8p^I=-Woo6j zw#r5&y*Af;1;nEK6#ja_lyXVOyQ|$bJRPI>b9cbJe|--G-&H^ZYrZeQD+}Eiq-ouMvhqe4ijnRD3{AqVwMRk1F?7qr@ARw(qo-{!HO4iM^aXhacy9`L5_x z4x_@nvivc6o4H#p|5c5idU>bl$s_{9kDs3YR{4ZpSnG=pvC#(uw!|;cJJ)mE&tO3t zMQY?-xLfCHVWHaPON;NV+nnj_92B-NKC^VU(Z$9!U%v+A9@=ay)Se&7pFn)q z0u|!c^Q|tMFJ)F`>-iBTr9p{v_O2$1IcN*}-!C8fJzk9iEJjAsa%tEX-?26v|E ziiJ>jlK=<%Ma#{}Zx=Fu0?P&dkXp0lJ{$hyq$+Hpz0-W|9j!q0{E*uqwh^wD~-@x$RJKOZf&E7N_h!6jOZVs1a@LMuUI0;6*Ds7Uk0vFlorw*%2~oCAD{ z`r4ofaq&KVw@Y4o4pl6uEP4zY>2Qb4@*;E!bjC+rSH}|OW@%;6A%#LO-cmjlnaf(( zOzH2`%;L_YXrw2ayAVoV=}M3M1Gz97;7fHA<{pGx>kK&BXfLEX>Yk5GDVq9LI{K2^ z*rXuI{P~o*rm=aR?LZ#=^PSmDncd&jx101vZxyW!-M*EDF5TnWJt4q|5~MFcfVo=2 zF7_XpW@82@zJa(#bU}3Iv0ZP$L4cQW$U9xBsH;MV0Y$VFYdw07$$ygv>asxfb$%^r zJ{jXC$hC_b9WZ4Mkjhf?03mp-s<*hLyvfEv{5gLeh2_x#QJlnhPowrQ0 z&{iD=tr^RLVtcla)?iQ<465vqBQ~j^#SA(9Z27UO!D+yL2^8cxGqbgf@G)I*RQ7?j zzg`~K*aP7y@n`rRdV)OG0B|VKK=^AvNsu+)cLccAkZ@tz zwo3_p=di?$JNu)%l?&RH$?ltTpW}&0MAh-IH;D4Hy2j%57``3oePlcQk-lKodnF0< zqtuH<PZAP%D@fiP<+z$|I7%BEsuwjheAgY;?1i2z2cti)jQw$&>v&$}f9$>1 z`6}dBXV+J~+C3Gv3)`v~^8nmw2~5u}rKNOnnV1jJn0&90TZl?lnuq2B-UA?x`U;@fEUT_c@l>J@_Q_Ujuv~d&>9HnrpxRD!nWyDP9TNP$WU4L z5a*iG`C@n{(?8G_y;Z^Fwcn| z(Z4|R-XZ2Eq7Iz;Rp_s&yH6zoNyKQ-_^wdbXuXGnafQ`$ytGXnp*H!eJWNtUTUzJ+m>U@u>IonLD4isi%BbRHCbQsV z)9q~}l`O!Ri=0VTJz{o^lnY;|KDxNAjPz~q#JA^oSBw?GPCxZGkGbx)l1r;LSeEZI z;o3q6)< zjo$-+$wR7O{4fvL!V*LR9zV7;|EQ}hFmrk+zNJFdL{gfX zm{q#8!S5+O=U_t;l1c+$Xk`@@H5VJ@;J){+GMbSW1eGb=`Fcfl6XZ(!9r8YwSQt?m z5C4W4g%_rBdHI~|Yc+a%Y5Tuhy5qfk9^#XO@*idyg(Y*%pg?si{zOYOAXhi*s(e%_ zs1nz}KOJ1G$v?>(Ht07S?BRT+o2t+#p5>tPb=F!@yS0*$odr=5y9_!Vit)`yqDxrQ z$f0Z^Eby;&_&P;QbGRu74qjfb;0lr_(7_rs^0g=xITzgoBy!22Hm{mBq0q7QW%c>R zO03LcU)t9^mtVL37$2y~UtYJ+Ma-RxCa`J}``nA%nuN?FpaD*?ED6BTiXn*%5(3~! z-cdrw=Ufxgaqgi#B_a4LONbDdN(e2gOT%kAH+UQO{0O__&jAkoO`dVd+?q6IXb3{hn*h8U zosd@MwOWg&M8P=Hw~{BN@e9J9G&;uc2oA z?I8m^2&99{RbhYkSHqFs&{z3*7}R|4NkG0&ML-_xDk{Xv(7sx^=&cDM-eK+~?s)4w z%&6HNc<*)$tzUw4c%t1+7tW4X!U6&+pib-=BRD-fg$F{X?M{7YTVwUh=wE~w2N*h! zoKvyuQMFURYOVj-0msU@^a>%&;2${5oJv#hXcR>Lg52EnP2w^s5BG(pyB#PJdA00URj zMa^G2-2P=5;pOxy^UIFVH@qVUmGsim-i4mDG@Tw7Gn`D}-uCAF7mk3Jd&8-?M`Pif+qte`*VK|d$z|4H zBOoarfB%r!fdH8fAw4s(fEy?TMKFkuY+}IHboLeh{T0xZG)bGrActGm?{H@{T_zYZ z7yfbKsSDTf50L#foX72YIvTEDY*MBacMXOWL2()n=N_cdBhk?kXDYTX7YNvm+KJ2b z!nR97_>HM}I!ftlO^07;dR)@}5eH7=RE1*82V25gy&4IoJDy)%1r@-C3un`(z(FCP z60llutVpaF!*K3?GYC*VmcI712ypVRsgjRDuQDEH6r;lU)zH*b9WLPnYzwP`&^ZbK z&8FZ)rN~2iVkFWQbOEzsefEF4^n;m8OdH>W+Dj5OUg#yWS!XGSMX26&`5wq9e{kdM z2j)Q4dj*xYOZbQS%x0qlU4W$0F(t^{i+sL&j@}rj<@cd(DQSK?NjO?ff68SFw_$VO zAY9#KS|bXa#AB)@QEta+5?&v;z&9B9BxiccWnAwNM0?BG(z!9;KmGVrhbjda3WV_0 zod$$ZHK?>0g8rO_C+QxNSOA<2K#j&^IPLEc{b|s|GO71ugA4=qUJ1h^coos&e73JO zq?I5@$K#pIueC-v?c0MQxNS=iBAXTc_ag4XbX$}8?-M{##BDcg)uY`U0B;OzYNdXy zHUQ@hN&%1>2UR4P`8D(KFCLYUJfaj(WP#hj)1a|Fhzfesf|=P1t*s5}YYUL>&f=$N zhC*ojq*4Loe84bXSV^BA2|y%qg_;tH3#VY~4?yqthkcw-Fz^6&N6@(X7+AG+f-mKL zkE-I&1x)6^8QK9-l@+@+mI;K5qC$Y4phtI}k_i>P=%KdF@H8!fK#f5AwYW#Iq~8JM z7H$-TJbH@-#P|w83WCuuv_Au&@hM${xljsGw1JB_c|nMLP1InS z_x8{ELB8BukZoT^NOzI|;q2c7=wxCzKd8RqYe6EB4Ri$C1Q^ilRSw7p#{Ue^Yv(no z=|vI~!_0ZEUvb$m2F>3)tJPEYd~vsh3bdg_`^|yv8h?4o4TgZq+0;^hPE3lFAN_wO z`PN!-D_#DJGu`X0ed9@zz@1;iJ;kPt5gY&Bq+A)OURo*Be7PR)ygxn(Id*_VJ*(-= z|6|~TTe|sTvtL(_cjT{^?UNrTnPwOOAG11BqI$%u@ zUooUZHVtm#o9~J*;DKk57%z}2!IECjMKaN3nI51 zCf_Vv{nxVzxDu>H-{+ZCIWjGN4>x~Nj*JJ6ouW+(oXtyraDHc8Z77&VXDfgyOoKCPiu20K= zR|;q$)sSq>gD%j)X$webY|G!Xz^Hm3&HPlA<*0h<#SQkvNr?8k@sB;WP@*rNr=mP} zQA|@z{~l~_KGAFp)VG2q-I_wqrZyj=^;rIv1@S2lsLva@uZ&oLZo$?M|C-qov@wip z3njc_XzyXIBHZiOkVg2<`I4U9Te=o3>vr3D`#ckuD=-5z+8hMKmVY)y$5{$WZI8d# zI9<8?N)fWT*ba& z{QGDtxIndfy0b_QzRkuq`s}J6!m}^<%f2GN5)>HvtH-xR} zy?cM?pG!#MS_eIv+MKc`GXY}5Nx4ksAVw5|m#FkWAN0QVBFW!vsRx1=q;Pe!i-aw* zuu@P^8Y;KcsCSwrsQP;@A|;p_SKbFbvBm-sUtvwC(5$%$SEsT2(Q*bT*(sjEEx*1g z17RM*vTHY5w1xtIDcGOS_D}qAr0lJW05255ueDueMf*u*#^6jk}K9arN4We@}EvpZ*DMc7X^Sc8^HI7Wv3b-|jT-PiyixUB$E zCSNXdq5rGuxupw@F z<_%6YQB+kG-%QJd@Ak@WZ-v9m0VXO08ngpmAF_=+yBv!5aOIGtfxNiEB4ZjzK6>e& zH(B&+-<+W(G>ZwM;`1Kpol&Ez&Y(UGHr2T2jWL3WInBsIw8e{}3^)miAaw5G*y`-Q zs&s7l*zN^9)GEBpby@51cz3Fd>#zQHASJ+iPP)N;66OOa>ObBW@UVJCMx&r}3})Vu zz37U&a27C;RtkykcO>#nt{*QyWZRz|?FZT?=Ag}DoI0YYDbY6@#ORD1&I`pN8dKY# z(wdqPHXBZ}kbmztRnq4Xc(04UsG8`XSq3)0gW?n()={!&6o+cC9B4yQ0!Y5}h7sV+ z?$Qb7W7n(Brr$$a1^x&jrcq_Po;egp>@69-4BJ^5jbq(gm+Ya#c_u7J`qz*Jn6C3p zcE^mJMz8_UxHUrgR}mtSx!?4Fw!+Ng6^p+~0I$*mBFVsCU51+jd%U1c#(f?j+RxGl zOIN?2g*WzCIfNj2?u$4jr4R^XC`g=x^Vxx{91wK123`SrX22_UN38o%K4;h3!NCHl zr%H(O*J8q#V8A=|ffnD3qUDYWs+vQX)}cWuJ9BeAS$ zPQ1856ru=xERJO~sizgL2>@Y zoWO4c?yy&KFgDz4$;Rwd9ps@Xk{Yx~CL z#`=Q#yE~wp{?|!BW7M~b-&Z&{q~m@m%91-S^}clnIYviP2=N&~zzR+~+2l74##~&N zov#mvra?|MO2DDyK(`wNj$(+D-Pjk+K!a<;&xxRXL2|;s`eskLG*N#G>Uz>(;a3d| zTiEUAz;_8Z|Fhkrv7sb%_dpY4m?ZxFDLfpsy4J`i;Kmy8Xx4fn+~R$#eLx;8gA z871BQ3C`35m1$4!_l2hB!@)Zd6$jD=jc`u?nQ_ntJN2;g?>9e>>ZuPJ(WT)+TM7Gm#95Z{j8 z9KqQhs&RYhf~fTWvjx=fV9mO?rkn>p$3ly$_S4_X8N(cNdqpCfN zM;%3Q5H$tt?$GEl_6&R7lg0-UY9ZsG%^>A+hMya)(JKO0q+*n$LDd+Ap zJ1E07?LM--q7L*O19#HFjTJd~`S{GXqNo(%&?&PSU+1eoh5=>}_@^YNE?a!tJm`M_ Do5vI_ literal 0 HcmV?d00001 diff --git a/data/public/upload/.gitkeep b/data/public/upload/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/data/ssl/.gitkeep b/data/ssl/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/data/testcase/.gitkeep b/data/testcase/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 55af5225..d7416a20 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -3,11 +3,10 @@ FROM python:3.6-alpine3.6 ENV OJ_ENV production RUN apk add --no-cache supervisor jpeg-dev zlib-dev postgresql-dev freetype-dev -ADD requirements.txt /tmp -RUN apk add --no-cache build-base && \ +ADD deploy/requirements.txt /tmp +RUN apk add --update --no-cache build-base nginx openssl && \ pip install --no-cache-dir -r /tmp/requirements.txt -i https://pypi.doubanio.com/simple && \ apk del build-base --purge -VOLUME [ "/app" ] - +ADD . /app CMD sh /app/deploy/run.sh diff --git a/deploy/oj.conf b/deploy/oj.conf new file mode 100644 index 00000000..504de814 --- /dev/null +++ b/deploy/oj.conf @@ -0,0 +1,62 @@ +user nobody; +daemon off; +pid /tmp/nginx.pid; +worker_processes auto; +pcre_jit on; +error_log /data/log/nginx_error.log warn; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + server_tokens off; + keepalive_timeout 65; + sendfile on; + tcp_nodelay on; + + gzip on; + gzip_vary on; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /app/data/log/nginx_access.log main; + + upstream backend { + server 127.0.0.1:8080; + keepalive 32; + } + + server { + listen 8000 default_server; + server_name _; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + client_max_body_size 200M; + + location /public { + root /app/data; + } + + location /api { + proxy_pass http://backend; + proxy_set_header Host $host; + } + + location /admin { + root /app/dist/admin; + try_files $uri $uri/ /index.html =404; + } + + location / { + root /app/dist; + try_files $uri $uri/ /index.html =404; + } + } + +} + diff --git a/deploy/run.sh b/deploy/run.sh index c8ca9d58..64a13b51 100644 --- a/deploy/run.sh +++ b/deploy/run.sh @@ -1,39 +1,28 @@ #!/bin/bash BASE=/app +DATA=$BASE/data -if [ ! -f "$BASE/custom_settings.py" ]; then - echo SECRET_KEY=\"$(cat /dev/urandom | head -1 | md5sum | head -c 32)\" >> /app/oj/custom_settings.py +if [ ! -f "$BASE/oj/custom_settings.py" ]; then + echo SECRET_KEY=\"$(cat /dev/urandom | head -1 | md5sum | head -c 32)\" >> $BASE/oj/custom_settings.py fi -if [ ! -d "$BASE/log" ]; then - mkdir -p $BASE/log -fi +mkdir -p $DATA/log $DATA/testcase $DATA/public/upload cd $BASE -find . -name "*.pyc" -delete - -# wait for postgresql start -sleep 6 n=0 -while [ $n -lt 3 ] +while [ $n -lt 5 ] do -python manage.py migrate -if [ $? -ne 0 ]; then - echo "Can't start server, try again in 3 seconds.." - sleep 3 - let "n+=1" - continue -fi -python manage.py initinstall -break + python manage.py migrate --no-input && + python manage.py initinstall && + break + n=$(($n+1)) + echo "Failed to migrate, going to retry..." + sleep 8 done -if [ $n -eq 3 ]; then - echo "Can't start server, please check log file for details." - exit 1 -fi +cp $BASE/deploy/oj.conf /etc/nginx/conf.d/default.conf -chown -R nobody:nogroup /data/log /data/test_case /data/avatar /data/upload +chown -R nobody:nogroup $DATA $BASE/dist exec supervisord -c /app/deploy/supervisor.conf diff --git a/deploy/supervisor.conf b/deploy/supervisor.conf index fc248d80..f36cf836 100644 --- a/deploy/supervisor.conf +++ b/deploy/supervisor.conf @@ -1,20 +1,32 @@ [supervisord] -logfile=/app/log/supervisord.log +logfile=/app/data/log/supervisord.log logfile_maxbytes=10MB logfile_backups=10 loglevel=info pidfile=/tmp/supervisord.pid nodaemon=true -childlogdir=/data/log/ +childlogdir=/app/data/log/ [supervisorctl] serverurl=unix:///tmp/supervisor.sock -[program:gunicorn] -command=sh -c "gunicorn oj.wsgi --user nobody -b 0.0.0.0:8080 --reload -w `grep -c ^processor /proc/cpuinfo`" +[program:nginx] +command=nginx -c /app/deploy/oj.conf directory=/app/ -stdout_logfile=/data/log/gunicorn.log -stderr_logfile=/data/log/gunicorn.log +stdout_logfile=/app/data/log/nginx.log +stderr_logfile=/app/data/log/nginx.log +autostart=true +autorestart=true +startsecs=5 +stopwaitsecs = 5 +killasgroup=true + +[program:gunicorn] +command=sh -c "gunicorn oj.wsgi --user nobody -b 127.0.0.1:8080 --reload -w `grep -c ^processor /proc/cpuinfo`" +directory=/app/ +user=nobody +stdout_logfile=/app/data/log/gunicorn.log +stderr_logfile=/app/data/log/gunicorn.log autostart=true autorestart=true startsecs=5 @@ -25,8 +37,8 @@ killasgroup=true command=celery -A oj worker -l warning directory=/app/ user=nobody -stdout_logfile=/data/log/celery.log -stderr_logfile=/data/log/celery.log +stdout_logfile=/app/data/log/celery.log +stderr_logfile=/app/data/log/celery.log autostart=true autorestart=true startsecs=5 diff --git a/oj/dev_settings.py b/oj/dev_settings.py index 5d9cb9e5..5c75a0b2 100644 --- a/oj/dev_settings.py +++ b/oj/dev_settings.py @@ -7,7 +7,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'HOST': '127.0.0.1', - 'PORT': 5432, + 'PORT': 5433, 'NAME': "onlinejudge", 'USER': "onlinejudge", 'PASSWORD': 'onlinejudge' @@ -24,16 +24,4 @@ DEBUG = True ALLOWED_HOSTS = ["*"] -TEST_CASE_DIR = "/tmp" - -LOG_PATH = f"{BASE_DIR}/log/" - -AVATAR_URI_PREFIX = "/static/avatar" -AVATAR_UPLOAD_DIR = f"{BASE_DIR}{AVATAR_URI_PREFIX}" - -UPLOAD_PREFIX = "/static/upload" -UPLOAD_DIR = f"{BASE_DIR}{UPLOAD_PREFIX}" - -STATICFILES_DIRS = [ - os.path.join(BASE_DIR, "static"), -] +DATA_DIR = f"{BASE_DIR}/data" diff --git a/oj/production_settings.py b/oj/production_settings.py index 3f9dee66..026c53a9 100644 --- a/oj/production_settings.py +++ b/oj/production_settings.py @@ -8,8 +8,8 @@ def get_env(name, default=""): DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'HOST': get_env("POSTGRES_HOST", "postgres"), - 'PORT': get_env("POSTGRES_PORT", "5433"), + 'HOST': get_env("POSTGRES_HOST", "oj-postgres"), + 'PORT': get_env("POSTGRES_PORT", "5432"), 'NAME': get_env("POSTGRES_DB"), 'USER': get_env("POSTGRES_USER"), 'PASSWORD': get_env("POSTGRES_PASSWORD") @@ -17,7 +17,7 @@ DATABASES = { } REDIS_CONF = { - "host": get_env("REDIS_HOST", "redis"), + "host": get_env("REDIS_HOST", "oj-redis"), "port": get_env("REDIS_PORT", "6379") } @@ -25,12 +25,4 @@ DEBUG = False ALLOWED_HOSTS = ['*'] -AVATAR_URI_PREFIX = "/static/avatar" -AVATAR_UPLOAD_DIR = "/data/avatar" - -UPLOAD_PREFIX = "/static/upload" -UPLOAD_DIR = "/data/upload" - -TEST_CASE_DIR = "/data/test_case" -LOG_PATH = "/data/log" -DEFAULT_JUDGE_SERVER_SERVICE_URL = "http://judge-server:8080/" +DATA_DIR = "/data" diff --git a/oj/settings.py b/oj/settings.py index 0c6e7687..e1de79bf 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -109,10 +109,22 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.8/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = '/storage/' AUTH_USER_MODEL = 'account.User' +TEST_CASE_DIR = os.path.join(DATA_DIR, "testcase") +LOG_PATH = os.path.join(DATA_DIR, "log") + +AVATAR_URI_PREFIX = "/public/avatar" +AVATAR_UPLOAD_DIR = f"{DATA_DIR}{AVATAR_URI_PREFIX}" + +UPLOAD_PREFIX = "/public/upload" +UPLOAD_DIR = f"{DATA_DIR}{UPLOAD_PREFIX}" + +STATICFILES_DIRS = [os.path.join(DATA_DIR, "public")] + + LOGGING = { 'version': 1, 'disable_existing_loggers': False, From cf40deb97c81e0a793001b69154b86efa66e2753 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 25 Nov 2017 12:30:00 +0800 Subject: [PATCH 05/14] add ssl cert --- .gitignore | 4 +- data/{testcase => test_case}/.gitkeep | 0 deploy/nginx/common.conf | 20 +++++++++ deploy/nginx/nginx.conf | 56 ++++++++++++++++++++++++ deploy/oj.conf | 62 --------------------------- deploy/run.sh | 22 ++++++---- deploy/supervisor.conf | 2 +- oj/settings.py | 2 +- 8 files changed, 93 insertions(+), 75 deletions(-) rename data/{testcase => test_case}/.gitkeep (100%) create mode 100644 deploy/nginx/common.conf create mode 100644 deploy/nginx/nginx.conf delete mode 100644 deploy/oj.conf diff --git a/.gitignore b/.gitignore index 822575d7..66e09105 100644 --- a/.gitignore +++ b/.gitignore @@ -61,8 +61,8 @@ custom_settings.py data/log/* !data/log/.gitkeep -data/testcase/* -!data/testcase/.gitkeep +data/test_case/* +!data/test_case/.gitkeep data/ssl/* !data/ssl/.gitkeep data/static/upload/* diff --git a/data/testcase/.gitkeep b/data/test_case/.gitkeep similarity index 100% rename from data/testcase/.gitkeep rename to data/test_case/.gitkeep diff --git a/deploy/nginx/common.conf b/deploy/nginx/common.conf new file mode 100644 index 00000000..478e2883 --- /dev/null +++ b/deploy/nginx/common.conf @@ -0,0 +1,20 @@ +location /public { + root /app/data; +} + +location /api { + proxy_pass http://backend; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + client_max_body_size 200M; +} + +location /admin { + root /app/dist/admin; + try_files $uri $uri/ /index.html =404; +} + +location / { + root /app/dist; + try_files $uri $uri/ /index.html =404; +} \ No newline at end of file diff --git a/deploy/nginx/nginx.conf b/deploy/nginx/nginx.conf new file mode 100644 index 00000000..54eb7ed2 --- /dev/null +++ b/deploy/nginx/nginx.conf @@ -0,0 +1,56 @@ +user nobody; +daemon off; +pid /tmp/nginx.pid; +worker_processes auto; +pcre_jit on; +error_log /data/log/nginx_error.log warn; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + server_tokens off; + keepalive_timeout 65; + sendfile on; + tcp_nodelay on; + + gzip on; + gzip_vary on; + gzip_types application/javascript text/css; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /data/log/nginx_access.log main; + + upstream backend { + server 127.0.0.1:8080; + keepalive 32; + } + + server { + listen 8000 default_server; + server_name _; + + include common.conf; + } + + server { + listen 1443 ssl http2 default_server; + server_name _; + ssl_certificate /data/ssl/server.crt; + ssl_certificate_key /data/ssl/server.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:ECDHE-RSA-AES128-GCM-SHA256:AES256+EECDH:DHE-RSA-AES128-GCM-SHA256:AES256+EDH:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + + include common.conf; + } + +} + diff --git a/deploy/oj.conf b/deploy/oj.conf deleted file mode 100644 index 504de814..00000000 --- a/deploy/oj.conf +++ /dev/null @@ -1,62 +0,0 @@ -user nobody; -daemon off; -pid /tmp/nginx.pid; -worker_processes auto; -pcre_jit on; -error_log /data/log/nginx_error.log warn; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - server_tokens off; - keepalive_timeout 65; - sendfile on; - tcp_nodelay on; - - gzip on; - gzip_vary on; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /app/data/log/nginx_access.log main; - - upstream backend { - server 127.0.0.1:8080; - keepalive 32; - } - - server { - listen 8000 default_server; - server_name _; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $http_host; - client_max_body_size 200M; - - location /public { - root /app/data; - } - - location /api { - proxy_pass http://backend; - proxy_set_header Host $host; - } - - location /admin { - root /app/dist/admin; - try_files $uri $uri/ /index.html =404; - } - - location / { - root /app/dist; - try_files $uri $uri/ /index.html =404; - } - } - -} - diff --git a/deploy/run.sh b/deploy/run.sh index 64a13b51..356ee949 100644 --- a/deploy/run.sh +++ b/deploy/run.sh @@ -1,15 +1,21 @@ #!/bin/bash -BASE=/app -DATA=$BASE/data +APP=/app +DATA=/data -if [ ! -f "$BASE/oj/custom_settings.py" ]; then - echo SECRET_KEY=\"$(cat /dev/urandom | head -1 | md5sum | head -c 32)\" >> $BASE/oj/custom_settings.py +if [ ! -f "$APP/oj/custom_settings.py" ]; then + echo SECRET_KEY=\"$(cat /dev/urandom | head -1 | md5sum | head -c 32)\" >> $APP/oj/custom_settings.py fi -mkdir -p $DATA/log $DATA/testcase $DATA/public/upload +mkdir -p $DATA/log $DATA/ssl $DATA/test_case $DATA/public/upload -cd $BASE +SSL="$DATA/ssl" +if [ ! -f "$SSL/server.key" ]; then + openssl req -x509 -newkey rsa:2048 -keyout "$SSL/server.key" -out "$SSL/server.crt" -days 1000 \ + -subj "/C=CN/ST=Beijing/L=Beijing/O=Beijing OnlineJudge Technology Co., Ltd./OU=Service Infrastructure Department/CN=`hostname`" -nodes +fi + +cd $APP n=0 while [ $n -lt 5 ] @@ -22,7 +28,5 @@ do sleep 8 done -cp $BASE/deploy/oj.conf /etc/nginx/conf.d/default.conf - -chown -R nobody:nogroup $DATA $BASE/dist +chown -R nobody:nogroup $DATA $APP/dist exec supervisord -c /app/deploy/supervisor.conf diff --git a/deploy/supervisor.conf b/deploy/supervisor.conf index f36cf836..c9e50327 100644 --- a/deploy/supervisor.conf +++ b/deploy/supervisor.conf @@ -11,7 +11,7 @@ childlogdir=/app/data/log/ serverurl=unix:///tmp/supervisor.sock [program:nginx] -command=nginx -c /app/deploy/oj.conf +command=nginx -c /app/deploy/nginx/nginx.conf directory=/app/ stdout_logfile=/app/data/log/nginx.log stderr_logfile=/app/data/log/nginx.log diff --git a/oj/settings.py b/oj/settings.py index e1de79bf..52c3f2c2 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -113,7 +113,7 @@ STATIC_URL = '/storage/' AUTH_USER_MODEL = 'account.User' -TEST_CASE_DIR = os.path.join(DATA_DIR, "testcase") +TEST_CASE_DIR = os.path.join(DATA_DIR, "test_case") LOG_PATH = os.path.join(DATA_DIR, "log") AVATAR_URI_PREFIX = "/public/avatar" From 1f9eca8b7d32979124f3ec5487112a12a4dd25c9 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 25 Nov 2017 12:54:29 +0800 Subject: [PATCH 06/14] fix initadmin script --- .travis.yml | 2 -- deploy/run.sh | 2 +- utils/management/commands/initadmin.py | 38 -------------------- utils/management/commands/initinstall.py | 17 --------- utils/management/commands/inituser.py | 44 ++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 58 deletions(-) delete mode 100644 utils/management/commands/initadmin.py delete mode 100644 utils/management/commands/initinstall.py create mode 100644 utils/management/commands/inituser.py diff --git a/.travis.yml b/.travis.yml index 7782d02c..5a01bfae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,9 @@ before_install: - docker run -it -d -e POSTGRES_DB=onlinejudge -e POSTGRES_USER=onlinejudge -e POSTGRES_PASSWORD=onlinejudge -p 127.0.0.1:5433:5432 postgres:10 install: - pip install -r deploy/requirements.txt - - mkdir log test_case upload - cp oj/custom_settings.example.py oj/custom_settings.py - echo "SECRET_KEY=\"`cat /dev/urandom | head -1 | md5sum | head -c 32`\"" >> oj/custom_settings.py - python manage.py migrate - - python manage.py initadmin script: - docker ps -a - flake8 . diff --git a/deploy/run.sh b/deploy/run.sh index 356ee949..bcf74767 100644 --- a/deploy/run.sh +++ b/deploy/run.sh @@ -21,7 +21,7 @@ n=0 while [ $n -lt 5 ] do python manage.py migrate --no-input && - python manage.py initinstall && + python manage.py inituser --username=root --password=rootroot --action=create_super_admin && break n=$(($n+1)) echo "Failed to migrate, going to retry..." diff --git a/utils/management/commands/initadmin.py b/utils/management/commands/initadmin.py deleted file mode 100644 index 78e8a458..00000000 --- a/utils/management/commands/initadmin.py +++ /dev/null @@ -1,38 +0,0 @@ -from django.core.management.base import BaseCommand - -from account.models import AdminType, ProblemPermission, User, UserProfile -from utils.shortcuts import rand_str # NOQA - - -class Command(BaseCommand): - def handle(self, *args, **options): - try: - admin = User.objects.get(username="root") - if admin.admin_type == AdminType.SUPER_ADMIN: - self.stdout.write(self.style.WARNING("Super admin user 'root' already exists, " - "would you like to reset it's password?\n" - "Input yes to confirm: ")) - if input() == "yes": - rand_password = "rootroot" - admin.save() - self.stdout.write(self.style.SUCCESS("Successfully created super admin user password.\n" - "Username: root\nPassword: %s\n" - "Remember to change password and turn on two factors auth " - "after installation." % rand_password)) - else: - self.stdout.write(self.style.SUCCESS("Nothing happened")) - else: - self.stdout.write(self.style.ERROR("User 'root' is not super admin.")) - except User.DoesNotExist: - user = User.objects.create(username="root", email="root@oj.com", admin_type=AdminType.SUPER_ADMIN, - problem_permission=ProblemPermission.ALL) - # for dev - # rand_password = rand_str(length=6) - rand_password = "rootroot" - user.set_password(rand_password) - user.save() - UserProfile.objects.create(user=user) - self.stdout.write(self.style.SUCCESS("Successfully created super admin user.\n" - "Username: root\nPassword: %s\n" - "Remember to change password and turn on two factors auth " - "after installation." % rand_password)) diff --git a/utils/management/commands/initinstall.py b/utils/management/commands/initinstall.py deleted file mode 100644 index bdb0f0a3..00000000 --- a/utils/management/commands/initinstall.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -from account.models import User -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - def handle(self, *args, **options): - if User.objects.exists(): - self.stdout.write(self.style.WARNING("Nothing happened\n")) - return - try: - if os.system("python manage.py initadmin") != 0: - self.stdout.write(self.style.ERROR("Failed to execute command 'initadmin'")) - exit(1) - self.stdout.write(self.style.SUCCESS("Done")) - except Exception as e: - self.stdout.write(self.style.ERROR("Failed to initialize, error: " + str(e))) diff --git a/utils/management/commands/inituser.py b/utils/management/commands/inituser.py new file mode 100644 index 00000000..c3f08270 --- /dev/null +++ b/utils/management/commands/inituser.py @@ -0,0 +1,44 @@ +from django.core.management.base import BaseCommand + +from account.models import AdminType, ProblemPermission, User, UserProfile +from utils.shortcuts import rand_str # NOQA + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument("--username", type=str) + parser.add_argument("--password", type=str) + parser.add_argument("--action", type=str) + + def handle(self, *args, **options): + username = options["username"] + password = options["password"] + action = options["action"] + + if not(username and password and action): + self.stdout.write(self.style.ERROR("Invalid args")) + exit(1) + + if action == "create_super_admin": + if User.objects.filter(username=username).exists(): + self.stdout.write(self.style.SUCCESS(f"User {username} exists, operation ignored")) + exit() + + user = User.objects.create(username=username, admin_type=AdminType.SUPER_ADMIN, + problem_permission=ProblemPermission.ALL) + user.set_password(password) + user.save() + UserProfile.objects.create(user=user) + + self.stdout.write(self.style.SUCCESS("User created")) + elif action == "reset": + try: + user = User.objects.get(username=username) + user.set_password(password) + user.save() + self.stdout.write(self.style.SUCCESS(f"Password is rested")) + except User.DoesNotExist: + self.stdout.write(self.style.ERROR(f"User {username} doesnot exist, operation ignored")) + exit(1) + else: + raise ValueError("Invalid action") From 05475fb161189bfc858e6cd287d2b727ab1d0967 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 25 Nov 2017 13:11:24 +0800 Subject: [PATCH 07/14] download frontend dist from github release --- deploy/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index d7416a20..b7f7997f 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -4,9 +4,14 @@ ENV OJ_ENV production RUN apk add --no-cache supervisor jpeg-dev zlib-dev postgresql-dev freetype-dev ADD deploy/requirements.txt /tmp -RUN apk add --update --no-cache build-base nginx openssl && \ - pip install --no-cache-dir -r /tmp/requirements.txt -i https://pypi.doubanio.com/simple && \ +RUN printf "https://mirrors.tuna.tsinghua.edu.cn/alpine/v3.6/community/\nhttps://mirrors.tuna.tsinghua.edu.cn/alpine/v3.6/main/" > /etc/apk/repositories && \ + apk add --update --no-cache build-base nginx openssl curl unzip && \ + pip install --no-cache-dir -r /tmp/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple && \ apk del build-base --purge ADD . /app +WORKDIR /app +RUN curl -L $(curl -s https://api.github.com/repos/QingdaoU/OnlineJudgeFE/releases/latest | grep /dist.zip | cut -d '"' -f 4) -o dist.zip && \ + unzip dist.zip && \ + rm dist.zip CMD sh /app/deploy/run.sh From 7fce29cb717b0b758bf090a7d93c9b99cfe9562c Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 25 Nov 2017 15:47:56 +0800 Subject: [PATCH 08/14] fix static file path --- .gitignore | 8 ++++---- oj/settings.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 66e09105..3a8cb907 100644 --- a/.gitignore +++ b/.gitignore @@ -65,7 +65,7 @@ data/test_case/* !data/test_case/.gitkeep data/ssl/* !data/ssl/.gitkeep -data/static/upload/* -!data/static/upload/.gitkeep -data/static/avatar/* -!data/static/avatar/default.png +data/public/upload/* +!data/public/upload/.gitkeep +data/public/avatar/* +!data/public/avatar/default.png diff --git a/oj/settings.py b/oj/settings.py index 52c3f2c2..d1ab2115 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -109,7 +109,7 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.8/howto/static-files/ -STATIC_URL = '/storage/' +STATIC_URL = '/public/' AUTH_USER_MODEL = 'account.User' From 79717c82b1869bab0fa6139c81843911854467d9 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 25 Nov 2017 19:12:37 +0800 Subject: [PATCH 09/14] move Dockerfile --- deploy/Dockerfile => Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) rename deploy/Dockerfile => Dockerfile (70%) diff --git a/deploy/Dockerfile b/Dockerfile similarity index 70% rename from deploy/Dockerfile rename to Dockerfile index b7f7997f..9e3f69c4 100644 --- a/deploy/Dockerfile +++ b/Dockerfile @@ -1,16 +1,14 @@ FROM python:3.6-alpine3.6 ENV OJ_ENV production -RUN apk add --no-cache supervisor jpeg-dev zlib-dev postgresql-dev freetype-dev - -ADD deploy/requirements.txt /tmp -RUN printf "https://mirrors.tuna.tsinghua.edu.cn/alpine/v3.6/community/\nhttps://mirrors.tuna.tsinghua.edu.cn/alpine/v3.6/main/" > /etc/apk/repositories && \ - apk add --update --no-cache build-base nginx openssl curl unzip && \ - pip install --no-cache-dir -r /tmp/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple && \ - apk del build-base --purge ADD . /app WORKDIR /app + +RUN printf "https://mirrors.tuna.tsinghua.edu.cn/alpine/v3.6/community/\nhttps://mirrors.tuna.tsinghua.edu.cn/alpine/v3.6/main/" > /etc/apk/repositories && \ + apk add --update --no-cache build-base nginx openssl curl unzip supervisor jpeg-dev zlib-dev postgresql-dev freetype-dev && \ + pip install --no-cache-dir -r /app/deploy/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple && \ + apk del build-base --purge RUN curl -L $(curl -s https://api.github.com/repos/QingdaoU/OnlineJudgeFE/releases/latest | grep /dist.zip | cut -d '"' -f 4) -o dist.zip && \ unzip dist.zip && \ rm dist.zip From 00eb3b19674dffb57b5102320b0a2cbba30758b8 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 25 Nov 2017 21:47:51 +0800 Subject: [PATCH 10/14] add api to reset openapi appkey and related middleware --- account/middleware.py | 12 ++++++++++++ account/tests.py | 16 ++++++++++++++++ account/urls/oj.py | 5 +++-- account/views/oj.py | 12 ++++++++++++ oj/settings.py | 1 + 5 files changed, 44 insertions(+), 2 deletions(-) diff --git a/account/middleware.py b/account/middleware.py index 1ef78ccf..245c32a0 100644 --- a/account/middleware.py +++ b/account/middleware.py @@ -3,6 +3,18 @@ from django.utils.timezone import now from django.utils.deprecation import MiddlewareMixin from utils.api import JSONResponse +from account.models import User + + +class APITokenAuthMiddleware(MiddlewareMixin): + def process_request(self, request): + appkey = request.META.get("HTTP_APPKEY") + if appkey: + try: + request.user = User.objects.get(open_api_appkey=appkey, open_api=True, is_disabled=False) + request.csrf_processing_done = True + except User.DoesNotExist: + pass class SessionRecordMiddleware(MiddlewareMixin): diff --git a/account/tests.py b/account/tests.py index 43a09fe2..97880aac 100644 --- a/account/tests.py +++ b/account/tests.py @@ -611,3 +611,19 @@ class GenerateUserAPITest(APITestCase): resp = self.client.post(self.url, data=self.data) self.assertSuccess(resp) mock_workbook.assert_called() + + +class OpenAPIAppkeyAPITest(APITestCase): + def setUp(self): + self.user = self.create_super_admin() + self.url = self.reverse("open_api_appkey_api") + + def test_reset_appkey(self): + resp = self.client.post(self.url, data={}) + self.assertFailed(resp) + + self.user.open_api = True + self.user.save() + resp = self.client.post(self.url, data={}) + self.assertSuccess(resp) + self.assertEqual(resp.data["data"]["appkey"], User.objects.get(username=self.user.username).open_api_appkey) diff --git a/account/urls/oj.py b/account/urls/oj.py index a92dd5be..1b26e14c 100644 --- a/account/urls/oj.py +++ b/account/urls/oj.py @@ -5,7 +5,7 @@ from ..views.oj import (ApplyResetPasswordAPI, ResetPasswordAPI, UserLoginAPI, UserLogoutAPI, UsernameOrEmailCheck, AvatarUploadAPI, TwoFactorAuthAPI, UserProfileAPI, UserRankAPI, CheckTFARequiredAPI, SessionManagementAPI, - ProfileProblemDisplayIDRefreshAPI) + ProfileProblemDisplayIDRefreshAPI, OpenAPIAppkeyAPI) from utils.captcha.views import CaptchaAPIView @@ -25,5 +25,6 @@ urlpatterns = [ url(r"^tfa_required/?$", CheckTFARequiredAPI.as_view(), name="tfa_required_check"), url(r"^two_factor_auth/?$", TwoFactorAuthAPI.as_view(), name="two_factor_auth_api"), url(r"^user_rank/?$", UserRankAPI.as_view(), name="user_rank_api"), - url(r"^sessions/?$", SessionManagementAPI.as_view(), name="session_management_api") + url(r"^sessions/?$", SessionManagementAPI.as_view(), name="session_management_api"), + url(r"^open_api_appkey/?$", OpenAPIAppkeyAPI.as_view(), name="open_api_appkey_api"), ] diff --git a/account/views/oj.py b/account/views/oj.py index 344a4196..5f258178 100644 --- a/account/views/oj.py +++ b/account/views/oj.py @@ -401,3 +401,15 @@ class ProfileProblemDisplayIDRefreshAPI(APIView): v["_id"] = id_map[k] profile.save(update_fields=["acm_problems_status", "oi_problems_status"]) return self.success() + + +class OpenAPIAppkeyAPI(APIView): + @login_required + def post(self, request): + user = request.user + if not user.open_api: + return self.error("Permission denied") + api_appkey = rand_str() + user.open_api_appkey = api_appkey + user.save() + return self.success({"appkey": api_appkey}) diff --git a/oj/settings.py b/oj/settings.py index d1ab2115..43aa1d6f 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -49,6 +49,7 @@ MIDDLEWARE_CLASSES = ( 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'account.middleware.APITokenAuthMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.security.SecurityMiddleware', From 1bf497364881523d4f66a7e5d6f3b0d9a89ec028 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sun, 26 Nov 2017 10:40:56 +0800 Subject: [PATCH 11/14] fix deploy issues - make supervisorctl happy - fix avatar path - refine logging config - add nginx buffer path --- deploy/nginx/common.conf | 2 +- deploy/nginx/nginx.conf | 1 + deploy/run.sh | 2 +- deploy/{supervisor.conf => supervisord.conf} | 24 +++--- oj/settings.py | 78 ++++++++------------ 5 files changed, 50 insertions(+), 57 deletions(-) rename deploy/{supervisor.conf => supervisord.conf} (60%) diff --git a/deploy/nginx/common.conf b/deploy/nginx/common.conf index 478e2883..ecfd2513 100644 --- a/deploy/nginx/common.conf +++ b/deploy/nginx/common.conf @@ -1,5 +1,5 @@ location /public { - root /app/data; + root /data; } location /api { diff --git a/deploy/nginx/nginx.conf b/deploy/nginx/nginx.conf index 54eb7ed2..0942890a 100644 --- a/deploy/nginx/nginx.conf +++ b/deploy/nginx/nginx.conf @@ -20,6 +20,7 @@ http { gzip on; gzip_vary on; gzip_types application/javascript text/css; + client_body_temp_path /tmp 1 2; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' diff --git a/deploy/run.sh b/deploy/run.sh index bcf74767..91ac2e40 100644 --- a/deploy/run.sh +++ b/deploy/run.sh @@ -29,4 +29,4 @@ do done chown -R nobody:nogroup $DATA $APP/dist -exec supervisord -c /app/deploy/supervisor.conf +exec supervisord -c /app/deploy/supervisord.conf diff --git a/deploy/supervisor.conf b/deploy/supervisord.conf similarity index 60% rename from deploy/supervisor.conf rename to deploy/supervisord.conf index c9e50327..6b231665 100644 --- a/deploy/supervisor.conf +++ b/deploy/supervisord.conf @@ -1,20 +1,26 @@ [supervisord] -logfile=/app/data/log/supervisord.log +logfile=/data/log/supervisord.log logfile_maxbytes=10MB logfile_backups=10 loglevel=info pidfile=/tmp/supervisord.pid nodaemon=true -childlogdir=/app/data/log/ +childlogdir=/data/log/ + +[inet_http_server] +port=127.0.0.1:9005 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface [supervisorctl] -serverurl=unix:///tmp/supervisor.sock +serverurl=http://127.0.0.1:9005 [program:nginx] command=nginx -c /app/deploy/nginx/nginx.conf directory=/app/ -stdout_logfile=/app/data/log/nginx.log -stderr_logfile=/app/data/log/nginx.log +stdout_logfile=/data/log/nginx.log +stderr_logfile=/data/log/nginx.log autostart=true autorestart=true startsecs=5 @@ -25,8 +31,8 @@ killasgroup=true command=sh -c "gunicorn oj.wsgi --user nobody -b 127.0.0.1:8080 --reload -w `grep -c ^processor /proc/cpuinfo`" directory=/app/ user=nobody -stdout_logfile=/app/data/log/gunicorn.log -stderr_logfile=/app/data/log/gunicorn.log +stdout_logfile=/data/log/gunicorn.log +stderr_logfile=/data/log/gunicorn.log autostart=true autorestart=true startsecs=5 @@ -37,8 +43,8 @@ killasgroup=true command=celery -A oj worker -l warning directory=/app/ user=nobody -stdout_logfile=/app/data/log/celery.log -stderr_logfile=/app/data/log/celery.log +stdout_logfile=/data/log/celery.log +stderr_logfile=/data/log/celery.log autostart=true autorestart=true startsecs=5 diff --git a/oj/settings.py b/oj/settings.py index 43aa1d6f..40335a1e 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -125,54 +125,40 @@ UPLOAD_DIR = f"{DATA_DIR}{UPLOAD_PREFIX}" STATICFILES_DIRS = [os.path.join(DATA_DIR, "public")] - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'standard': { - 'format': '%(asctime)s [%(threadName)s:%(thread)d] [%(name)s:%(lineno)d] [%(module)s:%(funcName)s] [%(levelname)s]- %(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S' - } - }, - 'handlers': { - 'django_error': { - 'level': 'WARNING', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os.path.join(LOG_PATH, 'django.log'), - 'formatter': 'standard' - }, - 'app_info': { - 'level': 'INFO', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os.path.join(LOG_PATH, 'app_info.log'), - 'formatter': 'standard' - }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'standard' - } - }, - 'loggers': { - 'django.request': { - 'handlers': ['django_error', 'console'], - 'level': 'WARNING', - 'propagate': True, - }, - 'django.db.backends': { - 'handlers': ['django_error', 'console'], - 'level': 'WARNING', - 'propagate': True, - }, - }, + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '[%(asctime)s] - [%(levelname)s] - [%(name)s:%(lineno)d] - %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'standard' + } + }, + 'loggers': { + 'django.request': { + 'handlers': ['console'], + 'level': 'ERROR', + 'propagate': True, + }, + 'django.db.backends': { + 'handlers': ['console'], + 'level': 'ERROR', + 'propagate': True, + }, + '': { + 'handlers': ['console'], + 'level': 'WARNING', + 'propagate': True, + } + }, } -app_logger = { - 'handlers': ['app_info', 'console'], - 'level': 'DEBUG', - 'propagate': False -} -LOGGING["loggers"].update({app: deepcopy(app_logger) for app in LOCAL_APPS}) REST_FRAMEWORK = { 'TEST_REQUEST_DEFAULT_FORMAT': 'json', From 324535474eb2997f43230b429bfa180933e75975 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sun, 26 Nov 2017 13:30:07 +0800 Subject: [PATCH 12/14] fix template mark --- judge/languages.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/judge/languages.py b/judge/languages.py index cc5f09b2..e7f35ffb 100644 --- a/judge/languages.py +++ b/judge/languages.py @@ -1,7 +1,7 @@ _c_lang_config = { - "template": """//PREPEND START + "template": """//PREPEND BEGIN #include //PREPEND END @@ -12,7 +12,7 @@ int add(int a, int b) { } //TEMPLATE END -//APPEND START +//APPEND BEGIN int main() { printf("%d", add(1, 2)); return 0; @@ -48,12 +48,23 @@ _c_lang_spj_config = { } _cpp_lang_config = { - "template": """/*--PREPEND START--*/ -/*--PREPEND END--*/ -/*--TEMPLATE BEGIN--*/ -/*--TEMPLATE END--*/ -/*--APPEND START--*/ -/*--APPEND END--*/""", + "template": """//PREPEND BEGIN +#include +//PREPEND END + +//TEMPLATE BEGIN +int add(int a, int b) { + // Please fill this blank + return ___________; +} +//TEMPLATE END + +//APPEND BEGIN +int main() { + std::cout << add(1, 2); + return 0; +} +//APPEND END""", "compile": { "src_name": "main.cpp", "exe_name": "main", From 945ea5e4e04b121f710b5c62b3d912386892e95c Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sun, 26 Nov 2017 13:42:38 +0800 Subject: [PATCH 13/14] parse problem code template --- judge/dispatcher.py | 13 +++++++-- problem/serializers.py | 63 +++++++++++++++++++++++++++++++++++++++++- problem/tests.py | 46 +++++++++++++++++++++++++++++- problem/utils.py | 10 +++++++ 4 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 problem/utils.py diff --git a/judge/dispatcher.py b/judge/dispatcher.py index 7c040297..0f5e3cbc 100644 --- a/judge/dispatcher.py +++ b/judge/dispatcher.py @@ -14,6 +14,7 @@ from contest.models import ContestRuleType, ACMContestRank, OIContestRank, Conte from judge.languages import languages, spj_languages from options.options import SysOptions from problem.models import Problem, ProblemRuleType +from problem.utils import parse_problem_template from submission.models import JudgeStatus, Submission from utils.cache import cache from utils.constants import CacheKey @@ -123,16 +124,24 @@ class JudgeDispatcher(DispatcherBase): cache.lpush(CacheKey.waiting_queue, json.dumps(data)) return - sub_config = list(filter(lambda item: self.submission.language == item["name"], languages))[0] + language = self.submission.language + sub_config = list(filter(lambda item: language == item["name"], languages))[0] spj_config = {} if self.problem.spj_code: for lang in spj_languages: if lang["name"] == self.problem.spj_language: spj_config = lang["spj"] break + + if language in self.problem.template: + template = parse_problem_template(self.problem.template[language]) + code = f"{template['prepend']}\n{self.submission.code}\n{template['append']}" + else: + code = self.submission.code + data = { "language_config": sub_config["config"], - "src": self.submission.code, + "src": code, "max_cpu_time": self.problem.time_limit, "max_memory": 1024 * 1024 * self.problem.memory_limit, "test_case_id": self.problem.test_case_id, diff --git a/problem/serializers.py b/problem/serializers.py index 57a034ce..46b2e222 100644 --- a/problem/serializers.py +++ b/problem/serializers.py @@ -4,6 +4,7 @@ from judge.languages import language_names, spj_language_names from utils.api import DateTimeTZField, UsernameSerializer, serializers from .models import Problem, ProblemRuleType, ProblemTag +from .utils import parse_problem_template class TestCaseUploadForm(forms.Form): @@ -110,9 +111,18 @@ class ContestProblemAdminSerializer(BaseProblemSerializer): class ProblemSerializer(BaseProblemSerializer): + template = serializers.SerializerMethodField() + + def get_template(self, obj): + ret = {} + for lang, code in obj.template.items(): + ret[lang] = parse_problem_template(code)["template"] + return ret + class Meta: model = Problem - exclude = ("contest", "test_case_score", "test_case_id", "visible", "is_public") + exclude = ("contest", "test_case_score", "test_case_id", "visible", "is_public", + "template", "spj_code", "spj_version", "spj_compile_ok") class ContestProblemSerializer(BaseProblemSerializer): @@ -131,3 +141,54 @@ class ContestProblemSafeSerializer(BaseProblemSerializer): class ContestProblemMakePublicSerializer(serializers.Serializer): id = serializers.IntegerField() display_id = serializers.CharField(max_length=32) + + +class ExportProblemSerializer(serializers.ModelSerializer): + description = serializers.SerializerMethodField() + input_description = serializers.SerializerMethodField() + output_description = serializers.SerializerMethodField() + test_case_score = serializers.SerializerMethodField() + hint = serializers.SerializerMethodField() + time_limit = serializers.SerializerMethodField() + memory_limit = serializers.SerializerMethodField() + spj = serializers.SerializerMethodField() + template = serializers.SerializerMethodField() + + def get_description(self, obj): + return {"format": "html", "value": obj.description} + + def get_input_description(self, obj): + return {"format": "html", "value": obj.input_description} + + def get_output_description(self, obj): + return {"format": "html", "value": obj.output_description} + + def get_hint(self, obj): + return {"format": "html", "value": obj.hint} + + def get_test_case_score(self, obj): + return obj.test_case_score if obj.rule_type == ProblemRuleType.OI else [] + + def get_time_limit(self, obj): + return {"unit": "ms", "value": obj.time_limit} + + def get_memory_limit(self, obj): + return {"unit": "MB", "value": obj.memory_limit} + + def get_spj(self, obj): + return {"enabled": obj.spj, + "code": obj.spj_code if obj.spj else None, + "language": obj.spj_language if obj.spj else None} + + def get_template(self, obj): + ret = {} + for k, v in obj.template.items(): + ret[k] = parse_problem_template(v) + return ret + + class Meta: + model = Problem + fields = ("_id", "title", "description", + "input_description", "output_description", + "test_case_score", "hint", "time_limit", "memory_limit", "samples", + "template", "spj", "rule_type", "source", "template") diff --git a/problem/tests.py b/problem/tests.py index dbd01b7f..93cd0585 100644 --- a/problem/tests.py +++ b/problem/tests.py @@ -11,10 +11,13 @@ from utils.api.tests import APITestCase from .models import ProblemTag from .models import Problem, ProblemRuleType -from .views.admin import TestCaseAPI from contest.models import Contest from contest.tests import DEFAULT_CONTEST_DATA +from .views.admin import TestCaseAPI +from .utils import parse_problem_template + + DEFAULT_PROBLEM_DATA = {"_id": "A-110", "title": "test", "description": "

test

", "input_description": "test", "output_description": "test", "time_limit": 1000, "memory_limit": 256, "difficulty": "Low", "visible": True, "tags": ["test"], "languages": ["C", "C++", "Java", "Python2"], "template": {}, @@ -257,3 +260,44 @@ class ContestProblemTest(ProblemCreateTestBase): contest.save() resp = self.client.get(self.url + "?contest_id=" + str(self.contest["id"])) self.assertSuccess(resp) + + +class ParseProblemTemplateTest(APITestCase): + def test_parse(self): + template_str = """ +//PREPEND BEGIN +aaa +//PREPEND END + +//TEMPLATE BEGIN +bbb +//TEMPLATE END + +//APPEND BEGIN +ccc +//APPEND END +""" + + ret = parse_problem_template(template_str) + self.assertEqual(ret["prepend"], "aaa\n") + self.assertEqual(ret["template"], "bbb\n") + self.assertEqual(ret["append"], "ccc\n") + + def test_parse1(self): + template_str = """ +//PREPEND BEGIN +aaa +//PREPEND END + +//APPEND BEGIN +ccc +//APPEND END +//APPEND BEGIN +ddd +//APPEND END +""" + + ret = parse_problem_template(template_str) + self.assertEqual(ret["prepend"], "aaa\n") + self.assertEqual(ret["template"], "") + self.assertEqual(ret["append"], "ccc\n") diff --git a/problem/utils.py b/problem/utils.py new file mode 100644 index 00000000..f8243099 --- /dev/null +++ b/problem/utils.py @@ -0,0 +1,10 @@ +import re + + +def parse_problem_template(template_str): + prepend = re.findall("//PREPEND BEGIN\n([\s\S]+?)//PREPEND END", template_str) + template = re.findall("//TEMPLATE BEGIN\n([\s\S]+?)//TEMPLATE END", template_str) + append = re.findall("//APPEND BEGIN\n([\s\S]+?)//APPEND END", template_str) + return {"prepend": prepend[0] if prepend else "", + "template": template[0] if template else "", + "append": append[0] if append else ""} From 86ca138b59a6c621f12a1be757aa9e2d8b2a5d11 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sun, 26 Nov 2017 15:55:33 +0800 Subject: [PATCH 14/14] copy default avatar --- deploy/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/run.sh b/deploy/run.sh index 91ac2e40..b710cba1 100644 --- a/deploy/run.sh +++ b/deploy/run.sh @@ -7,7 +7,7 @@ if [ ! -f "$APP/oj/custom_settings.py" ]; then echo SECRET_KEY=\"$(cat /dev/urandom | head -1 | md5sum | head -c 32)\" >> $APP/oj/custom_settings.py fi -mkdir -p $DATA/log $DATA/ssl $DATA/test_case $DATA/public/upload +mkdir -p $DATA/log $DATA/ssl $DATA/test_case $DATA/public/upload $DATA/public/avatar SSL="$DATA/ssl" if [ ! -f "$SSL/server.key" ]; then @@ -28,5 +28,6 @@ do sleep 8 done +cp data/public/avatar/default.png /data/public/avatar chown -R nobody:nogroup $DATA $APP/dist exec supervisord -c /app/deploy/supervisord.conf