QingdaoU
/
JudgeServer
mirror of https://github.com/QingdaoU/JudgeServer.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Python3 server

This commit is contained in:
virusdefender 2018-03-18 08:19:44 +08:00
parent e27c662a66
commit 48abe6dacb
11 changed files with 101 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -6,8 +6,9 @@ before_install:
- sudo mkdir -p /data/log
- pip install requests
script:
- docker pull qduoj/judge_server
- docker pull qduoj/judge-server
- cp docker-compose.example.yml docker-compose.yml
- docker-compose up -d
- docker ps -a
- sleep 5
- python tests/tests.py

13
Dockerfile
View File

@ -1,21 +1,20 @@
FROM registry.docker-cn.com/library/ubuntu:16.04
ENV DEBIAN_FRONTEND noninteractive
COPY build/java_policy /etc
RUN buildDeps='software-properties-common git libtool cmake python-dev python-pip libseccomp-dev' && \
apt-get update && apt-get install -y python python3.5 python-pkg-resources gcc g++ $buildDeps && \
add-apt-repository ppa:openjdk-r/ppa && apt-get update && apt-get install -y openjdk-7-jdk && \
pip install --no-cache-dir futures psutil gunicorn web.py requests && \
RUN buildDeps='software-properties-common git libtool cmake python-dev python3-pip python-pip libseccomp-dev' && \
apt-get update && apt-get install -y python python3.5 python-pkg-resources python3-pkg-resources gcc g++ $buildDeps && \
add-apt-repository ppa:openjdk-r/ppa && apt-get update && apt-get install -y openjdk-9-jdk-headless && \
pip3 install --no-cache-dir psutil gunicorn flask requests && \
cd /tmp && git clone -b newnew --depth 1 https://github.com/QingdaoU/Judger && cd Judger && \
mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python setup.py install && \
mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python3 setup.py install && \
apt-get purge -y --auto-remove $buildDeps && \
apt-get clean && rm -rf /var/lib/apt/lists/*
RUN mkdir -p /judger_run /test_case /log /code && \
useradd -r compiler
HEALTHCHECK --interval=5s --retries=3 CMD python /code/service.py
HEALTHCHECK --interval=5s --retries=3 CMD python3 /code/service.py
ADD server /code
WORKDIR /code
EXPOSE 8080

5
client/Python/client.py
View File

@ -1,6 +1,3 @@
# coding=utf-8
from __future__ import unicode_literals, print_function
import hashlib
import json
@ -27,7 +24,7 @@ class JudgeServerClient(object):
try:
return requests.post(url, **kwargs).json()
except Exception as e:
raise JudgeServerClientError(e.message)
raise JudgeServerClientError(str(e))
def ping(self):
return self._request(self.server_base_url + "/ping")

20
server/compiler.py
View File

@ -1,11 +1,7 @@
# coding=utf-8
from __future__ import unicode_literals
import _judger
import json
import os
import _judger
from config import COMPILER_LOG_PATH, COMPILER_USER_UID, COMPILER_GROUP_GID
from exception import CompileError
@ -24,13 +20,13 @@ class Compiler(object):
max_stack=128 * 1024 * 1024,
max_output_size=1024 * 1024,
max_process_number=_judger.UNLIMITED,
exe_path=_command[0].encode("utf-8"),
exe_path=_command[0],
# /dev/null is best, but in some system, this will call ioctl system call
input_path=src_path.encode("utf-8"),
output_path=compiler_out.encode("utf-8"),
error_path=compiler_out.encode("utf-8"),
args=[item.encode("utf-8") for item in _command[1::]],
env=[("PATH=" + os.getenv("PATH")).encode("utf-8")],
input_path=src_path,
output_path=compiler_out,
error_path=compiler_out,
args=_command[1::],
env=["PATH=" + os.getenv("PATH")],
log_path=COMPILER_LOG_PATH,
seccomp_rule_name=None,
uid=COMPILER_USER_UID,
@ -43,7 +39,7 @@ class Compiler(object):
os.remove(compiler_out)
if error:
raise CompileError(error)
raise CompileError("Compiler runtime error, info: %s" % json.dumps(result).decode("utf-8"))
raise CompileError("Compiler runtime error, info: %s" % json.dumps(result))
else:
os.remove(compiler_out)
return exe_path

11
server/config.py
View File

@ -1,15 +1,14 @@
# coding=utf-8
from __future__ import unicode_literals
import grp
import os
import pwd
import grp
JUDGER_WORKSPACE_BASE = "/judger_run"
LOG_BASE = "/log"
COMPILER_LOG_PATH = os.path.join(LOG_BASE, "compile.log").encode("utf-8")
JUDGER_RUN_LOG_PATH = os.path.join(LOG_BASE, "judger.log").encode("utf-8")
COMPILER_LOG_PATH = os.path.join(LOG_BASE, "compile.log")
JUDGER_RUN_LOG_PATH = os.path.join(LOG_BASE, "judger.log")
SERVER_LOG_PATH = os.path.join(LOG_BASE, "judge_server.log")
RUN_USER_UID = pwd.getpwnam("nobody").pw_uid
RUN_GROUP_GID = grp.getgrnam("nogroup").gr_gid

2
server/entrypoint.sh
View File

@ -2,4 +2,4 @@
chown compiler:compiler /spj
core=$(grep --count ^processor /proc/cpuinfo)
n=$(($core*2))
exec gunicorn --workers $n --threads $n --error-logfile /log/gunicorn.log --time 600 --bind 0.0.0.0:8080 server:wsgiapp
exec gunicorn --workers $n --threads $n --error-logfile /log/gunicorn.log --time 600 --bind 0.0.0.0:8080 server:app

15
server/exception.py
View File

@ -1,21 +1,24 @@
from __future__ import unicode_literals
class JudgeServerException(Exception):
def __init__(self, message):
super().__init__()
self.message = message
class CompileError(Exception):
class CompileError(JudgeServerException):
pass
class SPJCompileError(CompileError):
class SPJCompileError(JudgeServerException):
pass
class TokenVerificationFailed(Exception):
class TokenVerificationFailed(JudgeServerException):
pass
class JudgeClientError(Exception):
class JudgeClientError(JudgeServerException):
pass
class JudgeServiceError(Exception):
class JudgeServiceError(JudgeServerException):
pass

51
server/judge_client.py
View File

@ -1,17 +1,14 @@
# coding=utf-8
from __future__ import unicode_literals
import _judger
import psutil
import os
import json
import hashlib
import json
import os
from multiprocessing import Pool
import psutil
from config import TEST_CASE_DIR, JUDGER_RUN_LOG_PATH, RUN_GROUP_GID, RUN_USER_UID, SPJ_EXE_DIR
from exception import JudgeClientError
SPJ_WA = 1
SPJ_AC = 0
SPJ_ERROR = -1
@ -40,14 +37,15 @@ class JudgeClient(object):
self._spj_config = spj_config
self._output = output
if self._spj_version and self._spj_config:
self._spj_exe = os.path.join(SPJ_EXE_DIR, self._spj_config["exe_name"].format(spj_version=self._spj_version))
self._spj_exe = os.path.join(SPJ_EXE_DIR,
self._spj_config["exe_name"].format(spj_version=self._spj_version))
if not os.path.exists(self._spj_exe):
raise JudgeClientError("spj exe not found")
def _load_test_case_info(self):
try:
with open(os.path.join(self._test_case_dir, "info")) as f:
return json.loads(f.read())
return json.load(f)
except IOError:
raise JudgeClientError("Test case not found")
except ValueError:
@ -60,7 +58,7 @@ class JudgeClient(object):
user_output_file = os.path.join(self._submission_dir, str(test_case_file_id) + ".out")
with open(user_output_file, "r") as f:
content = f.read()
output_md5 = hashlib.md5(content.rstrip()).hexdigest()
output_md5 = hashlib.md5(content.rstrip().encode("utf-8")).hexdigest()
result = output_md5 == self._get_test_case_file_info(test_case_file_id)["stripped_output_md5"]
return output_md5, result
@ -68,19 +66,19 @@ class JudgeClient(object):
command = self._spj_config["command"].format(exe_path=self._spj_exe,
in_file_path=in_file_path,
user_out_file_path=user_out_file_path).split(" ")
seccomp_rule_name = self._spj_config["seccomp_rule"].encode("utf-8") if self._spj_config["seccomp_rule"] else None
seccomp_rule_name = self._spj_config["seccomp_rule"]
result = _judger.run(max_cpu_time=self._max_cpu_time * 3,
max_real_time=self._max_cpu_time * 9,
max_memory=self._max_memory * 3,
max_stack=128 * 1024 * 1024,
max_output_size=1024 * 1024 * 1024,
max_process_number=_judger.UNLIMITED,
exe_path=command[0].encode("utf-8"),
input_path=in_file_path.encode("utf-8"),
output_path="/tmp/spj.out".encode("utf-8"),
error_path="/tmp/spj.out".encode("utf-8"),
args=[item.encode("utf-8") for item in command[1::]],
env=[("PATH=" + os.environ.get("PATH", "")).encode("utf-8")],
exe_path=command[0],
input_path=in_file_path,
output_path="/tmp/spj.out",
error_path="/tmp/spj.out",
args=command[1::],
env=["PATH=" + os.environ.get("PATH", "")],
log_path=JUDGER_RUN_LOG_PATH,
seccomp_rule_name=seccomp_rule_name,
uid=RUN_USER_UID,
@ -88,19 +86,18 @@ class JudgeClient(object):
if result["result"] == _judger.RESULT_SUCCESS or \
(result["result"] == _judger.RESULT_RUNTIME_ERROR and
result["exit_code"] in [SPJ_WA, SPJ_ERROR] and result["signal"] == 0):
result["exit_code"] in [SPJ_WA, SPJ_ERROR] and result["signal"] == 0):
return result["exit_code"]
else:
return SPJ_ERROR
def _judge_one(self, test_case_file_id):
in_file = os.path.join(self._test_case_dir, self._get_test_case_file_info(test_case_file_id)["input_name"]).encode("utf-8")
user_output_file = os.path.join(self._submission_dir, test_case_file_id + ".out").encode("utf-8")
in_file = os.path.join(self._test_case_dir, self._get_test_case_file_info(test_case_file_id)["input_name"])
user_output_file = os.path.join(self._submission_dir, test_case_file_id + ".out")
command = self._run_config["command"].format(exe_path=self._exe_path, exe_dir=os.path.dirname(self._exe_path),
max_memory=self._max_memory / 1024).split(" ")
seccomp_rule_name = self._run_config["seccomp_rule"].encode("utf-8") if self._run_config["seccomp_rule"] else None
env = [item.encode("utf-8") for item in ["PATH=" + os.environ.get("PATH", "")] + self._run_config.get("env", [])]
env = ["PATH=" + os.environ.get("PATH", "")] + self._run_config.get("env", [])
run_result = _judger.run(max_cpu_time=self._max_cpu_time,
max_real_time=self._max_real_time,
@ -108,14 +105,14 @@ class JudgeClient(object):
max_stack=128 * 1024 * 1024,
max_output_size=1024 * 1024 * 1024,
max_process_number=_judger.UNLIMITED,
exe_path=command[0].encode("utf-8"),
exe_path=command[0],
input_path=in_file,
output_path=user_output_file,
error_path=user_output_file,
args=[item.encode("utf-8") for item in command[1::]],
args=command[1::],
env=env,
log_path=JUDGER_RUN_LOG_PATH,
seccomp_rule_name=seccomp_rule_name,
seccomp_rule_name=self._run_config["seccomp_rule"],
uid=RUN_USER_UID,
gid=RUN_GROUP_GID)
run_result["test_case"] = test_case_file_id
@ -144,7 +141,7 @@ class JudgeClient(object):
if self._output:
try:
with open(user_output_file, "r") as f:
run_result["output"] = f.read().decode("utf-8")
run_result["output"] = f.read()
except Exception:
pass
@ -153,7 +150,7 @@ class JudgeClient(object):
def run(self):
tmp_result = []
result = []
for test_case_file_id, _ in self._test_case_info["test_cases"].iteritems():
for test_case_file_id, _ in self._test_case_info["test_cases"].items():
tmp_result.append(self._pool.apply_async(_run, (self, test_case_file_id)))
self._pool.close()
self._pool.join()

91
server/server.py
View File

@ -1,21 +1,19 @@
# coding=utf-8
from __future__ import unicode_literals
import json
import os
import shutil
import uuid
import web
from flask import Flask, request, Response
from compiler import Compiler
from config import JUDGER_WORKSPACE_BASE, SPJ_SRC_DIR, SPJ_EXE_DIR
from exception import TokenVerificationFailed, CompileError, SPJCompileError,JudgeClientError
from exception import TokenVerificationFailed, CompileError, SPJCompileError, JudgeClientError
from judge_client import JudgeClient
from utils import server_info, logger, token
app = Flask(__name__)
DEBUG = os.environ.get("judger_debug") == "1"
app.debug = DEBUG
class InitSubmissionEnv(object):
@ -25,7 +23,7 @@ class InitSubmissionEnv(object):
def __enter__(self):
try:
os.mkdir(self.path)
os.chmod(self.path, 0777)
os.chmod(self.path, 0o777)
except Exception as e:
logger.exception(e)
raise JudgeClientError("failed to create runtime dir")
@ -40,26 +38,28 @@ class InitSubmissionEnv(object):
raise JudgeClientError("failed to clean runtime dir")
class JudgeServer(object):
def pong(self):
class JudgeServer:
@classmethod
def ping(cls):
data = server_info()
data["action"] = "pong"
return data
def judge(self, language_config, src, max_cpu_time, max_memory, test_case_id,
@classmethod
def judge(cls, language_config, src, max_cpu_time, max_memory, test_case_id,
spj_version=None, spj_config=None, spj_compile_config=None, spj_src=None, output=False):
# init
compile_config = language_config.get("compile")
run_config = language_config["run"]
submission_id = str(uuid.uuid4())
submission_id = uuid.uuid4().hex
if spj_version and spj_config:
spj_exe_path = os.path.join(SPJ_EXE_DIR, spj_config["exe_name"].format(spj_version=spj_version))
# spj src has not been compiled
if not os.path.isfile(spj_exe_path):
logger.warning("%s does not exists, spj src will be recompiled")
self.compile_spj(spj_version=spj_version, src=spj_src,
spj_compile_config=spj_compile_config)
cls.compile_spj(spj_version=spj_version, src=spj_src,
spj_compile_config=spj_compile_config)
with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=str(submission_id)) as submission_dir:
if compile_config:
@ -67,7 +67,7 @@ class JudgeServer(object):
# write source code into file
with open(src_path, "w") as f:
f.write(src.encode("utf-8"))
f.write(src)
# compile source code, return exe file path
exe_path = Compiler().compile(compile_config=compile_config,
@ -76,7 +76,7 @@ class JudgeServer(object):
else:
exe_path = os.path.join(submission_dir, run_config["exe_name"])
with open(exe_path, "w") as f:
f.write(src.encode("utf-8"))
f.write(src)
judge_client = JudgeClient(run_config=language_config["run"],
exe_path=exe_path,
@ -91,7 +91,8 @@ class JudgeServer(object):
return run_result
def compile_spj(self, spj_version, src, spj_compile_config):
@classmethod
def compile_spj(cls, spj_version, src, spj_compile_config):
spj_compile_config["src_name"] = spj_compile_config["src_name"].format(spj_version=spj_version)
spj_compile_config["exe_name"] = spj_compile_config["exe_name"].format(spj_version=spj_version)
@ -100,7 +101,7 @@ class JudgeServer(object):
# if spj source code not found, then write it into file
if not os.path.exists(spj_src_path):
with open(spj_src_path, "w") as f:
f.write(src.encode("utf-8"))
f.write(src)
try:
Compiler().compile(compile_config=spj_compile_config,
src_path=spj_src_path,
@ -110,56 +111,34 @@ class JudgeServer(object):
raise SPJCompileError(e.message)
return "success"
def POST(self):
_token = web.ctx.env.get("HTTP_X_JUDGE_SERVER_TOKEN", None)
@app.route('/', defaults={'path': ''})
@app.route('/<path:path>', methods=["POST"])
def server(path):
if path in ("judge", "ping", "compile_spj"):
_token = request.headers.get("X-Judge-Server-Token")
try:
if _token != token:
raise TokenVerificationFailed("invalid token")
if web.data():
try:
data = json.loads(web.data())
except Exception as e:
logger.info(web.data())
return {"ret": "ServerError", "data": "invalid json"}
else:
try:
data = request.json
except Exception:
data = {}
if web.ctx["path"] == "/judge":
callback = self.judge
elif web.ctx["path"] == "/ping":
callback = self.pong
elif web.ctx["path"] == "/compile_spj":
callback = self.compile_spj
else:
return json.dumps({"err": "InvalidMethod", "data": None})
return json.dumps({"err": None, "data": callback(**data)})
ret = {"err": None, "data": getattr(JudgeServer, path)(**data)}
except (CompileError, TokenVerificationFailed, SPJCompileError, JudgeClientError) as e:
logger.exception(e)
ret = dict()
ret["err"] = e.__class__.__name__
ret["data"] = e.message
return json.dumps(ret)
ret = {"err": e.__class__.__name__, "data": e.message}
except Exception as e:
logger.exception(e)
ret = dict()
ret["err"] = "JudgeClientError"
ret["data"] =e.__class__.__name__ + ":" + e.message
return json.dumps(ret)
urls = (
"/judge", "JudgeServer",
"/ping", "JudgeServer",
"/compile_spj", "JudgeServer"
)
ret = {"err": "JudgeClientError", "data": e.__class__.__name__ + " :" + str(e)}
else:
ret = {"err": "InvalidRequest", "data": "404"}
return Response(json.dumps(ret), mimetype='application/json')
if DEBUG:
logger.info("DEBUG=ON")
app = web.application(urls, globals())
wsgiapp = app.wsgifunc()
# gunicorn -w 4 -b 0.0.0.0:8080 server:wsgiapp
# gunicorn -w 4 -b 0.0.0.0:8080 server:app
if __name__ == "__main__":
app.run()
app.run(debug=DEBUG)

7
server/service.py
View File

@ -1,9 +1,7 @@
# coding=utf-8
from __future__ import unicode_literals
import os
import json
import os
import requests
import hashlib
from exception import JudgeServiceError
from utils import server_info, logger, token
@ -29,7 +27,6 @@ class JudgeService(object):
except Exception as e:
logger.exception("Heartbeat failed, response is {}".format(resp))
raise JudgeServiceError("Invalid heartbeat response")
def heartbeat(self):
data = server_info()

18
server/utils.py
View File

@ -1,17 +1,16 @@
# coding=utf-8
from __future__ import unicode_literals
import _judger
import psutil
import socket
import logging
import hashlib
import logging
import os
import socket
import psutil
from config import SERVER_LOG_PATH
from exception import JudgeClientError
logger = logging.getLogger(__name__)
handler = logging.FileHandler("/log/judge_server.log")
handler = logging.FileHandler(SERVER_LOG_PATH)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
@ -32,8 +31,7 @@ def get_token():
if token:
return token
else:
raise JudgeClientError("env 'token' not found")
raise JudgeClientError("env 'TOKEN' not found")
token = hashlib.sha256(get_token()).hexdigest()
token = hashlib.sha256(get_token().encode("utf-8")).hexdigest()