orosmatthew/bw-hspc-contest-env
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Include detailed run result info in DB & show in admin site (#10)

Browse Source 
* Send detailed submission run result info to site/database

* Show SubmissionStateReason on site

* Mark Build/TLE errors as graded immediately

* Remove superfluous log
This commit is contained in:
David Poeschl 2023-12-19 13:40:31 -08:00 committed by GitHub
parent 9175386c87
commit 17515af25c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 145 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
sandbox/src/index.ts
View File

@ -9,6 +9,26 @@ import { runJava } from './run/java.js';
export const timeoutSeconds = 30;
const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]);
export type RunResultKind = z.infer<typeof RunResultKind>;
const RunResult = z
.object({
kind: RunResultKind,
output: z.string().optional(),
exitCode: z.number().optional(),
runtimeMilliseconds: z.number().optional(),
resultKindReason: z.string().optional()
})
.strict();
const submissionPostData = z
.object({
submissionId: z.number(),
result: RunResult
})
.strict();
const submissionGetData = z
.object({
success: z.boolean(),
@ -30,21 +50,9 @@ const submissionGetData = z
})
.strict();
export type RunResult = {
kind: RunResultKind,
teamOutput?: string,
exitCode?: number,
runtimeMilliseconds?: number
buildErrors?: string,
sandboxErrorText?: string,
}
export enum RunResultKind {
CompileFailed,
TimeLimitExceeded,
Completed,
SandboxError
}
export type RunResult = z.infer<typeof RunResult>;
type SubmissionGetData = z.infer<typeof submissionGetData>;
type SubmissionPostData = z.infer<typeof submissionPostData>;
enum SubmissionProcessingResult {
NoSubmissions,
@ -52,8 +60,6 @@ enum SubmissionProcessingResult {
Error
}
type SubmissionGetData = z.infer<typeof submissionGetData>;
async function fetchQueuedSubmission(): Promise<SubmissionGetData | undefined> {
const res = await fetch(submissionApiUrl, { method: 'GET' });
if (res.status !== 200) {
@ -107,16 +113,17 @@ async function cloneAndRun(submissionData: SubmissionGetData) {
);
} catch (error) {
runResult = {
kind: RunResultKind.SandboxError,
sandboxErrorText: `An unexpected error occurred: ${EOL} ${error}`};
kind: 'SandboxError',
resultKindReason: `An unexpected error occurred: ${EOL} ${error}`};
}
printRunResult(runResult);
const postBodyObject: SubmissionPostData = { submissionId: submissionData.submission.id, result: runResult };
const res = await fetch(urlJoin(adminUrl, 'api/submission'), {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(makePostBody(submissionData.submission.id, runResult))
body: JSON.stringify(postBodyObject)
});
if (res.status !== 200) {
console.error('- POST: Failed with error code: ' + res.status + " " + res.statusText);
@ -136,42 +143,22 @@ function printRunResult(runResult: RunResult) {
console.log(`- RESULT: ${getRunResultDisplayText()}`);
function getRunResultDisplayText() {
if (runResult.kind == RunResultKind.SandboxError) {
return "Sandbox error: " + runResult.sandboxErrorText;
if (runResult.kind == 'SandboxError') {
return "Sandbox error: " + runResult.resultKindReason;
}
if (runResult.kind == RunResultKind.CompileFailed) {
if (runResult.kind == 'CompileFailed') {
return "Failed to compile";
}
if (runResult.kind == RunResultKind.TimeLimitExceeded) {
return `Time limit exceeded. Output Length: ${runResult.teamOutput?.length}.`;
if (runResult.kind == 'TimeLimitExceeded') {
return `Time limit exceeded. Output Length: ${runResult.output?.length}.`;
}
return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.teamOutput?.length}. Exit Code: ${runResult.exitCode}`;
return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.output?.length}. Exit Code: ${runResult.exitCode}`;
}
}
function makePostBody(submissionId: number, runResult: RunResult): { submissionId: number, output: string } {
let output: string;
switch (runResult.kind) {
case RunResultKind.CompileFailed:
output = `[Build Errors] ${EOL} ${runResult.buildErrors}`;
break;
case RunResultKind.TimeLimitExceeded:
output = `${runResult.teamOutput} ${EOL} [Timeout after ${timeoutSeconds} seconds]`;
break;
case RunResultKind.Completed:
output = runResult.teamOutput!;
break;
case RunResultKind.SandboxError:
output = `[Sandbox Error] ${EOL} ${runResult.sandboxErrorText}`;
break;
}
return { submissionId, output }
}
function validateEnv(): boolean {
return (
process.env.ADMIN_URL !== undefined &&

6
sandbox/src/run/java.ts
View File

@ -23,7 +23,7 @@ export async function runJava(
} catch(e) {
const buildErrorText = e?.toString() ?? "Unknown build errors.";
console.log("Build errors: " + buildErrorText);
return {kind: RunResultKind.CompileFailed, buildErrors: buildErrorText};
return {kind: 'CompileFailed', resultKindReason: buildErrorText};
}
console.log(`- RUN: ${mainClass}`);
@ -55,12 +55,12 @@ export async function runJava(
if (completedNormally) {
clearTimeout(timeoutHandle);
resolve({kind: RunResultKind.Completed, teamOutput: outputBuffer,
resolve({kind: 'Completed', output: outputBuffer,
exitCode: child.exitCode!, runtimeMilliseconds});
}
else {
console.log(`Process terminated, total sandbox time: ${runtimeMilliseconds}ms`);
resolve({kind: RunResultKind.TimeLimitExceeded, teamOutput: outputBuffer});
resolve({kind: 'TimeLimitExceeded', output: outputBuffer, resultKindReason: `Timeout after ${timeoutSeconds} seconds`});
}
});

9
web/prisma/schema.prisma
View File

@ -29,11 +29,20 @@ enum SubmissionState {
Incorrect
}
enum SubmissionStateReason {
BuildError
TimeLimitExceeded
IncorrectOverriddenAsCorrect
SandboxError
}
model Submission {
id Int @id @default(autoincrement())
createdAt DateTime @default(now())
gradedAt DateTime?
state SubmissionState
stateReason SubmissionStateReason?
stateReasonDetails String?
actualOutput String?
commitHash String
diff String?

3
web/src/routes/admin/diff/[submissionId]/+page.server.ts
View File

@ -1,7 +1,7 @@
import type { Actions, PageServerLoad } from './$types';
import { error, redirect } from '@sveltejs/kit';
import { db } from '$lib/server/prisma';
import { SubmissionState } from '@prisma/client';
import { SubmissionState, SubmissionStateReason } from '@prisma/client';
export const load = (async ({ params }) => {
const submissionId = parseInt(params.submissionId);
@ -38,6 +38,7 @@ export const actions = {
where: { id: submissionId },
data: {
state: correctBool ? SubmissionState.Correct : SubmissionState.Incorrect,
stateReason : correctBool ? SubmissionStateReason.IncorrectOverriddenAsCorrect : null,
message: message ? message.toString() : '',
gradedAt: gradedTime
}

1
web/src/routes/admin/submissions/+page.server.ts
View File

@ -13,6 +13,7 @@ export const load = (async () => {
createdAt: row.createdAt,
gradedAt: row.gradedAt,
state: row.state,
stateReason: row.stateReason,
problemName: problems.find((problem) => {
return problem.id == row.problemId;
})?.friendlyName,

8
web/src/routes/admin/submissions/+page.svelte
View File

@ -75,6 +75,14 @@
{:else if submission.state === 'Incorrect'}
<span class="badge bg-danger">Incorrect</span>
{/if}
{#if submission.stateReason === 'BuildError'}
<span class="badge bg-danger opacity-50">Build Error</span>
{:else if submission.stateReason === 'TimeLimitExceeded'}
<span class="badge bg-danger opacity-50">Time Limit Exceeded</span>
{:else if submission.stateReason === 'IncorrectOverriddenAsCorrect'}
<span class="badge bg-success opacity-50">Manually Graded</span>
{/if}
</td>
<td
>{submission.createdAt.toLocaleDateString() +

2
web/src/routes/admin/submissions/[submissionId]/+page.server.ts
View File

@ -22,6 +22,8 @@ export const load = (async ({ params }) => {
return {
id: submission.id,
state: submission.state,
stateReason: submission.stateReason,
stateReasonDetails: submission.stateReasonDetails,
teamName: team.name,
problemName: problem.friendlyName,
submitTime: submission.createdAt,

16
web/src/routes/admin/submissions/[submissionId]/+page.svelte
View File

@ -100,6 +100,14 @@
{:else if data.state === 'Incorrect'}
<span class="badge bg-danger">Incorrect</span>
{/if}
{#if data.stateReason === 'BuildError'}
<span class="badge bg-danger opacity-50">Build Error</span>
{:else if data.stateReason === 'TimeLimitExceeded'}
<span class="badge bg-danger opacity-50">Time Limit Exceeded</span>
{:else if data.stateReason === 'IncorrectOverriddenAsCorrect'}
<span class="badge bg-success opacity-50">Manually Graded</span>
{/if}
</td>
<td>{data.submitTime.toLocaleDateString() + ' ' + data.submitTime.toLocaleTimeString()}</td>
<td>
@ -119,7 +127,13 @@
<a href={'/admin/diff/' + data.id} class="btn btn-warning">Review Submission</a>
</div>
</div>
{:else if data.state == 'Incorrect'}
{:else if data.state == 'Incorrect' && data.stateReason == 'BuildError'}
<h3 style="text-align:center">Build Output</h3>
<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.stateReasonDetails}</textarea>
{:else if data.state == 'Incorrect' && data.stateReason == 'TimeLimitExceeded'}
<h3 style="text-align:center">Details</h3>
<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.stateReasonDetails}</textarea>
{:else}
<h3 style="text-align:center">Output</h3>
<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.output}</textarea>
<h3 style="text-align:center">Diff</h3>

65
web/src/routes/api/submission/+server.ts
View File

@ -1,5 +1,5 @@
import { db } from '$lib/server/prisma';
import { SubmissionState } from '@prisma/client';
import { SubmissionState, SubmissionStateReason } from '@prisma/client';
import { error, json } from '@sveltejs/kit';
import { z } from 'zod';
import type { RequestHandler } from './$types';
@ -34,32 +34,56 @@ export const GET = (async () => {
}
}) satisfies RequestHandler;
const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]);
export type RunResultKind = z.infer<typeof RunResultKind>;
const RunResult = z
.object({
kind: RunResultKind,
output: z.string().optional(),
exitCode: z.number().optional(),
runtimeMilliseconds: z.number().optional(),
resultKindReason: z.string().optional()
})
.strict();
const submissionPostData = z
.object({
submissionId: z.number(),
output: z.string()
result: RunResult
})
.strict();
export const POST = (async ({ request }) => {
const data = submissionPostData.safeParse(await request.json());
const requestJson = await request.json();
const data = submissionPostData.safeParse(requestJson);
if (!data.success) {
console.log("Error: POST to Submission API failed to parse given object: " + JSON.stringify(requestJson));
throw error(400);
}
const submission = await db.submission.findUnique({
where: { id: data.data.submissionId },
include: { problem: true }
});
if (!submission) {
console.log("Error: POST to Submission API for unknown submissionId: " + data.data.submissionId);
return json({ success: false });
}
if (submission.state !== SubmissionState.Queued) {
console.log("Error: POST to Submission API for already judged submissionId: " + data.data.submissionId);
return json({ success: false });
}
if (data.data.output.trimEnd() === submission.problem.realOutput.trimEnd()) {
switch (data.data.result.kind) {
case 'Completed':
if (data.data.result.output!.trimEnd() === submission.problem.realOutput.trimEnd()) {
await db.submission.update({
where: { id: data.data.submissionId },
data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.output }
data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.result.output,
stateReason: null, stateReasonDetails: null }
});
return json({ success: true });
} else {
@ -67,11 +91,38 @@ export const POST = (async ({ request }) => {
'expected',
'actual',
submission.problem.realOutput,
data.data.output
data.data.result.output!
);
await db.submission.update({
where: { id: data.data.submissionId },
data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.output }
data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.result.output,
stateReason: null, stateReasonDetails: null }
});
return json({ success: true });
}
case 'CompileFailed':
console.log('compile failed...');
await db.submission.update({
where: { id: data.data.submissionId },
data: { state: SubmissionState.Incorrect, gradedAt: new Date(),
stateReason: SubmissionStateReason.BuildError, stateReasonDetails: data.data.result.resultKindReason }
});
return json({ success: true });
case 'TimeLimitExceeded':
await db.submission.update({
where: { id: data.data.submissionId },
data: { state: SubmissionState.Incorrect, gradedAt: new Date(), actualOutput: data.data.result.output,
stateReason: SubmissionStateReason.TimeLimitExceeded, stateReasonDetails: data.data.result.resultKindReason }
});
return json({ success: true });
case 'SandboxError':
// TODO: Raise to admins somehow. For now, just mark stateReason so it *could* be observed
await db.submission.update({
where: { id: data.data.submissionId },
data: { stateReason: SubmissionStateReason.SandboxError, stateReasonDetails: data.data.result.resultKindReason }
});
return json({ success: true });
}