Include detailed run result info in DB & show in admin site (#10)

* Send detailed submission run result info to site/database

* Show SubmissionStateReason on site

* Mark Build/TLE errors as graded immediately

* Remove superfluous log
This commit is contained in:
David Poeschl 2023-12-19 13:40:31 -08:00 committed by GitHub
parent 9175386c87
commit 17515af25c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 145 additions and 72 deletions

View File

@ -9,6 +9,26 @@ import { runJava } from './run/java.js';
export const timeoutSeconds = 30; export const timeoutSeconds = 30;
const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]);
export type RunResultKind = z.infer<typeof RunResultKind>;
const RunResult = z
.object({
kind: RunResultKind,
output: z.string().optional(),
exitCode: z.number().optional(),
runtimeMilliseconds: z.number().optional(),
resultKindReason: z.string().optional()
})
.strict();
const submissionPostData = z
.object({
submissionId: z.number(),
result: RunResult
})
.strict();
const submissionGetData = z const submissionGetData = z
.object({ .object({
success: z.boolean(), success: z.boolean(),
@ -30,21 +50,9 @@ const submissionGetData = z
}) })
.strict(); .strict();
export type RunResult = { export type RunResult = z.infer<typeof RunResult>;
kind: RunResultKind, type SubmissionGetData = z.infer<typeof submissionGetData>;
teamOutput?: string, type SubmissionPostData = z.infer<typeof submissionPostData>;
exitCode?: number,
runtimeMilliseconds?: number
buildErrors?: string,
sandboxErrorText?: string,
}
export enum RunResultKind {
CompileFailed,
TimeLimitExceeded,
Completed,
SandboxError
}
enum SubmissionProcessingResult { enum SubmissionProcessingResult {
NoSubmissions, NoSubmissions,
@ -52,8 +60,6 @@ enum SubmissionProcessingResult {
Error Error
} }
type SubmissionGetData = z.infer<typeof submissionGetData>;
async function fetchQueuedSubmission(): Promise<SubmissionGetData | undefined> { async function fetchQueuedSubmission(): Promise<SubmissionGetData | undefined> {
const res = await fetch(submissionApiUrl, { method: 'GET' }); const res = await fetch(submissionApiUrl, { method: 'GET' });
if (res.status !== 200) { if (res.status !== 200) {
@ -107,16 +113,17 @@ async function cloneAndRun(submissionData: SubmissionGetData) {
); );
} catch (error) { } catch (error) {
runResult = { runResult = {
kind: RunResultKind.SandboxError, kind: 'SandboxError',
sandboxErrorText: `An unexpected error occurred: ${EOL} ${error}`}; resultKindReason: `An unexpected error occurred: ${EOL} ${error}`};
} }
printRunResult(runResult); printRunResult(runResult);
const postBodyObject: SubmissionPostData = { submissionId: submissionData.submission.id, result: runResult };
const res = await fetch(urlJoin(adminUrl, 'api/submission'), { const res = await fetch(urlJoin(adminUrl, 'api/submission'), {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(makePostBody(submissionData.submission.id, runResult)) body: JSON.stringify(postBodyObject)
}); });
if (res.status !== 200) { if (res.status !== 200) {
console.error('- POST: Failed with error code: ' + res.status + " " + res.statusText); console.error('- POST: Failed with error code: ' + res.status + " " + res.statusText);
@ -136,42 +143,22 @@ function printRunResult(runResult: RunResult) {
console.log(`- RESULT: ${getRunResultDisplayText()}`); console.log(`- RESULT: ${getRunResultDisplayText()}`);
function getRunResultDisplayText() { function getRunResultDisplayText() {
if (runResult.kind == RunResultKind.SandboxError) { if (runResult.kind == 'SandboxError') {
return "Sandbox error: " + runResult.sandboxErrorText; return "Sandbox error: " + runResult.resultKindReason;
} }
if (runResult.kind == RunResultKind.CompileFailed) { if (runResult.kind == 'CompileFailed') {
return "Failed to compile"; return "Failed to compile";
} }
if (runResult.kind == RunResultKind.TimeLimitExceeded) { if (runResult.kind == 'TimeLimitExceeded') {
return `Time limit exceeded. Output Length: ${runResult.teamOutput?.length}.`; return `Time limit exceeded. Output Length: ${runResult.output?.length}.`;
} }
return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.teamOutput?.length}. Exit Code: ${runResult.exitCode}`; return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.output?.length}. Exit Code: ${runResult.exitCode}`;
} }
} }
function makePostBody(submissionId: number, runResult: RunResult): { submissionId: number, output: string } {
let output: string;
switch (runResult.kind) {
case RunResultKind.CompileFailed:
output = `[Build Errors] ${EOL} ${runResult.buildErrors}`;
break;
case RunResultKind.TimeLimitExceeded:
output = `${runResult.teamOutput} ${EOL} [Timeout after ${timeoutSeconds} seconds]`;
break;
case RunResultKind.Completed:
output = runResult.teamOutput!;
break;
case RunResultKind.SandboxError:
output = `[Sandbox Error] ${EOL} ${runResult.sandboxErrorText}`;
break;
}
return { submissionId, output }
}
function validateEnv(): boolean { function validateEnv(): boolean {
return ( return (
process.env.ADMIN_URL !== undefined && process.env.ADMIN_URL !== undefined &&

View File

@ -23,7 +23,7 @@ export async function runJava(
} catch(e) { } catch(e) {
const buildErrorText = e?.toString() ?? "Unknown build errors."; const buildErrorText = e?.toString() ?? "Unknown build errors.";
console.log("Build errors: " + buildErrorText); console.log("Build errors: " + buildErrorText);
return {kind: RunResultKind.CompileFailed, buildErrors: buildErrorText}; return {kind: 'CompileFailed', resultKindReason: buildErrorText};
} }
console.log(`- RUN: ${mainClass}`); console.log(`- RUN: ${mainClass}`);
@ -55,12 +55,12 @@ export async function runJava(
if (completedNormally) { if (completedNormally) {
clearTimeout(timeoutHandle); clearTimeout(timeoutHandle);
resolve({kind: RunResultKind.Completed, teamOutput: outputBuffer, resolve({kind: 'Completed', output: outputBuffer,
exitCode: child.exitCode!, runtimeMilliseconds}); exitCode: child.exitCode!, runtimeMilliseconds});
} }
else { else {
console.log(`Process terminated, total sandbox time: ${runtimeMilliseconds}ms`); console.log(`Process terminated, total sandbox time: ${runtimeMilliseconds}ms`);
resolve({kind: RunResultKind.TimeLimitExceeded, teamOutput: outputBuffer}); resolve({kind: 'TimeLimitExceeded', output: outputBuffer, resultKindReason: `Timeout after ${timeoutSeconds} seconds`});
} }
}); });

View File

@ -29,11 +29,20 @@ enum SubmissionState {
Incorrect Incorrect
} }
enum SubmissionStateReason {
BuildError
TimeLimitExceeded
IncorrectOverriddenAsCorrect
SandboxError
}
model Submission { model Submission {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
createdAt DateTime @default(now()) createdAt DateTime @default(now())
gradedAt DateTime? gradedAt DateTime?
state SubmissionState state SubmissionState
stateReason SubmissionStateReason?
stateReasonDetails String?
actualOutput String? actualOutput String?
commitHash String commitHash String
diff String? diff String?

View File

@ -1,7 +1,7 @@
import type { Actions, PageServerLoad } from './$types'; import type { Actions, PageServerLoad } from './$types';
import { error, redirect } from '@sveltejs/kit'; import { error, redirect } from '@sveltejs/kit';
import { db } from '$lib/server/prisma'; import { db } from '$lib/server/prisma';
import { SubmissionState } from '@prisma/client'; import { SubmissionState, SubmissionStateReason } from '@prisma/client';
export const load = (async ({ params }) => { export const load = (async ({ params }) => {
const submissionId = parseInt(params.submissionId); const submissionId = parseInt(params.submissionId);
@ -38,6 +38,7 @@ export const actions = {
where: { id: submissionId }, where: { id: submissionId },
data: { data: {
state: correctBool ? SubmissionState.Correct : SubmissionState.Incorrect, state: correctBool ? SubmissionState.Correct : SubmissionState.Incorrect,
stateReason : correctBool ? SubmissionStateReason.IncorrectOverriddenAsCorrect : null,
message: message ? message.toString() : '', message: message ? message.toString() : '',
gradedAt: gradedTime gradedAt: gradedTime
} }

View File

@ -13,6 +13,7 @@ export const load = (async () => {
createdAt: row.createdAt, createdAt: row.createdAt,
gradedAt: row.gradedAt, gradedAt: row.gradedAt,
state: row.state, state: row.state,
stateReason: row.stateReason,
problemName: problems.find((problem) => { problemName: problems.find((problem) => {
return problem.id == row.problemId; return problem.id == row.problemId;
})?.friendlyName, })?.friendlyName,

View File

@ -75,6 +75,14 @@
{:else if submission.state === 'Incorrect'} {:else if submission.state === 'Incorrect'}
<span class="badge bg-danger">Incorrect</span> <span class="badge bg-danger">Incorrect</span>
{/if} {/if}
{#if submission.stateReason === 'BuildError'}
<span class="badge bg-danger opacity-50">Build Error</span>
{:else if submission.stateReason === 'TimeLimitExceeded'}
<span class="badge bg-danger opacity-50">Time Limit Exceeded</span>
{:else if submission.stateReason === 'IncorrectOverriddenAsCorrect'}
<span class="badge bg-success opacity-50">Manually Graded</span>
{/if}
</td> </td>
<td <td
>{submission.createdAt.toLocaleDateString() + >{submission.createdAt.toLocaleDateString() +

View File

@ -22,6 +22,8 @@ export const load = (async ({ params }) => {
return { return {
id: submission.id, id: submission.id,
state: submission.state, state: submission.state,
stateReason: submission.stateReason,
stateReasonDetails: submission.stateReasonDetails,
teamName: team.name, teamName: team.name,
problemName: problem.friendlyName, problemName: problem.friendlyName,
submitTime: submission.createdAt, submitTime: submission.createdAt,

View File

@ -100,6 +100,14 @@
{:else if data.state === 'Incorrect'} {:else if data.state === 'Incorrect'}
<span class="badge bg-danger">Incorrect</span> <span class="badge bg-danger">Incorrect</span>
{/if} {/if}
{#if data.stateReason === 'BuildError'}
<span class="badge bg-danger opacity-50">Build Error</span>
{:else if data.stateReason === 'TimeLimitExceeded'}
<span class="badge bg-danger opacity-50">Time Limit Exceeded</span>
{:else if data.stateReason === 'IncorrectOverriddenAsCorrect'}
<span class="badge bg-success opacity-50">Manually Graded</span>
{/if}
</td> </td>
<td>{data.submitTime.toLocaleDateString() + ' ' + data.submitTime.toLocaleTimeString()}</td> <td>{data.submitTime.toLocaleDateString() + ' ' + data.submitTime.toLocaleTimeString()}</td>
<td> <td>
@ -119,7 +127,13 @@
<a href={'/admin/diff/' + data.id} class="btn btn-warning">Review Submission</a> <a href={'/admin/diff/' + data.id} class="btn btn-warning">Review Submission</a>
</div> </div>
</div> </div>
{:else if data.state == 'Incorrect'} {:else if data.state == 'Incorrect' && data.stateReason == 'BuildError'}
<h3 style="text-align:center">Build Output</h3>
<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.stateReasonDetails}</textarea>
{:else if data.state == 'Incorrect' && data.stateReason == 'TimeLimitExceeded'}
<h3 style="text-align:center">Details</h3>
<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.stateReasonDetails}</textarea>
{:else}
<h3 style="text-align:center">Output</h3> <h3 style="text-align:center">Output</h3>
<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.output}</textarea> <textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.output}</textarea>
<h3 style="text-align:center">Diff</h3> <h3 style="text-align:center">Diff</h3>

View File

@ -1,5 +1,5 @@
import { db } from '$lib/server/prisma'; import { db } from '$lib/server/prisma';
import { SubmissionState } from '@prisma/client'; import { SubmissionState, SubmissionStateReason } from '@prisma/client';
import { error, json } from '@sveltejs/kit'; import { error, json } from '@sveltejs/kit';
import { z } from 'zod'; import { z } from 'zod';
import type { RequestHandler } from './$types'; import type { RequestHandler } from './$types';
@ -34,32 +34,56 @@ export const GET = (async () => {
} }
}) satisfies RequestHandler; }) satisfies RequestHandler;
const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]);
export type RunResultKind = z.infer<typeof RunResultKind>;
const RunResult = z
.object({
kind: RunResultKind,
output: z.string().optional(),
exitCode: z.number().optional(),
runtimeMilliseconds: z.number().optional(),
resultKindReason: z.string().optional()
})
.strict();
const submissionPostData = z const submissionPostData = z
.object({ .object({
submissionId: z.number(), submissionId: z.number(),
output: z.string() result: RunResult
}) })
.strict(); .strict();
export const POST = (async ({ request }) => { export const POST = (async ({ request }) => {
const data = submissionPostData.safeParse(await request.json()); const requestJson = await request.json();
const data = submissionPostData.safeParse(requestJson);
if (!data.success) { if (!data.success) {
console.log("Error: POST to Submission API failed to parse given object: " + JSON.stringify(requestJson));
throw error(400); throw error(400);
} }
const submission = await db.submission.findUnique({ const submission = await db.submission.findUnique({
where: { id: data.data.submissionId }, where: { id: data.data.submissionId },
include: { problem: true } include: { problem: true }
}); });
if (!submission) { if (!submission) {
console.log("Error: POST to Submission API for unknown submissionId: " + data.data.submissionId);
return json({ success: false }); return json({ success: false });
} }
if (submission.state !== SubmissionState.Queued) { if (submission.state !== SubmissionState.Queued) {
console.log("Error: POST to Submission API for already judged submissionId: " + data.data.submissionId);
return json({ success: false }); return json({ success: false });
} }
if (data.data.output.trimEnd() === submission.problem.realOutput.trimEnd()) {
switch (data.data.result.kind) {
case 'Completed':
if (data.data.result.output!.trimEnd() === submission.problem.realOutput.trimEnd()) {
await db.submission.update({ await db.submission.update({
where: { id: data.data.submissionId }, where: { id: data.data.submissionId },
data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.output } data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.result.output,
stateReason: null, stateReasonDetails: null }
}); });
return json({ success: true }); return json({ success: true });
} else { } else {
@ -67,11 +91,38 @@ export const POST = (async ({ request }) => {
'expected', 'expected',
'actual', 'actual',
submission.problem.realOutput, submission.problem.realOutput,
data.data.output data.data.result.output!
); );
await db.submission.update({ await db.submission.update({
where: { id: data.data.submissionId }, where: { id: data.data.submissionId },
data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.output } data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.result.output,
stateReason: null, stateReasonDetails: null }
});
return json({ success: true });
}
case 'CompileFailed':
console.log('compile failed...');
await db.submission.update({
where: { id: data.data.submissionId },
data: { state: SubmissionState.Incorrect, gradedAt: new Date(),
stateReason: SubmissionStateReason.BuildError, stateReasonDetails: data.data.result.resultKindReason }
});
return json({ success: true });
case 'TimeLimitExceeded':
await db.submission.update({
where: { id: data.data.submissionId },
data: { state: SubmissionState.Incorrect, gradedAt: new Date(), actualOutput: data.data.result.output,
stateReason: SubmissionStateReason.TimeLimitExceeded, stateReasonDetails: data.data.result.resultKindReason }
});
return json({ success: true });
case 'SandboxError':
// TODO: Raise to admins somehow. For now, just mark stateReason so it *could* be observed
await db.submission.update({
where: { id: data.data.submissionId },
data: { stateReason: SubmissionStateReason.SandboxError, stateReasonDetails: data.data.result.resultKindReason }
}); });
return json({ success: true }); return json({ success: true });
} }