From 17515af25c1dd25c6ccc626583c0cc5c11ed08a5 Mon Sep 17 00:00:00 2001 From: David Poeschl Date: Tue, 19 Dec 2023 13:40:31 -0800 Subject: [PATCH] Include detailed run result info in DB & show in admin site (#10) * Send detailed submission run result info to site/database * Show SubmissionStateReason on site * Mark Build/TLE errors as graded immediately * Remove superfluous log --- sandbox/src/index.ts | 79 +++++++--------- sandbox/src/run/java.ts | 6 +- web/prisma/schema.prisma | 9 ++ .../admin/diff/[submissionId]/+page.server.ts | 3 +- .../routes/admin/submissions/+page.server.ts | 1 + web/src/routes/admin/submissions/+page.svelte | 8 ++ .../[submissionId]/+page.server.ts | 2 + .../submissions/[submissionId]/+page.svelte | 16 +++- web/src/routes/api/submission/+server.ts | 93 ++++++++++++++----- 9 files changed, 145 insertions(+), 72 deletions(-) diff --git a/sandbox/src/index.ts b/sandbox/src/index.ts index 3ec4564..d517139 100644 --- a/sandbox/src/index.ts +++ b/sandbox/src/index.ts @@ -9,6 +9,26 @@ import { runJava } from './run/java.js'; export const timeoutSeconds = 30; +const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]); +export type RunResultKind = z.infer; + +const RunResult = z + .object({ + kind: RunResultKind, + output: z.string().optional(), + exitCode: z.number().optional(), + runtimeMilliseconds: z.number().optional(), + resultKindReason: z.string().optional() + }) + .strict(); + +const submissionPostData = z + .object({ + submissionId: z.number(), + result: RunResult + }) + .strict(); + const submissionGetData = z .object({ success: z.boolean(), @@ -30,21 +50,9 @@ const submissionGetData = z }) .strict(); -export type RunResult = { - kind: RunResultKind, - teamOutput?: string, - exitCode?: number, - runtimeMilliseconds?: number - buildErrors?: string, - sandboxErrorText?: string, -} - -export enum RunResultKind { - CompileFailed, - TimeLimitExceeded, - Completed, - SandboxError -} +export type RunResult = z.infer; +type SubmissionGetData = z.infer; +type SubmissionPostData = z.infer; enum SubmissionProcessingResult { NoSubmissions, @@ -52,8 +60,6 @@ enum SubmissionProcessingResult { Error } -type SubmissionGetData = z.infer; - async function fetchQueuedSubmission(): Promise { const res = await fetch(submissionApiUrl, { method: 'GET' }); if (res.status !== 200) { @@ -107,16 +113,17 @@ async function cloneAndRun(submissionData: SubmissionGetData) { ); } catch (error) { runResult = { - kind: RunResultKind.SandboxError, - sandboxErrorText: `An unexpected error occurred: ${EOL} ${error}`}; + kind: 'SandboxError', + resultKindReason: `An unexpected error occurred: ${EOL} ${error}`}; } printRunResult(runResult); + const postBodyObject: SubmissionPostData = { submissionId: submissionData.submission.id, result: runResult }; const res = await fetch(urlJoin(adminUrl, 'api/submission'), { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(makePostBody(submissionData.submission.id, runResult)) + body: JSON.stringify(postBodyObject) }); if (res.status !== 200) { console.error('- POST: Failed with error code: ' + res.status + " " + res.statusText); @@ -136,42 +143,22 @@ function printRunResult(runResult: RunResult) { console.log(`- RESULT: ${getRunResultDisplayText()}`); function getRunResultDisplayText() { - if (runResult.kind == RunResultKind.SandboxError) { - return "Sandbox error: " + runResult.sandboxErrorText; + if (runResult.kind == 'SandboxError') { + return "Sandbox error: " + runResult.resultKindReason; } - if (runResult.kind == RunResultKind.CompileFailed) { + if (runResult.kind == 'CompileFailed') { return "Failed to compile"; } - if (runResult.kind == RunResultKind.TimeLimitExceeded) { - return `Time limit exceeded. Output Length: ${runResult.teamOutput?.length}.`; + if (runResult.kind == 'TimeLimitExceeded') { + return `Time limit exceeded. Output Length: ${runResult.output?.length}.`; } - return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.teamOutput?.length}. Exit Code: ${runResult.exitCode}`; + return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.output?.length}. Exit Code: ${runResult.exitCode}`; } } -function makePostBody(submissionId: number, runResult: RunResult): { submissionId: number, output: string } { - let output: string; - switch (runResult.kind) { - case RunResultKind.CompileFailed: - output = `[Build Errors] ${EOL} ${runResult.buildErrors}`; - break; - case RunResultKind.TimeLimitExceeded: - output = `${runResult.teamOutput} ${EOL} [Timeout after ${timeoutSeconds} seconds]`; - break; - case RunResultKind.Completed: - output = runResult.teamOutput!; - break; - case RunResultKind.SandboxError: - output = `[Sandbox Error] ${EOL} ${runResult.sandboxErrorText}`; - break; - } - - return { submissionId, output } -} - function validateEnv(): boolean { return ( process.env.ADMIN_URL !== undefined && diff --git a/sandbox/src/run/java.ts b/sandbox/src/run/java.ts index 137105a..d5c5c1a 100644 --- a/sandbox/src/run/java.ts +++ b/sandbox/src/run/java.ts @@ -23,7 +23,7 @@ export async function runJava( } catch(e) { const buildErrorText = e?.toString() ?? "Unknown build errors."; console.log("Build errors: " + buildErrorText); - return {kind: RunResultKind.CompileFailed, buildErrors: buildErrorText}; + return {kind: 'CompileFailed', resultKindReason: buildErrorText}; } console.log(`- RUN: ${mainClass}`); @@ -55,12 +55,12 @@ export async function runJava( if (completedNormally) { clearTimeout(timeoutHandle); - resolve({kind: RunResultKind.Completed, teamOutput: outputBuffer, + resolve({kind: 'Completed', output: outputBuffer, exitCode: child.exitCode!, runtimeMilliseconds}); } else { console.log(`Process terminated, total sandbox time: ${runtimeMilliseconds}ms`); - resolve({kind: RunResultKind.TimeLimitExceeded, teamOutput: outputBuffer}); + resolve({kind: 'TimeLimitExceeded', output: outputBuffer, resultKindReason: `Timeout after ${timeoutSeconds} seconds`}); } }); diff --git a/web/prisma/schema.prisma b/web/prisma/schema.prisma index 5741eab..2e88403 100644 --- a/web/prisma/schema.prisma +++ b/web/prisma/schema.prisma @@ -29,11 +29,20 @@ enum SubmissionState { Incorrect } +enum SubmissionStateReason { + BuildError + TimeLimitExceeded + IncorrectOverriddenAsCorrect + SandboxError +} + model Submission { id Int @id @default(autoincrement()) createdAt DateTime @default(now()) gradedAt DateTime? state SubmissionState + stateReason SubmissionStateReason? + stateReasonDetails String? actualOutput String? commitHash String diff String? diff --git a/web/src/routes/admin/diff/[submissionId]/+page.server.ts b/web/src/routes/admin/diff/[submissionId]/+page.server.ts index 5cad745..f37ea1a 100644 --- a/web/src/routes/admin/diff/[submissionId]/+page.server.ts +++ b/web/src/routes/admin/diff/[submissionId]/+page.server.ts @@ -1,7 +1,7 @@ import type { Actions, PageServerLoad } from './$types'; import { error, redirect } from '@sveltejs/kit'; import { db } from '$lib/server/prisma'; -import { SubmissionState } from '@prisma/client'; +import { SubmissionState, SubmissionStateReason } from '@prisma/client'; export const load = (async ({ params }) => { const submissionId = parseInt(params.submissionId); @@ -38,6 +38,7 @@ export const actions = { where: { id: submissionId }, data: { state: correctBool ? SubmissionState.Correct : SubmissionState.Incorrect, + stateReason : correctBool ? SubmissionStateReason.IncorrectOverriddenAsCorrect : null, message: message ? message.toString() : '', gradedAt: gradedTime } diff --git a/web/src/routes/admin/submissions/+page.server.ts b/web/src/routes/admin/submissions/+page.server.ts index 8d6e7db..a8e2180 100644 --- a/web/src/routes/admin/submissions/+page.server.ts +++ b/web/src/routes/admin/submissions/+page.server.ts @@ -13,6 +13,7 @@ export const load = (async () => { createdAt: row.createdAt, gradedAt: row.gradedAt, state: row.state, + stateReason: row.stateReason, problemName: problems.find((problem) => { return problem.id == row.problemId; })?.friendlyName, diff --git a/web/src/routes/admin/submissions/+page.svelte b/web/src/routes/admin/submissions/+page.svelte index 9e5eb4a..577f465 100644 --- a/web/src/routes/admin/submissions/+page.svelte +++ b/web/src/routes/admin/submissions/+page.svelte @@ -75,6 +75,14 @@ {:else if submission.state === 'Incorrect'} Incorrect {/if} + + {#if submission.stateReason === 'BuildError'} + Build Error + {:else if submission.stateReason === 'TimeLimitExceeded'} + Time Limit Exceeded + {:else if submission.stateReason === 'IncorrectOverriddenAsCorrect'} + Manually Graded + {/if} {submission.createdAt.toLocaleDateString() + diff --git a/web/src/routes/admin/submissions/[submissionId]/+page.server.ts b/web/src/routes/admin/submissions/[submissionId]/+page.server.ts index 44725a3..ef94ec5 100644 --- a/web/src/routes/admin/submissions/[submissionId]/+page.server.ts +++ b/web/src/routes/admin/submissions/[submissionId]/+page.server.ts @@ -22,6 +22,8 @@ export const load = (async ({ params }) => { return { id: submission.id, state: submission.state, + stateReason: submission.stateReason, + stateReasonDetails: submission.stateReasonDetails, teamName: team.name, problemName: problem.friendlyName, submitTime: submission.createdAt, diff --git a/web/src/routes/admin/submissions/[submissionId]/+page.svelte b/web/src/routes/admin/submissions/[submissionId]/+page.svelte index 05db0f9..93b1ea1 100644 --- a/web/src/routes/admin/submissions/[submissionId]/+page.svelte +++ b/web/src/routes/admin/submissions/[submissionId]/+page.svelte @@ -100,6 +100,14 @@ {:else if data.state === 'Incorrect'} Incorrect {/if} + + {#if data.stateReason === 'BuildError'} + Build Error + {:else if data.stateReason === 'TimeLimitExceeded'} + Time Limit Exceeded + {:else if data.stateReason === 'IncorrectOverriddenAsCorrect'} + Manually Graded + {/if} {data.submitTime.toLocaleDateString() + ' ' + data.submitTime.toLocaleTimeString()} @@ -119,7 +127,13 @@ Review Submission -{:else if data.state == 'Incorrect'} +{:else if data.state == 'Incorrect' && data.stateReason == 'BuildError'} +

Build Output

+ +{:else if data.state == 'Incorrect' && data.stateReason == 'TimeLimitExceeded'} +

Details

+ +{:else}

Output

Diff

diff --git a/web/src/routes/api/submission/+server.ts b/web/src/routes/api/submission/+server.ts index 93b2a24..acfb58a 100644 --- a/web/src/routes/api/submission/+server.ts +++ b/web/src/routes/api/submission/+server.ts @@ -1,5 +1,5 @@ import { db } from '$lib/server/prisma'; -import { SubmissionState } from '@prisma/client'; +import { SubmissionState, SubmissionStateReason } from '@prisma/client'; import { error, json } from '@sveltejs/kit'; import { z } from 'zod'; import type { RequestHandler } from './$types'; @@ -34,45 +34,96 @@ export const GET = (async () => { } }) satisfies RequestHandler; +const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]); +export type RunResultKind = z.infer; + +const RunResult = z + .object({ + kind: RunResultKind, + output: z.string().optional(), + exitCode: z.number().optional(), + runtimeMilliseconds: z.number().optional(), + resultKindReason: z.string().optional() + }) + .strict(); + const submissionPostData = z .object({ submissionId: z.number(), - output: z.string() + result: RunResult }) .strict(); export const POST = (async ({ request }) => { - const data = submissionPostData.safeParse(await request.json()); + const requestJson = await request.json(); + const data = submissionPostData.safeParse(requestJson); if (!data.success) { + console.log("Error: POST to Submission API failed to parse given object: " + JSON.stringify(requestJson)); throw error(400); } + const submission = await db.submission.findUnique({ where: { id: data.data.submissionId }, include: { problem: true } }); + if (!submission) { + console.log("Error: POST to Submission API for unknown submissionId: " + data.data.submissionId); return json({ success: false }); } + if (submission.state !== SubmissionState.Queued) { + console.log("Error: POST to Submission API for already judged submissionId: " + data.data.submissionId); return json({ success: false }); } - if (data.data.output.trimEnd() === submission.problem.realOutput.trimEnd()) { - await db.submission.update({ - where: { id: data.data.submissionId }, - data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.output } - }); - return json({ success: true }); - } else { - const diff = Diff.createTwoFilesPatch( - 'expected', - 'actual', - submission.problem.realOutput, - data.data.output - ); - await db.submission.update({ - where: { id: data.data.submissionId }, - data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.output } - }); - return json({ success: true }); + + switch (data.data.result.kind) { + case 'Completed': + if (data.data.result.output!.trimEnd() === submission.problem.realOutput.trimEnd()) { + await db.submission.update({ + where: { id: data.data.submissionId }, + data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.result.output, + stateReason: null, stateReasonDetails: null } + }); + return json({ success: true }); + } else { + const diff = Diff.createTwoFilesPatch( + 'expected', + 'actual', + submission.problem.realOutput, + data.data.result.output! + ); + await db.submission.update({ + where: { id: data.data.submissionId }, + data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.result.output, + stateReason: null, stateReasonDetails: null } + }); + return json({ success: true }); + } + + case 'CompileFailed': + console.log('compile failed...'); + await db.submission.update({ + where: { id: data.data.submissionId }, + data: { state: SubmissionState.Incorrect, gradedAt: new Date(), + stateReason: SubmissionStateReason.BuildError, stateReasonDetails: data.data.result.resultKindReason } + }); + return json({ success: true }); + + case 'TimeLimitExceeded': + await db.submission.update({ + where: { id: data.data.submissionId }, + data: { state: SubmissionState.Incorrect, gradedAt: new Date(), actualOutput: data.data.result.output, + stateReason: SubmissionStateReason.TimeLimitExceeded, stateReasonDetails: data.data.result.resultKindReason } + }); + return json({ success: true }); + + case 'SandboxError': + // TODO: Raise to admins somehow. For now, just mark stateReason so it *could* be observed + await db.submission.update({ + where: { id: data.data.submissionId }, + data: { stateReason: SubmissionStateReason.SandboxError, stateReasonDetails: data.data.result.resultKindReason } + }); + return json({ success: true }); } }) satisfies RequestHandler;