Include detailed run result info in DB & show in admin site (#10)

* Send detailed submission run result info to site/database

* Show SubmissionStateReason on site

* Mark Build/TLE errors as graded immediately

* Remove superfluous log

sandbox/src/index.ts

@@ -9,6 +9,26 @@ import { runJava } from './run/java.js';
 
 export const timeoutSeconds = 30;
 
+const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]);
+export type RunResultKind = z.infer<typeof RunResultKind>;
+  
+const RunResult = z
+	.object({
+		kind: RunResultKind,
+		output: z.string().optional(),
+		exitCode: z.number().optional(),
+		runtimeMilliseconds: z.number().optional(),
+		resultKindReason: z.string().optional()
+	})
+	.strict();
+
+const submissionPostData = z
+	.object({
+		submissionId: z.number(),
+		result: RunResult
+	})
+	.strict();
+
 const submissionGetData = z
 	.object({
 		success: z.boolean(),
@@ -30,21 +50,9 @@ const submissionGetData = z
 	})
 	.strict();
 
-export type RunResult = {
+export type RunResult = z.infer<typeof RunResult>;
-	kind: RunResultKind,
+type SubmissionGetData = z.infer<typeof submissionGetData>;
-	teamOutput?: string,
+type SubmissionPostData = z.infer<typeof submissionPostData>;
-	exitCode?: number,
-	runtimeMilliseconds?: number
-	buildErrors?: string,
-	sandboxErrorText?: string,
-}
-
-export enum RunResultKind {
-	CompileFailed,
-	TimeLimitExceeded,
-	Completed,
-	SandboxError
-}
 
 enum SubmissionProcessingResult {
 	NoSubmissions,
@@ -52,8 +60,6 @@ enum SubmissionProcessingResult {
 	Error
 }
 
-type SubmissionGetData = z.infer<typeof submissionGetData>;
-
 async function fetchQueuedSubmission(): Promise<SubmissionGetData | undefined> {
 	const res = await fetch(submissionApiUrl, { method: 'GET' });
 	if (res.status !== 200) {
@@ -107,16 +113,17 @@ async function cloneAndRun(submissionData: SubmissionGetData) {
 		);
 	} catch (error) {
 		runResult = {
-			kind: RunResultKind.SandboxError, 
+			kind: 'SandboxError', 
-			sandboxErrorText: `An unexpected error occurred: ${EOL} ${error}`};
+			resultKindReason: `An unexpected error occurred: ${EOL} ${error}`};
 	}
 
 	printRunResult(runResult);
 
+	const postBodyObject: SubmissionPostData = { submissionId: submissionData.submission.id, result: runResult };
 	const res = await fetch(urlJoin(adminUrl, 'api/submission'), {
 		method: 'POST',
 		headers: { 'Content-Type': 'application/json' },
-		body: JSON.stringify(makePostBody(submissionData.submission.id, runResult))
+		body: JSON.stringify(postBodyObject)
 	});
 	if (res.status !== 200) {
 		console.error('- POST: Failed with error code: ' + res.status + " " + res.statusText);
@@ -136,42 +143,22 @@ function printRunResult(runResult: RunResult) {
 	console.log(`- RESULT: ${getRunResultDisplayText()}`);
 
 	function getRunResultDisplayText() {
-		if (runResult.kind == RunResultKind.SandboxError) {
+		if (runResult.kind == 'SandboxError') {
-			return "Sandbox error: " + runResult.sandboxErrorText;
+			return "Sandbox error: " + runResult.resultKindReason;
 		}
 
-		if (runResult.kind == RunResultKind.CompileFailed) {
+		if (runResult.kind == 'CompileFailed') {
 			return "Failed to compile";
 		}
 
-		if (runResult.kind == RunResultKind.TimeLimitExceeded) {
+		if (runResult.kind == 'TimeLimitExceeded') {
-			return `Time limit exceeded. Output Length: ${runResult.teamOutput?.length}.`;
+			return `Time limit exceeded. Output Length: ${runResult.output?.length}.`;
 		}
 
-		return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.teamOutput?.length}. Exit Code: ${runResult.exitCode}`;
+		return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.output?.length}. Exit Code: ${runResult.exitCode}`;
 	}
 }
 
-function makePostBody(submissionId: number, runResult: RunResult): { submissionId: number, output: string } {
-	let output: string;
-	switch (runResult.kind) {
-		case RunResultKind.CompileFailed:
-			output = `[Build Errors] ${EOL} ${runResult.buildErrors}`;
-			break;
-		case RunResultKind.TimeLimitExceeded:
-			output = `${runResult.teamOutput} ${EOL} [Timeout after ${timeoutSeconds} seconds]`;
-			break;
-		case RunResultKind.Completed:
-			output = runResult.teamOutput!;
-			break;
-		case RunResultKind.SandboxError:
-			output = `[Sandbox Error] ${EOL} ${runResult.sandboxErrorText}`;
-			break;
-	}
-
-	return { submissionId, output }
-}
-
 function validateEnv(): boolean {
 	return (
 		process.env.ADMIN_URL !== undefined &&

sandbox/src/run/java.ts

@@ -23,7 +23,7 @@ export async function runJava(
 	} catch(e) {
 		const buildErrorText = e?.toString() ?? "Unknown build errors.";
 		console.log("Build errors: " + buildErrorText);
-		return {kind: RunResultKind.CompileFailed, buildErrors: buildErrorText};
+		return {kind: 'CompileFailed', resultKindReason: buildErrorText};
 	}
 
 	console.log(`- RUN: ${mainClass}`);
@@ -55,12 +55,12 @@ export async function runJava(
 
 			if (completedNormally) {
 				clearTimeout(timeoutHandle);
-				resolve({kind: RunResultKind.Completed, teamOutput: outputBuffer, 
+				resolve({kind: 'Completed', output: outputBuffer, 
 					exitCode: child.exitCode!, runtimeMilliseconds});
 			}
 			else {
 				console.log(`Process terminated, total sandbox time: ${runtimeMilliseconds}ms`);
-				resolve({kind: RunResultKind.TimeLimitExceeded, teamOutput: outputBuffer});
+				resolve({kind: 'TimeLimitExceeded', output: outputBuffer, resultKindReason: `Timeout after ${timeoutSeconds} seconds`});
 			}
 		});
 

web/prisma/schema.prisma

@@ -29,11 +29,20 @@ enum SubmissionState {
   Incorrect
 }
 
+enum SubmissionStateReason {
+  BuildError
+  TimeLimitExceeded
+  IncorrectOverriddenAsCorrect
+  SandboxError
+}
+
 model Submission {
   id           Int             @id @default(autoincrement())
   createdAt    DateTime        @default(now())
   gradedAt     DateTime?
   state        SubmissionState
+  stateReason  SubmissionStateReason?
+  stateReasonDetails  String?
   actualOutput String?
   commitHash   String
   diff         String?

web/src/routes/admin/diff/[submissionId]/+page.server.ts

@@ -1,7 +1,7 @@
 import type { Actions, PageServerLoad } from './$types';
 import { error, redirect } from '@sveltejs/kit';
 import { db } from '$lib/server/prisma';
-import { SubmissionState } from '@prisma/client';
+import { SubmissionState, SubmissionStateReason } from '@prisma/client';
 
 export const load = (async ({ params }) => {
 	const submissionId = parseInt(params.submissionId);
@@ -38,6 +38,7 @@ export const actions = {
 			where: { id: submissionId },
 			data: {
 				state: correctBool ? SubmissionState.Correct : SubmissionState.Incorrect,
+				stateReason : correctBool ? SubmissionStateReason.IncorrectOverriddenAsCorrect : null,
 				message: message ? message.toString() : '',
 				gradedAt: gradedTime
 			}

web/src/routes/admin/submissions/+page.server.ts

@@ -13,6 +13,7 @@ export const load = (async () => {
 				createdAt: row.createdAt,
 				gradedAt: row.gradedAt,
 				state: row.state,
+				stateReason: row.stateReason,
 				problemName: problems.find((problem) => {
 					return problem.id == row.problemId;
 				})?.friendlyName,

web/src/routes/admin/submissions/+page.svelte

@@ -75,6 +75,14 @@
 						{:else if submission.state === 'Incorrect'}
 							<span class="badge bg-danger">Incorrect</span>
 						{/if}
+
+						{#if submission.stateReason === 'BuildError'}
+							<span class="badge bg-danger opacity-50">Build Error</span>
+						{:else if submission.stateReason === 'TimeLimitExceeded'}
+							<span class="badge bg-danger opacity-50">Time Limit Exceeded</span>
+						{:else if submission.stateReason === 'IncorrectOverriddenAsCorrect'}
+							<span class="badge bg-success opacity-50">Manually Graded</span>
+						{/if}
 					</td>
 					<td
 						>{submission.createdAt.toLocaleDateString() +

web/src/routes/admin/submissions/[submissionId]/+page.server.ts

@@ -22,6 +22,8 @@ export const load = (async ({ params }) => {
 	return {
 		id: submission.id,
 		state: submission.state,
+		stateReason: submission.stateReason,
+		stateReasonDetails: submission.stateReasonDetails,
 		teamName: team.name,
 		problemName: problem.friendlyName,
 		submitTime: submission.createdAt,

web/src/routes/admin/submissions/[submissionId]/+page.svelte

@@ -100,6 +100,14 @@
 					{:else if data.state === 'Incorrect'}
 						<span class="badge bg-danger">Incorrect</span>
 					{/if}
+
+					{#if data.stateReason === 'BuildError'}
+						<span class="badge bg-danger opacity-50">Build Error</span>
+					{:else if data.stateReason === 'TimeLimitExceeded'}
+						<span class="badge bg-danger opacity-50">Time Limit Exceeded</span>
+					{:else if data.stateReason === 'IncorrectOverriddenAsCorrect'}
+						<span class="badge bg-success opacity-50">Manually Graded</span>
+					{/if}
 				</td>
 				<td>{data.submitTime.toLocaleDateString() + ' ' + data.submitTime.toLocaleTimeString()}</td>
 				<td>
@@ -119,7 +127,13 @@
 			<a href={'/admin/diff/' + data.id} class="btn btn-warning">Review Submission</a>
 		</div>
 	</div>
-{:else if data.state == 'Incorrect'}
+{:else if data.state == 'Incorrect' && data.stateReason == 'BuildError'}
+	<h3 style="text-align:center">Build Output</h3>
+	<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.stateReasonDetails}</textarea>
+{:else if data.state == 'Incorrect' && data.stateReason == 'TimeLimitExceeded'}
+	<h3 style="text-align:center">Details</h3>
+	<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.stateReasonDetails}</textarea>
+{:else}
 	<h3 style="text-align:center">Output</h3>
 	<textarea use:stretchTextarea class="code mb-3 form-control" disabled>{data.output}</textarea>
 	<h3 style="text-align:center">Diff</h3>

web/src/routes/api/submission/+server.ts

@@ -1,5 +1,5 @@
 import { db } from '$lib/server/prisma';
-import { SubmissionState } from '@prisma/client';
+import { SubmissionState, SubmissionStateReason } from '@prisma/client';
 import { error, json } from '@sveltejs/kit';
 import { z } from 'zod';
 import type { RequestHandler } from './$types';
@@ -34,32 +34,56 @@ export const GET = (async () => {
 	}
 }) satisfies RequestHandler;
 
+const RunResultKind = z.enum(["CompileFailed", "TimeLimitExceeded", "Completed", "SandboxError"]);
+export type RunResultKind = z.infer<typeof RunResultKind>;
+  
+const RunResult = z
+	.object({
+		kind: RunResultKind,
+		output: z.string().optional(),
+		exitCode: z.number().optional(),
+		runtimeMilliseconds: z.number().optional(),
+		resultKindReason: z.string().optional()
+	})
+	.strict();
+
 const submissionPostData = z
 	.object({
 		submissionId: z.number(),
-		output: z.string()
+		result: RunResult
 	})
 	.strict();
 
 export const POST = (async ({ request }) => {
-	const data = submissionPostData.safeParse(await request.json());
+	const requestJson = await request.json();
+	const data = submissionPostData.safeParse(requestJson);
 	if (!data.success) {
+		console.log("Error: POST to Submission API failed to parse given object: " + JSON.stringify(requestJson));
 		throw error(400);
 	}
+
 	const submission = await db.submission.findUnique({
 		where: { id: data.data.submissionId },
 		include: { problem: true }
 	});
+
 	if (!submission) {
+		console.log("Error: POST to Submission API for unknown submissionId: " + data.data.submissionId);
 		return json({ success: false });
 	}
+
 	if (submission.state !== SubmissionState.Queued) {
+		console.log("Error: POST to Submission API for already judged submissionId: " + data.data.submissionId);
 		return json({ success: false });
 	}
-	if (data.data.output.trimEnd() === submission.problem.realOutput.trimEnd()) {
+
+	switch (data.data.result.kind) {
+		case 'Completed':
+			if (data.data.result.output!.trimEnd() === submission.problem.realOutput.trimEnd()) {
 				await db.submission.update({
 					where: { id: data.data.submissionId },
-			data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.output }
+					data: { state: SubmissionState.Correct, gradedAt: new Date(), actualOutput: data.data.result.output,
+						stateReason: null, stateReasonDetails: null }
 				});
 				return json({ success: true });
 			} else {
@@ -67,11 +91,38 @@ export const POST = (async ({ request }) => {
 					'expected',
 					'actual',
 					submission.problem.realOutput,
-			data.data.output
+					data.data.result.output!
 				);
 				await db.submission.update({
 					where: { id: data.data.submissionId },
-			data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.output }
+					data: { state: SubmissionState.InReview, diff: diff, actualOutput: data.data.result.output,
+						stateReason: null, stateReasonDetails: null }
+				});
+				return json({ success: true });
+			}
+
+		case 'CompileFailed':
+			console.log('compile failed...');
+			await db.submission.update({
+				where: { id: data.data.submissionId },
+				data: { state: SubmissionState.Incorrect, gradedAt: new Date(),
+					stateReason: SubmissionStateReason.BuildError, stateReasonDetails: data.data.result.resultKindReason }
+			});
+			return json({ success: true });
+
+		case 'TimeLimitExceeded':
+			await db.submission.update({
+				where: { id: data.data.submissionId },
+				data: { state: SubmissionState.Incorrect, gradedAt: new Date(), actualOutput: data.data.result.output,
+					stateReason: SubmissionStateReason.TimeLimitExceeded, stateReasonDetails: data.data.result.resultKindReason }
+			});
+			return json({ success: true });
+
+		case 'SandboxError':
+			// TODO: Raise to admins somehow. For now, just mark stateReason so it *could* be observed
+			await db.submission.update({
+				where: { id: data.data.submissionId },
+				data: { stateReason: SubmissionStateReason.SandboxError, stateReasonDetails: data.data.result.resultKindReason }
 			});
 			return json({ success: true });
 	}