Sandbox: Ensure runs are serialized, add detailed logging (#9)

Specific fixes to ensure runs are serialized:

- Added missing await when starting clone/run
- The runJava method now has a single exit path that can only execute when the process has guaranteed been 'close'd. The timeout detection previously sent SIGKILL and assumed it worked, but it was not working on my machine and the student code would continue to run indefinitely after returning.
- To get the SIGKILL to kill the submitted app's process, I added end/destroy calls to the process stdin/stdout/stderr streams.

Added lots of logging while diagnosing these issues, and it seems useful to keep. As part of this, the contest/team names are now included in the 'submissions' web api.

Also, I've structured the result data in more detail within the Sandbox, tracking whether a run succeeded or the way in which it failed. I'm collapsing these back down to just a string "output" with the preexisting failure markers (e.g. "[Timeout after 30 seconds]"), but would like to pass this over to the site/DB in a structured form eventually.
This commit is contained in:
David Poeschl 2023-12-17 08:30:16 -08:00 committed by GitHub
parent 6ee97baa2f
commit 79fd96e1d3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 217 additions and 49 deletions

View File

@ -1,12 +1,14 @@
import dotenv from 'dotenv'; import dotenv from 'dotenv';
import fs from 'fs-extra'; import fs from 'fs-extra';
import urlJoin from 'url-join'; import urlJoin from 'url-join';
import { z } from 'zod'; import { string, z } from 'zod';
import os from 'os'; import os, { EOL } from 'os';
import { join } from 'path'; import { join } from 'path';
import { simpleGit, SimpleGit } from 'simple-git'; import { simpleGit, SimpleGit } from 'simple-git';
import { runJava } from './run/java.js'; import { runJava } from './run/java.js';
export const timeoutSeconds = 30;
const submissionGetData = z const submissionGetData = z
.object({ .object({
success: z.boolean(), success: z.boolean(),
@ -14,7 +16,9 @@ const submissionGetData = z
.object({ .object({
id: z.number(), id: z.number(),
contestId: z.number(), contestId: z.number(),
contestName: z.string(),
teamId: z.number(), teamId: z.number(),
teamName: z.string(),
problem: z.object({ problem: z.object({
id: z.number(), id: z.number(),
pascalName: z.string(), pascalName: z.string(),
@ -26,14 +30,37 @@ const submissionGetData = z
}) })
.strict(); .strict();
export type RunResult = {
kind: RunResultKind,
teamOutput?: string,
exitCode?: number,
runtimeMilliseconds?: number
buildErrors?: string,
sandboxErrorText?: string,
}
export enum RunResultKind {
CompileFailed,
TimeLimitExceeded,
Completed,
SandboxError
}
enum SubmissionProcessingResult {
NoSubmissions,
SubmissionProcessed,
Error
}
type SubmissionGetData = z.infer<typeof submissionGetData>; type SubmissionGetData = z.infer<typeof submissionGetData>;
async function fetchQueuedSubmission(): Promise<SubmissionGetData | undefined> { async function fetchQueuedSubmission(): Promise<SubmissionGetData | undefined> {
const res = await fetch(urlJoin(adminUrl, 'api/submission'), { method: 'GET' }); const res = await fetch(submissionApiUrl, { method: 'GET' });
if (res.status !== 200) { if (res.status !== 200) {
console.error('Failed to fetch submission'); console.error(`Failed to fetch from ${submissionApiUrl} with status: ${res.status} ${res.statusText}`);
return undefined; return undefined;
} }
const data = submissionGetData.parse(await res.json()); const data = submissionGetData.parse(await res.json());
if (!data.success) { if (!data.success) {
return undefined; return undefined;
@ -54,20 +81,24 @@ async function cloneAndRun(submissionData: SubmissionGetData) {
const repoDir = join(buildDir, 'src'); const repoDir = join(buildDir, 'src');
fs.mkdirSync(repoDir); fs.mkdirSync(repoDir);
const git: SimpleGit = simpleGit({ baseDir: repoDir }); const teamRepoUrl = urlJoin(
await git.clone(
urlJoin(
repoUrl, repoUrl,
submissionData.submission.contestId.toString(), submissionData.submission.contestId.toString(),
submissionData.submission.teamId.toString() + '.git' submissionData.submission.teamId.toString() + '.git'
), );
console.log(`- CLONE: from ${teamRepoUrl}`);
const git: SimpleGit = simpleGit({ baseDir: repoDir });
await git.clone(
teamRepoUrl,
'.' '.'
); );
await git.checkout(submissionData.submission.commitHash); await git.checkout(submissionData.submission.commitHash);
const problemName = submissionData.submission.problem.pascalName; const problemName = submissionData.submission.problem.pascalName;
let output: string; let runResult: RunResult;
try { try {
output = await runJava( runResult = await runJava(
javaBinPath, javaBinPath,
buildDir, buildDir,
join(repoDir, problemName, problemName + '.java'), join(repoDir, problemName, problemName + '.java'),
@ -75,21 +106,70 @@ async function cloneAndRun(submissionData: SubmissionGetData) {
submissionData.submission.problem.realInput submissionData.submission.problem.realInput
); );
} catch (error) { } catch (error) {
output = `[An error occurred while running]\n${error}`; runResult = {
kind: RunResultKind.SandboxError,
sandboxErrorText: `An unexpected error occurred: ${EOL} ${error}`};
} }
printRunResult(runResult);
const res = await fetch(urlJoin(adminUrl, 'api/submission'), { const res = await fetch(urlJoin(adminUrl, 'api/submission'), {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ submissionId: submissionData.submission.id, output: output }) body: JSON.stringify(makePostBody(submissionData.submission.id, runResult))
}); });
if (res.status !== 200) { if (res.status !== 200) {
console.error('Failed to POST output'); console.error('- POST: Failed with error code: ' + res.status + " " + res.statusText);
return;
} }
const data = (await res.json()) as { success: boolean }; const data = (await res.json()) as { success: boolean };
if (!data.success) { if (!data.success) {
console.error('Output POST unsuccessful'); console.error('- POST: Failed with response: ' + JSON.stringify(data));
return;
} }
console.log(`- POST: Succeeded`);
}
function printRunResult(runResult: RunResult) {
console.log(`- RESULT: ${getRunResultDisplayText()}`);
function getRunResultDisplayText() {
if (runResult.kind == RunResultKind.SandboxError) {
return "Sandbox error: " + runResult.sandboxErrorText;
}
if (runResult.kind == RunResultKind.CompileFailed) {
return "Failed to compile";
}
if (runResult.kind == RunResultKind.TimeLimitExceeded) {
return `Time limit exceeded. Output Length: ${runResult.teamOutput?.length}.`;
}
return `Run completed. Time: ${runResult.runtimeMilliseconds}ms. Output Length: ${runResult.teamOutput?.length}. Exit Code: ${runResult.exitCode}`;
}
}
function makePostBody(submissionId: number, runResult: RunResult): { submissionId: number, output: string } {
let output: string;
switch (runResult.kind) {
case RunResultKind.CompileFailed:
output = `[Build Errors] ${EOL} ${runResult.buildErrors}`;
break;
case RunResultKind.TimeLimitExceeded:
output = `${runResult.teamOutput} ${EOL} [Timeout after ${timeoutSeconds} seconds]`;
break;
case RunResultKind.Completed:
output = runResult.teamOutput!;
break;
case RunResultKind.SandboxError:
output = `[Sandbox Error] ${EOL} ${runResult.sandboxErrorText}`;
break;
}
return { submissionId, output }
} }
function validateEnv(): boolean { function validateEnv(): boolean {
@ -110,30 +190,88 @@ const adminUrl = process.env.ADMIN_URL as string;
const repoUrl = process.env.REPO_URL as string; const repoUrl = process.env.REPO_URL as string;
const javaBinPath = process.env.JAVA_PATH as string; const javaBinPath = process.env.JAVA_PATH as string;
async function loop() { const submissionApiUrl = urlJoin(adminUrl, 'api/submission');
async function processNextSubmission(): Promise<SubmissionProcessingResult> {
let submissionData: SubmissionGetData | undefined; let submissionData: SubmissionGetData | undefined;
try { try {
submissionData = await fetchQueuedSubmission(); submissionData = await fetchQueuedSubmission();
} catch { } catch (e) {
console.error('Failed to fetch submission'); console.error(`Failed to fetch from ${submissionApiUrl} with error: ${e}`);
return; return SubmissionProcessingResult.Error;
} }
if (!submissionData) { if (!submissionData) {
console.error('Unable to fetch submission data'); console.error('Unable to fetch submission data');
} else { return SubmissionProcessingResult.Error;
}
if (!submissionData.submission) {
return SubmissionProcessingResult.NoSubmissions;
}
printSubmissionHeader(submissionData);
let processingResult: SubmissionProcessingResult;
try { try {
cloneAndRun(submissionData); await cloneAndRun(submissionData);
processingResult = SubmissionProcessingResult.SubmissionProcessed;
} catch { } catch {
console.error('Unable to clone and run'); console.error('Failed to clone/run.');
processingResult = SubmissionProcessingResult.Error;
} }
printSubmissionFooter(submissionData);
return processingResult;
}
function printSubmissionHeader(submissionData: SubmissionGetData) {
const submission = submissionData.submission;
if (!submission) {
return;
} }
console.log(`--- Submission ${submission.id} ---`);
console.log(`- INFO: Contest ${submission.contestId} '${submission.contestName}', ` +
`Team ${submission.teamId} '${submission.teamName}', ` +
`Problem ${submission.problem.id} '${submission.problem.pascalName}', ` +
`SHA '${submission.commitHash}'`);
}
function printSubmissionFooter(submissionData: SubmissionGetData) {
const submission = submissionData.submission;
if (!submission) {
return;
}
console.log(`--- End Submission ${submission.id} ---`);
} }
async function run() { async function run() {
console.log("Sandbox started. Periodically checking for submissions.");
let iterationsSinceProcessedSubmission = 0;
let anySubmissionsProcessed = false;
while (true) { while (true) {
await loop(); switch (await processNextSubmission()) {
case SubmissionProcessingResult.SubmissionProcessed:
iterationsSinceProcessedSubmission = 0;
anySubmissionsProcessed = true;
break;
case SubmissionProcessingResult.NoSubmissions:
if (iterationsSinceProcessedSubmission > 0 && iterationsSinceProcessedSubmission % 6 == 0) {
const numMinutes = iterationsSinceProcessedSubmission / 6;
console.log(`${numMinutes} minute${(numMinutes > 1 ? 's' : '')} since ` +
`${(anySubmissionsProcessed ? `last submission processed` : `sandbox startup with no submissions`)}`);
}
await new Promise((resolve) => setTimeout(resolve, 10000)); await new Promise((resolve) => setTimeout(resolve, 10000));
iterationsSinceProcessedSubmission++;
break;
case SubmissionProcessingResult.Error:
await new Promise((resolve) => setTimeout(resolve, 10000));
break;
}
} }
} }

View File

@ -1,6 +1,7 @@
import { join } from 'path'; import { join } from 'path';
import { exec, spawn } from 'child_process'; import { exec, spawn } from 'child_process';
import util from 'util'; import util from 'util';
import { RunResult, RunResultKind, timeoutSeconds } from '../index.js';
const execPromise = util.promisify(exec); const execPromise = util.promisify(exec);
@ -10,15 +11,23 @@ export async function runJava(
mainFile: string, mainFile: string,
mainClass: string, mainClass: string,
input: string input: string
): Promise<string> { ): Promise<RunResult> {
console.log(`- BUILD: ${mainFile}`);
const compileCommand = `${join(javaBinPath, 'javac')} -cp ${join( const compileCommand = `${join(javaBinPath, 'javac')} -cp ${join(
buildDir, buildDir,
'src' 'src'
)} ${mainFile} -d ${join(buildDir, 'build')}`; )} ${mainFile} -d ${join(buildDir, 'build')}`;
try {
await execPromise(compileCommand); await execPromise(compileCommand);
} catch(e) {
const buildErrorText = e?.toString() ?? "Unknown build errors.";
console.log("Build errors: " + buildErrorText);
return {kind: RunResultKind.CompileFailed, buildErrors: buildErrorText};
}
console.log(`- RUN: ${mainClass}`);
const runCommand = `${join(javaBinPath, 'java')} -cp "${join(buildDir, 'build')}" ${mainClass}`; const runCommand = `${join(javaBinPath, 'java')} -cp "${join(buildDir, 'build')}" ${mainClass}`;
return new Promise((resolve) => { return new Promise((resolve) => {
let outputBuffer = ''; let outputBuffer = '';
const child = spawn(runCommand, { shell: true }); const child = spawn(runCommand, { shell: true });
@ -30,25 +39,44 @@ export async function runJava(
child.stderr.on('data', (data) => { child.stderr.on('data', (data) => {
outputBuffer += data.toString(); outputBuffer += data.toString();
}); });
let runStartTime = performance.now();
child.stdin.write(input); child.stdin.write(input);
child.stdin.end(); child.stdin.end();
let resolved = false; let timeLimitExceeded = false;
let completedNormally = false;
child.on('close', () => { child.on('close', () => {
if (!resolved) { completedNormally = !timeLimitExceeded;
resolved = true;
resolve(outputBuffer); let runEndTime = performance.now();
const runtimeMilliseconds = Math.floor(runEndTime - runStartTime);
if (completedNormally) {
clearTimeout(timeoutHandle);
resolve({kind: RunResultKind.Completed, teamOutput: outputBuffer,
exitCode: child.exitCode!, runtimeMilliseconds});
}
else {
console.log(`Process terminated, total sandbox time: ${runtimeMilliseconds}ms`);
resolve({kind: RunResultKind.TimeLimitExceeded, teamOutput: outputBuffer});
} }
}); });
setTimeout(() => { let timeoutHandle = setTimeout(() => {
if (!resolved) { if (completedNormally) {
console.log('30 seconds reached, killing process'); return;
resolved = true;
child.kill('SIGKILL');
resolve(outputBuffer + '\n[Timeout after 30 seconds]');
} }
}, 30000);
console.log(`Run timed out after ${timeoutSeconds} seconds, killing process...`);
timeLimitExceeded = true;
child.stdin.end();
child.stdin.destroy();
child.stdout.destroy();
child.stderr.destroy();
child.kill('SIGKILL');
}, timeoutSeconds * 1000);
}); });
} }

View File

@ -9,7 +9,7 @@ export const GET = (async () => {
const submissions = await db.submission.findMany({ const submissions = await db.submission.findMany({
where: { state: SubmissionState.Queued }, where: { state: SubmissionState.Queued },
orderBy: { createdAt: 'asc' }, orderBy: { createdAt: 'asc' },
include: { problem: true }, include: { problem: true, contest: true, team: true },
take: 1 take: 1
}); });
if (submissions.length !== 0) { if (submissions.length !== 0) {
@ -17,8 +17,10 @@ export const GET = (async () => {
success: true, success: true,
submission: { submission: {
id: submissions[0].id, id: submissions[0].id,
contestId: submissions[0].contestId, contestId: submissions[0].contest.id,
teamId: submissions[0].teamId, contestName: submissions[0].contest.name,
teamId: submissions[0].team.id,
teamName: submissions[0].team.name,
problem: { problem: {
id: submissions[0].problemId, id: submissions[0].problemId,
pascalName: submissions[0].problem.pascalName, pascalName: submissions[0].problem.pascalName,