255 lines
9.1 KiB
JavaScript
255 lines
9.1 KiB
JavaScript
let jsPsychVerbalResponse = (function (jspsych) {
|
|
'use strict';
|
|
|
|
const info = {
|
|
name: "verbal-response",
|
|
parameters: {
|
|
transcription_server_url: {
|
|
type: jspsych.ParameterType.STRING,
|
|
default: null,
|
|
},
|
|
recording_key: {
|
|
type: jspsych.ParameterType.INT,
|
|
default: 71,
|
|
},
|
|
stimulus: {
|
|
type: jspsych.ParameterType.STRING,
|
|
default: null,
|
|
},
|
|
start_image: {
|
|
type: jspsych.ParameterType.STRING,
|
|
default: null,
|
|
},
|
|
video_html: {
|
|
type: jspsych.ParameterType.STRING,
|
|
default: null,
|
|
},
|
|
red_points_html: {
|
|
type: jspsych.ParameterType.STRING,
|
|
default: null,
|
|
},
|
|
blue_points_html: {
|
|
type: jspsych.ParameterType.STRING,
|
|
default: null,
|
|
},
|
|
locale: {
|
|
type: jspsych.ParameterType.STRING,
|
|
default: "en-US",
|
|
}
|
|
},
|
|
};
|
|
|
|
class VerbalResponsePlugin {
|
|
constructor(jsPsych) {
|
|
this.jsPsych = jsPsych;
|
|
this.rt = null;
|
|
this.recorded_data_chunks = [];
|
|
this.recording_key_down = false;
|
|
}
|
|
|
|
trial(display_element, trial) {
|
|
|
|
display_element.innerHTML = trial.video_html;
|
|
|
|
const pointsContainer = document.getElementById("points-container");
|
|
pointsContainer.innerHTML = pointsContainer.innerHTML + trial.red_points_html + trial.blue_points_html;
|
|
|
|
|
|
document.getElementById("prompts").innerHTML = trial.stimulus;
|
|
|
|
this.recorder = this.jsPsych.pluginAPI.getMicrophoneRecorder();
|
|
|
|
this.setupRecordingEvents(display_element, trial);
|
|
|
|
window.addEventListener("keydown", this.startRecording, false);
|
|
window.addEventListener("keyup", this.stopRecording, false);
|
|
|
|
}
|
|
|
|
setupRecordingEvents(display_element, trial) {
|
|
this.data_available_handler = (e) => {
|
|
if (e.data.size > 0) {
|
|
this.recorded_data_chunks.push(e.data);
|
|
}
|
|
};
|
|
this.stop_event_handler = () => {
|
|
const data = new Blob(this.recorded_data_chunks, {type: "audio/ogg"});
|
|
const reader = new FileReader();
|
|
reader.addEventListener("load", () => {
|
|
this.response = reader.result.split(",")[1];
|
|
this.load_resolver();
|
|
this.checkResponse(this.response, trial, display_element)
|
|
});
|
|
reader.readAsDataURL(data);
|
|
};
|
|
this.start_event_handler = (e) => {
|
|
this.recorded_data_chunks.length = 0;
|
|
this.recorder_start_time = e.timeStamp;
|
|
};
|
|
this.recorder.addEventListener("dataavailable", this.data_available_handler);
|
|
this.recorder.addEventListener("stop", this.stop_event_handler);
|
|
this.recorder.addEventListener("start", this.start_event_handler);
|
|
}
|
|
|
|
checkRecordingKey(e) {
|
|
return e.keyCode === 71;
|
|
|
|
}
|
|
|
|
startRecording(e) {
|
|
if (!this.checkRecordingKey(e)) {
|
|
return;
|
|
}
|
|
|
|
if (this.recording_key_down) {
|
|
return;
|
|
}
|
|
|
|
document.getElementById("recording").innerHTML = "Listening..."
|
|
this.recording_key_down = true;
|
|
this.recorder.start();
|
|
this.stimulus_start_time = Date.now();
|
|
}
|
|
|
|
delay = ms => new Promise(res => setTimeout(res, ms));
|
|
|
|
async stopRecording(e) {
|
|
if (!this.checkRecordingKey(e)) {
|
|
return;
|
|
}
|
|
|
|
if (!this.recording_key_down) {
|
|
|
|
return;
|
|
}
|
|
|
|
await this.delay(200);
|
|
this.recorder.stop();
|
|
this.recording_key_down = false;
|
|
document.getElementById("recording").innerHTML = "Processing..."
|
|
|
|
return new Promise((resolve) => {
|
|
this.load_resolver = resolve;
|
|
});
|
|
}
|
|
|
|
async checkResponse(responseAudio, trial, display_element) {
|
|
|
|
let transcription = await this.transcribeResponse(responseAudio, trial);
|
|
document.getElementById("recording").innerHTML = ""
|
|
//console.log(transcription);
|
|
|
|
if (!transcription) {
|
|
document.getElementById("prompts").innerHTML = `Your request wasn't detected. Hold down the 'G' key and try again`;
|
|
return;
|
|
}
|
|
|
|
const processedResponse = nlp(transcription)
|
|
|
|
const requested = processedResponse.match('~give~', null, {fuzzy: 0.75})
|
|
const negated = processedResponse.match('(~not~|~dont~|~keep~)', null, {fuzzy: 0.75})
|
|
const objectMentioned = processedResponse.match('~object~', null, {fuzzy: 0.75})
|
|
|
|
if (negated.found | !requested.found || !objectMentioned.found) {
|
|
document.getElementById("prompts").innerHTML = `Request not registered Be sure to say 'Give me the (red/blue) object' in full. Hold down the 'G' key and try again.`;
|
|
document.getElementById("recording").innerHTML = "Your request: " + transcription;
|
|
return;
|
|
}
|
|
|
|
const redMentioned = processedResponse.match('~red~', null, {fuzzy: 0.75})
|
|
const blueMentioned = processedResponse.match('~blue~', null, {fuzzy: 0.75})
|
|
|
|
if (redMentioned.found && blueMentioned.found) {
|
|
document.getElementById("prompts").innerHTML = `You mentioned both colours of objects. Please ask for only one. Hold down the 'G' key and try again`;
|
|
document.getElementById("recording").innerHTML = "Your request: " + transcription;
|
|
return;
|
|
}
|
|
|
|
if (!redMentioned.found && !blueMentioned.found) {
|
|
document.getElementById("prompts").innerHTML = `You mentioned neither colour of object. Hold down the 'G' key and try again`;
|
|
document.getElementById("recording").innerHTML = "Your request: " + transcription;
|
|
return;
|
|
}
|
|
|
|
let requestedColour;
|
|
|
|
if (redMentioned.found) {
|
|
requestedColour = "red";
|
|
}
|
|
|
|
if (blueMentioned.found) {
|
|
requestedColour = "blue";
|
|
}
|
|
|
|
document.getElementById("prompts").innerHTML = `You asked for the ` + requestedColour + ` object.`
|
|
this.transribedResponse = transcription;
|
|
this.chosenObject = requestedColour;
|
|
|
|
|
|
//await this.delay(100);
|
|
|
|
this.endTrial(display_element, trial);
|
|
|
|
|
|
}
|
|
|
|
|
|
transcribeResponse(responseAudio, trial) {
|
|
return new Promise(function (resolve, reject) {
|
|
const xhr = new XMLHttpRequest();
|
|
const requestBody = "clip=" + encodeURIComponent(responseAudio) + "&locale=" + trial.locale;
|
|
xhr.open("POST", trial.transcription_server_url);
|
|
xhr.onload = function () {
|
|
if (this.status >= 200 && this.status < 300) {
|
|
resolve(xhr.response);
|
|
} else {
|
|
reject({
|
|
status: this.status,
|
|
statusText: xhr.statusText
|
|
});
|
|
}
|
|
};
|
|
xhr.onerror = function () {
|
|
reject({
|
|
status: this.status,
|
|
statusText: xhr.statusText
|
|
});
|
|
};
|
|
xhr.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
|
|
xhr.send(requestBody);
|
|
});
|
|
}
|
|
|
|
endTrial(display_element, trial) {
|
|
this.recorder.removeEventListener("dataavailable", this.data_available_handler);
|
|
this.recorder.removeEventListener("start", this.start_event_handler);
|
|
this.recorder.removeEventListener("stop", this.stop_event_handler);
|
|
|
|
window.removeEventListener("keydown", this.startRecording);
|
|
window.removeEventListener("keyup", this.stopRecording);
|
|
|
|
document.getElementById("redpoints").innerHTML = "";
|
|
document.getElementById("bluepoints").innerHTML = "";
|
|
|
|
// kill any remaining setTimeout handlers
|
|
this.jsPsych.pluginAPI.clearAllTimeouts();
|
|
// gather the data to store for the trial
|
|
let trial_data = {
|
|
rt: this.rt,
|
|
stimulus: trial.stimulus,
|
|
response: this.response,
|
|
estimated_stimulus_onset: Math.round(this.stimulus_start_time - this.recorder_start_time),
|
|
transcribed_response: this.transribedResponse,
|
|
chosen_object: this.chosenObject,
|
|
};
|
|
|
|
this.jsPsych.finishTrial(trial_data);
|
|
}
|
|
}
|
|
|
|
VerbalResponsePlugin.info = info;
|
|
|
|
return VerbalResponsePlugin;
|
|
|
|
})(jsPsychModule);
|