Instructions:
Rank each constituent translation from Best to Worst relative to the other choices (ties are allowed). Grade only the highlighted part of each translation.
Please note that segments are selected automatically, and they should be taken as an approximate guide. They might include extra words, or miss words on either end that are not in the actual alignment. |
|
".$word_list[$start_word];
$word_list[$end_word-1] = $word_list[$end_word-1]."";
return(join(" ",$word_list));
}
function process_judgements() {
global $translation_id, $assignments;
if ($_POST["ASSIGNMENTID"]) {
$success = false;
list($assignment_type,$sentence_number,$assignment_info_string) = split("-",$_POST["ASSIGNMENTID"]);
///NIST-Style
if ($assignment_type == "N") {
$types = array("ADEQUACY","FLUENCY");
$score_hash = array();
while (list($key, $value) = each($_POST)) {
foreach ($types as $type) {
if (preg_match("/${type}_(.+)/",$key,$reg)) {
//at least one judgement is present, mark as a successful attempt
$success = true;
$score_hash[$translation_id[$reg[1]]][$type]=$value;
}
}
}
$nist_scores = array();
foreach($score_hash as $system => $score_list) {
$nist_scores[] = $system.":".($score_list["ADEQUACY"] ? $score_list["ADEQUACY"] : "-").",".($score_list["FLUENCY"] ? $score_list["FLUENCY"] : "-");
}
if ($success) {
$f = fopen("judgements","a");
fputs($f,"$_POST[TASK] ||| $_POST[ANNOTATOR] ||| $_POST[ASSIGNMENTID] ||| NIST ||| ".join(" ",$nist_scores)." ||| ".(time()-$_POST[TIME])."\n");
fclose($f);
}
//RANK
} elseif ($assignment_type == "R") {
$ranks = array();
$systems = array();
while (list($key, $value) = each($_POST)) {
if (preg_match("/RANK_(.+)/",$key,$reg)) {
//at least one judgement is present, mark as a successful attempt
$success = true;
$ranks[] = $translation_id[$reg[1]].":".$value;
}
}
if ($success) {
$f = fopen("judgements","a");
fputs($f,"$_POST[TASK] ||| $_POST[ANNOTATOR] ||| $_POST[ASSIGNMENTID] ||| RANK ||| ".join(" ",$ranks)." ||| ".(time()-$_POST[TIME])."\n");
fclose($f);
}
//CONSTITUENT RANK
} elseif ($assignment_type == "C") {
$ranks = array();
$systems = array();
while (list($key, $value) = each($_POST)) {
if (preg_match("/C_RANK_(.+)/",$key,$reg)) {
//at least one judgement is present, mark as a successful attempt
$success = true;
$ranks[] = $translation_id[$reg[1]].":".$value;
}
}
if ($success) {
$f = fopen("judgements","a");
fputs($f,"$_POST[TASK] ||| $_POST[ANNOTATOR] ||| $_POST[ASSIGNMENTID] ||| CONSTITUENT ||| ".join(" ",$ranks)." ||| ".(time()-$_POST[TIME])."\n");
fclose($f);
}
}
if($_POST["ASSIGNMENTID"] == $assignments[0] && $success) {
//remove element 0, they finished it
array_shift($assignments);
} else {
//LAZY BASTARD!
print("No judgements scored. Please try again.
");
}
}
}
function get_next_assignment() {
global $assignments;
return $assignments[0];
}
function load_task_info() {
global $annotator_conflict;
$line = file("task");
$index = 0;
for($i=0;$i 0) {
$assignments = split(";",$assignment_string);
if (count($assignments) > 0) {
return true;
}
}
else {
//look through finished tasks, remember tasks as we find them
$line = file("judgements");
for($i=0;$i0) {
$total_time += $time; $count_time++;
}
}
}
$line = file("assignments");
$index = 0;
for($i=0;$i array("random"=>$NIST_RANDOM,
"global_repeat"=>$NIST_GLOBAL_REPEAT,
"local_repeat"=>$NIST_LOCAL_REPEAT,
"switch_size"=>$NIST_SWITCH_SIZE,
"long_name"=>"NIST"),
"R" => array("random"=>$RANK_RANDOM,
"global_repeat"=>$RANK_GLOBAL_REPEAT,
"local_repeat"=>$RANK_LOCAL_REPEAT,
"switch_size"=>$RANK_SWITCH_SIZE,
"long_name"=>"RANK"),
"C" => array("random"=>$CONSTITUENT_RANDOM,
"global_repeat"=>$CONSTITUENT_GLOBAL_REPEAT,
"local_repeat"=>$CONSTITUENT_LOCAL_REPEAT,
"switch_size"=>$CONSTITUENT_SWITCH_SIZE,
"long_name"=>"CONSTITUENT"));
$task_order = array_keys($task_setup);
shuffle($task_order);
$toAssign = array();
$new_assignment_count = 0;
foreach($task_order as $task_type) {
$type_assignments = array();
$num_non_local_repeat = $task_setup[$task_type]["random"]+$task_setup[$task_type]["global_repeat"];
//load the repeat list and shuffle it
$repeat_list = $GLOBALS["repeat_list"][$task_setup[$task_type]["long_name"]];
shuffle($repeat_list);
//First assign all the tasks that require randomly pulling from the whole list or just the repeats
for ($i=0;$i<$num_non_local_repeat;$i++) {
$n = 0;
//Random first
if ($i<$task_setup[$task_type]["random"]) {
$max_number = 0;
if ($task_type == "N" || $task_type == "R") {
$max_number = $GLOBALS["max_sentence_number"]-1;
} elseif ($task_type == "C") {
$max_number = $GLOBALS["max_constituent_number"]-1;
}
$n = rand(0,$max_number);
//Then repeat list
} else {
//load the list of repeatable sentences
#$repeat_list = $GLOBALS["repeat_list"][$task_setup[$task_type]["long_name"]];
//randomly choose one
#$max_number = sizeof($repeat_list)-1;
#$index_n = rand(0,$max_number);
//get its sentence number
if (count($repeat_list > 0)) {
$n = array_pop($repeat_list);
} else {
//not enough repeats!
continue;
}
}
//generate the new task
$assignmentid = $task_type."-".$n;
$systems = array();
$already = array();
do {
$sys = rand(0,sizeof($GLOBALS["translation_id"])-1);
if (!$already[$sys] && !$annotator_conflict[$_POST["ANNOTATOR"]][$GLOBALS["translation_id"][$sys]]) {
$already[$sys] = 1;
$systems[] = $sys;
}
} while (sizeof($systems) < $max_per_page);
$sysList=join(",",$systems);
$assignmentid = $assignmentid."-".$sysList;
$type_assignments[] = $assignmentid;
}
//OK, shuffle up this list, then tack local repeats onto the end
shuffle($type_assignments);
$type_assignments_size = sizeof($type_assignments);
for ($i=0;$i<$task_setup[$task_type]["local_repeat"];$i++) {
$n = rand(0,$type_assignments_size-1);
$type_assignments[] = $type_assignments[$n];
}
//Instead of just appending to global array in this way, place each type alternating
$toAssign[$task_type]= $type_assignments;
$new_assignment_count += sizeof($type_assignments);
}
$new_assignments = array();
//assign things in groups
if (!$TEST_SWITCHING) {
foreach($task_order as $task_type) {
$new_assignments = array_merge($new_assignments,$toAssign[$task_type]);
}
} else {
while(sizeof($new_assignments) < $new_assignment_count) {
foreach($task_order as $task_type) {
for($i=0;$i<$task_setup[$task_type]["switch_size"] && sizeof($toAssign[$task_type]) > 0;$i++) {
$new_assignments[] = array_shift($toAssign[$task_type]);
}
}
}
}
$assignments = $new_assignments;
$assignment_list = join(";",$assignments);
$f = fopen("assignments","a");
fputs($f,$_POST["TASK"]." ||| ".$_POST["ANNOTATOR"]." ||| ".$assignment_list."\n");
fclose($f);
}
if (check_password()) {
load_task_info();
if ($_POST["GENERATEASSIGNMENT"]) {
//user has requested a new work unit
generate_new_assignments();
if (sizeof($assignments) == 0) {
//nothing to generate for this task
print("You have completed all required judgements for task $_POST[TASK]. Please choose another task.");
include("select_task_contents.php");
tail();
exit;
}
} else {
load_assignment_list();
process_judgements();
}
if (sizeof($assignments) == 0) {
//Decide if this assignments = 0 is because they've never done anything for this task
head("Choose Next Task");
?>
You have completed your current assignments for the task . Please choose your next task (or continue with your current task):
|