<?php
	$start = time();
	function get_parameter($param) {
		if (key_exists($param,$_POST)) return $_POST[$param];
		else if(key_exists($param,$_GET)) return $_GET[$param];
		else return null;
	}

	ini_set('memory_limit',"88M");
	
	$DUP_DEBUG = false;

	include_once("common.php");
	
	$file_suffix = ".official-wmt08";


	global $banned_systems;
	//$banned_systems = array("WMT08 All-English News"=>array("uedin-combo_xx"=>1));

	load_global_task_info();
	
	$annotator_id_map = array();
	$annotator_judgement_count = array();
	
	$task_filter = get_parameter("TASK");
	$user_filter = get_parameter("USER");
	$test_filter = get_parameter("TEST_TYPE");
	$use_percent = get_parameter("PERCENT");
	$learning_count = get_parameter("LEARNING");
	//$do_annotator = get_parameter("ANNOTATOR");
	$do_reuse = get_parameter("REUSE_ALL");
	$remove_identical = get_parameter("REMOVE_IDENTICAL");
	$group_type = get_parameter("GROUPING");
	if ($group_type == 'lang') $banned_systems = array("WMT08 All-English News"=>array("uedin-combo_xx"=>1));
	if ($task_filter) $do_reuse = false; 
	$get_raw = get_parameter("RAW");
	if ($do_intra_annotator) $do_annotator = 1;
	
	//Load all assignments, and index to keep track of what a user has done
	$assignments_map = array();
	$assignments_index = array();
	
	
	global $summary_tables;
	$summary_tables = array();

	$fh = fopen(get_assignments_file().$file_suffix,'r');
	while($line = fgets($fh)) {
		list($task,$user,$assignments_str) = split(" \|\|\| ",$line);

		//skip if filtering
		//if ($task_filter && ($task != $task_filter)) continue;
		//if ($user_filter && ($user != $user_filter)) continue;

		$assignments = split(';',trim($assignments_str));

		foreach ($assignments as $assignment_str) {
			list($test_type,$item_num,$systems_str) = split('-',trim($assignment_str));

			//skip if filtering
			//if ($test_filter && ($test_type != $test_filter)) continue;

			$assignments_map[$task][$test_type][$user][] = array('item_num'=>$item_num,'systems_list'=>split(',',trim($systems_str)));	
			$assignments_index[$task][$test_type][$user] = 0;
		}
	}
	fclose($fh);
	
	
	//load all judgements, correct for duplicates (pressing submit twice)
	$fh = fopen(get_judgements_file().$file_suffix,'r');
	
	$judgements_map = array();
	
	$num_corrected = 0;
	$num_skipped = 0;
	$num_judgements = 0;
	$judgement_id_num = 0;
	$missed_assignments_map = array();
	
	$user_num_type_judgements = array();
	
	function rename_system($system,$group_type) {
		if ($group_type == 'lang') {
			unset($regs);
			if (ereg("^(.+)_([a-z]{2})$",$system,$regs)) {
				$source_lang = $regs[2];
				$new_system = $regs[1];
				return $source_lang;				
			}
		} else if ($group_type == 'systype') {
			if (strpos($system,'rbmt')!==false) return "rbmt";
			elseif (strpos($system,'combo')!==false) return "combo";
			else return "unknown";
		}
		return $system;
	}
	
	while($line = fgets($fh)) {
		list($task, $user, $judgement_id, $test_type, $judgements_str, $time) = split(" \|\|\| ",$line);
		
		//we always build up user filters the same way
		if (!key_exists($user,$annotator_id_map)) $annotator_id_map[$user] = count($annotator_id_map);
		
		//skip if filtering
		//if ($task_filter && ($task != $task_filter)) continue;
		//if ($user_filter && ($user != $user_filter)) continue;

		list($test_type,$item_num,$systems_str) = split('-',trim($judgement_id));

		//skip if filtering
		//if ($test_filter && ($test_type != $test_filter)) continue;
		
		$scores_list = split(' ',trim($judgements_str));
		$scores_hash = array();
		foreach($scores_list as $score) {
			list($system,$values) = split(':',$score);
			if(!$banned_systems[$task][$system]) $scores_hash[$system]=$values;
		}
		$scores = array();
		foreach($scores_hash as $system=>$values) {
			if ($group_type) $system = rename_system($system,$group_type);
			$scores[] = $system.':'.$values;
		}
		$judgement = array('item_num'=>$item_num,
							'systems_list'=>split(',',trim($systems_str)),
							'scores'=>$scores,
							'time'=>trim($time),
							'user_judgement_num' => $user_num_type_judgements[$test_type][$user],
							'judgement_id' => $judgement_id_num++,
							'identical'=>false);
		
		//check if we should keep it, based on assigned judgement order
		$last_assigned_index = $assignments_index[$task][$test_type][$user];
		$assignment = $assignments_map[$task][$test_type][$user][$last_assigned_index];
		
		if ($judgement['item_num'] != $assignment['item_num']) {
			if ($DUP_DEBUG) print("mismatch: $task $user $test_type ".$judgement['item_num']." ".$assignment['item_num']."<br>");
			$last_judgement = null;
			if (count($judgements_map[$task][$test_type][$user])>=1) $last_judgement = end($judgements_map[$task][$test_type][$user]);
			if ($DUP_DEBUG) print("last extracted was: ".$last_judgement['item_num']."<br>");
			if($judgement['item_num'] == $last_judgement['item_num'] &&
			   $judgement['systems_list'] == $last_judgement['systems_list']) {
				if ($DUP_DEBUG) print("last extracted matched, skipping for $task $user $test_type.<br>");
				$num_corrected++;
				continue;
			} else {
				//check for 2-back jump 
				$fixed = false;
				$penultimate_judgement = null;
				if (count($judgements_map[$task][$test_type][$user])>=2) 
					$penultimate_judgement = $judgements_map[$task][$test_type][$user][(count($judgements_map[$task][$test_type][$user])-2)];
				if($judgement['item_num'] == $penultimate_judgement['item_num'] &&
				   $judgement['systems_list'] == $penultimate_judgement['systems_list']) {
				   		if ($DUP_DEBUG) print("second to last extracted matched, skipping and removing last recorded to move index back 2<br>"); 
						array_pop($judgements_map[$task][$test_type][$user]);
						$num_judgements--;
						$user_num_type_judgements[$test_type][$user]--;
						$assignments_index[$task][$test_type][$user]--;	
						$num_corrected+=2;
						continue;
				} 
				
				//check for forward jump 
				if (!$fixed) {
					$next_assignment = null;
					if (count($assignments_map[$task][$test_type][$user])>$last_assigned_index) {
						$next_assignment = $assignments_map[$task][$test_type][$user][$last_assigned_index+1];
					}
					if ($judgement['item_num'] == $next_assignment['item_num'] &&
						$judgement['systems_list'] == $next_assignment['systems_list']) {
						
				   			if ($DUP_DEBUG) print("judgement matched upcoming assignment, skipping & storing missed assignent and moving up 1<br>"); 
							$missed_assignments_map[$task][$test_type][$user][] = $assignment;
							$assignments_index[$task][$test_type][$user]++;	
							$num_skipped++;

							$fixed = true;
							
					}
				}
				
				//check old missing assignments
				//these will only be recovered when we restart a session, the first should always be pulled
				if (!$fixed) {
					$old_assignment = null;
					if (count($missed_assignments_map[$task][$test_type][$user])>0) $old_assignment = $missed_assignments_map[$task][$test_type][$user][0];
					if ($judgement['item_num'] == $old_assignment['item_num'] &&
						$judgement['systems_list'] == $old_assignment['systems_list']) {
						
				   			if ($DUP_DEBUG) print("judgement matched old missed assignment, matched and scoring<br>"); 
							array_pop($missed_assignments_map[$task][$test_type][$user]);
							//we'll increment this later, but we don't want it to move for now
							$assignments_index[$task][$test_type][$user]--;	
							$num_skipped--;
							$fixed = true;
							
					}
				}

				if (!$fixed) {
					print("SERIOUS WEIRDNESS ENCOUNTERED<br>");
				}
			}
		}
		$assignments_index[$task][$test_type][$user]++;	
		$judgements_map[$task][$test_type][$user][] = $judgement;
		$num_judgements++;
		$user_num_type_judgements[$test_type][$user]++;
		//$judgements = split(';',trim($judgements_str));
		//$judgements_map[$task][$user] = array_merge($judgements_map[$task][$user],$judgements);	

	}
	fclose($fh);
	
	if (!$get_raw) print("Corrected $num_corrected duplicates and $num_skipped skipped, found $num_judgements valid judgements and ".count($annotator_id_map)." users. Beginning to filter and remove learning items...<br>");

	
	//dump assignments
	unset($assignments_map);
	unset($assignments_index);

	$user_counts = array();

	$num_filtered_judgements = 0;
	
	$num_learning = 0;
	//dump judgements less than learning, filter down judgements for only the tasks we're concerned with
	foreach ($judgements_map as $task=>$test_types) {
		if ($task_filter && ($task != $task_filter)) {
			unset($judgements_map[$task]);
			continue;
		}
		foreach($test_types as $test_type=>$users) {
			if ($test_filter && ($test_type != $test_filter)) {
				unset($judgements_map[$task][$test_type]);
				continue;
			}
			foreach($users as $user=>$judgements) {
				if ($user_filter && ($user != $user_filter)) {
					unset($judgements_map[$task][$test_type][$user]);
					continue;
				}
			
				if ($learning_count) {
					for($i=0;$i<count($judgements);$i++) {
						if ($judgements[$i]['user_judgement_num'] >=$learning_count) {
							break;
						}
					}
					if ($i > 0) {
						$num_learning += $i;
						$judgements_map[$task][$test_type][$user] = array_slice($judgements,$i);
						$num_judgements -= $i;
					}
				}
				if (count($judgements_map[$task][$test_type][$user])>0) $user_counts[$user] += count($judgements_map[$task][$test_type][$user]);
				$num_filtered_judgements += count($judgements_map[$task][$test_type][$user]);
			}
		}
	}
	

	if (!$get_raw) print("Filtered judgements, ended up with ".count($user_counts)." users, $num_filtered_judgements judgements, leaving out $num_learning for learning<br>");
	
	//check for duplicate constituents

	$num_identical=array();
	if($remove_identical) {
		foreach($judgements_map as $task=>$test_types) {
			$constituent_mappings = get_constituent_mappings(get_global_task_property("constituent",null,null,$task).".verbose");
			if (!$constituent_mappings) continue;
			foreach($test_types as $test_type=>$users) {
				if($test_type == "R") continue;
				foreach($users as $user=>$judgements) {
					foreach ($judgements as $index=>$judgement) {
						$mapping = $constituent_mappings[$judgement['item_num']];
						$phrase_choice=-1;
						//CA is only all identical if it matches the ref
						if ($test_type == "CA") $phrase_choice=$mappings['ref'];
						$all_identical = true;
						foreach($judgement['scores'] as $score_str) {
							list($system,$scores) = split(':',$score_str);
							if($phrase_choice<0) $phrase_choice = $mapping[$system];
							if ($mapping[$system] != $phrase_choice) {
								$all_identical = false;
								//break;
							}							
						}
						if ($all_identical) {
							$num_identical[$test_type]++;
							$judgements_map[$task][$test_type][$user][$index]['identical'] = true;
						}
					
					}
				
				}
			}
		
		}
	}
	unset($constituent_mappings);
	if (!$num_identical['CA']) $num_identical['CA'] = 0;
	if (!$num_identical['C']) $num_identical['C'] = 0;
	if (!$get_raw) print("marked ".$num_identical['CA']."-CA (all systems and ref equal) ".$num_identical['C']."-C (all systems equal) constituent judgements as identical<br>");

	//now that we've recorded stats on annotators, we can add in the All-English judgements to paired tasks
	$num_added=0;
	$num_all=0;
	$num_added_points=0;
	if ($do_reuse) {
		global $LANG_CODES;
		print_r($lang_names);
		foreach($judgements_map['WMT08 All-English News'] as $test_type=>$users) {
			foreach($users as $user=>$judgements) {
				foreach($judgements as $all_judgement) {
					$all_scores = $all_judgement['scores'];
					$sorted_scores = array();
					foreach($all_scores as $score_str) {
						list($system,$score) = split(':',$score_str);
						unset($regs);
						ereg("^(.+)_([a-z]{2})$",$system,$regs);
						$source_lang = $regs[2];
						$new_system = $regs[1];
						$sorted_scores[$source_lang][]="$new_system".':'."$score";				
					}
					//ok, we've sorted by source lang. so now find times when we had more than 1 of the same source lang,
					//make a new fake judgement and add it to the right task
					foreach($sorted_scores as $source_lang=>$systems) {
						$found_one=false;
						if (count($systems)<2) continue;
						else {
							$found_one=true;
							$new_judgement = array('item_num'=>$all_judgement['item_num'],
													'scores'=>$sorted_scores[$source_lang],
													'time'=>0,
													'judgement_id' => $all_judgement['judgement_id'],
													'duplicated' => true,
													'identical'=>$all_judgement['identical']);
							$lang_name = $LANG_CODES[$source_lang];
							$new_task = "WMT08 $lang_name-English News";
							$judgements_map[$new_task][$test_type][$user][] = $new_judgement;
							$num_added++;
							$num_added_points+=count($sorted_scores[$source_lang]);
						}
					}
					if ($found_one) $num_all++;
				}
			}
	
		}
	}	
	if (!$get_raw) print("repurposed $num_all from All-English to $num_added new judgements, $num_added_points data points<br>");

	//go through, grouping and creating new tasks
	$grouped_judgements_map=array();
	foreach($judgements_map as $task=>$test_types) {
		foreach($test_types as $test_type=>$users) {
			foreach($users as $user=>$judgements) {
			
			}
		}
	}
	

	//ok, now we can process these judgements

	if ($get_raw) {
		output_raw(&$judgements_map,&$annotator_id_map);
		exit();
	}
	

	head("Statistics");
		?>Get a <a href='<?php print('?'.$_SERVER['QUERY_STRING']."&RAW=1");?>'>comma-delimited raw</a> version of the filtered data (right click and Save As).<?php
		
		
	?><hr><?php
		
		
		print_task_name("Annotators");
		//calculate_annotator_agreement($judgements_map,false,true);
		//calculate_annotator_agreement($judgements_map,true,true);
		calculate_annotator_agreement(&$judgements_map,false,false);
		calculate_annotator_agreement(&$judgements_map,true,false);

		
	?><hr><?php		
	
	
	//build raw data summary table
	foreach($judgements_map as $task=>$test_types) {
		foreach($test_types as $test_type=>$users) {
			$num_data = 0;
			$num_reused = 0;
			foreach($users as $user=>$judgements) {
				foreach($judgements as $judgement) {
					$num_data += count($judgement['scores']);
					if($judgement['duplicated']) $num_reused += count($judgement['scores']);
				}
			}
			$summary_tables['Total Items Judged'][$task][get_test_type_name($test_type)]=$num_data;
			$summary_tables['Total Items Judged'][$task]['Total']+=$num_data;
			$summary_tables['Total Items Judged']['Total_temp'][get_test_type_name($test_type)]+=$num_data;
			$summary_tables['Total Items Judged']['Total_temp']['Total']+=$num_data;
			
			if ($do_reuse && ereg("News",$task) && ($task != "WMT08 All-English News") && ($num_reused>0)) {
				$summary_tables['Total Items Judged']["$task (native)"][get_test_type_name($test_type)]=($num_data-$num_reused);
				//$summary_tables['Total Items Judged']["$task (native)"]['Total']+=($num_data-$num_reused);
	
				$summary_tables['Total Items Judged']["$task (reused from All)"][get_test_type_name($test_type)]=$num_reused;
				//$summary_tables['Total Items Judged']["$task (reused from All)"]['Total']+=$num_reused;
			}

		}
	
	}
	
	$total_row = $summary_tables['Total Items Judged']['Total_temp'];
	unset($summary_tables['Total Items Judged']['Total_temp']);
	$summary_tables['Total Items Judged']['Total'] = $total_row;
		

		print_task_name("Tasks");	
		calculate_task_results(&$judgements_map,$use_percent) ;

	?><hr><?php
		print_task_name("Summary Tables");
		
		foreach($summary_tables as $name=>$table) {
			print_table($name,$table);
		
		}
	$end = time();
	print("finished in ".($end-$start)."<BR>");
	tail();
	

function output_raw(&$judgements_map,&$annotator_id_map) {
	$headers = array('TASK','TYPE','ITEM_ID','ANNOTATOR_ID','TIME','SYSTEM_0','SCORE_0_A','SCORE_0_B','SYSTEM_1','SCORE_1_A','SCORE_1_B','SYSTEM_2','SCORE_2_A','SCORE_2_B','SYSTEM_3','SCORE_3_A','SCORE_3_B','SYSTEM_4','SCORE_4_A','SCORE_4_B');
	print(join(',',$headers));
	print("\n");
	foreach($judgements_map as $task=>$test_types) {
		foreach($test_types as $test_type=>$users) {
			foreach($users as $user=>$judgements) {
				foreach($judgements as $judgement) {
				 $results = array($task,get_test_type_name($test_type),$judgement['item_num'],$annotator_id_map[$user],$judgement['time']);
				 foreach($judgement['scores'] as $score_str) {
					list($system,$value_str) = split(':',$score_str);
				 	list($scoreA,$scoreB) = split(',',$value_str);
				 	$results[] = $system;
				 	$results[] = $scoreA;
				 	$results[] = $scoreB;
				 }
				 print(join(',',$results));
				 print("\n");
				}
			}
		}
	}

}

function calculate_task_results(&$judgements_map,$use_percent) {
	global $summary_tables;
	global $banned_systems;
	$summary_tables['All Systems Performance']['header'] = array("Type","Task","System","Score");
	foreach ($judgements_map as $task=>$test_types) {
		$rankings_map = array();
		$systems_list = array();
		$summary_tables["$task Summary"]['header'] = array('System','Rank','Constituent','Constituent Acceptance');
		foreach($test_types as $test_type=>$users) {
			foreach($users as $user=>$judgements) {
				foreach($judgements as $judgement) {
					$system_scores = array();
					foreach($judgement['scores'] as $score_str) {
						list($system,$score) = split(':',$score_str);
						$system_scores[$system] = $score;
						$systems_list[$test_type][$system] = 1;
					}
					
					if ($test_type == "R" || $test_type == "C") {

						foreach($system_scores as $systemA=>$scoreA) {
							foreach($system_scores as $systemB=>$scoreB) {
								//in ranks, 5 is lower than 1
								if ($systemA != $systemB) {
									if ($scoreA < $scoreB) {
										$rankings_map[$test_type][$systemA][$systemB]['A better']++;
										//$rankings_map[$test_type][$systemA]['ALL']['A better']++;
										$rankings_map[$test_type]['&lt; others'][$systemB]['B better']++;
									}
									elseif ($scoreA > $scoreB) {
										$rankings_map[$test_type][$systemA][$systemB]['B better']++;
										//$rankings_map[$test_type][$systemA]['ALL']['B better']++;
										$rankings_map[$test_type]['&gt; others'][$systemB]['B better']++;
										$rankings_map[$test_type]['&gt;= others'][$systemB]['B better']++;
									} else {
										$rankings_map[$test_type][$systemA][$systemB]['Equal']++;
										//$rankings_map[$test_type][$systemA]['ALL']['Equal']++;
										$rankings_map[$test_type]['&gt; others'][$systemB]['Equal']++;
										//$rankings_map[$test_type]['&gt;= others'][$systemB]['Equal']++;
										$rankings_map[$test_type]['&lt; others'][$systemB]['Equal']++;
										$rankings_map[$test_type]['&gt;= others'][$systemB]['B better']++;

									}
									$rankings_map[$test_type][$systemA][$systemB]['Total']++;
									//$rankings_map[$test_type][$systemA]['ALL']['Total']++;
									$rankings_map[$test_type]['&gt; others'][$systemB]['Total']++;
									$rankings_map[$test_type]['&gt;= others'][$systemB]['Total']++;
									$rankings_map[$test_type]['&lt; others'][$systemB]['Total']++;

									//$rankings_map[$test_type]['ALL']['ALL']['Total']++;

								}
							}
						}
					} else if ($test_type == "CA") {
						foreach($system_scores as $system=>$score) {
							if ($banned_systems[$system]) continue;
							if ($score == 1) {
								$rankings_map[$test_type][$system]['Yes']++;
								//$rankings_map[$test_type]['ALL']['Yes']++;
							} elseif ($score == 2) {
								$rankings_map[$test_type][$system]['No']++;
								//$rankings_map[$test_type]['ALL']['No']++;
							} elseif ($score == 3) {
								$rankings_map[$test_type][$system]['Not Sure']++;
								//$rankings_map[$test_type]['ALL']['Not Sure']++;
							}
							$rankings_map[$test_type][$system]['Total']++;
							//$rankings_map[$test_type]['ALL']['Total']++;

						}
					}				
				}
			}
		}
		
		//TODO change best calculation and move
		$best_in_column = array();
		$best_in_row = array();

		foreach ($rankings_map as $test_type=>$rows) {
			foreach($rows as $row=>$columns) {
				$row_comp_value = -100;
				foreach($columns as $column=>$values) {
					$column_comp_value = -100;
					if ($test_type == "R" || $test_type == "C") {
						$total = $values['Total'];
						foreach($values as $name=>$value) {
							if ($name == 'Total') continue;
							$rankings_map[$test_type][$row][$column]["$name %"] = $value/$total;
						}
						if (!array_key_exists($row,$systems_list[$test_type]) && ($row_comp_value < $rankings_map[$test_type][$row][$column]["B better %"])) {
							$best_in_row[$test_type][$row] = $column;
							$row_comp_value = $rankings_map[$test_type][$row][$column]["B better %"];
						}
					}	
					else if ($test_type == "CA") {
						if ($column == 'Not Sure' || $column == 'Total') continue;
						$rankings_map[$test_type][$row]["$column %"] = $values/($rankings_map[$test_type][$row]['Yes']+$rankings_map[$test_type][$row]['No']);
						$column_comp_value = $rankings_map[$test_type][$row]["$column %"];
						if ($column != 'Yes') $column_comp_value *= -1;
						if (!$best_in_column[$test_type][$column] || 
							$best_in_column[$test_type][$column]['value'] < $column_comp_value) {
							$best_in_column[$test_type][$column]['value'] = $column_comp_value;
							$best_in_column[$test_type][$column]['row'] = $row;
	
						}

					}
					
				}
			}		
		}

		//print the new way
		
		print_task_name($task);
		
		$performance_table = array();
		
		foreach ($systems_list as $test_type=>$systems) {
			$ordered_systems = array_keys($systems);
			sort($ordered_systems);
			print_test_type($test_type);
			print_notes($test_type);
			print_table_start();
			//array_push($ordered_systems,'ALL');
			$row_keys = $ordered_systems;
			$column_keys = null;
			if ($test_type == "C" || $test_type == "R") {
				$column_keys = $ordered_systems;
				array_push($row_keys,'&gt; others');
				array_push($row_keys,'&gt;= others');
				array_push($row_keys,'&lt; others');

				array_unshift($column_keys,"");
			} else if ($test_type = "CA") {
				$column_keys = array('System','Yes','No','Not Sure','Total');
				//if ($use_percent) $column_keys = array('Yes %','No %','Not Sure %');
				
			}
			print_header_row($column_keys);
			array_shift($column_keys);
			foreach($row_keys as $row_key) {
				print_row_start();
				print_header_cell($row_key);
				foreach($column_keys as $column_key) {
					$column = $rankings_map[$test_type][$row_key][$column_key];
					$display = new Display_Cell($column);
					$p = 1;
					if ($test_type == "C" || $test_type == "R") {
						if ($row_key==$column_key) $display->contents = "";
						else if ($use_percent) {
							$display->contents = $column['B better %'];
						} else {
							$col_better = $column['B better'];
							$total =  $column['Total'];
							$equal = $column['Equal'];
							$display->contents = sprintf("%d / %d (%d)",$col_better,$total,$equal);
						}
						$display->p = sign_test($column['B better']+0.0,$column['Total']-$column['Equal']-$column['B better']+0.0);
						
					} else if ($test_type == "CA") {
						if (($column_key != "Total") && ($column_key != "Not Sure")) {
							if ($use_percent) $display->contents = $rankings_map[$test_type][$row_key]["$column_key %"];
							if ($column_key == "Yes") {
								$display->p = sign_test($rankings_map[$test_type][$row_key]['Yes'],$rankings_map[$test_type][$row_key]['No']);
								//this is our summary column
								$summary_tables['All Systems Performance'][] = array($test_type,$task,$row_key,$display);
								$performance_table[$row_key][$test_type]=$display;
							
							}
						}
					}
					$display->is_best = ($best_in_column[$test_type][$column_key]['row'] == $row_key) || ($best_in_row[$test_type][$row_key] == $column_key);
					if ($test_type == "C" || $test_type == "R") {
						if (array_key_exists($row_key,$systems)) {
							$display->is_best =  ($column['B better %'] > $rankings_map[$test_type][$column_key][$row_key]['B better %']);
						} elseif ($row_key=='&gt;= others') {
							//this is our summary row
							$summary_tables['All Systems Performance'][] = array($test_type,$task,$column_key,$display);
							$performance_table[$column_key][$test_type]=$display;
						}
					
					}
					
					print_cell_obj($display);
				}
				print_row_end();
			}
			print_table_end();
		}
		$summary_tables["$task Summary"] = $performance_table;
	}
}

function print_cell_obj($cell,$precision=3,$is_summary=false) {
	$display = $cell->contents;
	if (trim($display) == "") $display = '&nbsp;';
	if (is_float($display)) $display = sprintf("%0.".$precision."f",$display);
	if ($cell->p <= 0.05) {
		$display = "$display*";
	}
	if ($cell->p <=0.01) {
		$display = "$display*";
	}

	if ($cell->is_best) $display = "<b>+ $display</b>";
	print("<td>$display</td>");
}

class Display_Cell {
	//glorified struct

	public $contents;
	public $is_best;
	public $p;
	
	public function __construct($contents,$is_best=false,$p=1) {
		$this->contents = $contents;
		$this->is_best = $is_best;
		$this->p = $p;
	}
}

function calculate_annotator_agreement(&$judgements_map,$do_intra_annotator = false,$include_unsure=true) {
	//ok, we have all of our judgements, we need to do comparisons by user & judgement_id
	global $summary_tables;
	$type_agreements = array();
	$user_agreements=array();
	$user_total = array();


	foreach ($judgements_map as $task=>$test_types) {
		$task_type_agreements = array();
		foreach($test_types as $test_type=>$users) {
			$user_rankings = array();
			$user_judgements = array();
			foreach($users as $user=>$judgements) {
				$coverage = array();
				foreach($judgements as $judgement) {
					$system_scores = array();
					if ($judgement['duplicated']) continue;
					if ($judgement['identical']) {
						//print('skipping identical<br>');
						continue;
					}
					foreach($judgement['scores'] as $score_str) {
						list($system,$score) = split(':',$score_str);
						$system_scores[$system] = $score;
						$systems_list[$test_type][$system] = 1;
					}
					$item_num = $judgement['item_num'];
					$judgement_id = $judgement['judgement_id'];
					foreach($system_scores as $systemA=>$scoreA) {
						if ($test_type == 'R' || $test_type == 'C') {
							foreach($system_scores as $systemB=>$scoreB) {
								if ($systemA == $systemB) continue;
								if($coverage[$systemB][$systemA]) continue;
								$coverage[$systemA][$systemB] = true;
								//in ranks, 5 is lower than 1

								if ($scoreA < $scoreB) {
									$user_rankings[$user][$item_num][$judgement_id][$systemA][$systemB]=1;
								} elseif ($scoreB < $scoreA) {
									$user_rankings[$user][$item_num][$judgement_id][$systemA][$systemB]=-1;
								} else {
									$user_rankings[$user][$item_num][$judgement_id][$systemA][$systemB]=0;
								}
							}
						} elseif ($test_type == "CA") {
							if ($scoreA == '1') { //Yes
								$user_rankings[$user][$item_num][$judgement_id][$systemA] = 1;
							} elseif ($scoreA == '2') { //No
								$user_rankings[$user][$item_num][$judgement_id][$systemA] = -1;
							} elseif  ($scoreA == '3') { //Not Sure
								if($include_unsure) $user_rankings[$user][$item_num][$judgement_id][$systemA] = 0;
								else continue;
							}
							$user_judgements[$user][$item_num][$judgement_id] = $judgement;

						}
						
					}
				}
			}
			$user_coverage = array();
			$agreements = 0;
			$total = 0;

			foreach ($user_rankings as $user1=>$item_nums1) {
				//print_r($item_nums1);
				foreach($user_rankings as $user2=>$item_nums2) {
					if (!$do_intra_annotator && ($user1 == $user2)) continue;
					if ($do_intra_annotator && ($user1 != $user2)) continue;
					if ($user_coverage[$user2][$user1]) continue;
					$user_coverage[$user1][$user2] = 1;
					foreach($item_nums1 as $item_num1=>$judgements1) {
						$judgements2 = $item_nums2[$item_num1];
						if (!$judgements2) continue;
						//print("$user1 to $user2 for $item_num1 ".count($judgements1)." and ".count($judgements2)."<br>");
						foreach($judgements1 as $judgement1=>$systemA1s) {
							if ($test_type == 'R' || $test_type == 'C') {

								foreach($systemA1s as $systemA1=>$systemB1s) {
									foreach($systemB1s as $systemB1=>$comparison1) {
										foreach($judgements2 as $j2key=>$judgement2) {
											if ($do_intra_annotator && $j2key == $judgement1) {
												//print("skipping agreement with $j2key and $judgement1<br>");
												continue;
											}
											$comparsion2 = null;
											if (array_key_exists($systemA1,$judgement2) && array_key_exists($systemB1,$judgement2[$systemA1])) {
												$comparison2 = $judgement2[$systemA1][$systemB1];
												//print("system match $systemA1 $systemB1<br>");
											} else if (array_key_exists($systemB1,$judgement2) && array_key_exists($systemA1,$judgement2[$systemB1])) {
												$comparison2 = $judgement2[$systemB1][$systemA1];
												//print("reverse system match $systemA1 $systemB1<br>");
												$comparison2 *= -1;
											} else {
												continue;
											}
											if ($comparison1 == $comparison2) {
												$agreements++;
												$user_agreements[$user1][$test_type]++;
												if (!$do_intra_annotator) $user_agreements[$user2][$test_type]++;
												//print("$test_type $j2key vs $judgement1<br>");
												//print_r($systemA1s);
												//print("<BR>");
												//print_r($judgement2);
												//print("<BR>");
												
											}
											$total++;
											$user_total[$user1][$test_type]++;
											if (!$do_intra_annotator) $user_total[$user2][$test_type]++;
											//print("$user1 said $comparison1, $user2 said $comparison2 on $judgement1 and $j2key for $item_num1<br>");
											//print_r($user_judgements[$user1][$item_num1][$judgement1]);
											//print("<BR>");
											//print_r($user_judgements[$user2][$item_num1][$j2key]);
											//print("<BR>");

										}
									}
								}
							} elseif ($test_type == "CA") {
								foreach($systemA1s as $system1=>$comparison1) {
									foreach($judgements2 as $j2key=>$judgement2) {
										if ($do_intra_annotator && $j2key == $judgement1) {
											//print("skipping agreement with $j2key and $judgement1<br>");
											continue;
										}
										$comparison2 = null;
										if (array_key_exists($system1,$judgement2)) {
											$comparison2 = $judgement2[$system1];
											//print("system match $system1<br>");
										} else {
											continue;
										}
										if ($comparison1 == $comparison2) {
											$agreements++;	
											$user_agreements[$user1][$test_type]++;
											if (!$do_intra_annotator) $user_agreements[$user2][$test_type]++;
										}
										$total++;
										$user_total[$user1][$test_type]++;
										if (!$do_intra_annotator) $user_total[$user2][$test_type]++;
										//print("$user1 said $comparison1, $user2 said $comparison2 on $judgement1 and $j2key for $item_num1<br>");
										//print_r($user_judgements[$user1][$item_num1][$judgement1]);
										//print("<BR>");
										//print_r($user_judgements[$user2][$item_num1][$j2key]);
										//print("<BR>");
									}
								}
							}
							
						}

					}
				
				}
			
			}
			//print("$task $test_type agreed $agreements out of $total comparisons.<BR>");
			
			//store global info
			$type_agreements[$test_type]['agree']+=$agreements;
			$type_agreements[$test_type]['total']+=$total;
			//ok, user rankings is populated for all users, so double-iterate
			if ($test_type == "R" || $test_type == "C")  {
				$type_agreements[$test_type]['type'] = "relative";
				$type_agreements[$test_type]['chance'] = 1.0/3.0;
			} elseif ($test_type == "CA") {
				$type_agreements[$test_type]['type'] = "absolute";
				$type_agreements[$test_type]['chance'] = 1.0/3.0;
				if (!$include_unsure) $type_agreements[$test_type]['chance'] = 1.0/2.0;
			}
			//do local
			//$task_type_agreements[$test_type]['agree']=$agreements;
			//$task_type_agreements[$test_type]['total']=$total;
			//$task_type_agreements[$test_type]['type']=$type_agreements[$test_type]['type'];
			//$task_type_agreements[$test_type]['chance']=$type_agreements[$test_type]['chance'];
			
			
		}
		//do_annotator_table($task_type_agreements,$include_unsure,$do_intra_annotator,$task);
			
	}
	do_annotator_table($type_agreements,$include_unsure,$do_intra_annotator,"");

	$prefix="Inter-";
	if ($do_intra_annotator) $prefix="Intra-";
	$summary_tables[$prefix."Annotator Agreement per User"]['header']=array('','Agree','Total','P(A)','P(E)','K');
	foreach($user_agreements as $user=>$test_types) {
		foreach($test_types as $test_type=>$value)	{
			$summary_tables[$prefix."Annotator Agreement per User"][$user." ".get_test_type_name($test_type)][]=$user_agreements[$user][$test_type];		
			$summary_tables[$prefix."Annotator Agreement per User"][$user." ".get_test_type_name($test_type)][]=$user_total[$user][$test_type];		
			$chance = 1.0/3.0;
			if (($test_type == "CA") && (!$include_unsure)) $chance = 1.0/2.0;
			$summary_tables[$prefix."Annotator Agreement per User"][$user." ".get_test_type_name($test_type)][]=$user_agreements[$user][$test_type]/$user_total[$user][$test_type];		
			$summary_tables[$prefix."Annotator Agreement per User"][$user." ".get_test_type_name($test_type)][]=$chance;		
			$summary_tables[$prefix."Annotator Agreement per User"][$user." ".get_test_type_name($test_type)][]=kappa($user_agreements[$user][$test_type]/$user_total[$user][$test_type],$chance);			
		}
	
	}


}

function do_annotator_table(&$type_agreements,$include_unsure=true,$do_intra_annotator,$task="") {
	$prefix = "Inter-";
	if ($do_intra_annotator) $prefix="Intra-";
	print_table_title("$task $prefix"."Annotator Agreement ".($include_unsure ? "(with 'Not Sure' for CA)" : ""));
	print_table_start();
	print_header_row(array("Test Type","Agree","Total","P(A)","P(E)","K"));
	foreach($type_agreements as $test_type=>$values) {
		print_row_start();
		print_header_cell(get_test_type_name($test_type)." (".$values['type'].")");
		print_cell_txt($values['agree']);
		print_cell_txt($values['total']);
		print_cell_txt($values['agree']/$values['total'],false,1,3);
		print_cell_txt($values['chance'],false,1,3);
		print_cell_txt(kappa($values['agree']/$values['total'],$values['chance']),false,1,3);
		print_row_end(); 
	}
	print_table_end();
}


function get_test_type_name($test_type) {
	$test_type_ids = get_global_task_property('test_type_ids');
	return $test_type_ids[$test_type];
}


function print_task_name($task) {
	print("<h1>$task</h1>\n");
}

function print_table_title($title) {
	print("<h2>$title</h2>\n");
}

function print_test_type($test_type) {
	print_table_title(get_test_type_name($test_type));
}

function print_notes($test_type) {
	if ($test_type == R || $test_type == C) {
		print("<p>Showing column better than row / total (equal in parentheses)</p\n");
	
	}
}

function print_table_start() {
	print("<table border=\"1\">\n");
}

function print_row_start() {
	print("<tr>\n");

}

function print_row_end() {
	print("</tr>\n");

}

function print_table($name,$rows) {
	print_table_title($name);
	print_table_start();
	
	//get all column keys for header row
	$column_headers = array();
	$table_headers = array();
	$text_header = false;
	$first_row_key = array_shift(array_keys($rows));
	if ($first_row_key == 'header') {
		$table_headers = array_shift($rows);
		$text_header = true;
	} else {
		foreach ($rows as $row_key=>$columns) {
			foreach($columns as $column_key=>$ignore) {
				$column_headers[$column_key]=1;
			}
		}
		$column_headers = array_keys($column_headers);
		sort($column_headers);
		$table_headers = $column_headers;
		if (is_string($row_key)) array_unshift($table_headers,"");
		
	}
	print_header_row($table_headers);
	//$first_row = array_pop($rows);
	foreach($rows as $row_key=>$columns) {
		print_row_start();
		if(is_string($row_key)) print_header_cell($row_key);
		if ($text_header) {
			foreach($columns as $ignored_key=>$cell) {
				print_cell($cell);
			}
		} else {
			foreach($column_headers as $column_key) {
				print_cell($columns[$column_key]);
			}
		}
		print_row_end();
	}
	
	print_table_end();
}

function print_cell($cell,$precision=3) {
	if (is_object($cell)) print_cell_obj($cell,$precision);
	else print_cell_txt($cell,false,1,$precision);
}

function print_cell_txt($contents,$best=false,$p=1,$precision=3) {
	if (trim($contents) == "") $contents = '&nbsp;';
	if (is_float($contents)) $contents = sprintf("%0.".$precision."f",$contents);
	if ($p <= 0.05) {
		$contents = "$contents*";
	}
	if ($p <=0.01) {
		$contents = "$contents*";
	}

	if ($best) $contents = "<b>+ $contents</b>";
	print("<td>$contents</td>");
}

function print_header_row($row_contents) {
	print_row_start();
	foreach($row_contents as $column) {
		print_header_cell($column);
	}
	print_row_end();
}

function print_header_cell($contents) {
	if (trim($contents) == "") $contents = '&nbsp;';
	print("<th>$contents</th>");
}

function print_table_end() {
	print("</table>");
}

#below taken from: http://www.fon.hum.uva.nl/rob/Statistics-tools/SignTest.pl and http://www.fon.hum.uva.nl/rob/Statistics-tools/Normal-Z.pl

function sign_test($n1,$n2) {
	$N = $n1 + $n2;
	$i = ($n1 < $n2) ? $n1 : $n2;
	$p = 0;
	if ($N <= 100) {
		while($i>=0) {
			$p += k_out_n($i,$N);
			$i--;
		}
		$p /= exp(log(2)*$N);
		$p *= 2;
	} else {
		$Z = abs($n1 - $n2);
		$Z = ($Z-1)/sqrt($N);
		$p = normal_z($Z);
	}
	if ($p > 1) $p = 1;
	//print("ptest for $n1 and $n2 is $p<br>");
	return $p;
}

function k_out_n($k,$n) {
	$kn = 1.0;
	for(; $k>0; --$k, --$n) {
		$kn *= $n/$k;
	}
	return $kn;
}

function normal_z($x) {
	$b = array(0.319381530, -0.356563782, 1.781477937, -1.821255978, 1.330274429);
	$p = 0.2316419;
	$t = 1/(1+$p*$x);

	$fact = $t;
	$sum = 0.0;
	
	# Sum polynomial
	foreach($b as $value){ 
	  $sum += $value*$fact;
	  $fact *= $t;
	};
	# Calculate probability
	$p = 2*$sum*exp(-$x*$x/2.0)/(sqrt(2*pi()));
	return $p;
}

function kappa($agree,$chance) {
	return(($agree-$chance)/(1-$chance));
}

function get_constituent_mappings($constituent_file) {
	//we iterate through the verbose file, loading array of system=>string_num for each line
	if (!file_exists($constituent_file)) return false;
	$mappings=array();
	$fh = fopen($constituent_file,'r');
	while($line = fgets($fh)) {
		$contents = split(' \|\|\| ',$line);
		$system_map = array();
		$phrase_map = array();
		array_shift($contents); //sentence num 
		array_shift($contents);  //part of speech
		array_shift($contents); //src
		//array_shift($contents); //reference
		foreach ($contents as $info_str) {
			list($system_str,$phrase) = split("\t",$info_str);
			list($system,$pos) = split(':',$system_str);
			if(!array_key_exists($phrase,$phrase_map)) $phrase_map[$phrase]=count($phrase_map);
			$system_map[$system]=$phrase_map[$phrase];
		}
		$mappings[] = $system_map;
	}
	fclose($fh);
	return $mappings;
}