//
// Core Program for TETDM 
// TextData.java Version 0.14
// Copyright(C) 2011 2012 TETDM CORE MEMBER ALL RIGHTS RESERVED.
// This program is provided under the license.
//


package source.TextData;

import source.*;

import java.io.*;
import java.util.*;

import java.util.regex.*;

public class TextData
{
	public String originalText;		//Loaded text data	
	public String sourceFilename;	//Load text filename
	public String absolutePath;		//Absolute path of execution directory
	public String defaultCode;
	
	//Segment Data
	public SegmentData segment[];
	public int segmentNumber;	

	//Sentence Data
	public SentenceData sentence[];		//pointer to segment[].sentence[]
	public int sentenceNumber;	
	
	public int sentenceToSegment[];		//segment number of i-th sentence
	
	//Keywords Data
	public KeywordData keyword[];
	public int keywordNumber;

	public final int maxKeywordNumber = 10000;	//Maximum number of keywords, The process is NOT implemented when number is above 10000.

    //Word Data
    public int wordNumber;
	
	//Text Relation Data
	public RelationalData segmentRelation, sentenceRelation, keywordRelationBySegment, keywordRelationBySentence;
	
	
	//display for visualization
	public int fontSize = 14;

	public class Focus
	{
		public boolean focusKeywords[];
		public boolean focusSegments[];
		public boolean focusSentences[];

		public int mainFocusKeyword, subFocusKeyword;
		public int mainFocusSegment, subFocusSegment;
		public int mainFocusSentence, subFocusSentence;
		
		public String mainFocusString, subFocusString;
		public double mainFocusDouble, subFocusDouble;
		public int mainFocusInteger, subFocusInteger;		
		
		Focus(int keywordNumber, int sentenceNumber, int segmentNumber)
		{
			focusKeywords = new boolean[keywordNumber];
			focusSegments = new boolean[segmentNumber];
			focusSentences = new boolean[sentenceNumber];
			
			clear();
		}
		
		public void clear()
		{
			mainFocusKeyword = subFocusKeyword = mainFocusSegment = subFocusSegment = mainFocusSentence = subFocusSentence = -1;
			mainFocusString = subFocusString = "";
			mainFocusDouble = subFocusDouble = 0.0;
			mainFocusInteger = subFocusInteger = 0;
			
			for(int i=0;i<keywordNumber;i++)
				focusKeywords[i] = false;

			for(int i=0;i<segmentNumber;i++)
				focusSegments[i] = false;		
			
			for(int i=0;i<sentenceNumber;i++)
				focusSentences[i] = false;
		}
	}

	public Focus focus;
	
	public Control control;
	
	public SettingData settingData;
	
	public Connection connection;	
	
	public TextData(){}

	public TextData(String sourcefile, String chafile, SettingData sp1, String absolute, Control cont)
	{
		sourceFilename = sourcefile;
		defaultCode = sp1.defaultCode;
		settingData = sp1;
		control = cont;//		this.cont = cont;	
		absolutePath = absolute;
		
		connection = new Connection(this);
		
		originalText = FILEIO.TextFileAllReadCode(new File(sourcefile));//Prepare the original text

		//Keywords Data
		keyword = new KeywordData[maxKeywordNumber];

		for(int i=0;i<maxKeywordNumber;i++)
			keyword[i] = new KeywordData();
		keywordNumber = 0;

		//Segment Data
		segmentNumber = FILEIO.CountTags(originalText, sp1.segment_tag);
		segment = new SegmentData[segmentNumber];
		String cut_text[] = new String[segmentNumber];
		FILEIO.CutText(originalText, cut_text, sp1.segment_tag);
		for(int i=0;i<segmentNumber;i++)
			segment[i] = new SegmentData(cut_text[i], i);

		//Load words information from the result of ChaSen
		readChasenFile(chafile, sp1, sp1.isEnglish);	//Read word data from a cha-file	//count sentenceNumber
		
		System.out.println("SEGMENT NUMBER = "+segmentNumber);
		System.out.println("SENTENCE NUMBER = "+sentenceNumber);
		System.out.println("KEYWORD NUMBER = "+keywordNumber);		

		//Copy sentence, Calculate: sentenceToSegment[]
		sentenceToSegment = new int[sentenceNumber];
		sentence = new SentenceData[sentenceNumber];
		
		for(int i=0,s=0;i<segmentNumber;i++)
			for(int j=0;j<segment[i].sentenceNumber;j++)
			{
				sentence[s] = segment[i].sentence[j];
				sentenceToSegment[s++] = i;
			}	

		//Count word Number
		wordNumber=0;
		for(int i=0;i<sentenceNumber;i++)
		    wordNumber+=sentence[i].wordNumber;

		//Calculate keyword appearance data: appearingSentenceTable[], appearingSentence[], sentenceFrequency, seg_times[], seg_sentimes[], appearingSegmentTable[], appearingSegment[], segmentFrequency 
		createKeywordAppearance();
		
		//Calculate segment[].kind_num, sentence[].kind_num
		createKindNumbers();
		
		//PreProcess 1, Calculate cos similarity between segments, sentences, keywords by segment frequencies, and keywords by sentences frequencies
		createTextRelation();
		
		
		boolean active[] = new boolean[keywordNumber];
		for(int i=0;i<keywordNumber;i++)
			if(keyword[i].segmentFrequency < 2)
				active[i] = false;
			else
				active[i] = true;

//		keywordRelationBySegment.optimizeLink(keywordRelationBySegment.cos, 0.5, active);
		
/*		MakeCluster mk = new MakeCluster(keywordRelationBySegment);   CLUSTERING */
		
//		boolean active[] = new boolean[segmentNumber];		
		
//		segr.optimizeLink(segr.cos, 0.5, active);
//		MakeCluster mk = new MakeCluster(segr);
		
		
		focus = new Focus(keywordNumber,sentenceNumber,segmentNumber);
	}

//---------------------------	
	
	public void fileSave(String alltext)
	{
		File savefile = new File(sourceFilename);
		
		try
		{
			BufferedWriter bw1 = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(savefile), defaultCode));
			
			bw1.write(alltext);	
			bw1.close();			
			System.out.println("Save text to the file: "+savefile.getName());
		}
		catch(Exception e){
			System.out.println("File Writing ERROR Text data");
		}		
	}
	
	public void fileSave(String subtexts[])
	{
		File savefile = new File(sourceFilename);
		
		try
		{
			BufferedWriter bw1 = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(savefile), defaultCode));

			for(int i=0;i<subtexts.length;i++)
			{	
				bw1.write(subtexts[i]);
			}	
			bw1.close();
			System.out.println("Save text to the file: "+savefile.getName());			
		}
		catch(Exception e){
			System.out.println("File Writing ERROR Text data");
		}		
	}	


	public void createKeywordAppearance()
	{
		int id;
		
		for(int i=0;i<keywordNumber;i++)
			keyword[i].initializeTables(segmentNumber, sentenceNumber);
		
		for(int i=0, k=0;i<segmentNumber;i++)
			for(int j=0;j<segment[i].sentenceNumber;j++,k++)
				for(int n=0;n<segment[i].sentence[j].keywordNumber;n++)
				{
					id = segment[i].sentence[j].keywordIDList[n];

					if(keyword[id].lastAppearingSegment != i && keyword[id].frequency > 0)
						// Avoid from counting twice in a segment, Noise may have id but frequency must be zero					
					{
						keyword[id].appearingSegmentTable[i] = 1;
						keyword[id].lastAppearingSegment = keyword[id].appearingSegment[keyword[id].segmentFrequency++] = i;
					}
					
					if(keyword[id].lastAppearingSentence != k && keyword[id].frequency > 0)
						// Avoid from counting twice in a sentence, Noise may have id but frequency must be zero					
					{
						keyword[id].appearingSentenceTable[k] = 1;
						keyword[id].lastAppearingSentence = keyword[id].appearingSentence[keyword[id].sentenceFrequency++] = k;
						
						keyword[id].sentenceFrequencyInSegment[i]++;
					}
					keyword[id].frequencyInSegment[i]++;
				}
	}
	
	public void createKindNumbers()
	{
		int kindlist[];
		
		kindlist = new int[segmentNumber];		
		for(int i=0;i<segmentNumber;i++)
		{
			kindlist[i] = 0;

			for(int j=0;j<keywordNumber;j++)
				for(int k=0;k<segmentNumber && keyword[j].appearingSegment[k] != -1;k++)
					kindlist[keyword[j].appearingSegment[k]]++;
			
			segment[i].kindKeywordNumber = kindlist[i];
		}
		
		kindlist = new int[sentenceNumber];		
		for(int i=0;i<sentenceNumber;i++)
		{
			kindlist[i] = 0;
			
			for(int j=0;j<keywordNumber;j++)
				for(int k=0;k<sentenceNumber && keyword[j].appearingSentence[k] != -1;k++)
					kindlist[keyword[j].appearingSentence[k]]++;
			
			sentence[i].kindKeywordNumber = kindlist[i];
		}		
	}

//---------------------------
	
	String nameList[];
	int frequencyList[];
	int coFrequencyList[][];	
	
	public void createTextRelation()
	{
		int limit = 1000;

		if(segmentNumber > limit)
			System.out.println("NOT CREATED SEGMENT RELATION segmentNumber/max =:"+segmentNumber +"/"+limit);
		else
		{
			Pattern pattern = Pattern.compile("([A-Za-z0-9_]+.txt)");		//TEXT FILE NAME
			
			segmentRelation = new RelationalData(segmentNumber);
			initializeLists(segmentNumber);
			for(int i=0;i<segmentNumber;i++)
				if(segment[i].sentenceNumber > 0)				
				{
					Matcher matcher = pattern.matcher(segment[i].sentence[0].sentenceText);
					if(matcher.find())
						nameList[i] = matcher.group(1);
					else
						nameList[i] = "SEG"+(i+1);
				}			
			for(int i=0;i<segmentNumber;i++)
				frequencyList[i] = segment[i].kindKeywordNumber;
			for(int i=0;i<segmentNumber;i++)
				for(int j=0;j<=i;j++)
				{
					coFrequencyList[i][j] = coFrequencyList[j][i] = 0;
					for(int k=0;k<keywordNumber;k++)
						coFrequencyList[i][j] = coFrequencyList[j][i] += (keyword[k].appearingSegmentTable[i] * keyword[k].appearingSegmentTable[j]);
				}
			calcurateValues(segmentRelation);
		}
		
		if(sentenceNumber > limit)
			System.out.println("NOT CREATED SENTENCE RELATION sentenceNumber/max =:"+sentenceNumber +"/"+limit);
		else
		{
			sentenceRelation = new RelationalData(sentenceNumber);
			initializeLists(sentenceNumber);
			for(int i=0;i<sentenceNumber;i++)
				nameList[i] = "SEN"+(i+1);
			for(int i=0;i<sentenceNumber;i++)
				frequencyList[i] = sentence[i].kindKeywordNumber;
			for(int i=0;i<sentenceNumber;i++)
				for(int j=0;j<=i;j++)
				{
					coFrequencyList[i][j] = coFrequencyList[j][i] = 0;
					for(int k=0;k<keywordNumber;k++)
						coFrequencyList[i][j] = coFrequencyList[j][i] += (keyword[k].appearingSentenceTable[i] * keyword[k].appearingSentenceTable[j]);
				}
			calcurateValues(sentenceRelation);
		}
		
		if(keywordNumber > limit)
			System.out.println("NOT CREATED KEYWORD RELATION keywordNumber/max =:"+keywordNumber +"/"+limit);
		else
		{
			keywordRelationBySegment = new RelationalData(keywordNumber);
			initializeLists(keywordNumber);
			for(int i=0;i<keywordNumber;i++)
				nameList[i] = keyword[i].word;
			for(int i=0;i<keywordNumber;i++)
				frequencyList[i] = keyword[i].segmentFrequency;
			for(int i=0;i<keywordNumber;i++)
				for(int j=0;j<=i;j++)
				{
					coFrequencyList[i][j] = coFrequencyList[j][i] = 0;
					for(int k=0;k<segmentNumber;k++)
						coFrequencyList[i][j] = coFrequencyList[j][i] += (keyword[i].appearingSegmentTable[k] * keyword[j].appearingSegmentTable[k]);
				}
			calcurateValues(keywordRelationBySegment);
			
			keywordRelationBySentence = new RelationalData(keywordNumber);
			initializeLists(keywordNumber);
			for(int i=0;i<keywordNumber;i++)
				nameList[i] = keyword[i].word;
			for(int i=0;i<keywordNumber;i++)
				frequencyList[i] = keyword[i].sentenceFrequency;
			for(int i=0;i<keywordNumber;i++)
				for(int j=0;j<=i;j++)
				{
					coFrequencyList[i][j] = coFrequencyList[j][i] = 0;
					for(int k=0;k<sentenceNumber;k++)
						coFrequencyList[i][j] = coFrequencyList[j][i] += (keyword[i].appearingSentenceTable[k] * keyword[j].appearingSentenceTable[k]);
				}
			calcurateValues(keywordRelationBySentence);
		}
	}
	
	void initializeLists(int num)
	{
		nameList = new String[num];
		frequencyList = new int[num];
		coFrequencyList = new int[num][num];
	}
	
	void calcurateValues(RelationalData rd)
	{
		rd.setName(nameList);
		rd.setFrequency(frequencyList);
		rd.setCoFrequency(coFrequencyList);
		rd.createCondValue();
		rd.createCosValue();
	}

	
	
//---------------------------	
	

	//Save load data to buffer and copies after all
	public void readChasenFile(String chaf, SettingData sp1, boolean english)
	{
		File chafile = new File(chaf);
		String ss;
		String tempword, gomi1, endformword, hinshi;
		int notr = 0;		//Flag for eliminating new-lines without words at the end of a segment

		System.out.println("##chafile = "+chaf);
		
		
		String keywords = "";

		try
		{
			int countw1 = 0;	//Number of keywords in a sentence
			int countw2 = 0;	//Number of all words in a sentence
			int counts = 0;		//Number of lines
			int total_counts = 0;	//Number of all lines in original text
			int seg_n = 0;		//Number of segment that is currently loaded

			int count = 0;		//For search subjects(Nominative)
			int last_id = 0;	//For search subjects, Subject ID

			BufferedReader br1 = new BufferedReader(new InputStreamReader(new FileInputStream(chafile), "JISAutoDetect"));
			StringWriter sw1 = new StringWriter();			//For words
			BufferedWriter bw1 = new BufferedWriter(sw1);
			StringWriter sw2 = new StringWriter();			//For word IDs
			BufferedWriter bw2 = new BufferedWriter(sw2);
			StringWriter sw3 = new StringWriter();			//For keywords
			BufferedWriter bw3 = new BufferedWriter(sw3);
			StringWriter sw4 = new StringWriter();			//For end-form of words
			BufferedWriter bw4 = new BufferedWriter(sw4);

			while((ss = br1.readLine()) != null)
			{
				if(ss.equals(""))
					continue;
			
				String oneline[] = ss.split("	"); //TAB
				tempword = oneline[0];
				
				if(tempword.equals("EOS"))
				{

					bw1.newLine();
					bw1.write(" ");

					bw4.newLine();
					bw4.write(" ");
					bw2.write("-8 ");
					countw2++;
					continue;
				}

				gomi1 = oneline[1];
				endformword = oneline[2];
				hinshi = oneline[3];
				
				if(hinshi.equals(sp1.unknown))	//UNKNOWN word
				{
					if(tempword.equals(sp1.segment_tag))	//The end of the segment [sunaribarafuto]
					{
						//Process at the end of a sentence//Do not count as sentence that is only new-line
						if(countw2 > 0 && notr == 1)
						{
							counts++;
							bw1.write(" @@");
							bw2.write(" @");
							bw3.write(countw1+" ");
							bw4.write(" @@");
//							System.out.println("SEN:"+counts+" "+countw1+"/"+countw2);
							countw1 = countw2 = 0;
							notr = 0;
						}
						//Process at the end of a segment
						segment[seg_n].sentenceNumber = counts;
						segment[seg_n].positionOfFirstSentence = total_counts;
						total_counts += counts;
						bw1.flush();
						bw2.flush();
						bw3.flush();
						bw4.flush();
						segment[seg_n].initializeSentence(sw1.toString(), sw2.toString(), sw3.toString(), sw4.toString(), english);
						sw1.getBuffer().delete(0,sw1.getBuffer().length());
						sw2.getBuffer().delete(0,sw2.getBuffer().length());
						sw3.getBuffer().delete(0,sw3.getBuffer().length());
						sw4.getBuffer().delete(0,sw4.getBuffer().length());
						seg_n++;
//						System.out.println("SEG-"+seg_n+" sen:"+counts);
						counts = 0;
					}
					else
					{
						bw1.write(tempword+" ");
						bw4.write(endformword+" ");
						bw2.write("-1 ");
						countw2++;
						notr = 1;
					}
					continue;
				}

//System.out.println(tempword);

				if(oneline[1].equals(""))	//For ChaSen bug { Symbol opening parenthesis
				{
					bw1.write(tempword+" ");
					bw4.write(endformword+" ");
					bw2.write("-2 ");
					countw2++;
					notr = 1;
					continue;
				}

				bw1.write(tempword+" ");	//All other words
				bw4.write(endformword+" ");	//All other words
				notr = 1;
				countw2++;

				int k=0;	//For ID

				// End of a sentence
				for(int i=0;i<sp1.touns;i++)
					if(tempword.equals(sp1.kind_touns[i]))
					{
						counts++;
						bw1.write(" @@");
						bw2.write("-3 @");
						bw3.write(countw1+" ");
						bw4.write(" @@");
//						System.out.println("SEN:"+counts+" "+countw1+"/"+countw2);
						countw1 = countw2 = 0;
						notr = 0;
						k=1;
					}

				count++;

				//Particles for finding subject
				for(int i=0;i<sp1.objects;i++)
					if(tempword.equals(sp1.kind_objects[i]) && hinshi.indexOf(sp1.kind_objects_speech[i]) == 0)
					{
						if(count <= 2) //Permit including [SAMA, MONO(SHA), TACHI],[DE, NI, NIOITE, NITSUITE]
						{
//							System.out.println(count+":"+tempword);

/*							keyword[last_id].subject++;
							if(keyword[last_id].lastseg != seg_n)
							{
								keyword[last_id].subject_sn++;
								keyword[last_id].lastseg = seg_n;
							}												SHUGO HINDO */
						}
						break;
					}

				//Process for noise-1: Elimination of stop words
				{
					int i;
					for(i=0;i<sp1.noises;i++)
						if(endformword.equals(sp1.kind_noises[i]))
							break;
					if(i < sp1.noises)
					{
						bw2.write("-4 ");
						continue;
					}
				}

				//Process for noise-2: Elimination of HIRAGANA, one letter except for periods(k==0)
				if(endformword.length() == 1 && k == 0)//For One character words
					if(sp1.excludeOne)
					{
						bw2.write("-5 ");
						continue;
					}

				char code = tempword.charAt(0);
				if(code>=0x3041 && code<=0x3093)	//For HIRAGANA words
					if(sp1.excludeHiragana)					
					{
						bw2.write("-6 ");
						continue;
					}
				
				if(code>=0x30a1 && code<=0x30f3)	//For KATAKANA words
					if(sp1.excludeKatakana)					
					{
						bw2.write("-6 ");
						continue;
					}				

				//Keywords without conjugation
				for(int i=0;i<sp1.nouns;i++)
					if(hinshi.indexOf(sp1.kind_nouns[i]) == 0)
					{
						//Entry for Keyword list
						countw1++;
						int j;
						for(j=0;j<keywordNumber;j++)
							if(endformword.equals(keyword[j].word))
								break;
						if(j == keywordNumber)
						{
							keyword[keywordNumber].word = endformword;
							keywordNumber++;
							keywords += (tempword+" ");
						}

						keyword[j].frequency++;	//Count frequency
						count=0;			//Subject candidate
						last_id = j;		//Subject candidate ID

						bw2.write(j+" ");
						k=1;

						if(hinshi.indexOf(sp1.kind_speech[0]) ==0)
							keyword[j].partOfSpeech = 1;//NOUN
						else if(hinshi.indexOf(sp1.kind_speech[1]) ==0)
							keyword[j].partOfSpeech = 4;//CONJUNCTION
						else if(hinshi.indexOf(sp1.kind_speech[2]) ==0)
							keyword[j].partOfSpeech = 5;//ADVERB
						else if(hinshi.indexOf(sp1.kind_speech[3]) ==0)
							keyword[j].partOfSpeech = 6;//INTERJECTION
						else if(hinshi.indexOf(sp1.kind_speech[4]) ==0)
							keyword[j].partOfSpeech = 7;//PARTICLE
					}

				//Keywords with conjugation
				for(int i=0;i<sp1.verbs;i++)
					if(hinshi.indexOf(sp1.kind_verbs[i]) == 0)
					{
						//Entry for keyword list
						countw1++;
						int j;
						for(j=0;j<keywordNumber;j++)
							if(endformword.equals(keyword[j].word))
								break;
						if(j == keywordNumber)
						{
							keyword[keywordNumber].word = endformword;
							keywordNumber++;
							keywords += (" "+tempword);
						}
						keyword[j].frequency++;	//Count frequency

						bw2.write(j+" ");
						k=1;

						if(hinshi.indexOf(sp1.kind_speech[5]) ==0)
							keyword[j].partOfSpeech = 2;//VERB
						else if(hinshi.indexOf(sp1.kind_speech[6]) ==0)
							keyword[j].partOfSpeech = 3;//ADJECTIVE
						else if(hinshi.indexOf(sp1.kind_speech[7]) ==0)
							keyword[j].partOfSpeech = 8;//AUXILIARY VERB
					}
				if(k == 0)
					bw2.write("-7 ");
			}

			br1.close();

			if(countw2 > 1 && notr == 1)	//When text does not end with a period
											//Process at the end of sentences//Do not count as sentence which has only new-line
			{
				counts++;
				bw3.write(countw1+" ");
//				System.out.println("SEN:"+counts+" "+countw1+"/"+countw2);
			}

			//AT THE END OF A SEGMENT

			segment[seg_n].sentenceNumber = counts;
			segment[seg_n].positionOfFirstSentence = total_counts;
			total_counts += counts;
			bw1.flush();
			bw2.flush();
			bw3.flush();
			bw4.flush();
			segment[seg_n].initializeSentence(sw1.toString(), sw2.toString(), sw3.toString(), sw4.toString(), english);
			sw1.getBuffer().delete(0,sw1.getBuffer().length());
			sw2.getBuffer().delete(0,sw2.getBuffer().length());
			sw3.getBuffer().delete(0,sw3.getBuffer().length());
			sw4.getBuffer().delete(0,sw4.getBuffer().length());
			seg_n++;
//			System.out.println("SEG-"+seg_n+" sen:"+counts);

			sentenceNumber = total_counts;
		}
		catch(Exception e){
			System.out.println("File Reading ERROR chasen data");
		}
	}
}



