
/*******************************************************************************
 * Copyright 2014 Centre for Development of Advanced Computing(C-DAC), Pune
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package in.cdac.medinfo.csnotk.csnolib.icdmap;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
import java.util.Map;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MMapDirectory;

import in.cdac.medinfo.csnotk.csnolib.commons.Constants;
import in.cdac.medinfo.csnotk.csnolib.commons.EnumProperties;
import in.cdac.medinfo.csnotk.csnolib.db.ConnectionManager;
import in.cdac.medinfo.csnotk.csnolib.db.DBUtil;
import in.cdac.medinfo.csnotk.csnolib.util.PropertyReader;
//Logs
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import in.cdac.medinfo.csnotk.csnolib.util.LoggerUtility;
class LuceneICDMapAgent {
	private List<MappingComponent> mappingComponentList=null;
	private ListIterator<MappingComponent> mappingComponentListIterator=null;
	private Map<String,MappingComponent> mapMC=null;

		@SuppressWarnings("unused")
		private static final long serialVersionUID = 1L;

		/**
		 * static Fields
		 */
		private  IndexWriter indexWriter = null;
		private  IndexSearcher searcher = null;
		private  Analyzer analyzer = null;

		private  IndexWriterConfig indexConfig = null;
		private  Directory indexDirectory = null;
		private  IndexReader indexReader = null;
		private static Object mutex= new Object();


		/**
		 * Returns {@link IndexWriter} object.
		 * @param refsetId Identifier of Refset
		 * @return object of {@link IndexWriter}.
		 * @throws IOException if any error occurs while processing.
		 */
		private  IndexWriter getIndexWriter(String refsetId) throws IOException
		{
			if(this.indexWriter==null)
				this.indexWriter=new IndexWriter(getIndexDirectoryToWrite(refsetId),getIndexConfig());
			return indexWriter;
		}

		/**
		 * Returns {@link IndexReader} object.
		 * @param refsetId Identifier of Refset
		 * @return instance of {@link IndexReader}.
		 * @throws IOException if any error occurs while processing.
		 */
		private  IndexReader getIndexReader(String refsetId) throws IOException{
			if(indexReader==null){
				synchronized (mutex){
					if(getIndexDirectoryToRead(refsetId)!=null)
					indexReader = DirectoryReader.open(getIndexDirectoryToRead(refsetId));
				}
			}
			return indexReader;
		}

		/**
		 * Returns {@link IndexSearcher} object.
		 * @param reader instance of {@link IndexReader} .
		 * @return {@link IndexSearcher}.
		 */
		private IndexSearcher getIndexSearcher(IndexReader reader){
			if(this.searcher==null)
				this.searcher = new IndexSearcher(reader);
			return this.searcher;
		}

		/**
		 * Returns {@link Analyzer} object.
		 * @return {@link Analyzer}.
		 */
		private  Analyzer getAnalyzer(){
			if(this.analyzer==null)
				this.analyzer = new SpecialCharAnalyzer();
			return this.analyzer;
		}



		/**
		 * Returns {@link IndexWriterConfig} object.
		 * @return {@link IndexWriterConfig}.
		 */
		private IndexWriterConfig getIndexConfig(){
			if(this.indexConfig==null)
				this.indexConfig=new IndexWriterConfig(this.getAnalyzer());
			return this.indexConfig;
		}

		/**
		 * Returns the index directory.
		 * @param refsetId Identifier of Refset
		 * @return {@link Directory}.
		 * @throws IOException if any error occurs while processing.
		 */
		private Directory getIndexDirectoryToRead(String refsetId){
            Logger logger = LogManager.getLogger(LuceneICDMapAgent.class);
			if(indexDirectory == null){
				synchronized (mutex){
					PropertyReader.loadSystemProperties();
					try{
						if (refsetId==null)
							//File-based Directory implementation that uses mmap for reading, and FSDirectory.FSIndexOutput for writing.
							indexDirectory = new MMapDirectory(Paths.get(new File(PropertyReader.getProperty(Constants.DATA_DIR)+ File.separator +"SnoIndex").getAbsolutePath()));
						else
							indexDirectory =  new MMapDirectory(Paths.get(new File(PropertyReader.getProperty(Constants.DATA_DIR)+File.separator+refsetId).getAbsolutePath()));
						}catch (IOException e){
						//Log the Exception
                        logger.error(LoggerUtility.MapICDException, LoggerUtility.createErrorMessage(e));
					}

					//public RAMDirectory(Directory dir,IOContext context) throws IOException
					//Creates a new RAMDirectory instance from a different Directory implementation. This can be used to
					//load a disk-based index into memory.

					//Warning: This class is not intended to work with huge indexes. Everything beyond several hundred
					//megabytes will waste resources (GC cycles), because it uses an internal buffer size of 1024 bytes,
					//producing millions of byte[1024] arrays. This class is optimized for small memory-resident indexes.
					//It also has bad concurrency on multithreaded environments.
					//Note that the resulting RAMDirectory instance is fully independent from the original Directory (it is a complete copy).
					//Any subsequent changes to the original Directory will not be visible in the RAMDirectory instance.
				}
			}
			return indexDirectory;
		}
		/**
		 * Returns index directory location to write the files.
		 * @param refsetId Identifier of Refset
		 * @return {@link Directory}.
		 * @throws IOException if any error occurs while processing.
		 */
		private Directory getIndexDirectoryToWrite(String refsetId)throws IOException{
			if(indexDirectory == null){
				PropertyReader.loadSystemProperties();
				try{
					if(refsetId==null)
						indexDirectory = FSDirectory.open(Paths.get(new File(PropertyReader.getProperty(Constants.DATA_DIR)+File.separator+MappingConstants.MAP_INDEX).getAbsolutePath()));
					else
						indexDirectory = FSDirectory.open(Paths.get(new File(PropertyReader.getProperty(Constants.DATA_DIR)+File.separator+refsetId.trim()).getAbsolutePath()));
			}catch(IOException e){
				StackTraceElement arrStackTraceElement[]= e.getStackTrace();
				if(arrStackTraceElement[0].getClassName().toString().equals("java.io.WinNTFileSystem")){
					if(refsetId==null)
					new File(PropertyReader.getProperty(Constants.DATA_DIR)+File.separator+MappingConstants.MAP_INDEX).mkdir();
					else
					new File(PropertyReader.getProperty((Constants.DATA_DIR)+File.separator+refsetId.trim())).mkdir();
					getIndexDirectoryToWrite(refsetId);
				}
			}
}
			return indexDirectory;
		}

		/**
		 * For Deleting the old index.
		 *  @param refsetId	Identifier of MapIndex for deleting the old created index.
		 **/
		boolean deleteIndex(String refsetId){
            Logger logger = LogManager.getLogger(LuceneICDMapAgent.class);
				try{
					this.setIndexDirectory(null);
					indexDirectory= getIndexDirectoryToWrite(refsetId);
						if(indexDirectory != null){
							String[] list = indexDirectory.listAll();
							if(list != null )
								for(int i=0;i<list.length;i++)
									indexDirectory.deleteFile(list[i].toString());
						}
						else{
							logger.info(LoggerUtility.ConsoleWithLog, "Path is invalid");
							return false;
						}
					}catch(IOException e){
						//Log the Exception
                        logger.error(LoggerUtility.MapICDException, LoggerUtility.createErrorMessage(e));
						}
					this.setIndexDirectory(null);
					return true;
		}

		private void setIndexDirectory(Directory indexDirectory){
			this.indexDirectory=indexDirectory;
		}



		/**
		 * Releases Lucene resource like indexWriter, indexDirectory, indexConfig for multiple read/write operations.
		 */
		private void releaseLuceneResources(){
			this.indexWriter=null;
			this.indexDirectory=null;
			this.indexConfig=null; //we need to make it null otherwise it throws exception
		}

		/**
		 * Lucene index creation utility.<br>
		 * @param refsetId Identifier of Refset.
		 */
		boolean createIndex(String refsetId){
            Logger logger = LogManager.getLogger(LuceneICDMapAgent.class);
			Statement statement = null;
			Connection connection = null;
			try{
				long indexOffset = 10;
				logger.info(LoggerUtility.ConsoleWithLog, "Loading System Properties ...");
				PropertyReader.loadSystemProperties();
				indexOffset = Constants.INDEX_OFFSET;
				this.getIndexWriter(refsetId);
				QueryManager qm = new QueryManager();
				long offset = 0;
				logger.info(LoggerUtility.ConsoleWithLog, "Reading database properties ...");
				try{
					connection = ConnectionManager.getConnection();
					statement = connection.createStatement();
				logger.info(LoggerUtility.ConsoleWithLog, "Database Path : {}",PropertyReader.getProperty(Constants.DATA_DIR));
				}catch(SQLException e){
					logger.error(LoggerUtility.ConsoleWithMapICDException, "Database connection Failed ...");
					return false;
				}
				if(refsetId!=null && refsetId!="" && refsetId==MappingConstants.MAP_INDEX){
						long totalRecords = DBUtil.getTotalRows(MappingConstants.ICD_MAP_TABLE,statement,null);
						logger.info(LoggerUtility.ConsoleWithLog, "Total number of ICD_Map records needs to be indexed : {}",totalRecords);
						qm.setOffset(offset);
						qm.setNumberOfRecords(indexOffset);
						logger.info(LoggerUtility.ConsoleWithLog, "Indexes are going to generate ... Please wait while index generation completes ...");
						if(totalRecords>0){
							do{
								logger.info(LoggerUtility.ConsoleWithLog, "Number of records which are indexed till now : {} / {}",offset,totalRecords);
								this.addICDMapDocument(qm.getDomain(MappingConstants.ICD_MAP_TABLE, offset, totalRecords));
								offset = offset+indexOffset;
								qm.setOffset(offset);
								qm.setNumberOfRecords(indexOffset);
							}while(totalRecords > offset);
							if(offset>totalRecords){
							logger.info(LoggerUtility.ConsoleWithLog, "Number of records which are indexed till now : {} / {}",totalRecords,totalRecords);
							logger.info(LoggerUtility.ConsoleWithLog, "ICD_Map index generation process completed !!!");
							}else{
								logger.info(LoggerUtility.ConsoleWithLog, "ICD_Map index generation process interrupted !!!");
								return false;
							}
						}
						this.getIndexWriter(refsetId).commit();
						this.getIndexWriter(refsetId).close();
						this.releaseLuceneResources();
				}
			}catch (IOException|SQLException|java.text.ParseException e){
					//Log the Exception
					if(e instanceof IOException)
						logger.error(LoggerUtility.ConsoleWithMapICDException, "Reading writing issue ...");
					else
						logger.error(LoggerUtility.ConsoleWithMapICDException, "Database connection issue ... ");
                    logger.error(LoggerUtility.MapICDException, LoggerUtility.createErrorMessage(e));
					return false;
				}finally{
					try{
						if(statement!=null)
							statement.close();
						if(connection!=null)
							connection.close();
						}catch(SQLException e){
							logger.error(LoggerUtility.ConsoleWithMapICDException, "Database connection is not closed");
                            logger.error(LoggerUtility.MapICDException, LoggerUtility.createErrorMessage(e));
						}
					}
				return true;
			}



		/**
		 * Return list of {@link MappingComponent} from the SNOMED CT repository of SNOMED to ICD Map based on provided results.
		 * @param results - {@link ScoreDoc[]} results searched from the indexed documents.<br>
		 * @throws IOException If any error occurs while processing.
		 * @throws java.text.ParseException If error occurs at the parsing time.
		 * @return List of {@link MappingComponent} matching the given parameter of {@link ScoreDoc[]}.<br>
		 */
		private List<MappingComponent> getICDMap(ScoreDoc[] results) throws IOException, java.text.ParseException{
			List<MappingComponent> mappingComponentList = new ArrayList<MappingComponent>();
			SimpleDateFormat simpleDateFormatterObj=new SimpleDateFormat("E MMM dd HH:mm:ss Z yyyy",Locale.ENGLISH);
			Document document = null;
			IndexSearcher indexSearcher=null;
			if(getIndexReader(MappingConstants.MAP_INDEX)!=null)
				indexSearcher=this.getIndexSearcher(indexReader);
			else
				return mappingComponentList;
			try{
			for(int i = 0; i< results.length;i++){
				int docId = results[i].doc;
				 document = indexSearcher.doc(docId);
				MappingComponent mappingComponentObj=new MappingComponent();
					mappingComponentObj.setId(document.get(MappingConstants.ID));
					mappingComponentObj.setEffectiveTime(simpleDateFormatterObj.parse(document.get(MappingConstants.EFFECTIVE_TIME)));
					mappingComponentObj.setActive(Byte.parseByte(document.get(MappingConstants.ICD_ACTIVE)));
					mappingComponentObj.setModuleId(Long.parseLong(document.get(MappingConstants.MODULE_ID)));
					mappingComponentObj.setRefsetId(Long.parseLong(document.get(MappingConstants.REFSET_ID)));
					mappingComponentObj.setReferencedComponentId(Long.parseLong(document.get(MappingConstants.REFERENCED_COMPONENT_ID)));
					mappingComponentObj.setMapGroup(Integer.parseInt(document.get(MappingConstants.MAP_GROUP)));
					mappingComponentObj.setMapPriority(Integer.parseInt(document.get(MappingConstants.MAP_PRIORITY)));
					mappingComponentObj.setMapRule(document.get(MappingConstants.MAP_RULE));
					mappingComponentObj.setMapAdvice(document.get(MappingConstants.MAP_ADVICE));
					mappingComponentObj.setMapTarget(document.get(MappingConstants.MAP_TARGET));
					mappingComponentObj.setCorrelationId(Long.parseLong(document.get(MappingConstants.CORRELATION_ID)));
					mappingComponentObj.setMapCategoryId(Long.parseLong(document.get(MappingConstants.MAPCATEGORY_ID)));
					mappingComponentList.add(mappingComponentObj);
			}
			}finally{
				indexSearcher=null;
				document = null;
				simpleDateFormatterObj=null;
			}
			return mappingComponentList;
		}

		/**
		 * Adds documents to lucene index.
		 * @param mappingComponentList list of {@link MapppingComponent}.
		 * @return {@link IndexWriter}.
		 * @throws IOException if any error occurs while processing.
		 */
		private IndexWriter addICDMapDocument(List<MappingComponent> mappingComponentList) throws IOException{
			Field id = new StringField(MappingConstants.ID,"", Field.Store.YES);
			Field effectiveTime = new StringField(MappingConstants.EFFECTIVE_TIME,"", Field.Store.YES);
			Field icdActive = new StringField(MappingConstants.ICD_ACTIVE,"", Field.Store.YES);
			Field moduleID = new StringField(MappingConstants.MODULE_ID,"", Field.Store.YES);
			Field refsetID = new StringField(MappingConstants.REFSET_ID,"", Field.Store.YES);
			Field referencedComponentID = new StringField(MappingConstants.REFERENCED_COMPONENT_ID,"", Field.Store.YES);
			Field mapGroup = new StringField(MappingConstants.MAP_GROUP, "", Field.Store.YES);
			Field mapPriority = new StringField(MappingConstants.MAP_PRIORITY, "", Field.Store.YES);
			Field mapRule = new StringField(MappingConstants.MAP_RULE,"", Field.Store.YES);
			Field mapAdvice = new StringField(MappingConstants.MAP_ADVICE,"", Field.Store.YES);
			Field mapTaget = new StringField(MappingConstants.MAP_TARGET, "", Field.Store.YES);
			Field correlationID = new StringField(MappingConstants.CORRELATION_ID,"", Field.Store.YES);
			Field mapCategoryID = new StringField(MappingConstants.MAPCATEGORY_ID,"", Field.Store.YES);

				for(MappingComponent mappingComponent:mappingComponentList){
				Document document = new Document();

				// setting values to field
				id.setStringValue(mappingComponent.getId());
				effectiveTime.setStringValue(mappingComponent.getEffectiveTime()+"");
				icdActive.setStringValue(mappingComponent.getActive()+"");
				moduleID.setStringValue(mappingComponent.getModuleId()+"");
				refsetID.setStringValue(mappingComponent.getRefsetId()+"");
				referencedComponentID.setStringValue(mappingComponent.getReferencedComponentId()+"");
				mapGroup.setStringValue(mappingComponent.getMapGroup()+"");
				mapPriority.setStringValue(mappingComponent.getMapPriority()+"");
				mapRule.setStringValue(mappingComponent.getMapRule()+"");
				mapAdvice.setStringValue(mappingComponent.getMapAdvice()+"");
				mapTaget.setStringValue(mappingComponent.getMapTarget()+"");
				correlationID.setStringValue(mappingComponent.getCorrelationId()+"");
				mapCategoryID.setStringValue(mappingComponent.getMapCategoryId()+"");

				// adding field to document
				document.add(id);
				document.add(effectiveTime);
				document.add(icdActive);
				document.add(moduleID);
				document.add(refsetID);
				document.add(referencedComponentID);
				document.add(mapGroup);
				document.add(mapPriority);
				document.add(mapRule);
				document.add(mapAdvice);
				document.add(mapTaget);
				document.add(correlationID);
				document.add(mapCategoryID);
				this.indexWriter.addDocument(document);
			}
			return this.indexWriter;
		}


		/**
		 * Return list of {@link MappingComponent} from the SNOMED CT repository of SNOMED to ICD Map based on provided conceptId.
		 * @param conceptId - {@link String} results searched from the indexed documents.<br>
		 * @throws IOException If any error occurs while processing.
		 * @return List of {@link MappingComponent} matching the given parameter of {@link ScoreDoc[]}.<br>
		 */
		List<MappingComponent> getICDMap(String conceptId){
            Logger logger = LogManager.getLogger(LuceneICDMapAgent.class);
			if(conceptId!=null){
			String inputQuery=null;
			StringBuilder queryCriteria=null;
			QueryParser queryParser =null;
			PropertyReader.loadSystemProperties();
			inputQuery=MappingConstants.REFERENCED_COMPONENT_ID;
			queryParser = new QueryParser(inputQuery,getAnalyzer());
			queryCriteria=new StringBuilder(inputQuery+":"+conceptId+" AND "+MappingConstants.ICD_ACTIVE+":"+MappingConstants.ACTIVE);
			queryParser.setAllowLeadingWildcard(true);
			try{
				Query query = null;
				ScoreDoc[] results = null;
				query =queryParser.parse(queryCriteria.toString().trim());
				if(EnumProperties.SEARCH_RETURNLIMIT_VAL.getValue()!=null)
					results = this.search(query,Integer.parseInt(EnumProperties.SEARCH_RETURNLIMIT_VAL.getValue()),true,MappingConstants.MAP_INDEX);
				else
					results = this.search(query,MappingConstants.MAP_ALL_RECORD,true,MappingConstants.MAP_INDEX);
				mappingComponentList=this.getICDMap(results);
				mappingComponentListIterator=mappingComponentList.listIterator();
				mapMC=new HashMap<String,MappingComponent>(); //to remove duplicate entries for same mapTarget
				while(mappingComponentListIterator.hasNext()){
					MappingComponent mappingComponent=mappingComponentListIterator.next();
					Long mapCategoryId=mappingComponent.getMapCategoryId();
					if(!(mapCategoryId==Long.parseLong(MappingConstants.CATAGORY_ID_MAP_SOURCE_PROPERLY_CLASSIFIED)||mapCategoryId==Long.parseLong(MappingConstants.CATAGORY_ID_MAP_SOURCE_CONTEXT_DEPENDENT))){
						mappingComponentListIterator.remove();
					}else{
						mapMC.put(mappingComponent.getMapTarget(),mappingComponent);
					}
				}
				Set<String> mappingComponentKeySet=mapMC.keySet();
				mappingComponentList=new ArrayList<MappingComponent>();
				for(String mapTargetKey:mappingComponentKeySet)
					mappingComponentList.add(mapMC.get(mapTargetKey));
				return mappingComponentList;
			}catch (IOException|java.text.ParseException e){
				//Log the Exception
				logger.error(LoggerUtility.ConsoleWithMapICDException, "MapIndex directory is not available");
                logger.error(LoggerUtility.MapICDException, LoggerUtility.createErrorMessage(e));
			}catch(Exception e){
				//Log the Exception
                logger.error(LoggerUtility.MapICDException, LoggerUtility.createErrorMessage(e));
			}finally{
				mappingComponentList=null;
				mappingComponentListIterator=null;
				mapMC=null;
			}
			}
			return null;
		}

		/**
		 * Searches for a particular input query in the index.
		 * @param query
		 * 				 input query i.e. the term to be searched.
		 * @param returnLimitPerPage
		 * 				 returnLimit per page.
		 * @param inOrder
		 * 				 whether the results should be in order.
		 * @param refsetId
		 * 				Identifier of Refset
		 * @return scoreDocs
		 * 				 ScoreDoc array that contains list of top scoring results.
		 * @throws IOException if any error occurs while processing.
		 */
		private ScoreDoc[] search(Query query,int returnLimitPerPage, boolean inOrder,String refsetId) throws IOException{
			int indexOffset = Constants.INDEX_OFFSET;
			PropertyReader.loadSystemProperties();
			TopScoreDocCollector topScoreDocCollector = null;
			if(returnLimitPerPage==0){
				returnLimitPerPage = returnLimitPerPage+indexOffset;
				topScoreDocCollector = TopScoreDocCollector.create(MappingConstants.MAP_ALL_RECORD, Constants.TOTAL_HIT_THRESHOLD);
			}else
				topScoreDocCollector = TopScoreDocCollector.create(returnLimitPerPage, Constants.TOTAL_HIT_THRESHOLD);
			if(this.getIndexReader(refsetId)!=null)
			this.getIndexSearcher(this.getIndexReader(refsetId)).search(query,topScoreDocCollector);
			TopDocs topDocs = topScoreDocCollector.topDocs();
			return topDocs.scoreDocs;
		}
		/**
		 * Close opened {@link IndexReader} created by getIndexReader()
		 */
		private void closeIndexReader(){
            Logger logger = LogManager.getLogger(LuceneICDMapAgent.class);
				try {
					if(indexReader!=null)
					this.indexReader.close();
					indexReader=null;
				} catch (IOException e) {
					logger.error(LoggerUtility.MapICDException, "Error occurred during closing indexReader");
				}

		}

		@Override
		public void finalize(){
				this.closeIndexReader();
		}


}
