// states 
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <fstream.h>
#include <iostream>
#include <string>
#include <vector>

#define IN_WHITE 0
#define IN_RTOKN 1   //regular token
#define IN_STOKN 2   //single character token

#define MAX_LINE_CHAR 2000  //the maximum number of characters per 
                            //line in CGIF file
#define TOKENSIZE    50
#define KINDSIZE     10
#define NAMESIZE     50
#define COMMENTSIZE 500

enum token_kind_type 
{
	     unknown,           // 0
		 identifier,        // 1
		 beginConcept,      // 2
		 endConcept,        // 3
         beginRelation,     // 4
		 endRelation,       // 5
		 beginComment,      // 6
		 endComment,        // 7
		 beginActor,        // 8
	     endActor,          // 9
		 beginSet,          //10
		 endSet,            //11
		 beginReferent,     //12
		 negation,          //13
		 listSeparator,     //14
	     defLabel,          //15
		 boundLabel,        //16
		 beginIndexical,    //17
		 beginQuantifier,   //18
		 beginStructure,    //19
	     disjunctSeparator, //20
		 conjunctSeparator, //21
		 whyExclamation,    //22
		 beginQuotedstr,    //23
	     endQuotedstr       //24
};

struct tokenType
{
	token_kind_type kind;
	std::string value;
};


typedef std::vector<tokenType> string_vector_type;


typedef struct simpleTokenType
{
	char value[TOKENSIZE];
	token_kind_type kind;
}simpletokenstruct;



// routine to find character in string 
int sindex(char ch,char *string)
{
  char *cp;
  for(cp=string;*cp;++cp)
  if(ch==*cp) return (int)(cp-string); //return postion of character
  return -1;             //no match found 
}


token_kind_type identifyTokenType(char ch)
{
   static int comment=0;
   token_kind_type type = unknown;

   switch (ch)
   {
	   case '[': type = beginConcept;
		   break;
	   case ']': type = endConcept;
		   break;
	   case '(': type = beginRelation;
		   break;
	   case ')': type = endRelation;
		   break;
	   case '<': type = beginActor;
		   break;
	   case '>': type = endActor;
		   break;
	   case '{': type = beginSet;
		   break;
	   case '}': type = endSet;
		   break;
	   case '%': type = beginStructure;
		   break;
	   case ';': if(comment ==0)
				 {
					 type = beginComment;
		             comment=1;
				 } else 
				 {
					 type = endComment;
					 comment =0;
				 }
				 break;
	   case '~': type = negation;
		   break;
	   case '@': type = beginQuantifier;
		   break;
	   case '#': type = beginIndexical;
		   break;
	   case '&': type = conjunctSeparator;
		   break;
	   case '|': type = disjunctSeparator;
		   break;
	   case '!': type = whyExclamation;
		   break;
	   case ':': type = beginReferent;
		   break;
	   case '*': type =defLabel;
		   break;
	   case '?': type = boundLabel;
		   break;
	   default: type =unknown;
		        break;
   }
   return type;
}


void getOneToken(tokenType *token, char *line, int *next, int *pstate)
{
	static char *whitesp=" \t\r\"'";	// blank and tab 
	static char *singlech="[]()<>{};:~,*?#@%|&!";
	
	char ch;
	int tokenFound=0;
	int tokpos=0;

	while((ch=line[*next]) && (!tokenFound))      
	{
	    tokenFound =0;
		if(sindex(ch,singlech)>=0)
		{
			switch(*pstate)
			{
			    case IN_WHITE: 
			    case IN_STOKN:  token->value += ch;
					            tokenFound =1;
								token->kind = identifyTokenType(ch);

                                *pstate = IN_STOKN;
					   		    break;
			     case IN_RTOKN: tokenFound =1;
					            token->kind = identifier;
							    (*next)--;
							    *pstate = IN_STOKN;
							    break;
			}
		}

	     else
		 {
		     if (sindex(ch,whitesp)>=0)   // white 
			 {
		        switch(*pstate)
				{
			       case IN_WHITE: *pstate = IN_WHITE;
					              tokenFound=0;
					              break;
			       case IN_STOKN: break;
			       case IN_RTOKN: tokenFound =1;
					              token->kind = identifier;
                                  *pstate = IN_WHITE;
							      break;
				}
			 }
		     else  /* anything else is just a real character */
			 {
			    token->value += ch;
				*pstate = IN_RTOKN;
			 }
		 }
		 (*next)++;
	} 
}

// the token analyzer 
int tokenAnalyzer(string_vector_type &tmp_vect, char *line)
{
    int next=0;
	int pstate=IN_WHITE;
	int i=0;
	
	while (line[next])
	{
		tokenType token;
		token.kind = unknown;
        		
        tokenType oneToken;
		oneToken.value="";

		getOneToken(&oneToken, line, &next, &pstate);
		
		if(strcmp(oneToken.value.c_str(), "") )
		{
			token.value = oneToken.value;
		    token.kind  = oneToken.kind;
	        tmp_vect.push_back(token);
  		    i++;
		}
	}
	return i;
}

/* open CGIF file and read into a string*/
int readCGIF(char *sourcefile, string_vector_type &str_vector)
{
    int tokenNumber=0;
	
	// open input stream
	ifstream cgif_file(sourcefile);
	if(cgif_file == NULL)
	{
		printf("can not open source file: %s\n",sourcefile);
		exit(1);
	}
	    
    char line[MAX_LINE_CHAR];

	do
	{   
		cgif_file.getline(line, MAX_LINE_CHAR);

		if(strcmp(line,".") )  // str does not contain only "."
		{
           tokenNumber += tokenAnalyzer(str_vector, line);
		} 
		
	} while (!cgif_file.eof());

    return tokenNumber;
}

 
	
//print the token into a file
void printTokens(char *outputFile,string_vector_type tokens)
{
	
	FILE *fp;
    //create a new file for output
    if( (fp = fopen( outputFile, "w" )) == NULL )
	{
	    printf("Can't create output file %s\n",outputFile);
	    exit(0);
	}

	string_vector_type::iterator iter;
    iter = tokens.begin();

    int j=0;
    while(iter != tokens.end()) 
    {   

      fprintf(fp," Token: %3d, Kind: %2d, Value: %s\n",++j,(*iter).kind,
		      (*iter).value.c_str());

	  iter++;
   }
   fclose(fp);
}
   
//convert tokens into simple token array
void convertTokens(string_vector_type tokens, simpleTokenType* simpleTokens,
				   int *newTokenNumber, int seg[8])
{
    seg[0] = 6;

	string_vector_type::iterator iter;
    iter = tokens.begin();

    int j=0;
	
    while(iter != tokens.end()) 
    {   
        if( !strcmp( (*iter).value.c_str(),"RelationHierarchy") )
		{
			seg[1]= j-3;
			seg[2]= j+2;
		}
        if( !strcmp( (*iter).value.c_str(),"CatalogOfIndividuals"))
		{
			seg[3]= j-3;
			seg[4]= j+2;
		}

		if( !strcmp( (*iter).value.c_str(),"Assertion"))
		{
			seg[5]= j-3;
			seg[6]= j+2;
		}

	    //if( !strcmp( (*iter).value.c_str(),"LambdaExpression") )
		if( (!strcmp( (*iter).value.c_str(),"def")) || 
			(!strcmp( (*iter).value.c_str(),"Def")) 
		  )
		{
	        int left=1,right=0;
		    do
			{
			    if( !strcmp( (*iter).value.c_str() ,"(") ) left++;
			    if( !strcmp( (*iter).value.c_str() ,")") ) right++;
			    iter++;
			} while (left != right);
		  
		    j--;
		}
      
        strcpy( (simpleTokens[j]).value, (*iter).value.c_str() );
        (simpleTokens[j]).kind = (*iter).kind;
      
	    j++;
	    iter++;
	}
    *newTokenNumber = j;
	seg[7]= j-2;
	//for (int h=0; h<8; h++) printf(" the seg: %4d\n", seg[h]);
}