>LOCAL loTokenizer AS Tokenizer >LOCAL lcToken AS String >LOCAL lcTest AS String > >m.loTokenizer = CREATEOBJECT("Tokenizer") >m.loTokenizer.AddTokenPattern("(\+|-)?(\.\d+|\d+(\.\d+)?)((e|E)(\+|-)?\d+)*", "N") >m.loTokenizer.AddTokenPattern('("[^"]*"|' + "'[^']*'|\[[^\[]*\])", "C") >m.loTokenizer.AddTokenPattern("{\^\d{1,4}-\d{1,2}-\d{1,2}}", "D") >m.loTokenizer.AddTokenPattern("{\^\d{1,4}-\d{1,2}-\d{1,2}\s+\d{1,2}:\d{1,2}(:\d{1,2})?(\s+?([aA]|[pP])[mM]?)?}", "T") >m.loTokenizer.AddTokenPattern("\.([tT]|[fF])\.", "L") >m.loTokenizer.AddTokenPattern("\.[nN][uU][lL]{2}\.", "U") >m.loTokenizer.AddTokenPattern("(==|!=|<=|>=|<>|!|=|>|<|\+|-|\*|\/|%|,|\)|\(|\$|[aA][nN][dD]|[oO][rR]|[nN][oO][tT])", "O") >m.loTokenizer.AddTokenPattern("[a-zA-Z][a-zA-Z0-9_]*\s*(\(|\[)", "F") >m.loTokenizer.AddTokenPattern("([mM]\.)?[a-zA-Z_][a-zA-Z0-9_]*", "V") >m.loTokenizer.AddTokenPattern("\.\s*[a-zA-Z_][a-zA-Z0-9_]*\s*(\(|\[)", "M") >m.loTokenizer.AddTokenPattern("\.\s*[a-zA-Z_][a-zA-Z0-9_]*", "P") > >CLEAR >ACCEPT "Test Expression: " TO m.loTest > >IF !m.loTokenizer.GetTokens(m.loTest) > > ? "Error @" + m.loTokenizer.ErrorPointer > >ELSE > > ? "Tokens found:" > FOR EACH m.loToken IN m.loTokenizer.Tokens > ? m.loToken.Type,m.loToken.Value > ENDFOR > >ENDIF > >DEFINE CLASS Tokenizer AS Custom > > RegExpr = .NULL. > TokenPatterns = .NULL. > Tokens = .NULL. > ErrorPointer = "" > > FUNCTION Init > > IF !"\_REGEXP.VCX" $ SET("Classlib") > SET CLASSLIB TO (ADDBS(HOME(1)) + "ffc\_regexp.vcx") ADDITIVE > ENDIF > > This.RegExpr = CREATEOBJECT("_regexp") > > This.TokenPatterns = CREATEOBJECT("collection") > This.Tokens = CREATEOBJECT("collection") > > ENDFUNC > > FUNCTION AddTokenPattern (tcPattern AS String, tcType AS String) > > LOCAL loTokenType AS TokenType > > m.loTokenType = CREATEOBJECT("TokenType", IIF(LEFT(m.tcPattern,1) != "^", "^", "") + m.tcPattern, m.tcType) > > This.TokenPatterns.Add(m.loTokenType) > > ENDFUNC > > FUNCTION GetTokens (tcExpression AS String) > > LOCAL lcExpression AS String > LOCAL loTokenType AS TokenType > LOCAL lnTokenPattern AS Integer > LOCAL lcToken AS String > LOCAL loToken AS Token > > m.lcExpression = ALLTRIM(m.tcExpression,0," ",CHR(13),CHR(10),CHR(9)) > This.Tokens.Remove(-1) > This.ErrorPointer = "" > > DO WHILE !EMPTY(m.lcExpression) > > m.lcToken = .NULL. > > FOR m.lnTokenPattern = 1 TO This.TokenPatterns.Count > > m.loTokenType = This.TokenPatterns.Item(m.lnTokenPattern) > > This.RegExpr.Clear() > This.RegExpr.Pattern = m.loTokenType.Pattern > > IF This.RegExpr.Execute(m.lcExpression,.F.) = 1 > > m.lcToken = This.RegExpr.Matches[1,2] > m.lcExpression = LTRIM(SUBSTR(m.lcExpression,LEN(m.lcToken) + 1),0," ",CHR(13),CHR(10),CHR(9)) > > m.loToken = CREATEOBJECT("Token", m.lcToken, m.loTokenType.Type) > > This.Tokens.Add(m.loToken) > EXIT > > ENDIF > ENDFOR > > IF ISNULL(m.lcToken) > This.ErrorPointer = m.lcExpression > RETURN .F. > ENDIF > ENDDO > > RETURN .T. > ENDFUNC > >ENDDEFINE > >DEFINE CLASS TokenType AS Custom > > Pattern = "" > Type = "" > > FUNCTION Init (tcPattern AS String, tcType AS String) > > This.Pattern = m.tcPattern > This.Type = m.tcType > > ENDFUNC > >ENDDEFINE > >DEFINE CLASS Token AS Custom > > Value = "" > Type = "" > > FUNCTION Init (tcValue AS String, tcType AS String) > > This.Value = m.tcValue > This.Type = m.tcType > > ENDFUNC > >ENDDEFINE >