From f622d5d9c8244b9787f4e992afc3400f64ca0a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AC=9D=E6=A1=82=E7=B6=B1?= Date: Tue, 1 Mar 2016 13:22:40 +0800 Subject: [PATCH] it seems there is no syntax error in the tokens parser --- LINQ/LINQ/LDM/Parser/ParserAPI/Extensions.vb | 2 + LINQ/LINQ/LDM/Parser/ParserAPI/Parser.vb | 250 +++++++++---------- LINQ/LINQ/LDM/Parser/ParserAPI/Tokenizer.vb | 104 ++++---- 3 files changed, 178 insertions(+), 178 deletions(-) diff --git a/LINQ/LINQ/LDM/Parser/ParserAPI/Extensions.vb b/LINQ/LINQ/LDM/Parser/ParserAPI/Extensions.vb index 650d573..f91c74b 100644 --- a/LINQ/LINQ/LDM/Parser/ParserAPI/Extensions.vb +++ b/LINQ/LINQ/LDM/Parser/ParserAPI/Extensions.vb @@ -8,6 +8,8 @@ Namespace LDM.Parser Public ReadOnly Property Primitive As Tokens = Tokens.String Public ReadOnly Property OpenParens As Tokens = Tokens.LPair Public ReadOnly Property CloseParens As Tokens = Tokens.RPair + Public ReadOnly Property Comma As Tokens = Tokens.ParamDeli + Public ReadOnly Property Dot As Tokens = Tokens.CallFunc ''' ''' Current tokens is a operator? diff --git a/LINQ/LINQ/LDM/Parser/ParserAPI/Parser.vb b/LINQ/LINQ/LDM/Parser/ParserAPI/Parser.vb index 0a5f5fb..d3dc44f 100644 --- a/LINQ/LINQ/LDM/Parser/ParserAPI/Parser.vb +++ b/LINQ/LINQ/LDM/Parser/ParserAPI/Parser.vb @@ -58,35 +58,34 @@ Namespace LDM.Parser Dim left As CodeExpression = Nothing, right As CodeExpression = Nothing Dim cont As Boolean = True, applyNot As Boolean = False, applyNegative As Boolean = False While cont - Select Case t.Current.Type - Case TokenType.Primitive - left = New CodePrimitiveExpression(t.Current.ParsedObject) - t.GetNextToken() - cont = False - Case TokenType.[Operator] - ' An operator here is considered a unary operator. - Select Case t.Current.Text - Case "-" - applyNegative = True - Case "!" - applyNot = True - Case Else - Throw New Exception("Unexpected operator: " & t.Current.Text) - End Select - t.GetNextToken() - Continue While - Case TokenType.Identifier - left = ReadIdentifier(t) - cont = False - Case TokenType.OpenParens - t.GetNextToken() - left = ReadExpression(t, TokenPriority.None) - t.GetNextToken() - If TypeOf left Is CodeTypeReferenceExpression Then - left = New CodeCastExpression(TryCast(left, CodeTypeReferenceExpression).Type, ReadExpression(t, TokenPriority.None)) - End If - cont = False - End Select + If t.Current.Type = Primitive Then + left = New CodePrimitiveExpression(t.Current.ParsedObject) + t.GetNextToken() + cont = False + ElseIf t.Current.Type.IsOperator Then + ' An operator here is considered a unary operator. + Select Case t.Current.Text + Case "-" + applyNegative = True + Case "!" + applyNot = True + Case Else + Throw New Exception("Unexpected operator: " & t.Current.Text) + End Select + t.GetNextToken() + Continue While + ElseIf t.Current.Type = Tokens.Identifier Then + left = ReadIdentifier(t) + cont = False + ElseIf t.Current.Type = OpenParens Then + t.GetNextToken() + left = ReadExpression(t, TokenPriority.None) + t.GetNextToken() + If TypeOf left Is CodeTypeReferenceExpression Then + left = New CodeCastExpression(TryCast(left, CodeTypeReferenceExpression).Type, ReadExpression(t, TokenPriority.None)) + End If + cont = False + End If If t.IsInvalid Then cont = False End If @@ -99,113 +98,112 @@ Namespace LDM.Parser ElseIf applyNegative Then left = New CodeBinaryOperatorExpression(New CodePrimitiveExpression(0), CodeBinaryOperatorType.Subtract, left) End If - If t.IsInvalid OrElse t.Current.Type = TokenType.CloseParens OrElse t.Current.Type = TokenType.Comma OrElse t.Current.Type = TokenType.CloseBracket Then + If t.IsInvalid OrElse t.Current.Type = CloseParens OrElse t.Current.Type = Comma OrElse t.Current.Type = Tokens.CloseBracket Then Return left End If cont = True While cont AndAlso Not t.IsInvalid Dim token As Token = t.Current - Select Case token.Type - Case TokenType.[Operator] - If t.Current.Priority < priority Then - cont = False - Else - ' In the case we have an operator, we'll assume it's a binary operator. - Dim binOp As CodeBinaryOperatorType - Dim notEquals As Boolean = False - Select Case token.Text - Case ">" - binOp = CodeBinaryOperatorType.GreaterThan - Case ">=" - binOp = CodeBinaryOperatorType.GreaterThanOrEqual - Case "<" - binOp = CodeBinaryOperatorType.LessThan - Case "<=" - binOp = CodeBinaryOperatorType.LessThanOrEqual - Case "=", "==" - binOp = CodeBinaryOperatorType.ValueEquality - Case "!=" - binOp = CodeBinaryOperatorType.ValueEquality - notEquals = True - Case "|" - binOp = CodeBinaryOperatorType.BitwiseOr - Case "||" - binOp = CodeBinaryOperatorType.BooleanOr - Case "&" - binOp = CodeBinaryOperatorType.BitwiseAnd - Case "&&" - binOp = CodeBinaryOperatorType.BooleanAnd - Case "-" - binOp = CodeBinaryOperatorType.Subtract - Case "+" - binOp = CodeBinaryOperatorType.Add - Case "/" - binOp = CodeBinaryOperatorType.Divide - Case "%" - binOp = CodeBinaryOperatorType.Modulus - Case "*" - binOp = CodeBinaryOperatorType.Multiply - Case Else - Throw New Exception("Unrecognized operator: " & t.Current.Text) - End Select - If t.IsInvalid Then - Throw New Exception("Expected token for right side of binary expression.") - End If - t.GetNextToken() - right = ReadExpression(t, token.Priority) - left = New CodeBinaryOperatorExpression(left, binOp, right) - - ' If the operator was the not equals operator, we just negate the previous binary expression. - If notEquals Then - left = New CodeBinaryOperatorExpression(left, binOp, New CodePrimitiveExpression(False)) - End If - End If - Case TokenType.CloseParens - 't.GetNextToken(); + If token.Type.IsOperator Then + If t.Current.Priority < priority Then cont = False - Case TokenType.Dot - ' A dot could appear after some parentheses. In this case we need to parse - ' what's after the dot as an identifier. + Else + ' In the case we have an operator, we'll assume it's a binary operator. + Dim binOp As CodeBinaryOperatorType + Dim notEquals As Boolean = False + Select Case token.Text + Case ">" + binOp = CodeBinaryOperatorType.GreaterThan + Case ">=" + binOp = CodeBinaryOperatorType.GreaterThanOrEqual + Case "<" + binOp = CodeBinaryOperatorType.LessThan + Case "<=" + binOp = CodeBinaryOperatorType.LessThanOrEqual + Case "=", "==" + binOp = CodeBinaryOperatorType.ValueEquality + Case "!=" + binOp = CodeBinaryOperatorType.ValueEquality + notEquals = True + Case "|" + binOp = CodeBinaryOperatorType.BitwiseOr + Case "||" + binOp = CodeBinaryOperatorType.BooleanOr + Case "&" + binOp = CodeBinaryOperatorType.BitwiseAnd + Case "&&" + binOp = CodeBinaryOperatorType.BooleanAnd + Case "-" + binOp = CodeBinaryOperatorType.Subtract + Case "+" + binOp = CodeBinaryOperatorType.Add + Case "/" + binOp = CodeBinaryOperatorType.Divide + Case "%" + binOp = CodeBinaryOperatorType.Modulus + Case "*" + binOp = CodeBinaryOperatorType.Multiply + Case Else + Throw New Exception("Unrecognized operator: " & t.Current.Text) + End Select + If t.IsInvalid Then + Throw New Exception("Expected token for right side of binary expression.") + End If t.GetNextToken() - right = ReadIdentifier(t) - Dim ceTemp As CodeExpression = right - While True - If TypeOf ceTemp Is CodeVariableReferenceExpression Then - left = New CodePropertyReferenceExpression(left, TryCast(ceTemp, CodeVariableReferenceExpression).VariableName) + right = ReadExpression(t, token.Priority) + left = New CodeBinaryOperatorExpression(left, binOp, right) + + ' If the operator was the not equals operator, we just negate the previous binary expression. + If notEquals Then + left = New CodeBinaryOperatorExpression(left, binOp, New CodePrimitiveExpression(False)) + End If + End If + ElseIf Token.Type = CloseParens Then + 't.GetNextToken(); + cont = False + ElseIf Token.Type = Dot Then + ' A dot could appear after some parentheses. In this case we need to parse + ' what's after the dot as an identifier. + t.GetNextToken() + right = ReadIdentifier(t) + Dim ceTemp As CodeExpression = right + While True + If TypeOf ceTemp Is CodeVariableReferenceExpression Then + left = New CodePropertyReferenceExpression(left, TryCast(ceTemp, CodeVariableReferenceExpression).VariableName) + Exit While + ElseIf TypeOf ceTemp Is CodePropertyReferenceExpression Then + Dim cpre As CodePropertyReferenceExpression = TryCast(ceTemp, CodePropertyReferenceExpression) + If TypeOf cpre.TargetObject Is CodeThisReferenceExpression Then + cpre.TargetObject = left + left = cpre Exit While - ElseIf TypeOf ceTemp Is CodePropertyReferenceExpression Then - Dim cpre As CodePropertyReferenceExpression = TryCast(ceTemp, CodePropertyReferenceExpression) - If TypeOf cpre.TargetObject Is CodeThisReferenceExpression Then - cpre.TargetObject = left - left = cpre - Exit While - Else - ceTemp = cpre.TargetObject - End If - ElseIf TypeOf ceTemp Is CodeFieldReferenceExpression Then - Dim cfre As CodeFieldReferenceExpression = TryCast(ceTemp, CodeFieldReferenceExpression) - If TypeOf cfre.TargetObject Is CodeThisReferenceExpression Then - cfre.TargetObject = left - left = cfre - Exit While - End If - ElseIf TypeOf ceTemp Is CodeMethodInvokeExpression Then - Dim cmie As CodeMethodInvokeExpression = TryCast(ceTemp, CodeMethodInvokeExpression) - If TypeOf cmie.Method.TargetObject Is CodeThisReferenceExpression Then - cmie.Method.TargetObject = left - left = cmie - Exit While - Else - ceTemp = cmie.Method.TargetObject - End If Else - Throw New Exception("Unexpected identifier found after .") + ceTemp = cpre.TargetObject End If - End While - cont = False - Case Else - Throw New Exception("Token not expected: " & token.Text) - End Select + ElseIf TypeOf ceTemp Is CodeFieldReferenceExpression Then + Dim cfre As CodeFieldReferenceExpression = TryCast(ceTemp, CodeFieldReferenceExpression) + If TypeOf cfre.TargetObject Is CodeThisReferenceExpression Then + cfre.TargetObject = left + left = cfre + Exit While + End If + ElseIf TypeOf ceTemp Is CodeMethodInvokeExpression Then + Dim cmie As CodeMethodInvokeExpression = TryCast(ceTemp, CodeMethodInvokeExpression) + If TypeOf cmie.Method.TargetObject Is CodeThisReferenceExpression Then + cmie.Method.TargetObject = left + left = cmie + Exit While + Else + ceTemp = cmie.Method.TargetObject + End If + Else + Throw New Exception("Unexpected identifier found after .") + End If + End While + cont = False + Else + Throw New Exception("Token not expected: " & token.Text) + End If End While Return left End Function diff --git a/LINQ/LINQ/LDM/Parser/ParserAPI/Tokenizer.vb b/LINQ/LINQ/LDM/Parser/ParserAPI/Tokenizer.vb index 25586bb..2832520 100644 --- a/LINQ/LINQ/LDM/Parser/ParserAPI/Tokenizer.vb +++ b/LINQ/LINQ/LDM/Parser/ParserAPI/Tokenizer.vb @@ -1,6 +1,7 @@ Imports System.CodeDom Imports System.Text Imports Microsoft.VisualBasic.ComponentModel.DataSourceModel +Imports Microsoft.VisualBasic.Linq.Statements Imports Microsoft.VisualBasic.Linq.Statements.TokenIcer Imports Microsoft.VisualBasic.Scripting.TokenIcer @@ -11,7 +12,7 @@ Namespace LDM.Parser ''' Public Class Tokenizer - ReadOnly Tokens As Iterator(Of Token(Of Tokens)) + ReadOnly _tokens As Iterator(Of Token(Of Tokens)) Dim _IsInvalid As Boolean = False Dim _PrevToken As Token = Token.NullToken @@ -21,12 +22,12 @@ Namespace LDM.Parser ''' ''' string to tokenize Public Sub New(s As String) - Call Me.New(Statements.TokenIcer.GetTokens(s)) + Call Me.New(TokenIcer.GetTokens(s)) End Sub Sub New(tokens As IEnumerable(Of Token(Of Tokens))) - Me.Tokens = New Iterator(Of Token(Of Tokens))(tokens) - MoveNext() + Me._tokens = New Iterator(Of Token(Of Tokens))(tokens) + ' MoveNext() End Sub ''' @@ -34,7 +35,7 @@ Namespace LDM.Parser ''' invalid. ''' Private Sub MoveNext() - If Not Tokens.MoveNext() Then + If Not _tokens.MoveNext() Then _IsInvalid = True End If End Sub @@ -42,7 +43,7 @@ Namespace LDM.Parser ''' ''' Allows access to the token most recently parsed. ''' - Public ReadOnly Property Current() As Token(Of Tokens) + Public ReadOnly Property Current() As Token Get Return _PrevToken End Get @@ -65,8 +66,8 @@ Namespace LDM.Parser If _IsInvalid Then Return False End If - Dim Current = Tokens.GetCurrent - Return Current.TokenName = Statements.TokenIcer.Tokens.String + Dim Current = _tokens.GetCurrent + Return Current.TokenName = Tokens.String End Get End Property @@ -78,7 +79,7 @@ Namespace LDM.Parser If _IsInvalid Then Return False End If - Return Tokens.GetCurrent.TokenName = Statements.TokenIcer.Tokens.CallFunc + Return _tokens.GetCurrent.TokenName = Tokens.CallFunc End Get End Property @@ -90,7 +91,7 @@ Namespace LDM.Parser If _IsInvalid Then Return False End If - Return Tokens.GetCurrent.TokenName = Statements.TokenIcer.Tokens.ParamDeli + Return _tokens.GetCurrent.TokenName = Tokens.ParamDeli End Get End Property @@ -102,8 +103,8 @@ Namespace LDM.Parser If _IsInvalid Then Return False End If - Dim t As Tokens = Tokens.GetCurrent.TokenName - Return t = Statements.TokenIcer.Tokens.Integer OrElse t = Statements.TokenIcer.Tokens.Float + Dim t As Tokens = _tokens.GetCurrent.TokenName + Return t = Tokens.Integer OrElse t = Tokens.Float End Get End Property @@ -115,7 +116,7 @@ Namespace LDM.Parser If _IsInvalid Then Return False End If - Return Tokens.GetCurrent.TokenName = Statements.TokenIcer.Tokens.WhiteSpace + Return _tokens.GetCurrent.TokenName = Tokens.WhiteSpace End Get End Property @@ -127,7 +128,7 @@ Namespace LDM.Parser If _IsInvalid Then Return False Else - Return Tokens.GetCurrent.TokenName.IsOperator + Return _tokens.GetCurrent.TokenName.IsOperator End If End Get End Property @@ -146,10 +147,10 @@ Namespace LDM.Parser If IsChar Then token__1 = GetString() ElseIf IsComma Then - token__1 = New Token(",", Statements.TokenIcer.Tokens.ParamDeli, TokenPriority.None) + token__1 = New Token(",", Tokens.ParamDeli, TokenPriority.None) MoveNext() ElseIf IsDot Then - token__1 = New Token(".", Statements.TokenIcer.Tokens.CallFunc, TokenPriority.None) + token__1 = New Token(".", Tokens.CallFunc, TokenPriority.None) MoveNext() ElseIf IsNumber Then token__1 = GetNumber() @@ -174,14 +175,14 @@ Namespace LDM.Parser ''' ''' Private Function GetString() As Token - Dim s As String = Tokens.GetCurrent.TokenValue + Dim s As String = _tokens.GetCurrent.TokenValue ' "false" or "true" is a primitive expression. If s = "false" OrElse s = "true" Then - Return New Token([Boolean].Parse(s), Statements.TokenIcer.Tokens.String, TokenPriority.None) + Return New Token([Boolean].Parse(s), Tokens.String, TokenPriority.None) End If ' The previous token was a quote, so this is a primitive string. - Return New Token(Tokens.GetCurrent, TokenPriority.None) + Return New Token(_tokens.GetCurrent, TokenPriority.None) End Function ''' @@ -194,13 +195,13 @@ Namespace LDM.Parser ''' Any numbers containing a dot (".") are considered doubles. ''' Private Function GetNumber() As Token - Dim Current = Tokens.GetCurrent + Dim Current = _tokens.GetCurrent Dim s As String = Current.TokenValue - If Current.TokenName = Statements.TokenIcer.Tokens.Float Then - Return New Token([Double].Parse(s), Statements.TokenIcer.Tokens.String, TokenPriority.None) + If Current.TokenName = Tokens.Float Then + Return New Token([Double].Parse(s), Tokens.String, TokenPriority.None) End If - Return New Token(Int32.Parse(s), Statements.TokenIcer.Tokens.String, TokenPriority.None) + Return New Token(Int32.Parse(s), Tokens.String, TokenPriority.None) End Function ''' @@ -209,58 +210,57 @@ Namespace LDM.Parser ''' ''' Private Function GetOperator() As Token - Dim Current = Tokens.GetCurrent + Dim Current = _tokens.GetCurrent Select Case Current.TokenName - Case Statements.TokenIcer.Tokens.Equals OrElse Statements.TokenIcer.Tokens.Is + Case Tokens.Equals OrElse Tokens.Is + Return New Token(Current.TokenValue, Tokens.Is, TokenPriority.Equality) + Case Tokens.Minus MoveNext() - Return New Token(Current.TokenValue, Statements.TokenIcer.Tokens.Is, TokenPriority.Equality) - Case Statements.TokenIcer.Tokens.Minus - MoveNext() - If _PrevToken.TokenName = Primitive OrElse _PrevToken.Type = Statements.TokenIcer.Tokens.Identifier Then - Return New Token(Current.TokenValue, Statements.TokenIcer.Tokens.Minus, TokenPriority.PlusMinus) + If _PrevToken.TokenName = Primitive OrElse _PrevToken.Type = Tokens.Identifier Then + Return New Token(Current.TokenValue, Tokens.Minus, TokenPriority.PlusMinus) Else - Return New Token(Current.TokenValue, Statements.TokenIcer.Tokens.Minus, TokenPriority.UnaryMinus) + Return New Token(Current.TokenValue, Tokens.Minus, TokenPriority.UnaryMinus) End If - Case Statements.TokenIcer.Tokens.Plus + Case Tokens.Plus MoveNext() - Return New Token(Current.TokenValue, Statements.TokenIcer.Tokens.Plus, TokenPriority.PlusMinus) - Case Statements.TokenIcer.Tokens.Not + Return New Token(Current.TokenValue, Tokens.Plus, TokenPriority.PlusMinus) + Case Tokens.Not MoveNext() - If Tokens.GetCurrent.TokenName = Statements.TokenIcer.Tokens.Equals Then + If _tokens.GetCurrent.TokenName = Tokens.Equals Then MoveNext() - Return New Token(Tokens.GetCurrent.TokenValue, Statements.TokenIcer.Tokens.Equals, TokenPriority.Equality) + Return New Token(_tokens.GetCurrent.TokenValue, Tokens.Equals, TokenPriority.Equality) Else - Return New Token(Tokens.GetCurrent.TokenValue, Statements.TokenIcer.Tokens.Not, TokenPriority.[Not]) + Return New Token(_tokens.GetCurrent.TokenValue, Tokens.Not, TokenPriority.[Not]) End If - Case Statements.TokenIcer.Tokens.Asterisk OrElse Statements.TokenIcer.Tokens.Slash + Case Tokens.Asterisk OrElse Tokens.Slash MoveNext() - Return New Token(Tokens.GetCurrent.TokenValue, Tokens.GetCurrent.TokenName, TokenPriority.MulDiv) - Case Statements.TokenIcer.Tokens.Mod + Return New Token(_tokens.GetCurrent.TokenValue, _tokens.GetCurrent.TokenName, TokenPriority.MulDiv) + Case Tokens.Mod MoveNext() - Return New Token("%", Statements.TokenIcer.Tokens.Mod, TokenPriority.[Mod]) - Case Statements.TokenIcer.Tokens.Or + Return New Token("%", Tokens.Mod, TokenPriority.[Mod]) + Case Tokens.Or MoveNext() - Return New Token(Tokens.GetCurrent.TokenValue, Statements.TokenIcer.Tokens.Or, TokenPriority.[Or]) - Case Statements.TokenIcer.Tokens.And + Return New Token(_tokens.GetCurrent.TokenValue, Tokens.Or, TokenPriority.[Or]) + Case Tokens.And MoveNext() - Return New Token(Tokens.GetCurrent, Statements.TokenIcer.Tokens.And, TokenPriority.[And]) - Case Statements.TokenIcer.Tokens.LPair + Return New Token(_tokens.GetCurrent, Tokens.And, TokenPriority.[And]) + Case Tokens.LPair MoveNext() Return New Token("(", OpenParens, TokenPriority.None) - Case Statements.TokenIcer.Tokens.RPair + Case Tokens.RPair MoveNext() Return New Token(")", CloseParens, TokenPriority.None) - Case Statements.TokenIcer.Tokens.OpenBracket + Case Tokens.OpenBracket MoveNext() - Return New Token("[", Statements.TokenIcer.Tokens.OpenBracket, TokenPriority.None) - Case Statements.TokenIcer.Tokens.CloseBracket + Return New Token("[", Tokens.OpenBracket, TokenPriority.None) + Case Tokens.CloseBracket MoveNext() - Return New Token("]", Statements.TokenIcer.Tokens.CloseBracket, TokenPriority.None) + Return New Token("]", Tokens.CloseBracket, TokenPriority.None) Case Else ' When we detect a quote, we can just ignore it since the user doesn't really need to know about it. MoveNext() - _PrevToken = New Token(Tokens.GetCurrent.TokenValue, Statements.TokenIcer.Tokens.String, TokenPriority.None) + _PrevToken = New Token(_tokens.GetCurrent.TokenValue, Tokens.String, TokenPriority.None) Return GetString() End Select Return Token.NullToken