From e462ff66176908e5f7bebd390922e8a189293eff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AC=9D=E6=A1=82=E7=B6=B1?= Date: Thu, 11 Feb 2016 07:22:16 +0800 Subject: [PATCH] change tokens parser using the token parser and stack parser in the app framework --- LINQ/LINQ/LDM/Expression/Closure.vb | 4 +- LINQ/LINQ/LDM/Expression/LetClosure.vb | 7 +- LINQ/LINQ/LDM/Parser/ClosureParser.vb | 30 +- LINQ/LINQ/LDM/Parser/ClosureTokens.vb | 21 +- LINQ/LINQ/LDM/Parser/Tokens/Closure.vb | 12 +- LINQ/LINQ/LDM/Parser/Tokens/FromClosure.vb | 2 +- LINQ/LINQ/LDM/Parser/Tokens/InClosure.vb | 4 +- LINQ/LINQ/LDM/Parser/Tokens/LetClosure.vb | 20 +- LINQ/LINQ/LDM/Parser/Tokens/SelectClosure.vb | 12 +- LINQ/LINQ/LDM/Parser/Tokens/WhereClosure.vb | 12 +- LINQ/LINQ/LDM/StackParser.vb | 94 ------ LINQ/LINQ/LDM/TokenIcer.vb | 109 +++++++ LINQ/LINQ/LINQ.vbproj | 3 +- LINQ/LINQ/TokenIcer.vb | 300 ------------------- 14 files changed, 173 insertions(+), 457 deletions(-) delete mode 100644 LINQ/LINQ/LDM/StackParser.vb create mode 100644 LINQ/LINQ/LDM/TokenIcer.vb delete mode 100644 LINQ/LINQ/TokenIcer.vb diff --git a/LINQ/LINQ/LDM/Expression/Closure.vb b/LINQ/LINQ/LDM/Expression/Closure.vb index bd90a39..0edb50a 100644 --- a/LINQ/LINQ/LDM/Expression/Closure.vb +++ b/LINQ/LINQ/LDM/Expression/Closure.vb @@ -1,6 +1,4 @@ -Imports Microsoft.VisualBasic.LINQ.TokenIcer - -Namespace LDM.Expression +Namespace LDM.Expression Public MustInherit Class Closure diff --git a/LINQ/LINQ/LDM/Expression/LetClosure.vb b/LINQ/LINQ/LDM/Expression/LetClosure.vb index 8c67819..f2a1ac6 100644 --- a/LINQ/LINQ/LDM/Expression/LetClosure.vb +++ b/LINQ/LINQ/LDM/Expression/LetClosure.vb @@ -2,8 +2,9 @@ Imports System.Text Imports System.CodeDom Imports Microsoft.VisualBasic.LINQ.Framework.Provider -Imports Microsoft.VisualBasic.LINQ.TokenIcer.TokenParser +Imports Microsoft.VisualBasic.LINQ.Statements Imports Microsoft.VisualBasic.LINQ.Extensions +Imports Microsoft.VisualBasic.Scripting.TokenIcer Namespace LDM.Expression @@ -45,8 +46,8 @@ Namespace LDM.Expression ''' ''' Protected Overrides Function __parsing() As CodeExpression - Dim init As Func = - DirectCast(_source, Statements.Tokens.LetClosure).Expression.Args.First + Dim init As Func(Of TokenIcer.Tokens) = + DirectCast(_source, Tokens.LetClosure).Expression.Args.First End Function End Class End Namespace \ No newline at end of file diff --git a/LINQ/LINQ/LDM/Parser/ClosureParser.vb b/LINQ/LINQ/LDM/Parser/ClosureParser.vb index 1720967..56fce93 100644 --- a/LINQ/LINQ/LDM/Parser/ClosureParser.vb +++ b/LINQ/LINQ/LDM/Parser/ClosureParser.vb @@ -1,22 +1,22 @@ -Imports Microsoft.VisualBasic.LINQ.TokenIcer +Imports Microsoft.VisualBasic.LINQ.Statements +Imports Microsoft.VisualBasic.Scripting.TokenIcer Public Module ClosureParser Public Function TryParse(st As String) As ClosureTokens() - Dim parser As TokenParser = New TokenParser With {.InputString = st} - Dim token As Token = Nothing + Dim source As Token(Of TokenIcer.Tokens)() = TokenIcer.GetTokens(st) Dim parts As New List(Of ClosureTokens) - Dim tmp As New List(Of Token) - Dim current As TokenParser.Tokens + Dim tmp As New List(Of Token(Of TokenIcer.Tokens)) + Dim current As TokenIcer.Tokens Dim closure As ClosureTokens - Do While Not parser.GetToken.ShadowCopy(token) Is Nothing + For Each token As Token(Of TokenIcer.Tokens) In source Select Case token.TokenName - Case TokenParser.Tokens.Imports, - TokenParser.Tokens.In, - TokenParser.Tokens.Let, - TokenParser.Tokens.Select, - TokenParser.Tokens.Where + Case TokenIcer.Tokens.Imports, + TokenIcer.Tokens.In, + TokenIcer.Tokens.Let, + TokenIcer.Tokens.Select, + TokenIcer.Tokens.Where closure = New ClosureTokens With { .Token = current, @@ -25,14 +25,14 @@ Public Module ClosureParser Call parts.Add(closure) Call tmp.Clear() current = token.TokenName - Case TokenParser.Tokens.From - current = TokenParser.Tokens.From - Case TokenParser.Tokens.WHITESPACE + Case TokenIcer.Tokens.From + current = TokenIcer.Tokens.From + Case TokenIcer.Tokens.WhiteSpace ' Do Nothing Case Else Call tmp.Add(token) End Select - Loop + Next closure = New ClosureTokens With { .Token = current, diff --git a/LINQ/LINQ/LDM/Parser/ClosureTokens.vb b/LINQ/LINQ/LDM/Parser/ClosureTokens.vb index 2f912c0..6a92a42 100644 --- a/LINQ/LINQ/LDM/Parser/ClosureTokens.vb +++ b/LINQ/LINQ/LDM/Parser/ClosureTokens.vb @@ -1,11 +1,14 @@ -Imports Microsoft.VisualBasic.LINQ.TokenIcer -Imports Microsoft.VisualBasic.LINQ.TokenIcer.TokenParser +Imports Microsoft.VisualBasic.LINQ.Statements +Imports Microsoft.VisualBasic.Scripting.TokenIcer -Public Class ClosureTokens - Public Property Token As Tokens - Public Property Tokens As Token() +Namespace Statements - Public Overrides Function ToString() As String - Return $"[{Token}] {Tokens.ToArray(Function(x) x.TokenValue).JoinBy(" ")}" - End Function -End Class \ No newline at end of file + Public Class ClosureTokens + Public Property Token As TokenIcer.Tokens + Public Property Tokens As Token(Of TokenIcer.Tokens)() + + Public Overrides Function ToString() As String + Return $"[{Token}] {Tokens.ToArray(Function(x) x.TokenValue).JoinBy(" ")}" + End Function + End Class +End Namespace \ No newline at end of file diff --git a/LINQ/LINQ/LDM/Parser/Tokens/Closure.vb b/LINQ/LINQ/LDM/Parser/Tokens/Closure.vb index 27e08b9..ef298ae 100644 --- a/LINQ/LINQ/LDM/Parser/Tokens/Closure.vb +++ b/LINQ/LINQ/LDM/Parser/Tokens/Closure.vb @@ -1,4 +1,4 @@ -Imports Microsoft.VisualBasic.LINQ.TokenIcer +Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer Namespace Statements.Tokens @@ -7,14 +7,14 @@ Namespace Statements.Tokens Protected _statement As LINQStatement Protected _source As ClosureTokens - Sub New(type As TokenParser.Tokens, tokens As ClosureTokens(), parent As LINQStatement) + Sub New(type As TokenIcer.Tokens, tokens As ClosureTokens(), parent As LINQStatement) _source = GetTokens(type, from:=tokens) _statement = parent If _source Is Nothing Then - If type = TokenParser.Tokens.From OrElse - type = TokenParser.Tokens.In OrElse - type = TokenParser.Tokens.Select Then + If type = TokenIcer.Tokens.From OrElse + type = TokenIcer.Tokens.In OrElse + type = TokenIcer.Tokens.Select Then ' LET and SELECT is optional ' But From, In and Select is required ' If missing, then syntax error, throw exception @@ -35,7 +35,7 @@ Namespace Statements.Tokens Return _source.ToString End Function - Public Shared Function GetTokens(type As TokenParser.Tokens, from As ClosureTokens()) As ClosureTokens + Public Shared Function GetTokens(type As TokenIcer.Tokens, from As ClosureTokens()) As ClosureTokens Dim LQuery = (From x As ClosureTokens In from Where x.Token = type Select x).FirstOrDefault Return LQuery End Function diff --git a/LINQ/LINQ/LDM/Parser/Tokens/FromClosure.vb b/LINQ/LINQ/LDM/Parser/Tokens/FromClosure.vb index 6ca6f45..1518f76 100644 --- a/LINQ/LINQ/LDM/Parser/Tokens/FromClosure.vb +++ b/LINQ/LINQ/LDM/Parser/Tokens/FromClosure.vb @@ -26,7 +26,7 @@ Namespace Statements.Tokens Public Property TypeId As String Sub New(tokens As ClosureTokens(), parent As LINQStatement) - Call MyBase.New(TokenIcer.TokenParser.Tokens.From, tokens, parent) + Call MyBase.New(TokenIcer.Tokens.From, tokens, parent) Name = _source.Tokens(Scan0).TokenValue TypeId = _source.Tokens(2).TokenValue diff --git a/LINQ/LINQ/LDM/Parser/Tokens/InClosure.vb b/LINQ/LINQ/LDM/Parser/Tokens/InClosure.vb index c917863..bafe967 100644 --- a/LINQ/LINQ/LDM/Parser/Tokens/InClosure.vb +++ b/LINQ/LINQ/LDM/Parser/Tokens/InClosure.vb @@ -75,9 +75,9 @@ Namespace Statements.Tokens End Sub Public Shared Function CreateObject(tokens As ClosureTokens(), parent As LINQStatement) As InClosure - Dim source As ClosureTokens = Closure.GetTokens(TokenIcer.TokenParser.Tokens.In, tokens) + Dim source As ClosureTokens = Closure.GetTokens(TokenIcer.Tokens.In, tokens) If source.Tokens.Length = 1 AndAlso - source.Tokens(Scan0).TokenName = TokenIcer.TokenParser.Tokens.String Then + source.Tokens(Scan0).TokenName = TokenIcer.Tokens.String Then Return New UriRef(source, parent) Else Return New Reference(source, parent) diff --git a/LINQ/LINQ/LDM/Parser/Tokens/LetClosure.vb b/LINQ/LINQ/LDM/Parser/Tokens/LetClosure.vb index e899a7f..bdd5093 100644 --- a/LINQ/LINQ/LDM/Parser/Tokens/LetClosure.vb +++ b/LINQ/LINQ/LDM/Parser/Tokens/LetClosure.vb @@ -1,7 +1,7 @@ Imports System.Text.RegularExpressions Imports System.Text -Imports Microsoft.VisualBasic.LINQ.LDM -Imports Microsoft.VisualBasic.LINQ.TokenIcer +Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer +Imports Microsoft.VisualBasic.Scripting.TokenIcer Namespace Statements.Tokens @@ -23,7 +23,7 @@ Namespace Statements.Tokens ''' Public Property Type As String - Public Property Expression As Func + Public Property Expression As Func(Of TokenIcer.Tokens) ''' ''' Let var = expression @@ -36,10 +36,10 @@ Namespace Statements.Tokens Dim sk As Integer - If _source.Tokens(1).TokenName = TokenParser.Tokens.Equals Then + If _source.Tokens(1).TokenName = TokenIcer.Tokens.Equals Then sk = 2 ' 没有申明类型 Else - If _source.Tokens(1).TokenName = TokenParser.Tokens.As Then + If _source.Tokens(1).TokenName = TokenIcer.Tokens.As Then Type = _source.Tokens(2).TokenValue sk = 4 Else @@ -47,8 +47,8 @@ Namespace Statements.Tokens End If End If - Dim expr As Token() = _source.Tokens.Skip(sk).ToArray - Expression = New Queue(Of Token)(expr).Parsing + Dim expr As IEnumerable(Of Token(Of TokenIcer.Tokens)) = _source.Tokens.Skip(sk) + Expression = expr.Parsing(stackt) End Sub Public Function ToFieldDeclaration() As CodeDom.CodeMemberField @@ -69,7 +69,7 @@ Namespace Statements.Tokens Dim current As ClosureTokens = Nothing Dim list As New List(Of ClosureTokens) - Do While tokens.Read(i, current).Token = TokenIcer.TokenParser.Tokens.Let + Do While tokens.Read(i, current).Token = TokenIcer.Tokens.Let Call list.Add(current) If i = tokens.Length Then Exit Do @@ -85,12 +85,12 @@ Namespace Statements.Tokens Dim current As ClosureTokens = Nothing Dim list As New List(Of ClosureTokens) - Do While tokens.Read(i, current).Token <> TokenIcer.TokenParser.Tokens.Where + Do While tokens.Read(i, current).Token <> TokenIcer.Tokens.Where If i = tokens.Length Then Exit Do End If Loop - Do While tokens.Read(i, current).Token = TokenIcer.TokenParser.Tokens.Let + Do While tokens.Read(i, current).Token = TokenIcer.Tokens.Let Call list.Add(current) If i = tokens.Length Then Exit Do diff --git a/LINQ/LINQ/LDM/Parser/Tokens/SelectClosure.vb b/LINQ/LINQ/LDM/Parser/Tokens/SelectClosure.vb index 5cb27a5..51e010b 100644 --- a/LINQ/LINQ/LDM/Parser/Tokens/SelectClosure.vb +++ b/LINQ/LINQ/LDM/Parser/Tokens/SelectClosure.vb @@ -2,6 +2,8 @@ Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode.VBC Imports Microsoft.VisualBasic.LINQ.LDM +Imports Microsoft.VisualBasic.Scripting.TokenIcer +Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer.Parser Namespace Statements.Tokens @@ -13,7 +15,7 @@ Namespace Statements.Tokens ''' 通过Select表达式所产生的数据投影 ''' ''' - Public ReadOnly Property Projects As Func() + Public ReadOnly Property Projects As Func(Of TokenIcer.Tokens)() ''' ''' @@ -21,12 +23,12 @@ Namespace Statements.Tokens ''' 使用逗号分隔数据投影 ''' Sub New(tokens As ClosureTokens(), parent As LINQStatement) - Call MyBase.New(TokenIcer.TokenParser.Tokens.Select, tokens, parent) - Projects = StackParser.Parsing(New Queue(Of TokenIcer.Token)(_source.Tokens)).Args + Call MyBase.New(TokenIcer.Tokens.Select, tokens, parent) + Projects = _source.Tokens.Parsing(stackT).Args End Sub - Private Shared Function __isDelimiter(x As TokenIcer.Token) As Boolean - Return x.TokenName = TokenIcer.TokenParser.Tokens.paramDeli + Private Shared Function __isDelimiter(x As Token(Of TokenIcer.Tokens)) As Boolean + Return x.TokenName = TokenIcer.Tokens.ParamDeli End Function Public Sub Initialzie() diff --git a/LINQ/LINQ/LDM/Parser/Tokens/WhereClosure.vb b/LINQ/LINQ/LDM/Parser/Tokens/WhereClosure.vb index c97bb02..3d34ab2 100644 --- a/LINQ/LINQ/LDM/Parser/Tokens/WhereClosure.vb +++ b/LINQ/LINQ/LDM/Parser/Tokens/WhereClosure.vb @@ -4,20 +4,18 @@ Imports System.Text Imports System.Text.RegularExpressions Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode.VBC -Imports Microsoft.VisualBasic.LINQ.LDM -Imports Microsoft.VisualBasic.LINQ.TokenIcer +Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer.Parser +Imports Microsoft.VisualBasic.Scripting.TokenIcer Namespace Statements.Tokens Public Class WhereClosure : Inherits Closure - Public ReadOnly Property Expression As Func + Public ReadOnly Property Expression As Func(Of TokenIcer.Tokens) Sub New(tokens As ClosureTokens(), parent As LINQStatement) - Call MyBase.New(TokenParser.Tokens.Where, tokens, parent) - - Dim source = New Queue(Of Token)(_source.Tokens) - Expression = source.Parsing + Call MyBase.New(TokenIcer.Tokens.Where, tokens, parent) + Expression = _source.Tokens.Parsing(stackt) End Sub Public Sub Initialize() diff --git a/LINQ/LINQ/LDM/StackParser.vb b/LINQ/LINQ/LDM/StackParser.vb deleted file mode 100644 index 88d9911..0000000 --- a/LINQ/LINQ/LDM/StackParser.vb +++ /dev/null @@ -1,94 +0,0 @@ -Imports System.Runtime.CompilerServices -Imports Microsoft.VisualBasic.LINQ.TokenIcer - -Namespace LDM - - Public Module StackParser - - ''' - ''' 返回顶层的根节点 - ''' - ''' - ''' - Public Function Parsing(source As Queue(Of Token)) As Func - Dim pretend As Token = New Token(TokenParser.Tokens.Pretend, "Pretend") - Dim root As Func = New Func With { - .Caller = New List(Of Token) From {pretend}, - .Args = __parsing(source) - } - Return root - End Function - - ''' - ''' 主要是解析当前的栈层之中的使用，逗号分隔的参数列表 - ''' - ''' - ''' - Private Function __parsing(source As Queue(Of Token)) As Func() - Dim list As New List(Of Func) - Dim current As Func = New Func With { - .Caller = New List(Of Token) - } - - Do While Not source.IsNullOrEmpty - Dim x As Token = source.Dequeue - - If Not x.TokenName = TokenParser.Tokens.ParamDeli Then - Call current.Caller.Add(x) - End If - - If source.Count = 0 Then - Call list.Add(current) - Exit Do - End If - - Dim peek As Token = source.Peek - - If peek.TokenName = TokenParser.Tokens.LPair Then ' 向下一层堆栈 - Call source.Dequeue() - current.Args = __parsing(source) - ElseIf peek.TokenName = TokenParser.Tokens.RPair Then ' 向上一层退栈 - Call source.Dequeue() - Call list.Add(current) - Exit Do - ElseIf x.TokenName = TokenParser.Tokens.ParamDeli Then - Call list.Add(current) - current = New Func With { - .Caller = New List(Of Token) - } - End If - Loop - - If list.Count = 0 Then - Call list.Add(current) - End If - - Return list.ToArray - End Function - End Module - - Public Class Func - - Public Property Caller As List(Of Token) - Public Property Args As Func() - - Public ReadOnly Property IsFuncCalls As Boolean - Get - Return Not Args.IsNullOrEmpty - End Get - End Property - - Public Overrides Function ToString() As String - If Args.IsNullOrEmpty Then - Return String.Join(" ", Caller.ToArray(Function(x) x.TokenValue)) - Else - Dim caller As String = String.Join(" ", Me.Caller.ToArray(Function(x) x.TokenValue)) - Dim params As String() = Me.Args.ToArray(Function(x) x.ToString) - Dim args As String = String.Join(", ", params) - Return $"{caller}({args})" - End If - End Function - End Class -End Namespace - - diff --git a/LINQ/LINQ/LDM/TokenIcer.vb b/LINQ/LINQ/LDM/TokenIcer.vb new file mode 100644 index 0000000..b524fc1 --- /dev/null +++ b/LINQ/LINQ/LDM/TokenIcer.vb @@ -0,0 +1,109 @@ +' This file was Auto Generated with TokenIcer +Imports System.Collections.Generic +Imports System.Text.RegularExpressions +Imports Microsoft.VisualBasic.Scripting.TokenIcer + +Namespace Statements.TokenIcer + + ' This is our token enumeration. It holds every token defined in the grammar + ''' + ''' Tokens is an enumeration of all possible token values. + ''' + Public Enum Tokens + UNDEFINED = 0 + CallFunc = 1 + [Integer] = 2 + Float = 3 + ArrayType = 4 + ParamDeli = 5 + WhiteSpace = 6 + [Let] = 7 + Equals = 8 + StringIntrop = 9 + [String] = 10 + VbCrLf = 11 + LPair = 12 + RPair = 13 + Asterisk = 14 + Slash = 15 + Plus = 16 + Minus = 17 + Pretend = 18 + cor = 19 + [And] = 20 + [Not] = 21 + [Or] = 22 + [Is] = 23 + [As] = 24 + From = 25 + Where = 26 + [Select] = 27 + [Imports] = 28 + [In] = 29 + var = 30 + varRef = 31 + End Enum + + Public Module Parser + + Public ReadOnly Property stackT As StackTokens(Of Tokens) = + New StackTokens(Of Tokens)(Function(a, b) a = b) With { + .LPair = Tokens.LPair, + .ParamDeli = Tokens.ParamDeli, + .Pretend = Tokens.Pretend, + .RPair = Tokens.RPair, + .WhiteSpace = Tokens.WhiteSpace + } + + Private Function __getParser() As TokenParser(Of Tokens) + Dim _tokens As New Dictionary(Of Tokens, String)() + + ' These lines add each grammar rule to the dictionary + _tokens.Add(Tokens.CallFunc, "->\s*[a-zA-Z_][a-zA-Z0-9_]*") + _tokens.Add(Tokens.[Integer], "[0-9]+") + _tokens.Add(Tokens.Float, "[0-9]+\.+[0-9]+") + _tokens.Add(Tokens.ArrayType, "[a-zA-Z_][a-zA-Z0-9_]*\(\)") + _tokens.Add(Tokens.ParamDeli, ",") + _tokens.Add(Tokens.WhiteSpace, "[ \t]+") + _tokens.Add(Tokens.[Let], "[Ll][Ee][Tt]") + _tokens.Add(Tokens.Equals, "=") + _tokens.Add(Tokens.StringIntrop, "\$"".*?""") + _tokens.Add(Tokens.[String], """.*?""") + _tokens.Add(Tokens.VbCrLf, "[\r\n]+") + _tokens.Add(Tokens.LPair, "\(") + _tokens.Add(Tokens.RPair, "\)") + _tokens.Add(Tokens.Asterisk, "\*") + _tokens.Add(Tokens.Slash, "\/") + _tokens.Add(Tokens.Plus, "\+") + _tokens.Add(Tokens.Minus, "\-") + _tokens.Add(Tokens.Pretend, "Pretend") + _tokens.Add(Tokens.cor, "[nN][eE][wW]") + _tokens.Add(Tokens.[And], "[aA][nN][dD]") + _tokens.Add(Tokens.[Not], "[nN][oO][tT]") + _tokens.Add(Tokens.[Or], "[oO][rR]") + _tokens.Add(Tokens.[Is], "[iI][sS]") + _tokens.Add(Tokens.[As], "[aA][sS]") + _tokens.Add(Tokens.From, "[fF][rR][oO][mM]") + _tokens.Add(Tokens.Where, "[wW][hH][eE][rR][eE]") + _tokens.Add(Tokens.[Select], "[sS][eE][lL][eE][cC][tT]") + _tokens.Add(Tokens.[Imports], "^[Ii][mM][pP][oO][rR][tT][sS]") + _tokens.Add(Tokens.[In], "[iI][nN]") + _tokens.Add(Tokens.var, "[a-zA-Z_][a-zA-Z0-9_]*") + _tokens.Add(Tokens.varRef, "\$[a-zA-Z0-9_]*") + + Return New TokenParser(Of Tokens)(_tokens, Tokens.UNDEFINED) + End Function + + ' Our Constructor, which simply initializes values + ''' + ''' Default Constructor + ''' + ''' + ''' The constructor initalizes memory and adds all of the tokens to the token dictionary. + ''' + Public Function GetTokens(expr As String) As Token(Of Tokens)() + Return __getParser.GetTokens(expr) + End Function + End Module +End Namespace + diff --git a/LINQ/LINQ/LINQ.vbproj b/LINQ/LINQ/LINQ.vbproj index e68e3fe..312105e 100644 --- a/LINQ/LINQ/LINQ.vbproj +++ b/LINQ/LINQ/LINQ.vbproj @@ -116,7 +116,6 @@ - @@ -126,7 +125,7 @@ - + diff --git a/LINQ/LINQ/TokenIcer.vb b/LINQ/LINQ/TokenIcer.vb deleted file mode 100644 index c2e7149..0000000 --- a/LINQ/LINQ/TokenIcer.vb +++ /dev/null @@ -1,300 +0,0 @@ -' This file was Auto Generated with TokenIcer -Imports System.Collections.Generic -Imports System.Text.RegularExpressions - -Namespace TokenIcer - ''' - ''' TokenParser - ''' - ''' - ''' TokenParser is the main parser engine for converting input into lexical tokens. - ''' - Public Class TokenParser - ' This dictionary will store our RegEx rules - Private ReadOnly _tokens As Dictionary(Of Tokens, String) - ' This dictionary will store our matches - Private ReadOnly _regExMatchCollection As Dictionary(Of Tokens, MatchCollection) - ' This input string will store the string to parse - Private _inputString As String - ' This index is used internally so the parser knows where it left off - Private _index As Integer - - ' This is our token enumeration. It holds every token defined in the grammar - ''' - ''' Tokens is an enumeration of all possible token values. - ''' - Public Enum Tokens - UNDEFINED = 0 - CallFunc = 1 - [Integer] = 2 - Float = 3 - ArrayType = 4 - ParamDeli = 5 - WhiteSpace = 6 - [Let] = 7 - Equals = 8 - StringIntrop = 9 - [String] = 10 - VbCrLf = 11 - LPair = 12 - RPair = 13 - Asterisk = 14 - Slash = 15 - Plus = 16 - Minus = 17 - Pretend = 18 - cor = 19 - [And] = 20 - [Not] = 21 - [Or] = 22 - [Is] = 23 - [As] = 24 - From = 25 - Where = 26 - [Select] = 27 - [Imports] = 28 - [In] = 29 - var = 30 - varRef = 31 - End Enum - - ' A public setter for our input string - ''' - ''' InputString Property - ''' - ''' - ''' The string value that holds the input string. - ''' - Public WriteOnly Property InputString() As String - Set - _inputString = Value - PrepareRegex() - End Set - End Property - - ' Our Constructor, which simply initializes values - ''' - ''' Default Constructor - ''' - ''' - ''' The constructor initalizes memory and adds all of the tokens to the token dictionary. - ''' - Public Sub New() - _tokens = New Dictionary(Of Tokens, String)() - _regExMatchCollection = New Dictionary(Of Tokens, MatchCollection)() - _index = 0 - _inputString = String.Empty - - ' These lines add each grammar rule to the dictionary - _tokens.Add(Tokens.CallFunc, "->\s*[a-zA-Z_][a-zA-Z0-9_]*") - _tokens.Add(Tokens.[Integer], "[0-9]+") - _tokens.Add(Tokens.Float, "[0-9]+\.+[0-9]+") - _tokens.Add(Tokens.ArrayType, "[a-zA-Z_][a-zA-Z0-9_]*\(\)") - _tokens.Add(Tokens.ParamDeli, ",") - _tokens.Add(Tokens.WhiteSpace, "[ \t]+") - _tokens.Add(Tokens.[Let], "[Ll][Ee][Tt]") - _tokens.Add(Tokens.Equals, "=") - _tokens.Add(Tokens.StringIntrop, "\$"".*?""") - _tokens.Add(Tokens.[String], """.*?""") - _tokens.Add(Tokens.VbCrLf, "[\r\n]+") - _tokens.Add(Tokens.LPair, "\(") - _tokens.Add(Tokens.RPair, "\)") - _tokens.Add(Tokens.Asterisk, "\*") - _tokens.Add(Tokens.Slash, "\/") - _tokens.Add(Tokens.Plus, "\+") - _tokens.Add(Tokens.Minus, "\-") - _tokens.Add(Tokens.Pretend, "Pretend") - _tokens.Add(Tokens.cor, "[nN][eE][wW]") - _tokens.Add(Tokens.[And], "[aA][nN][dD]") - _tokens.Add(Tokens.[Not], "[nN][oO][tT]") - _tokens.Add(Tokens.[Or], "[oO][rR]") - _tokens.Add(Tokens.[Is], "[iI][sS]") - _tokens.Add(Tokens.[As], "[aA][sS]") - _tokens.Add(Tokens.From, "[fF][rR][oO][mM]") - _tokens.Add(Tokens.Where, "[wW][hH][eE][rR][eE]") - _tokens.Add(Tokens.[Select], "[sS][eE][lL][eE][cC][tT]") - _tokens.Add(Tokens.[Imports], "^[Ii][mM][pP][oO][rR][tT][sS]") - _tokens.Add(Tokens.[In], "[iI][nN]") - _tokens.Add(Tokens.var, "[a-zA-Z_][a-zA-Z0-9_]*") - _tokens.Add(Tokens.varRef, "\$[a-zA-Z0-9_]*") - End Sub - - ' This function preloads the matches based on our rules and the input string - ''' - ''' PrepareRegex prepares the regex for parsing by pre-matching the Regex tokens. - ''' - Private Sub PrepareRegex() - _regExMatchCollection.Clear() - For Each pair As KeyValuePair(Of Tokens, String) In _tokens - _regExMatchCollection.Add(pair.Key, Regex.Matches(_inputString, pair.Value)) - Next - End Sub - - ' ResetParser() will reset the parser. - ' Keep in mind that you must set the input string again - ''' - ''' ResetParser resets the parser to its inital state. Reloading InputString is required. - ''' - ''' - Public Sub ResetParser() - _index = 0 - _inputString = String.Empty - _regExMatchCollection.Clear() - End Sub - - ' GetToken() retrieves the next token and returns a token object - ''' - ''' GetToken gets the next token in queue - ''' - ''' - ''' GetToken attempts to the match the next character(s) using the - ''' Regex rules defined in the dictionary. If a match can not be - ''' located, then an Undefined token will be created with an empty - ''' string value. In addition, the token pointer will be incremented - ''' by one so that this token doesn't attempt to get identified again by - ''' GetToken() - ''' - Public Function GetToken() As Token - ' If we are at the end of our input string then - ' we return null to signify the end of our input string. - ' While parsing tokens, you will undoubtedly be in a loop. - ' Having your loop check for a null token is a good way to end that - ' loop - If _index >= _inputString.Length Then - Return Nothing - End If - - ' Iterate through our prepared matches/Tokens dictionary - For Each pair As KeyValuePair(Of Tokens, MatchCollection) In _regExMatchCollection - ' Iterate through each prepared match - For Each match As Match In pair.Value - ' If we find a match, update our index pointer and return a new Token object - If match.Index = _index Then - _index += match.Length - Return New Token(pair.Key, match.Value) - ElseIf match.Index > _index Then - Exit For - End If - Next - Next - - ' If execution got here, then we increment our index pointer - ' and return an Undefined token. - _index += 1 - Return New Token(Tokens.UNDEFINED, String.Empty) - End Function - - ' Peek() will retrieve a PeekToken object and will allow you to see the next token - ' that GetToken() will retrieve. - ''' - ''' Returns the next token that GetToken() will return. - ''' - ''' - Public Function Peek() As PeekToken - Return Peek(New PeekToken(_index, New Token(Tokens.UNDEFINED, String.Empty))) - End Function - - ' This is an overload for Peek(). By passing in the last PeekToken object - ' received from Peek(), you can peek ahead to the next token, and the token after that, etc... - ''' - ''' Returns the next token after the Token passed here - ''' - ''' The PeekToken token returned from a previous Peek() call - ''' - Public Function Peek(peekToken As PeekToken) As PeekToken - Dim oldIndex As Integer = _index - - _index = peekToken.TokenIndex - - If _index >= _inputString.Length Then - _index = oldIndex - Return Nothing - End If - - For Each pair As KeyValuePair(Of Tokens, String) In _tokens - Dim r As New Regex(pair.Value) - Dim m As Match = r.Match(_inputString, _index) - - If m.Success AndAlso m.Index = _index Then - _index = _index + m.Length - Dim pt As New PeekToken(_index, New Token(pair.Key, m.Value)) - _index = oldIndex - Return pt - End If - Next - Dim pt2 As New PeekToken(_index + 1, New Token(Tokens.UNDEFINED, String.Empty)) - _index = oldIndex - Return pt2 - End Function - End Class - - ' This defines the PeekToken object - ''' - ''' A PeekToken object class - ''' - ''' - ''' A PeekToken is a special pointer object that can be used to Peek() several - ''' tokens ahead in the GetToken() queue. - ''' - Public Class PeekToken - Public Property TokenIndex() As Integer - Get - Return m_TokenIndex - End Get - Set - m_TokenIndex = Value - End Set - End Property - Private m_TokenIndex As Integer - - Public Property TokenPeek() As Token - Get - Return m_TokenPeek - End Get - Set - m_TokenPeek = Value - End Set - End Property - Private m_TokenPeek As Token - - Public Sub New(index As Integer, value As Token) - TokenIndex = index - TokenPeek = value - End Sub - End Class - - ' This defines the Token object - ''' - ''' a Token object class - ''' - ''' - ''' A Token object holds the token and token value. - ''' - Public Class Token - Public Property TokenName() As TokenParser.Tokens - Get - Return m_TokenName - End Get - Set - m_TokenName = Value - End Set - End Property - Private m_TokenName As TokenParser.Tokens - - Public Property TokenValue() As String - Get - Return m_TokenValue - End Get - Set - m_TokenValue = Value - End Set - End Property - Private m_TokenValue As String - - Public Sub New(name As TokenParser.Tokens, value As String) - TokenName = name - TokenValue = value - End Sub - End Class -End Namespace -