wandyine
/
Darwinism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change tokens parser

Browse Source 
using the token parser and stack parser in the app framework
master
謝桂綱 10 years ago
parent 19a82aff3a
commit e462ff6617
14 changed files with 173 additions and 457 deletions
Show Stats Download Patch File Download Diff File

4
LINQ/LINQ/LDM/Expression/Closure.vb
Unescape View File

@ -1,6 +1,4 @@
Imports Microsoft.VisualBasic.LINQ.TokenIcer
Namespace LDM.Expression
Namespace LDM.Expression
Public MustInherit Class Closure

7
LINQ/LINQ/LDM/Expression/LetClosure.vb
Unescape View File

@ -2,8 +2,9 @@
Imports System.Text
Imports System.CodeDom
Imports Microsoft.VisualBasic.LINQ.Framework.Provider
Imports Microsoft.VisualBasic.LINQ.TokenIcer.TokenParser
Imports Microsoft.VisualBasic.LINQ.Statements
Imports Microsoft.VisualBasic.LINQ.Extensions
Imports Microsoft.VisualBasic.Scripting.TokenIcer
Namespace LDM.Expression
@ -45,8 +46,8 @@ Namespace LDM.Expression
''' </summary>
''' <returns></returns>
Protected Overrides Function __parsing() As CodeExpression
Dim init As Func =
DirectCast(_source, Statements.Tokens.LetClosure).Expression.Args.First
Dim init As Func(Of TokenIcer.Tokens) =
DirectCast(_source, Tokens.LetClosure).Expression.Args.First
End Function
End Class
End Namespace

30
LINQ/LINQ/LDM/Parser/ClosureParser.vb
Unescape View File

@ -1,22 +1,22 @@
Imports Microsoft.VisualBasic.LINQ.TokenIcer
Imports Microsoft.VisualBasic.LINQ.Statements
Imports Microsoft.VisualBasic.Scripting.TokenIcer
Public Module ClosureParser
Public Function TryParse(st As String) As ClosureTokens()
Dim parser As TokenParser = New TokenParser With {.InputString = st}
Dim token As Token = Nothing
Dim source As Token(Of TokenIcer.Tokens)() = TokenIcer.GetTokens(st)
Dim parts As New List(Of ClosureTokens)
Dim tmp As New List(Of Token)
Dim current As TokenParser.Tokens
Dim tmp As New List(Of Token(Of TokenIcer.Tokens))
Dim current As TokenIcer.Tokens
Dim closure As ClosureTokens
Do While Not parser.GetToken.ShadowCopy(token) Is Nothing
For Each token As Token(Of TokenIcer.Tokens) In source
Select Case token.TokenName
Case TokenParser.Tokens.Imports,
TokenParser.Tokens.In,
TokenParser.Tokens.Let,
TokenParser.Tokens.Select,
TokenParser.Tokens.Where
Case TokenIcer.Tokens.Imports,
TokenIcer.Tokens.In,
TokenIcer.Tokens.Let,
TokenIcer.Tokens.Select,
TokenIcer.Tokens.Where
closure = New ClosureTokens With {
.Token = current,
@ -25,14 +25,14 @@ Public Module ClosureParser
Call parts.Add(closure)
Call tmp.Clear()
current = token.TokenName
Case TokenParser.Tokens.From
current = TokenParser.Tokens.From
Case TokenParser.Tokens.WHITESPACE
Case TokenIcer.Tokens.From
current = TokenIcer.Tokens.From
Case TokenIcer.Tokens.WhiteSpace
' Do Nothing
Case Else
Call tmp.Add(token)
End Select
Loop
Next
closure = New ClosureTokens With {
.Token = current,

21
LINQ/LINQ/LDM/Parser/ClosureTokens.vb
Unescape View File

@ -1,11 +1,14 @@
Imports Microsoft.VisualBasic.LINQ.TokenIcer
Imports Microsoft.VisualBasic.LINQ.TokenIcer.TokenParser
Imports Microsoft.VisualBasic.LINQ.Statements
Imports Microsoft.VisualBasic.Scripting.TokenIcer
Public Class ClosureTokens
Public Property Token As Tokens
Public Property Tokens As Token()
Namespace Statements
Public Overrides Function ToString() As String
Return $"[{Token}] {Tokens.ToArray(Function(x) x.TokenValue).JoinBy(" ")}"
End Function
End Class
Public Class ClosureTokens
Public Property Token As TokenIcer.Tokens
Public Property Tokens As Token(Of TokenIcer.Tokens)()
Public Overrides Function ToString() As String
Return $"[{Token}] {Tokens.ToArray(Function(x) x.TokenValue).JoinBy(" ")}"
End Function
End Class
End Namespace

12
LINQ/LINQ/LDM/Parser/Tokens/Closure.vb
Unescape View File

@ -1,4 +1,4 @@
Imports Microsoft.VisualBasic.LINQ.TokenIcer
Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer
Namespace Statements.Tokens
@ -7,14 +7,14 @@ Namespace Statements.Tokens
Protected _statement As LINQStatement
Protected _source As ClosureTokens
Sub New(type As TokenParser.Tokens, tokens As ClosureTokens(), parent As LINQStatement)
Sub New(type As TokenIcer.Tokens, tokens As ClosureTokens(), parent As LINQStatement)
_source = GetTokens(type, from:=tokens)
_statement = parent
If _source Is Nothing Then
If type = TokenParser.Tokens.From OrElse
type = TokenParser.Tokens.In OrElse
type = TokenParser.Tokens.Select Then
If type = TokenIcer.Tokens.From OrElse
type = TokenIcer.Tokens.In OrElse
type = TokenIcer.Tokens.Select Then
' LET and SELECT is optional
' But From, In and Select is required
' If missing, then syntax error, throw exception
@ -35,7 +35,7 @@ Namespace Statements.Tokens
Return _source.ToString
End Function
Public Shared Function GetTokens(type As TokenParser.Tokens, from As ClosureTokens()) As ClosureTokens
Public Shared Function GetTokens(type As TokenIcer.Tokens, from As ClosureTokens()) As ClosureTokens
Dim LQuery = (From x As ClosureTokens In from Where x.Token = type Select x).FirstOrDefault
Return LQuery
End Function

2
LINQ/LINQ/LDM/Parser/Tokens/FromClosure.vb
Unescape View File

@ -26,7 +26,7 @@ Namespace Statements.Tokens
Public Property TypeId As String
Sub New(tokens As ClosureTokens(), parent As LINQStatement)
Call MyBase.New(TokenIcer.TokenParser.Tokens.From, tokens, parent)
Call MyBase.New(TokenIcer.Tokens.From, tokens, parent)
Name = _source.Tokens(Scan0).TokenValue
TypeId = _source.Tokens(2).TokenValue

4
LINQ/LINQ/LDM/Parser/Tokens/InClosure.vb
Unescape View File

@ -75,9 +75,9 @@ Namespace Statements.Tokens
End Sub
Public Shared Function CreateObject(tokens As ClosureTokens(), parent As LINQStatement) As InClosure
Dim source As ClosureTokens = Closure.GetTokens(TokenIcer.TokenParser.Tokens.In, tokens)
Dim source As ClosureTokens = Closure.GetTokens(TokenIcer.Tokens.In, tokens)
If source.Tokens.Length = 1 AndAlso
source.Tokens(Scan0).TokenName = TokenIcer.TokenParser.Tokens.String Then
source.Tokens(Scan0).TokenName = TokenIcer.Tokens.String Then
Return New UriRef(source, parent)
Else
Return New Reference(source, parent)

20
LINQ/LINQ/LDM/Parser/Tokens/LetClosure.vb
Unescape View File

@ -1,7 +1,7 @@
Imports System.Text.RegularExpressions
Imports System.Text
Imports Microsoft.VisualBasic.LINQ.LDM
Imports Microsoft.VisualBasic.LINQ.TokenIcer
Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer
Imports Microsoft.VisualBasic.Scripting.TokenIcer
Namespace Statements.Tokens
@ -23,7 +23,7 @@ Namespace Statements.Tokens
''' <returns></returns>
Public Property Type As String
Public Property Expression As Func
Public Property Expression As Func(Of TokenIcer.Tokens)
''' <summary>
''' Let var = expression
@ -36,10 +36,10 @@ Namespace Statements.Tokens
Dim sk As Integer
If _source.Tokens(1).TokenName = TokenParser.Tokens.Equals Then
If _source.Tokens(1).TokenName = TokenIcer.Tokens.Equals Then
sk = 2 ' 没有申明类型
Else
If _source.Tokens(1).TokenName = TokenParser.Tokens.As Then
If _source.Tokens(1).TokenName = TokenIcer.Tokens.As Then
Type = _source.Tokens(2).TokenValue
sk = 4
Else
@ -47,8 +47,8 @@ Namespace Statements.Tokens
End If
End If
Dim expr As Token() = _source.Tokens.Skip(sk).ToArray
Expression = New Queue(Of Token)(expr).Parsing
Dim expr As IEnumerable(Of Token(Of TokenIcer.Tokens)) = _source.Tokens.Skip(sk)
Expression = expr.Parsing(stackt)
End Sub
Public Function ToFieldDeclaration() As CodeDom.CodeMemberField
@ -69,7 +69,7 @@ Namespace Statements.Tokens
Dim current As ClosureTokens = Nothing
Dim list As New List(Of ClosureTokens)
Do While tokens.Read(i, current).Token = TokenIcer.TokenParser.Tokens.Let
Do While tokens.Read(i, current).Token = TokenIcer.Tokens.Let
Call list.Add(current)
If i = tokens.Length Then
Exit Do
@ -85,12 +85,12 @@ Namespace Statements.Tokens
Dim current As ClosureTokens = Nothing
Dim list As New List(Of ClosureTokens)
Do While tokens.Read(i, current).Token <> TokenIcer.TokenParser.Tokens.Where
Do While tokens.Read(i, current).Token <> TokenIcer.Tokens.Where
If i = tokens.Length Then
Exit Do
End If
Loop
Do While tokens.Read(i, current).Token = TokenIcer.TokenParser.Tokens.Let
Do While tokens.Read(i, current).Token = TokenIcer.Tokens.Let
Call list.Add(current)
If i = tokens.Length Then
Exit Do

12
LINQ/LINQ/LDM/Parser/Tokens/SelectClosure.vb
Unescape View File

@ -2,6 +2,8 @@
Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode
Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode.VBC
Imports Microsoft.VisualBasic.LINQ.LDM
Imports Microsoft.VisualBasic.Scripting.TokenIcer
Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer.Parser
Namespace Statements.Tokens
@ -13,7 +15,7 @@ Namespace Statements.Tokens
''' 通过Select表达式所产生的数据投影
''' </summary>
''' <returns></returns>
Public ReadOnly Property Projects As Func()
Public ReadOnly Property Projects As Func(Of TokenIcer.Tokens)()
''' <summary>
'''
@ -21,12 +23,12 @@ Namespace Statements.Tokens
''' <param name="tokens">使用逗号分隔数据投影</param>
''' <param name="parent"></param>
Sub New(tokens As ClosureTokens(), parent As LINQStatement)
Call MyBase.New(TokenIcer.TokenParser.Tokens.Select, tokens, parent)
Projects = StackParser.Parsing(New Queue(Of TokenIcer.Token)(_source.Tokens)).Args
Call MyBase.New(TokenIcer.Tokens.Select, tokens, parent)
Projects = _source.Tokens.Parsing(stackT).Args
End Sub
Private Shared Function __isDelimiter(x As TokenIcer.Token) As Boolean
Return x.TokenName = TokenIcer.TokenParser.Tokens.paramDeli
Private Shared Function __isDelimiter(x As Token(Of TokenIcer.Tokens)) As Boolean
Return x.TokenName = TokenIcer.Tokens.ParamDeli
End Function
Public Sub Initialzie()

12
LINQ/LINQ/LDM/Parser/Tokens/WhereClosure.vb
Unescape View File

@ -4,20 +4,18 @@ Imports System.Text
Imports System.Text.RegularExpressions
Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode
Imports Microsoft.VisualBasic.LINQ.Framework.DynamicCode.VBC
Imports Microsoft.VisualBasic.LINQ.LDM
Imports Microsoft.VisualBasic.LINQ.TokenIcer
Imports Microsoft.VisualBasic.LINQ.Statements.TokenIcer.Parser
Imports Microsoft.VisualBasic.Scripting.TokenIcer
Namespace Statements.Tokens
Public Class WhereClosure : Inherits Closure
Public ReadOnly Property Expression As Func
Public ReadOnly Property Expression As Func(Of TokenIcer.Tokens)
Sub New(tokens As ClosureTokens(), parent As LINQStatement)
Call MyBase.New(TokenParser.Tokens.Where, tokens, parent)
Dim source = New Queue(Of Token)(_source.Tokens)
Expression = source.Parsing
Call MyBase.New(TokenIcer.Tokens.Where, tokens, parent)
Expression = _source.Tokens.Parsing(stackt)
End Sub
Public Sub Initialize()

94
LINQ/LINQ/LDM/StackParser.vb
Unescape View File

@ -1,94 +0,0 @@
Imports System.Runtime.CompilerServices
Imports Microsoft.VisualBasic.LINQ.TokenIcer
Namespace LDM
Public Module StackParser
''' <summary>
''' 返回顶层的根节点
''' </summary>
''' <param name="source"></param>
''' <returns></returns>
<Extension> Public Function Parsing(source As Queue(Of Token)) As Func
Dim pretend As Token = New Token(TokenParser.Tokens.Pretend, "Pretend")
Dim root As Func = New Func With {
.Caller = New List(Of Token) From {pretend},
.Args = __parsing(source)
}
Return root
End Function
''' <summary>
''' 主要是解析当前的栈层之中的使用，逗号分隔的参数列表
''' </summary>
''' <param name="source"></param>
''' <returns></returns>
Private Function __parsing(source As Queue(Of Token)) As Func()
Dim list As New List(Of Func)
Dim current As Func = New Func With {
.Caller = New List(Of Token)
}
Do While Not source.IsNullOrEmpty
Dim x As Token = source.Dequeue
If Not x.TokenName = TokenParser.Tokens.ParamDeli Then
Call current.Caller.Add(x)
End If
If source.Count = 0 Then
Call list.Add(current)
Exit Do
End If
Dim peek As Token = source.Peek
If peek.TokenName = TokenParser.Tokens.LPair Then ' 向下一层堆栈
Call source.Dequeue()
current.Args = __parsing(source)
ElseIf peek.TokenName = TokenParser.Tokens.RPair Then ' 向上一层退栈
Call source.Dequeue()
Call list.Add(current)
Exit Do
ElseIf x.TokenName = TokenParser.Tokens.ParamDeli Then
Call list.Add(current)
current = New Func With {
.Caller = New List(Of Token)
}
End If
Loop
If list.Count = 0 Then
Call list.Add(current)
End If
Return list.ToArray
End Function
End Module
Public Class Func
Public Property Caller As List(Of Token)
Public Property Args As Func()
Public ReadOnly Property IsFuncCalls As Boolean
Get
Return Not Args.IsNullOrEmpty
End Get
End Property
Public Overrides Function ToString() As String
If Args.IsNullOrEmpty Then
Return String.Join(" ", Caller.ToArray(Function(x) x.TokenValue))
Else
Dim caller As String = String.Join(" ", Me.Caller.ToArray(Function(x) x.TokenValue))
Dim params As String() = Me.Args.ToArray(Function(x) x.ToString)
Dim args As String = String.Join(", ", params)
Return $"{caller}({args})"
End If
End Function
End Class
End Namespace

109
LINQ/LINQ/LDM/TokenIcer.vb
Unescape View File

@ -0,0 +1,109 @@
' This file was Auto Generated with TokenIcer
Imports System.Collections.Generic
Imports System.Text.RegularExpressions
Imports Microsoft.VisualBasic.Scripting.TokenIcer
Namespace Statements.TokenIcer
' This is our token enumeration. It holds every token defined in the grammar
''' <summary>
''' Tokens is an enumeration of all possible token values.
''' </summary>
Public Enum Tokens
UNDEFINED = 0
CallFunc = 1
[Integer] = 2
Float = 3
ArrayType = 4
ParamDeli = 5
WhiteSpace = 6
[Let] = 7
Equals = 8
StringIntrop = 9
[String] = 10
VbCrLf = 11
LPair = 12
RPair = 13
Asterisk = 14
Slash = 15
Plus = 16
Minus = 17
Pretend = 18
cor = 19
[And] = 20
[Not] = 21
[Or] = 22
[Is] = 23
[As] = 24
From = 25
Where = 26
[Select] = 27
[Imports] = 28
[In] = 29
var = 30
varRef = 31
End Enum
Public Module Parser
Public ReadOnly Property stackT As StackTokens(Of Tokens) =
New StackTokens(Of Tokens)(Function(a, b) a = b) With {
.LPair = Tokens.LPair,
.ParamDeli = Tokens.ParamDeli,
.Pretend = Tokens.Pretend,
.RPair = Tokens.RPair,
.WhiteSpace = Tokens.WhiteSpace
}
Private Function __getParser() As TokenParser(Of Tokens)
Dim _tokens As New Dictionary(Of Tokens, String)()
' These lines add each grammar rule to the dictionary
_tokens.Add(Tokens.CallFunc, "->\s*[a-zA-Z_][a-zA-Z0-9_]*")
_tokens.Add(Tokens.[Integer], "[0-9]+")
_tokens.Add(Tokens.Float, "[0-9]+\.+[0-9]+")
_tokens.Add(Tokens.ArrayType, "[a-zA-Z_][a-zA-Z0-9_]*\(\)")
_tokens.Add(Tokens.ParamDeli, ",")
_tokens.Add(Tokens.WhiteSpace, "[ \t]+")
_tokens.Add(Tokens.[Let], "[Ll][Ee][Tt]")
_tokens.Add(Tokens.Equals, "=")
_tokens.Add(Tokens.StringIntrop, "\$"".*?""")
_tokens.Add(Tokens.[String], """.*?""")
_tokens.Add(Tokens.VbCrLf, "[\r\n]+")
_tokens.Add(Tokens.LPair, "\(")
_tokens.Add(Tokens.RPair, "\)")
_tokens.Add(Tokens.Asterisk, "\*")
_tokens.Add(Tokens.Slash, "\/")
_tokens.Add(Tokens.Plus, "\+")
_tokens.Add(Tokens.Minus, "\-")
_tokens.Add(Tokens.Pretend, "Pretend")
_tokens.Add(Tokens.cor, "[nN][eE][wW]")
_tokens.Add(Tokens.[And], "[aA][nN][dD]")
_tokens.Add(Tokens.[Not], "[nN][oO][tT]")
_tokens.Add(Tokens.[Or], "[oO][rR]")
_tokens.Add(Tokens.[Is], "[iI][sS]")
_tokens.Add(Tokens.[As], "[aA][sS]")
_tokens.Add(Tokens.From, "[fF][rR][oO][mM]")
_tokens.Add(Tokens.Where, "[wW][hH][eE][rR][eE]")
_tokens.Add(Tokens.[Select], "[sS][eE][lL][eE][cC][tT]")
_tokens.Add(Tokens.[Imports], "^[Ii][mM][pP][oO][rR][tT][sS]")
_tokens.Add(Tokens.[In], "[iI][nN]")
_tokens.Add(Tokens.var, "[a-zA-Z_][a-zA-Z0-9_]*")
_tokens.Add(Tokens.varRef, "\$[a-zA-Z0-9_]*")
Return New TokenParser(Of Tokens)(_tokens, Tokens.UNDEFINED)
End Function
' Our Constructor, which simply initializes values
''' <summary>
''' Default Constructor
''' </summary>
''' <remarks>
''' The constructor initalizes memory and adds all of the tokens to the token dictionary.
''' </remarks>
Public Function GetTokens(expr As String) As Token(Of Tokens)()
Return __getParser.GetTokens(expr)
End Function
End Module
End Namespace

3
LINQ/LINQ/LINQ.vbproj
Unescape View File

@ -116,7 +116,6 @@
</ItemGroup>
<ItemGroup>
<Compile Include="APP\CLI.vb" />
<Compile Include="LDM\StackParser.vb" />
<Compile Include="LDM\Expression\Expression.vb" />
<Compile Include="LDM\Expression\Closure.vb" />
<Compile Include="LDM\Expression\FromClosure.vb" />
@ -126,7 +125,7 @@
<Compile Include="LDM\Expression\WhereClosure.vb" />
<Compile Include="LDM\Parser\ClosureTokens.vb" />
<Compile Include="LDM\Parser\ClosureParser.vb" />
<Compile Include="TokenIcer.vb" />
<Compile Include="LDM\TokenIcer.vb" />
<Compile Include="Framewok\SDK_PATH.vb" />
<Compile Include="Framewok\ObjectModel\ParallelLINQ.vb" />
<Compile Include="Framewok\ObjectModel\LINQ.vb" />

300
LINQ/LINQ/TokenIcer.vb
Unescape View File

@ -1,300 +0,0 @@
' This file was Auto Generated with TokenIcer
Imports System.Collections.Generic
Imports System.Text.RegularExpressions
Namespace TokenIcer
''' <summary>
''' TokenParser
''' </summary>
''' <remarks>
''' TokenParser is the main parser engine for converting input into lexical tokens.
''' </remarks>
Public Class TokenParser
' This dictionary will store our RegEx rules
Private ReadOnly _tokens As Dictionary(Of Tokens, String)
' This dictionary will store our matches
Private ReadOnly _regExMatchCollection As Dictionary(Of Tokens, MatchCollection)
' This input string will store the string to parse
Private _inputString As String
' This index is used internally so the parser knows where it left off
Private _index As Integer
' This is our token enumeration. It holds every token defined in the grammar
''' <summary>
''' Tokens is an enumeration of all possible token values.
''' </summary>
Public Enum Tokens
UNDEFINED = 0
CallFunc = 1
[Integer] = 2
Float = 3
ArrayType = 4
ParamDeli = 5
WhiteSpace = 6
[Let] = 7
Equals = 8
StringIntrop = 9
[String] = 10
VbCrLf = 11
LPair = 12
RPair = 13
Asterisk = 14
Slash = 15
Plus = 16
Minus = 17
Pretend = 18
cor = 19
[And] = 20
[Not] = 21
[Or] = 22
[Is] = 23
[As] = 24
From = 25
Where = 26
[Select] = 27
[Imports] = 28
[In] = 29
var = 30
varRef = 31
End Enum
' A public setter for our input string
''' <summary>
''' InputString Property
''' </summary>
''' <value>
''' The string value that holds the input string.
''' </value>
Public WriteOnly Property InputString() As String
Set
_inputString = Value
PrepareRegex()
End Set
End Property
' Our Constructor, which simply initializes values
''' <summary>
''' Default Constructor
''' </summary>
''' <remarks>
''' The constructor initalizes memory and adds all of the tokens to the token dictionary.
''' </remarks>
Public Sub New()
_tokens = New Dictionary(Of Tokens, String)()
_regExMatchCollection = New Dictionary(Of Tokens, MatchCollection)()
_index = 0
_inputString = String.Empty
' These lines add each grammar rule to the dictionary
_tokens.Add(Tokens.CallFunc, "->\s*[a-zA-Z_][a-zA-Z0-9_]*")
_tokens.Add(Tokens.[Integer], "[0-9]+")
_tokens.Add(Tokens.Float, "[0-9]+\.+[0-9]+")
_tokens.Add(Tokens.ArrayType, "[a-zA-Z_][a-zA-Z0-9_]*\(\)")
_tokens.Add(Tokens.ParamDeli, ",")
_tokens.Add(Tokens.WhiteSpace, "[ \t]+")
_tokens.Add(Tokens.[Let], "[Ll][Ee][Tt]")
_tokens.Add(Tokens.Equals, "=")
_tokens.Add(Tokens.StringIntrop, "\$"".*?""")
_tokens.Add(Tokens.[String], """.*?""")
_tokens.Add(Tokens.VbCrLf, "[\r\n]+")
_tokens.Add(Tokens.LPair, "\(")
_tokens.Add(Tokens.RPair, "\)")
_tokens.Add(Tokens.Asterisk, "\*")
_tokens.Add(Tokens.Slash, "\/")
_tokens.Add(Tokens.Plus, "\+")
_tokens.Add(Tokens.Minus, "\-")
_tokens.Add(Tokens.Pretend, "Pretend")
_tokens.Add(Tokens.cor, "[nN][eE][wW]")
_tokens.Add(Tokens.[And], "[aA][nN][dD]")
_tokens.Add(Tokens.[Not], "[nN][oO][tT]")
_tokens.Add(Tokens.[Or], "[oO][rR]")
_tokens.Add(Tokens.[Is], "[iI][sS]")
_tokens.Add(Tokens.[As], "[aA][sS]")
_tokens.Add(Tokens.From, "[fF][rR][oO][mM]")
_tokens.Add(Tokens.Where, "[wW][hH][eE][rR][eE]")
_tokens.Add(Tokens.[Select], "[sS][eE][lL][eE][cC][tT]")
_tokens.Add(Tokens.[Imports], "^[Ii][mM][pP][oO][rR][tT][sS]")
_tokens.Add(Tokens.[In], "[iI][nN]")
_tokens.Add(Tokens.var, "[a-zA-Z_][a-zA-Z0-9_]*")
_tokens.Add(Tokens.varRef, "\$[a-zA-Z0-9_]*")
End Sub
' This function preloads the matches based on our rules and the input string
''' <summary>
''' PrepareRegex prepares the regex for parsing by pre-matching the Regex tokens.
''' </summary>
Private Sub PrepareRegex()
_regExMatchCollection.Clear()
For Each pair As KeyValuePair(Of Tokens, String) In _tokens
_regExMatchCollection.Add(pair.Key, Regex.Matches(_inputString, pair.Value))
Next
End Sub
' ResetParser() will reset the parser.
' Keep in mind that you must set the input string again
''' <summary>
''' ResetParser resets the parser to its inital state. Reloading InputString is required.
''' </summary>
''' <seealso cref="InputString">
Public Sub ResetParser()
_index = 0
_inputString = String.Empty
_regExMatchCollection.Clear()
End Sub
' GetToken() retrieves the next token and returns a token object
''' <summary>
''' GetToken gets the next token in queue
''' </summary>
''' <remarks>
''' GetToken attempts to the match the next character(s) using the
''' Regex rules defined in the dictionary. If a match can not be
''' located, then an Undefined token will be created with an empty
''' string value. In addition, the token pointer will be incremented
''' by one so that this token doesn't attempt to get identified again by
''' GetToken()
''' </remarks>
Public Function GetToken() As Token
' If we are at the end of our input string then
' we return null to signify the end of our input string.
' While parsing tokens, you will undoubtedly be in a loop.
' Having your loop check for a null token is a good way to end that
' loop
If _index >= _inputString.Length Then
Return Nothing
End If
' Iterate through our prepared matches/Tokens dictionary
For Each pair As KeyValuePair(Of Tokens, MatchCollection) In _regExMatchCollection
' Iterate through each prepared match
For Each match As Match In pair.Value
' If we find a match, update our index pointer and return a new Token object
If match.Index = _index Then
_index += match.Length
Return New Token(pair.Key, match.Value)
ElseIf match.Index > _index Then
Exit For
End If
Next
Next
' If execution got here, then we increment our index pointer
' and return an Undefined token.
_index += 1
Return New Token(Tokens.UNDEFINED, String.Empty)
End Function
' Peek() will retrieve a PeekToken object and will allow you to see the next token
' that GetToken() will retrieve.
''' <summary>
''' Returns the next token that GetToken() will return.
''' </summary>
''' <seealso cref="Peek(PeekToken)">
Public Function Peek() As PeekToken
Return Peek(New PeekToken(_index, New Token(Tokens.UNDEFINED, String.Empty)))
End Function
' This is an overload for Peek(). By passing in the last PeekToken object
' received from Peek(), you can peek ahead to the next token, and the token after that, etc...
''' <summary>
''' Returns the next token after the Token passed here
''' </summary>
''' <param name="peekToken">The PeekToken token returned from a previous Peek() call</param>
''' <seealso cref="Peek()">
Public Function Peek(peekToken As PeekToken) As PeekToken
Dim oldIndex As Integer = _index
_index = peekToken.TokenIndex
If _index >= _inputString.Length Then
_index = oldIndex
Return Nothing
End If
For Each pair As KeyValuePair(Of Tokens, String) In _tokens
Dim r As New Regex(pair.Value)
Dim m As Match = r.Match(_inputString, _index)
If m.Success AndAlso m.Index = _index Then
_index = _index + m.Length
Dim pt As New PeekToken(_index, New Token(pair.Key, m.Value))
_index = oldIndex
Return pt
End If
Next
Dim pt2 As New PeekToken(_index + 1, New Token(Tokens.UNDEFINED, String.Empty))
_index = oldIndex
Return pt2
End Function
End Class
' This defines the PeekToken object
''' <summary>
''' A PeekToken object class
''' </summary>
''' <remarks>
''' A PeekToken is a special pointer object that can be used to Peek() several
''' tokens ahead in the GetToken() queue.
''' </remarks>
Public Class PeekToken
Public Property TokenIndex() As Integer
Get
Return m_TokenIndex
End Get
Set
m_TokenIndex = Value
End Set
End Property
Private m_TokenIndex As Integer
Public Property TokenPeek() As Token
Get
Return m_TokenPeek
End Get
Set
m_TokenPeek = Value
End Set
End Property
Private m_TokenPeek As Token
Public Sub New(index As Integer, value As Token)
TokenIndex = index
TokenPeek = value
End Sub
End Class
' This defines the Token object
''' <summary>
''' a Token object class
''' </summary>
''' <remarks>
''' A Token object holds the token and token value.
''' </remarks>
Public Class Token
Public Property TokenName() As TokenParser.Tokens
Get
Return m_TokenName
End Get
Set
m_TokenName = Value
End Set
End Property
Private m_TokenName As TokenParser.Tokens
Public Property TokenValue() As String
Get
Return m_TokenValue
End Get
Set
m_TokenValue = Value
End Set
End Property
Private m_TokenValue As String
Public Sub New(name As TokenParser.Tokens, value As String)
TokenName = name
TokenValue = value
End Sub
End Class
End Namespace
Write Preview
Loading…
Cancel
Save