diff --git a/LINQ/LINQ/LDM/Parser/Token.vb b/LINQ/LINQ/LDM/Parser/Token.vb
index 1827e3d..3abec04 100644
--- a/LINQ/LINQ/LDM/Parser/Token.vb
+++ b/LINQ/LINQ/LDM/Parser/Token.vb
@@ -31,6 +31,10 @@ Namespace LDM.Parser
_ParsedObject = text
End Sub
+ Sub New(source As Token(Of Tokens), priority As TokenPriority)
+ Call Me.New(source.TokenValue, source.TokenName, priority)
+ End Sub
+
'''
''' Constructor for tokens that are parsed.
'''
diff --git a/LINQ/LINQ/LDM/Parser/Tokenizer.vb b/LINQ/LINQ/LDM/Parser/Tokenizer.vb
index 3029976..c9ebbb5 100644
--- a/LINQ/LINQ/LDM/Parser/Tokenizer.vb
+++ b/LINQ/LINQ/LDM/Parser/Tokenizer.vb
@@ -21,7 +21,11 @@ Namespace LDM.Parser
'''
''' string to tokenize
Public Sub New(s As String)
- _En = s.GetEnumerator()
+ Call Me.New(Statements.TokenIcer.GetTokens(s))
+ End Sub
+
+ Sub New(tokens As IEnumerable(Of Token(Of Tokens)))
+ Me.Tokens = New Iterator(Of Token(Of Tokens))(tokens)
MoveNext()
End Sub
@@ -30,7 +34,7 @@ Namespace LDM.Parser
''' invalid.
'''
Private Sub MoveNext()
- If Not _En.MoveNext() Then
+ If Not Tokens.MoveNext() Then
_IsInvalid = True
End If
End Sub
@@ -61,7 +65,8 @@ Namespace LDM.Parser
If _IsInvalid Then
Return False
End If
- Return ((_En.Current >= "A"c AndAlso _En.Current <= "Z"c) OrElse (_En.Current >= "a"c AndAlso _En.Current <= "z"c) OrElse _En.Current = "_"c)
+ Dim Current = Tokens.GetCurrent
+ Return Current.TokenName = Statements.TokenIcer.Tokens.String
End Get
End Property
@@ -73,7 +78,7 @@ Namespace LDM.Parser
If _IsInvalid Then
Return False
End If
- Return _En.Current = "."c
+ Return Tokens.GetCurrent.TokenName = Statements.TokenIcer.Tokens.CallFunc
End Get
End Property
@@ -85,7 +90,7 @@ Namespace LDM.Parser
If _IsInvalid Then
Return False
End If
- Return _En.Current = ","c
+ Return Tokens.GetCurrent.TokenName = Statements.TokenIcer.Tokens.ParamDeli
End Get
End Property
@@ -97,7 +102,8 @@ Namespace LDM.Parser
If _IsInvalid Then
Return False
End If
- Return (_En.Current >= "0"c AndAlso _En.Current <= "9"c)
+ Dim t As Tokens = Tokens.GetCurrent.TokenName
+ Return t = Statements.TokenIcer.Tokens.Integer OrElse t = Statements.TokenIcer.Tokens.Float
End Get
End Property
@@ -109,7 +115,7 @@ Namespace LDM.Parser
If _IsInvalid Then
Return False
End If
- Return (_En.Current = " "c OrElse _En.Current = ControlChars.Tab)
+ Return Tokens.GetCurrent.TokenName = Statements.TokenIcer.Tokens.WhiteSpace
End Get
End Property
@@ -121,10 +127,17 @@ Namespace LDM.Parser
If _IsInvalid Then
Return False
End If
- Select Case _En.Current
- Case ">"c, "<"c, "="c, "-"c, "+"c, "!"c,
- "/"c, "%"c, "*"c, "&"c, "|"c, "("c,
- ")"c, "["c, "]"c, """"c
+ Select Case Tokens.GetCurrent.TokenName
+ Case Statements.TokenIcer.Tokens.Slash,
+ Statements.TokenIcer.Tokens.RPair,
+ Statements.TokenIcer.Tokens.Plus,
+ Statements.TokenIcer.Tokens.Or,
+ Statements.TokenIcer.Tokens.Not,
+ Statements.TokenIcer.Tokens.Minus,
+ Statements.TokenIcer.Tokens.LPair,
+ Statements.TokenIcer.Tokens.Is,
+ Statements.TokenIcer.Tokens.Equals,
+ Statements.TokenIcer.Tokens.And
Return True
Case Else
Return False
@@ -137,19 +150,19 @@ Namespace LDM.Parser
''' that token.
'''
''' next token
- Public Function GetNextToken() As Token(Of Tokens)
+ Public Function GetNextToken() As Token
If _IsInvalid Then
- Return New Token(Of Tokens)(Tokens.UNDEFINED)
+ Return Token.NullToken
End If
Dim token__1 As Token(Of Tokens)
If IsChar Then
token__1 = GetString()
ElseIf IsComma Then
- token__1 = New Token(Of Tokens)(TokenType.Comma, ",", TokenPriority.None)
+ token__1 = New Token(",", Statements.TokenIcer.Tokens.ParamDeli, TokenPriority.None)
MoveNext()
ElseIf IsDot Then
- token__1 = New Token(".", TokenType.Dot, TokenPriority.None)
+ token__1 = New Token(".", Statements.TokenIcer.Tokens.CallFunc, TokenPriority.None)
MoveNext()
ElseIf IsNumber Then
token__1 = GetNumber()
@@ -173,61 +186,15 @@ Namespace LDM.Parser
''' primitive quoted string, a primitive expression, or an identifier
'''
'''
- Private Function GetString() As Token(Of Tokens)
- ' Handle empty strings
- If _PrevToken.Type = TokenType.Quote AndAlso _En.Current = """"c Then
- MoveNext()
- Return New Token(String.Empty, TokenType.Primitive, TokenPriority.None)
- End If
- Dim sb As New StringBuilder()
- sb.Append(_En.Current)
- While True
- If _IsInvalid Then
- Exit While
- End If
- MoveNext()
- If _IsInvalid Then
- Exit While
- End If
-
- If IsChar Then
- sb.Append(_En.Current)
- ElseIf IsNumber Then
- sb.Append(_En.Current)
- Else
- If _PrevToken.Type = TokenType.Quote Then
- If _En.Current = """"c Then
- MoveNext()
- Exit While
- ElseIf _En.Current = "\"c Then
- ' In the case of \, we'll add that character and whatever character follows it.
- sb.Append(_En.Current)
- MoveNext()
- If Not _IsInvalid Then
- sb.Append(_En.Current)
- End If
- Else
- sb.Append(_En.Current)
- End If
- Else
- Exit While
- End If
- End If
- End While
- Dim s As String = sb.ToString()
-
+ Private Function GetString() As Token
+ Dim s As String = Tokens.GetCurrent.TokenValue
' "false" or "true" is a primitive expression.
If s = "false" OrElse s = "true" Then
- Return New Token([Boolean].Parse(s), TokenType.Primitive, TokenPriority.None)
+ Return New Token([Boolean].Parse(s), Statements.TokenIcer.Tokens.String, TokenPriority.None)
End If
' The previous token was a quote, so this is a primitive string.
- If _PrevToken.Type = TokenType.Quote Then
- Return New Token(s, TokenType.Primitive, TokenPriority.None)
- End If
-
- ' The default is that the string indicates an identifier.
- Return New Token(s, TokenType.Identifier, TokenPriority.None)
+ Return New Token(Tokens.GetCurrent, TokenPriority.None)
End Function
'''
@@ -240,67 +207,13 @@ Namespace LDM.Parser
''' Any numbers containing a dot (".") are considered doubles.
'''
Private Function GetNumber() As Token
- Dim sb As New StringBuilder()
- sb.Append(_En.Current)
- Dim isDouble As Boolean = False
- Dim isLong As Boolean = False
- Dim cont As Boolean = True
- While cont
- If _IsInvalid Then
- Exit While
- End If
- MoveNext()
- If _IsInvalid Then
- Exit While
- End If
+ Dim Current = Tokens.GetCurrent
+ Dim s As String = Current.TokenValue
- If IsNumber Then
- sb.Append(_En.Current)
- ElseIf IsChar Then
- Select Case _En.Current
- Case "D"c, "d"c
- isDouble = True
- MoveNext()
- If IsChar OrElse IsNumber Then
- sb.Append(_En.Current)
- Throw New ArgumentException("Invalid number: " & sb.ToString())
- Else
- cont = False
- End If
- Case "L"c, "l"c
- isLong = True
- MoveNext()
- If IsChar OrElse IsNumber Then
- sb.Append(_En.Current)
- Throw New ArgumentException("Invalid number: " & sb.ToString())
- Else
- cont = False
- End If
- Case Else
- sb.Append(_En.Current)
- Throw New ArgumentException("Invalid number: " & sb.ToString())
- End Select
- ElseIf IsDot Then
- sb.Append(_En.Current)
- If isDouble Then
- ' The number has already been marked as a double, which means it already
- ' contains a number.
- Throw New ArgumentException("Invalid number: " & sb.ToString())
- Else
- isDouble = True
- End If
- Else
- Exit While
- End If
- End While
- Dim s As String = sb.ToString()
- If isLong Then
- Return New Token(Int64.Parse(s), TokenType.Primitive, TokenPriority.None)
- End If
- If isDouble Then
- Return New Token([Double].Parse(s), TokenType.Primitive, TokenPriority.None)
+ If Current.TokenName = Statements.TokenIcer.Tokens.Float Then
+ Return New Token([Double].Parse(s), Statements.TokenIcer.Tokens.String, TokenPriority.None)
End If
- Return New Token(Int32.Parse(s), TokenType.Primitive, TokenPriority.None)
+ Return New Token(Int32.Parse(s), Statements.TokenIcer.Tokens.String, TokenPriority.None)
End Function
'''
@@ -309,8 +222,9 @@ Namespace LDM.Parser
'''
'''
Private Function GetOperator() As Token
- Dim op As New String(_En.Current, 1)
- Select Case _En.Current
+ Dim Current = Tokens.GetCurrent
+
+ Select Case Current.TokenName
Case "<"c, "="c, ">"c
MoveNext()
If _En.Current = "="c Then