AU3Lexer.au3

11/22/2017 - 3:45 PM

#include <Math.au3>
#include <Array.au3>

#include "al_keywords.au3"
#include "al_funcs.au3"
#include "al_macros.au3"

Global Enum $AL_TOK_EOF = 0, _ ; End of File
		$AL_TOK_EOL, _ ; End of Line
		$AL_TOK_OP, _ ; Operator.
		$AL_TOK_ASSIGN, _ ; Operator.
		$AL_TOK_KEYWORD, _ ; Keyword. E.g. 'Func', 'Local' etc. (not used if $AL_FLAG_NORESOLVEKEYWORD is set)
		$AL_TOK_FUNC, _ ; Standard function (not used if $AL_FLAG_NORESOLVEKEYWORD is set)
		$AL_TOK_WORD, _ ; Word (but not keyword or standard function)
		$AL_TOK_OPAR, _ ; (
		$AL_TOK_EPAR, _ ; )
		$AL_TOK_OBRACK, _ ; [
		$AL_TOK_EBRACK, _ ; ]
		$AL_TOK_COMMA, _ ; ,
		$AL_TOK_STR, _ ; " ... "
		$AL_TOK_NUMBER, _ ; Integer, float, hex etc.
		$AL_TOK_MACRO, _ ; @...
		$AL_TOK_VARIABLE, _ ; $...
		$AL_TOK_PREPROC, _; Preprocessor statement. NB returns whole line and does no processing apart from #cs and #ce
		$AL_TOK_COMMENT, _ ; Comment. Includes multiline.
		$AL_TOK_LINECONT ; Line continuation _ (only used if $AL_FLAG_AUTOLINECONT not set)

Global Enum Step *2 _
		$AL_FLAG_AUTOLINECONT = 1, _
		$AL_FLAG_NORESOLVEKEYWORD, _
		$AL_FLAG_AUTOINCLUDE, _
		$__AL_FLAG_LINECONT

Global Enum $AL_LEXI_FILENAME = 0, _
		$AL_LEXI_DATA, _
		$AL_LEXI_FLAGS, _
		$AL_LEXI_ABS, _
		$AL_LEXI_LINE, _
		$AL_LEXI_COL, _
		$AL_LEXI_LASTTOK_ABS, _
		$AL_LEXI_LASTTOK_LINE, _
		$AL_LEXI_LASTTOK_COL, _
		$__AL_LEXI_COUNT

Global Enum $AL_ST_START = -1, _
		$AL_ST_NONE, _
		$AL_ST_INT, _
		$AL_ST_FLOAT, _
		$AL_ST_FLOATE, _
		$AL_ST_FLOATES, _
		$AL_ST_ZERO, _
		$AL_ST_HEX, _
		$AL_ST_STRINGS, _
		$AL_ST_STRINGD, _
		$AL_ST_MACRO, _
		$AL_ST_VARIABLE, _
		$AL_ST_COMMENT, _
		$AL_ST_COMMENTMULTI, _
		$AL_ST_COMMENTMULTINL, _
		$AL_ST_COMMENTMULTIEND, _
		$AL_ST_PREPROC, _
		$AL_ST_PREPROCLINE, _
		$AL_ST_INCLUDELINE, _
		$AL_ST_LINECONT, _
		$AL_ST_KEYWORD


; Test assertions:
;~ _AuLex_LexTestAssert("", $AL_TOK_EOF, "")
;~ _AuLex_LexTestAssert(@CRLF, $AL_TOK_EOL, @CRLF)
;~ _AuLex_LexTestAssert(@LF, $AL_TOK_EOL, @LF)
;~ _AuLex_LexTestAssert(@CR, $AL_TOK_EOL, @CR)
;~ _AuLex_LexTestAssert("123", $AL_TOK_NUMBER, "123")
;~ _AuLex_LexTestAssert("123.456", $AL_TOK_NUMBER, "123.456")
;~ _AuLex_LexTestAssert("123e4", $AL_TOK_NUMBER, "123e4")
;~ _AuLex_LexTestAssert("123e-4", $AL_TOK_NUMBER, "123e-4")
;~ _AuLex_LexTestAssert("123.456e4", $AL_TOK_NUMBER, "123.456e4")
;~ _AuLex_LexTestAssert("123.456e-4", $AL_TOK_NUMBER, "123.456e-4")
;~ _AuLex_LexTestAssert("0111", $AL_TOK_NUMBER, "0111")
;~ _AuLex_LexTestAssert("0xABCDEF", $AL_TOK_NUMBER, "0xABCDEF")
;~ _AuLex_LexTestAssert("'This is a test'", $AL_TOK_STR, "'This is a test'")
;~ _AuLex_LexTestAssert("'This is '' a test'", $AL_TOK_STR, "'This is '' a test'")
;~ _AuLex_LexTestAssert("""This is a test""", $AL_TOK_STR, """This is a test""")
;~ _AuLex_LexTestAssert("""This is """" a test""", $AL_TOK_STR, """This is """" a test""")
;~ _AuLex_LexTestAssert("@Testing", $AL_TOK_MACRO, "@Testing")
;~ _AuLex_LexTestAssert("$Testing", $AL_TOK_VARIABLE, "$Testing")
;~ _AuLex_LexTestAssert("; This is a comment", $AL_TOK_COMMENT, "; This is a comment")
;~ _AuLex_LexTestAssert("#include <Test>", $AL_TOK_PREPROC, "#include <Test>")
;~ _AuLex_LexTestAssert("#cs" & @CRLF & "Testing comment" & @CRLF & "#This won't match" & @CRLF & "#ce", $AL_TOK_COMMENT)
;~ _AuLex_LexTestAssert("(", $AL_TOK_OPAR)
;~ _AuLex_LexTestAssert(")", $AL_TOK_EPAR)
;~ _AuLex_LexTestAssert("[", $AL_TOK_OBRACK)
;~ _AuLex_LexTestAssert("]", $AL_TOK_EBRACK)
;~ _AuLex_LexTestAssert("*", $AL_TOK_OP)
;~ _AuLex_LexTestAssert("*=", $AL_TOK_OP)
;~ _AuLex_LexTestAssert("<=", $AL_TOK_OP)
;~ _AuLex_LexTestAssert("<>", $AL_TOK_OP)
;~ _AuLex_LexTestAssert("^", $AL_TOK_OP)
;~ _AuLex_LexTestAssert("_Test", $AL_TOK_WORD)
;~ _AuLex_LexTestAssert("Test", $AL_TOK_WORD)
;~ _AuLex_LexTestAssert("Local", $AL_TOK_KEYWORD)
;~ _AuLex_LexTestAssert("StringInStr", $AL_TOK_FUNC)
;~ _AuLex_LexTestAssert("_" & @CRLF & " _Test", $AL_TOK_LINECONT, "_", 0)
;~ _AuLex_LexTestAssert("_" & @CRLF & " _Test", $AL_TOK_WORD, "_Test", $AL_FLAG_AUTOLINECONT)
;~ _AuLex_LexTestAssert("_ ; Testing Comment" & @CRLF & " _Test", $AL_TOK_COMMENT, "; Testing Comment", $AL_FLAG_AUTOLINECONT)
;~
;~ Func _AuLex_LexTestAssert($sString, $iExpType, $sExpectData = Default, $iFlags = 0)
;~ 	If $sExpectData = Default Then $sExpectData = $sString
;~
;~ 	Local $l = _AuLex_StartLexS("Test", $sString, $iFlags)
;~
;~ 	Local $sData = _AuLex_NextToken($l)
;~ 	Local $iType = @extended
;~
;~ 	If $iExpType <> $iType Or $sData <> $sExpectData Then
;~ 		ConsoleWrite("Assertion Failed! " & @error & @LF & @TAB & "Input: {" & $sString & "} (" & StringToBinary($sData) & ")" & @LF & @TAB & _
;~ 				"Token Type: " & $iType & "  (" & $iExpType & ")" & @LF & @TAB & "Token Data: {" & $sData & "} (" & StringToBinary($sData) & ")" & @LF)
;~ 	EndIf
;~ EndFunc   ;==>_AuLex_LexTestAssert



; Example: Process this file:

;~ Local $l = _AuLex_StartLex(@ScriptName, $AL_FLAG_AUTOLINECONT)

;~ Local $sData, $iType
;~ Do
;~ 	$sData = _AuLex_NextToken($l)
;~ 	$iType = @extended

;~ 	ConsoleWrite(StringFormat("%-4.4i: %s\n", $iType, $sData))
;~ Until $iType = $AL_TOK_EOF



Func _AuLex_StartLex($sFile, $iFlags)
	Local $sData = FileRead($sFile)
	If @error Then Return SetError(1, 0, 0) ; File couldn't be read.

	Return _AuLex_StartLexS($sFile, $sData, $iFlags)
EndFunc   ;==>_AuLex_StartLex

; Starts a lexer for a data string.
Func _AuLex_StartLexS($sName, $sData, $iFlags)
	Local $lexRet[$__AL_LEXI_COUNT]

	$lexRet[$AL_LEXI_FILENAME] = $sName
	$lexRet[$AL_LEXI_DATA] = $sData & @CRLF
	$lexRet[$AL_LEXI_FLAGS] = $iFlags

	$lexRet[$AL_LEXI_ABS] = 1
	$lexRet[$AL_LEXI_LINE] = 1
	$lexRet[$AL_LEXI_COL] = 1

	$lexRet[$AL_LEXI_LASTTOK_ABS] = 1
	$lexRet[$AL_LEXI_LASTTOK_LINE] = 1
	$lexRet[$AL_LEXI_LASTTOK_COL] = 1

	Return $lexRet
EndFunc   ;==>_AuLex_StartLexS

; Returns the next token (as a string)
; The token type (an $AL_TOK_ constant) is returned in @extended
Func _AuLex_NextToken(ByRef $lex)
	Local $iState = $AL_ST_START
	Local $c, $c2, $anchor

	Local $iRetTok = 0
	Local $vRetData = ""

	While 1
		$c = __AuLex_NextChar($lex)

		Switch $iState
			Case $AL_ST_START
				Select
					Case $c = ""
						Return SetExtended($AL_TOK_EOF, "")
					Case _String_IsNewLine($c)
						If BitAND($lex[$AL_LEXI_FLAGS], $__AL_FLAG_LINECONT) Then
							$lex[$AL_LEXI_FLAGS] = BitXOR($lex[$AL_LEXI_FLAGS], $__AL_FLAG_LINECONT)
						Else
							$lex[$AL_LEXI_LASTTOK_ABS] = $lex[$AL_LEXI_ABS] - StringLen($c)
							$lex[$AL_LEXI_LASTTOK_LINE] = $lex[$AL_LEXI_LINE] - 1
							$lex[$AL_LEXI_LASTTOK_COL] = -1 ; TODO

							Return SetExtended($AL_TOK_EOL, $c)
						EndIf
					Case Not StringIsSpace($c)
						; Save token position
						$lex[$AL_LEXI_LASTTOK_ABS] = $lex[$AL_LEXI_ABS] - 1
						$lex[$AL_LEXI_LASTTOK_LINE] = $lex[$AL_LEXI_LINE]
						$lex[$AL_LEXI_LASTTOK_COL] = $lex[$AL_LEXI_COL] - 1

						$iState = $AL_ST_NONE
						__AuLex_PrevChar($lex)
				EndSelect
			Case $AL_ST_NONE
				Select
					Case $c = '0'
						$iState = $AL_ST_ZERO
					Case $c = "'"
						$iState = $AL_ST_STRINGS
					Case $c = '"'
						$iState = $AL_ST_STRINGD
					Case $c = "@"
						$iState = $AL_ST_MACRO
					Case $c = "$"
						$iState = $AL_ST_VARIABLE
					Case $c = ";"
						$iState = $AL_ST_COMMENT
					Case $c = "#"
						$iState = $AL_ST_PREPROC
					Case $c = "("
						$iRetTok = $AL_TOK_OPAR
						ExitLoop
					Case $c = ")"
						$iRetTok = $AL_TOK_EPAR
						ExitLoop
					Case $c = "["
						$iRetTok = $AL_TOK_OBRACK
						ExitLoop
					Case $c = "]"
						$iRetTok = $AL_TOK_EBRACK
						ExitLoop
					Case $c = ","
						$iRetTok = $AL_TOK_COMMA
						ExitLoop
					Case StringInStr("*/+-&", $c)
						If __AuLex_PeekChar($lex) = "=" Then
							__AuLex_NextChar($lex)
							$iRetTok = $AL_TOK_ASSIGN
						Else
							$iRetTok = $AL_TOK_OP
						EndIf

						ExitLoop
					Case StringInStr("=>", $c)
						If __AuLex_PeekChar($lex) = "=" Then
							__AuLex_NextChar($lex)
						EndIf

						$iRetTok = $AL_TOK_OP
						ExitLoop
					Case $c = "<"
						If StringInStr("=>", __AuLex_PeekChar($lex)) Then
							__AuLex_NextChar($lex)
						EndIf

						$iRetTok = $AL_TOK_OP
						ExitLoop
					Case $c = "^"
						$iRetTok = $AL_TOK_OP
						ExitLoop
					Case $c = "_"
						$c2 = __AuLex_PeekChar($lex)

						If StringIsAlNum($c2) Or $c2 = "_" Then
							$iState = $AL_ST_KEYWORD
						ElseIf BitAND($lex[$AL_LEXI_FLAGS], $AL_FLAG_AUTOLINECONT) Then
							$iState = $AL_ST_LINECONT
						Else
							$iRetTok = $AL_TOK_LINECONT
							ExitLoop
						EndIf
					Case StringIsDigit($c)
						$iState = $AL_ST_INT
					Case StringIsAlpha($c)
						$iState = $AL_ST_KEYWORD
					Case Else
						; ERROR: Invalid character
						Return SetError(@ScriptLineNumber, 0, 0)
				EndSelect
			Case $AL_ST_INT
				If $c = '.' Then
					$iState = $AL_ST_FLOAT
				ElseIf $c = 'e' Then
					$iState = $AL_ST_FLOATE
				ElseIf Not StringIsDigit($c) Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_NUMBER

					ExitLoop
				EndIf
			Case $AL_ST_FLOAT
				If $c = 'e' Then
					$iState = $AL_ST_FLOATE
				ElseIf Not StringIsDigit($c) Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_NUMBER

					ExitLoop
				EndIf
			Case $AL_ST_FLOATE
				If $c = '+' Or $c = '-' Or StringIsDigit($c) Then
					$iState = $AL_ST_FLOATES
				Else
					__AuLex_PrevChar($lex)

					; NB: Next token will be 'e' which is most likely an error.
					$iRetTok = $AL_TOK_NUMBER

					ExitLoop
				EndIf
			Case $AL_ST_FLOATES
				If Not StringIsDigit($c) Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_NUMBER

					ExitLoop
				EndIf
			Case $AL_ST_ZERO
				If StringInStr("Xx", $c) Then
					$iState = $AL_ST_HEX
				ElseIf $c = '.' Then
					$iState = $AL_ST_FLOAT
				ElseIf StringIsDigit($c) Then
					$iState = $AL_ST_INT
				Else
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_NUMBER

					ExitLoop
				EndIf
			Case $AL_ST_HEX
				If Not StringIsXDigit($c) Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_NUMBER

					ExitLoop
				EndIf
			Case $AL_ST_STRINGS
				If $c = "'" Then
					$c2 = __AuLex_PeekChar($lex)

					If $c2 <> "'" Then
						$iRetTok = $AL_TOK_STR

						ExitLoop
					Else
						__AuLex_NextChar($lex)
					EndIf
				ElseIf $c = "" Or $c = @CRLF Then
					; ERROR: String not terminated
					Return SetError(@ScriptLineNumber, 0, 0)
				EndIf
			Case $AL_ST_STRINGD
				If $c = '"' Then
					$c2 = __AuLex_PeekChar($lex)

					If $c2 <> '"' Then
						$iRetTok = $AL_TOK_STR

						ExitLoop
					Else
						__AuLex_NextChar($lex)
					EndIf
				ElseIf $c = "" Or $c = @CRLF Then
					; ERROR: String not terminated
					Return SetError(@ScriptLineNumber, 0, 0)
				EndIf
			Case $AL_ST_MACRO
				If $c <> "_" And Not StringIsAlNum($c) Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_MACRO
					ExitLoop
				EndIf
			Case $AL_ST_VARIABLE
				If $c <> "_" And Not StringIsAlNum($c) Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_VARIABLE
					ExitLoop
				EndIf
			Case $AL_ST_COMMENT
				If $c = "" Or _String_IsNewLine($c) Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_COMMENT
					ExitLoop
				EndIf
			Case $AL_ST_COMMENTMULTI
				If _String_IsNewLine($c) Then
					$iState = $AL_ST_COMMENTMULTINL
				ElseIf $c = "" Then
					; ERROR: Multiline comment not terminated
					Return SetError(@ScriptLineNumber, 0, 0)
				EndIf
			Case $AL_ST_COMMENTMULTINL
				If $c = "#" Then
					$iState = $AL_ST_COMMENTMULTIEND
					$anchor = $lex[$AL_LEXI_ABS]
				ElseIf $c = "" Then
					; ERROR: Multiline comment not terminated
					Return SetError(@ScriptLineNumber, 0, 0)
				ElseIf _String_IsNewLine($c) Then
					$iState = $AL_ST_COMMENTMULTINL
				Else
					$iState = $AL_ST_COMMENTMULTI
				EndIf
			Case $AL_ST_COMMENTMULTIEND
				If StringIsSpace($c) Or $c = "" Then
					Switch StringStripWS(StringMid($lex[$AL_LEXI_DATA], $anchor, $lex[$AL_LEXI_ABS] - $anchor), 2)
						Case "ce", "comments-end"
							__AuLex_PrevChar($lex)
							$iRetTok = $AL_TOK_COMMENT
							ExitLoop
						Case Else
							If _String_IsNewLine($c) Then
								$iState = $AL_ST_COMMENTMULTINL
							Else
								$iState = $AL_ST_COMMENTMULTI
							EndIf
					EndSwitch
				EndIf
			Case $AL_ST_PREPROC
				If StringIsSpace($c) Or $c = "" Then
					Switch StringStripWS(StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_LASTTOK_ABS], $lex[$AL_LEXI_ABS] - $lex[$AL_LEXI_LASTTOK_ABS]), 2)
						Case "#cs", "#comments-start"
							$iState = $AL_ST_COMMENTMULTI
						Case "#include"
							If Not BitAND($lex[$AL_LEXI_FLAGS], $AL_FLAG_AUTOINCLUDE) Then ContinueCase
							$iState = $AL_ST_INCLUDELINE
						Case Else
							If _String_IsNewLine($c) Then
								__AuLex_PrevChar($lex)
								$iRetTok = $AL_TOK_PREPROC
								ExitLoop
							Else
								$iState = $AL_ST_PREPROCLINE
							EndIf
					EndSwitch
				EndIf
			Case $AL_ST_PREPROCLINE
				If _String_IsNewLine($c) Or $c = "" Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_PREPROC
					ExitLoop
				EndIf
			Case $AL_ST_INCLUDELINE
				If _String_IsNewLine($c) Or $c = "" Then
					$vRetData = StringMid($lex[$AL_LEXI_DATA], _
							$lex[$AL_LEXI_LASTTOK_ABS], $lex[$AL_LEXI_ABS] - $lex[$AL_LEXI_LASTTOK_ABS])

					$c2 = StringStripWS(StringTrimLeft(StringStripWS($vRetData, 3), StringLen("#include")), 3)

					Switch StringLeft($c2, 1)
						Case '"'
							If StringRight($c2, 1) <> '"' Then
								; ERROR: Incorrect include line
								Return SetError(@ScriptLineNumber, 0, "")
							EndIf

							$c2 = StringTrimLeft(StringTrimRight($c2, 1), 1)

							; TODO
						Case '<'
							If StringRight($c2, 1) <> '>' Then
								; ERROR: Incorrect include line
								Return SetError(@ScriptLineNumber, 0, "")
							EndIf

							$c2 = StringTrimLeft(StringTrimRight($c2, 1), 1)

							; TODO
						Case Else
							; ERROR: Incorrect include line
							Return SetError(@ScriptLineNumber, 0, "")
					EndSwitch

					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_PREPROC
					ExitLoop
				EndIf
			Case $AL_ST_LINECONT
				If $c = ";" Then
					$lex[$AL_LEXI_LASTTOK_ABS] = $lex[$AL_LEXI_ABS] - 1
					$lex[$AL_LEXI_LASTTOK_LINE] = $lex[$AL_LEXI_LINE]
					$lex[$AL_LEXI_LASTTOK_COL] = $lex[$AL_LEXI_COL] - 1

					$iState = $AL_ST_COMMENT
					$lex[$AL_LEXI_FLAGS] = BitOR($lex[$AL_LEXI_FLAGS], $__AL_FLAG_LINECONT)
				ElseIf _String_IsNewLine($c) Then
					$iState = $AL_ST_START
				ElseIf $c = "" Then
					; Error: No line after a continuation
					Return SetError(@ScriptLineNumber, 0, 0)
				ElseIf Not StringIsSpace($c) Then
					; ERROR: Something after a line continuation
					Return SetError(@ScriptLineNumber, 0, 0)
				EndIf
			Case $AL_ST_KEYWORD
				If Not (StringIsAlNum($c) Or $c = "_") Then
					__AuLex_PrevChar($lex)
					$iRetTok = $AL_TOK_WORD
					ExitLoop
				EndIf
		EndSwitch
	WEnd

	$vRetData = StringMid($lex[$AL_LEXI_DATA], _
			$lex[$AL_LEXI_LASTTOK_ABS], $lex[$AL_LEXI_ABS] - $lex[$AL_LEXI_LASTTOK_ABS])

	$vRetData = StringStripWS($vRetData, 3)

	If $iRetTok = $AL_TOK_WORD And _
			Not BitAND($lex[$AL_LEXI_FLAGS], $AL_FLAG_NORESOLVEKEYWORD) Then
		If _AuLex_IsKeyword($vRetData) Then
			$iRetTok = $AL_TOK_KEYWORD
		ElseIf _AuLex_IsStandardFunc($vRetData) Then
			$iRetTok = $AL_TOK_FUNC
		EndIf
	EndIf

	Return SetExtended($iRetTok, $vRetData)
EndFunc   ;==>_AuLex_NextToken

Func _AuLex_IsKeyword($s)
	Return _ArrayBinarySearch($__AL_KEYWORDS, $s) >= 0
EndFunc   ;==>_AuLex_IsKeyword

Func _AuLex_IsStandardFunc($s)
	Return _ArrayBinarySearch($__AL_FUNCS, $s) >= 0
EndFunc   ;==>_AuLex_IsStandardFunc

; Returns the next character and increments the counters.
Func __AuLex_NextChar(ByRef $lex)
	Local $ret = __AuLex_PeekChar($lex)
	If $ret = "" Then Return ""

	$lex[$AL_LEXI_ABS] += StringLen($ret)

	If _String_IsNewLine($ret) Then
		$lex[$AL_LEXI_LINE] += 1
		$lex[$AL_LEXI_COL] = 1
	Else
		$lex[$AL_LEXI_COL] += StringLen($ret)
	EndIf

	Return $ret
EndFunc   ;==>__AuLex_NextChar

Func __AuLex_PrevChar(ByRef $lex)
	If $lex[$AL_LEXI_ABS] >= StringLen($lex[$AL_LEXI_DATA]) Then Return "" ; Don't step back from EOF

	Local $ret = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS] - 1, 1)

	If $ret = @LF Then
		Local $r2 = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS] - 2, 1)

		If $r2 = @CR Then $ret = @CRLF
	EndIf

	$lex[$AL_LEXI_ABS] -= StringLen($ret)

	If _String_IsNewLine($ret) Then
		$lex[$AL_LEXI_LINE] -= 1
		$lex[$AL_LEXI_COL] = $lex[$AL_LEXI_ABS] - _
				_Max(StringInStr($lex[$AL_LEXI_DATA], @LF, 2, -1, $lex[$AL_LEXI_ABS], $lex[$AL_LEXI_ABS]), _
				StringInStr($lex[$AL_LEXI_DATA], @CR, 2, -1, $lex[$AL_LEXI_ABS], $lex[$AL_LEXI_ABS]))
	Else
		$lex[$AL_LEXI_COL] -= StringLen($ret)
	EndIf

	Return $ret
EndFunc   ;==>__AuLex_PrevChar

; Returns the next character without incrementing any of the counters.
Func __AuLex_PeekChar(ByRef $lex)
	Local $ret = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS], 1)

	If $ret = @CR Then
		Local $r2 = StringMid($lex[$AL_LEXI_DATA], $lex[$AL_LEXI_ABS] + 1, 1)

		If $r2 = @LF Then $ret = @CRLF
	EndIf

	Return $ret
EndFunc   ;==>__AuLex_PeekChar

Func _String_IsNewLine($c)
	Return StringInStr(@CRLF, $c)
EndFunc   ;==>_String_IsNewLine
Cacher is the code snippet organizer for pro developers

We empower you and your team to get more done, faster

AU3Lexer.au3
