7c358cd5e6
* Modernize Whitespace token: basic lexer * Modernize Whitespace token: bibtex lexer * Modernize Whitespace token: boa lexer * Modernize Whitespace token: capnproto lexer + new example * Modernize Whitespace token: cddl lexer * Modernize Whitespace token: chapel lexer * Modernize Whitespace token: c_like lexer * Modernize Whitespace token: configs lexer * Modernize Whitespace token: console lexer * Modernize Whitespace token: crystal lexer * Modernize Whitespace token: csound lexer * Modernize Whitespace token: css lexer * Revert a change in basic lexer
63 lines
1.6 KiB
Text
63 lines
1.6 KiB
Text
---input---
|
|
This is a comment.
|
|
|
|
@ARTICLE{ruckenstein-diffusion,
|
|
author = "Liu, Hongquin" # and # "Ruckenstein, Eli",
|
|
year = 1997,
|
|
month = JAN,
|
|
pages = "888-895"
|
|
}
|
|
|
|
---tokens---
|
|
'This is a comment.' Comment
|
|
'\n\n' Text.Whitespace
|
|
|
|
'@ARTICLE' Name.Class
|
|
'{' Punctuation
|
|
'ruckenstein-diffusion' Name.Label
|
|
',' Punctuation
|
|
'\n ' Text.Whitespace
|
|
'author' Name.Attribute
|
|
' ' Text.Whitespace
|
|
'=' Punctuation
|
|
' ' Text.Whitespace
|
|
'"' Literal.String
|
|
'Liu, Hongquin' Literal.String
|
|
'"' Literal.String
|
|
' ' Text.Whitespace
|
|
'#' Punctuation
|
|
' ' Text.Whitespace
|
|
'and' Name.Variable
|
|
' ' Text.Whitespace
|
|
'#' Punctuation
|
|
' ' Text.Whitespace
|
|
'"' Literal.String
|
|
'Ruckenstein, Eli' Literal.String
|
|
'"' Literal.String
|
|
',' Punctuation
|
|
'\n ' Text.Whitespace
|
|
'year' Name.Attribute
|
|
' ' Text.Whitespace
|
|
'=' Punctuation
|
|
' ' Text.Whitespace
|
|
'1997' Literal.Number
|
|
',' Punctuation
|
|
'\n ' Text.Whitespace
|
|
'month' Name.Attribute
|
|
' ' Text.Whitespace
|
|
'=' Punctuation
|
|
' ' Text.Whitespace
|
|
'JAN' Name.Variable
|
|
',' Punctuation
|
|
'\n ' Text.Whitespace
|
|
'pages' Name.Attribute
|
|
' ' Text.Whitespace
|
|
'=' Punctuation
|
|
' ' Text.Whitespace
|
|
'"' Literal.String
|
|
'888-895' Literal.String
|
|
'"' Literal.String
|
|
'\n' Text.Whitespace
|
|
|
|
'}' Punctuation
|
|
'\n' Text.Whitespace
|