From 336dd5e5eabefe46ff4994f47fcb68a2376c4b81 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 24 Dec 2017 15:24:06 -0500 Subject: [PATCH] FEATURE: Parsing of comments. --- README.md | 7 ++++++ TODO.md | 1 - examples/test.ucg | 4 ++++ src/ast.rs | 3 +++ src/lib.rs | 2 ++ src/parse.rs | 58 +++++++++++++++++++++++++++-------------------- src/tokenizer.rs | 43 ++++++++++++++++++++++++++++++++++- 7 files changed, 91 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 7bdcb25..c2f5a84 100644 --- a/README.md +++ b/README.md @@ -160,3 +160,10 @@ We also support a limited macro facility with the macro expression. Macro's always output a tuple whose fields are evaluated at the location they are called from. You can acccess the generated fields from the resulting tuple like usual. + +### Comments + +Comments begin with `//` and go till the end of the line. + + // This is a comment. + let one = 1; diff --git a/TODO.md b/TODO.md index cad865f..fbfa3fc 100644 --- a/TODO.md +++ b/TODO.md @@ -16,6 +16,5 @@ organiztion for a given configuration structure. Some options here could be # Minor Fixes and Polish -* Comment syntax * JSON export * YAML export \ No newline at end of file diff --git a/examples/test.ucg b/examples/test.ucg index 93ac277..e4a728d 100644 --- a/examples/test.ucg +++ b/examples/test.ucg @@ -1,7 +1,9 @@ +// A few constants. let dbhost1 = "db1.prod.net"; let dbhost2 = "db2.prod.net"; let dbname = "testdb"; +// Constructor for database connection strings. let mk_db_conn = macro (host, port, db) => { host = host, port = port, @@ -12,8 +14,10 @@ let mk_db_conn = macro (host, port, db) => { let db_conn1 = mk_db_conn(dbhost1, 3306, dbname); let db_conn2 = mk_db_conn(dbhost2, 3306, dbname); +// We have two database connections in a list let db_conns = [db_conn1.conn_string, db_conn2.conn_string]; +// Our server configuration. let server_config = { dbconn_list = db_conns, db_conn1 = db_conns.0, diff --git a/src/ast.rs b/src/ast.rs index ffc4e69..0e64d3a 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -86,12 +86,14 @@ macro_rules! value_node { }; } +#[allow(unused_macros)] macro_rules! make_tok { ( $e: expr, $l:expr, $c:expr ) => { Token::new($e, $l, $c) }; } +#[allow(unused_macros)] macro_rules! make_expr { ( $e:expr ) => { make_expr!($e, 1, 1) @@ -113,6 +115,7 @@ macro_rules! make_expr { /// /// make_selector!(foo", ["bar"] => 1, 0); /// ``` +#[allow(unused_macros)] macro_rules! make_selector { ( $h:expr ) => { make_selector!($h, 1, 0) diff --git a/src/lib.rs b/src/lib.rs index c2dfb8d..97294ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +//#![feature(trace_macros,log_syntax)] + #[macro_use] extern crate nom; #[macro_use] diff --git a/src/parse.rs b/src/parse.rs index d5d4b4c..6a5f03e 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -23,6 +23,14 @@ use ast::*; use tokenizer::*; use error as E; +macro_rules! eat_space { + ($i:expr, $($args:tt)*) => ( + { + sep!($i, emptyspace, $($args)*) + } + ) +} + type ParseResult = Result>; fn symbol_to_value(s: Token) -> ParseResult { @@ -116,7 +124,7 @@ named!( field_value( Span ) -> (Token, Expression), do_parse!( field: barewordtok >> - ws!(equaltok) >> + eat_space!(equaltok) >> value: expression >> (field, value) ) @@ -128,7 +136,7 @@ fn vec_to_tuple(t: (Span, FieldList)) -> ParseResult { } named!(field_list( Span ) -> FieldList, - separated_list!(commatok, ws!(field_value))); + separated_list!(commatok, eat_space!(field_value))); named!( #[doc="Capture a tuple of named fields with values. {=,...}"], @@ -137,7 +145,7 @@ named!( do_parse!( pos: position!() >> v: delimited!(lbracetok, - ws!(field_list), + eat_space!(field_list), rbracetok) >> (pos, v) ), @@ -157,7 +165,7 @@ named!(list_value( Span ) -> Value, do_parse!( pos: position!() >> leftsquarebracket >> - elements: ws!(separated_list!(ws!(commatok), expression)) >> + elements: eat_space!(separated_list!(eat_space!(commatok), expression)) >> rightsquarebracket >> (pos, elements) ), @@ -204,7 +212,7 @@ macro_rules! do_binary_expr { $i, do_parse!( pos: position!() >> left: value >> - ws!($fn) >> + eat_space!($fn) >> right: expression >> (pos, $typ, left, right) ), @@ -329,7 +337,7 @@ named!(copy_expression( Span ) -> Expression, pos: position!() >> selector: selector_list >> lbracetok >> - fields: ws!(field_list) >> + fields: eat_space!(field_list) >> rbracetok >> (pos, SelectorDef::new(selector, pos.line as usize, pos.offset as usize), fields) ), @@ -366,17 +374,17 @@ fn tuple_to_macro(mut t: (Span, Vec, Value)) -> ParseResult { } } -named!(arglist( Span ) -> Vec, separated_list!(ws!(commatok), symbol)); +named!(arglist( Span ) -> Vec, separated_list!(eat_space!(commatok), symbol)); named!(macro_expression( Span ) -> Expression, map_res!( do_parse!( pos: position!() >> macrotok >> - ws!(lparentok) >> - arglist: ws!(arglist) >> + eat_space!(lparentok) >> + arglist: eat_space!(arglist) >> rparentok >> - ws!(fatcommatok) >> + eat_space!(fatcommatok) >> map: tuple >> (pos, arglist, map) ), @@ -410,9 +418,9 @@ named!(select_expression( Span ) -> Expression, do_parse!( pos: position!() >> selecttok >> - val: ws!(terminated!(expression, commatok)) >> - default: ws!(terminated!(expression, commatok)) >> - map: ws!(tuple) >> + val: eat_space!(terminated!(expression, commatok)) >> + default: eat_space!(terminated!(expression, commatok)) >> + map: eat_space!(tuple) >> (pos, val, default, map) ), tuple_to_select @@ -430,10 +438,10 @@ fn tuple_to_format(t: (Token, Vec)) -> ParseResult { named!(format_expression( Span ) -> Expression, map_res!( do_parse!( - tmpl: ws!(strtok) >> - ws!(pcttok) >> + tmpl: eat_space!(strtok) >> + eat_space!(pcttok) >> lparentok >> - args: ws!(separated_list!(ws!(commatok), expression)) >> + args: eat_space!(separated_list!(eat_space!(commatok), expression)) >> rparentok >> (tmpl, args) ), @@ -466,7 +474,7 @@ named!(selector_value( Span ) -> Value, map_res!( do_parse!( pos: position!() >> - sl: ws!(selector_list) >> + sl: eat_space!(selector_list) >> (pos, sl) ), vec_to_selector_value @@ -479,7 +487,7 @@ named!(call_expression( Span ) -> Expression, pos: position!() >> macroname: selector_value >> lparentok >> - args: ws!(separated_list!(ws!(commatok), expression)) >> + args: eat_space!(separated_list!(eat_space!(commatok), expression)) >> rparentok >> (pos, macroname, args) ), @@ -509,7 +517,7 @@ named!(expression( Span ) -> Expression, complete!(select_expression) | complete!(call_expression) | complete!(copy_expression) | - ws!(simple_expression) + eat_space!(simple_expression) ) ); @@ -519,7 +527,7 @@ fn expression_to_statement(v: Expression) -> ParseResult { named!(expression_statement( Span ) -> Statement, map_res!( - terminated!(ws!(expression), semicolontok), + terminated!(eat_space!(expression), semicolontok), expression_to_statement ) ); @@ -535,9 +543,9 @@ named!(let_statement( Span ) -> Statement, map_res!( terminated!(do_parse!( lettok >> - name: ws!(barewordtok) >> + name: eat_space!(barewordtok) >> equaltok >> - val: ws!(expression) >> + val: eat_space!(expression) >> (name, val) ), semicolontok), tuple_to_let @@ -555,9 +563,9 @@ named!(import_statement( Span ) -> Statement, map_res!( terminated!(do_parse!( importtok >> - path: ws!(strtok) >> + path: eat_space!(strtok) >> astok >> - name: ws!(barewordtok) >> + name: eat_space!(barewordtok) >> (name, path) ), semicolontok), tuple_to_import @@ -576,7 +584,7 @@ pub fn parse(input: Span) -> IResult> { let mut out = Vec::new(); let mut i = input; loop { - match ws!(i, statement) { + match eat_space!(i, statement) { IResult::Error(e) => { return IResult::Error(e); } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 76b3b42..7fe93f3 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. use nom_locate::LocatedSpan; -use nom::{alpha, is_alphanumeric, digit}; +use nom; +use nom::{alpha, is_alphanumeric, digit, InputLength, sp}; use ast::*; @@ -145,6 +146,10 @@ named!(pub rightsquarebracket( Span ) -> Token, do_tag_tok!("]") ); +named!(pub commentprefix( Span ) -> Token, + do_tag_tok!("//") +); + named!(pub fatcommatok( Span ) -> Token, do_tag_tok!("=>") ); @@ -168,3 +173,39 @@ named!(pub importtok( Span ) -> Token, named!(pub astok( Span ) -> Token, do_tag_tok!("as") ); + +pub fn end_of_input(input: Span) -> nom::IResult { + if input.input_len() == 0 { + return nom::IResult::Done(input, input); + } else { + return nom::IResult::Incomplete(nom::Needed::Unknown); + } +} + +fn comment(input: Span) -> nom::IResult { + match commentprefix(input) { + nom::IResult::Done(rest, _) => { + match alt!(rest, take_until!("\r\n") | take_until!("\n")) { + nom::IResult::Done(rest, cmt) => nom::IResult::Done(rest, cmt), + nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i), + nom::IResult::Error(e) => { + if let nom::ErrorKind::Eof = e { + return nom::IResult::Done(input, input) + } else { + return nom::IResult::Error(e) + } + } + } + } + nom::IResult::Incomplete(i) => { + return nom::IResult::Incomplete(i) + } + nom::IResult::Error(e) => { + return nom::IResult::Error(e) + } + } +} + +named!(pub emptyspace( Span ) -> Span, + alt!(sp | comment) +); \ No newline at end of file