diff --git a/src/parser/source_tree.rs b/src/parser/source_tree.rs index 6850d9ca0..f1e5efb79 100644 --- a/src/parser/source_tree.rs +++ b/src/parser/source_tree.rs @@ -264,6 +264,18 @@ mod tests { assert!(viz.starts_with("visualise")); } + #[test] + fn test_extract_sql_preserves_jinja_ref() { + let query = "SELECT order_date, region, revenue FROM {{ ref('fct_orders') }}\nVISUALISE order_date AS x, revenue AS y, region AS color\nDRAW point"; + let tree = SourceTree::new(query).unwrap(); + + let sql = tree.extract_sql().unwrap(); + assert_eq!( + sql, + "SELECT order_date, region, revenue FROM {{ ref('fct_orders') }}" + ); + } + #[test] fn test_extract_sql_no_visualise() { let query = "SELECT * FROM data WHERE x > 5"; @@ -289,6 +301,18 @@ mod tests { assert!(viz.starts_with("VISUALISE FROM mtcars")); } + #[test] + fn test_extract_sql_visualise_from_jinja_ref() { + let query = "VISUALISE FROM {{ ref('fct_orders') }} DRAW point MAPPING x AS x, y AS y"; + let tree = SourceTree::new(query).unwrap(); + + let sql = tree.extract_sql().unwrap(); + assert_eq!(sql, "SELECT * FROM {{ ref('fct_orders') }}"); + + let viz = tree.extract_visualise().unwrap(); + assert!(viz.starts_with("VISUALISE FROM {{ ref('fct_orders') }}")); + } + #[test] fn test_extract_sql_visualise_from_with_cte() { let query = @@ -407,6 +431,15 @@ mod tests { assert!(sql.contains("SELECT * FROM 'mtcars.csv'")); } + #[test] + fn test_extract_sql_from_first_jinja_ref() { + let query = "FROM {{ ref('fct_orders') }} VISUALISE DRAW point MAPPING x AS x, y AS y"; + let tree = SourceTree::new(query).unwrap(); + + let sql = tree.extract_sql().unwrap(); + assert_eq!(sql, "SELECT * FROM {{ ref('fct_orders') }}"); + } + #[test] fn test_extract_sql_from_first_case_insensitive() { let query = "from sales visualise DRAW point MAPPING x AS x, y AS y"; diff --git a/tree-sitter-ggsql/grammar.js b/tree-sitter-ggsql/grammar.js index d70335408..b6645f6df 100644 --- a/tree-sitter-ggsql/grammar.js +++ b/tree-sitter-ggsql/grammar.js @@ -17,6 +17,10 @@ function caseInsensitive(keyword) { module.exports = grammar({ name: 'ggsql', + inline: $ => [ + $.source_ref, + ], + conflicts: $ => [ [$.sql_portion], ], @@ -63,6 +67,7 @@ module.exports = grammar({ $.case_expression, $.cast_expression, $.function_call, + $.jinja_template, $.non_from_sql_keyword, $.string, $.number, @@ -84,6 +89,7 @@ module.exports = grammar({ $.case_expression, // CASE WHEN ... THEN ... END $.cast_expression, // CAST(expr AS type), TRY_CAST(expr AS type) $.function_call, // Regular function calls like COUNT(), SUM() + $.jinja_template, $.sql_keyword, $.string, $.number, @@ -128,6 +134,7 @@ module.exports = grammar({ $.identifier, $.string, $.number, + $.jinja_template, $.subquery, ',', '(', ')', '*', '.', '=', /[^\s;(),'"]+/ @@ -143,6 +150,7 @@ module.exports = grammar({ $.identifier, $.string, $.number, + $.jinja_template, $.subquery, ',', '(', ')', '*', '.', '=', /[^\s;(),'"]+/ @@ -157,6 +165,7 @@ module.exports = grammar({ $.identifier, $.string, $.number, + $.jinja_template, $.subquery, ',', '(', ')', '*', '.', '=', /[^\s;(),'"]+/ @@ -171,6 +180,7 @@ module.exports = grammar({ $.identifier, $.string, $.number, + $.jinja_template, $.subquery, ',', '(', ')', '*', '.', '=', /[^\s;(),'"]+/ @@ -179,6 +189,7 @@ module.exports = grammar({ other_sql_statement: $ => prec(-1, repeat1(choice( $.non_from_sql_keyword, + $.jinja_template, /[^\s;(),'"]+/, $.string, $.number, @@ -218,6 +229,7 @@ module.exports = grammar({ $.sql_keyword, $.string, $.number, + $.jinja_template, $.identifier, $.subquery, ',', '*', '.', '=', '<', '>', '!', '::', @@ -242,6 +254,7 @@ module.exports = grammar({ $.cast_expression, $.function_call, $.subquery, // also handles IN-lists like ('a', 'b') + $.jinja_template, token('='), token('!='), token('<>'), token('<='), token('>='), token('<'), token('>'), token('+'), token('-'), token('*'), token('/'), token('%'), token('||'), token('::'), @@ -396,6 +409,7 @@ module.exports = grammar({ $.qualified_name, // Handles both simple identifiers and table.column $.number, $.string, + $.jinja_template, '*', // CASE expression $.case_expression, @@ -470,9 +484,16 @@ module.exports = grammar({ repeat(seq('.', $.identifier)) )), + source_ref: $ => choice( + $.qualified_name, + $.string, + $.namespaced_identifier, + $.jinja_template + ), + table_ref: $ => prec.right(seq( choice( - field('table', choice($.qualified_name, $.string, $.namespaced_identifier)), + field('table', $.source_ref), $.subquery, ), optional(seq( @@ -591,14 +612,14 @@ module.exports = grammar({ // Option 1: Just FROM (inherit global mappings) seq( caseInsensitive('FROM'), - field('layer_source', choice($.qualified_name, $.string, $.namespaced_identifier)) + field('layer_source', $.source_ref) ), // Option 2: Mapping list (uses shared structure), optionally followed by FROM seq( $.mapping_list, optional(seq( caseInsensitive('FROM'), - field('layer_source', choice($.qualified_name, $.string, $.namespaced_identifier)) + field('layer_source', $.source_ref) )) ) ) @@ -928,6 +949,15 @@ module.exports = grammar({ $.quoted_identifier ), + // Jinja templates are opaque SQL-side tokens. dbt/fusion renders these + // before ggsql executes SQL, but the parser must preserve them while + // splitting SQL from VISUALISE. + jinja_template: $ => token(choice( + seq('{{', repeat(choice(/[^}]+/, /}[^}]/)), '}}'), + seq('{%', repeat(choice(/[^%]+/, /%[^%]/)), '%}'), + seq('{#', repeat(choice(/[^#]+/, /#[^#]/)), '#}') + )), + // Identifier for use in filter expressions - uses lower precedence so that // keywords like PARTITION and ORDER can take priority and end the filter filter_identifier: $ => token(prec(-1, /[a-zA-Z_][a-zA-Z0-9_]*/)), diff --git a/tree-sitter-ggsql/test/corpus/basic.txt b/tree-sitter-ggsql/test/corpus/basic.txt index 2afb1ce33..72b173176 100644 --- a/tree-sitter-ggsql/test/corpus/basic.txt +++ b/tree-sitter-ggsql/test/corpus/basic.txt @@ -3596,3 +3596,122 @@ SELECT grade, ROUND(COUNT(CASE WHEN status = 'Default' THEN 1 END) * 100.0 / COU (viz_clause (draw_clause (geom_type))))) + +================================================================================ +SQL source with Jinja ref +================================================================================ + +SELECT order_date, region, revenue FROM {{ ref('fct_orders') }} +VISUALISE order_date AS x, revenue AS y, region AS color +DRAW point + +-------------------------------------------------------------------------------- + +(query + (sql_portion + (sql_statement + (select_statement + (select_body + (identifier + (bare_identifier)) + (identifier + (bare_identifier)) + (identifier + (bare_identifier)) + (from_clause + (table_ref + table: (jinja_template))))))) + (visualise_statement + (visualise_keyword) + (global_mapping + (mapping_list + (mapping_element + (explicit_mapping + value: (mapping_value + (column_reference + (identifier + (bare_identifier)))) + name: (aesthetic_name))) + (mapping_element + (explicit_mapping + value: (mapping_value + (column_reference + (identifier + (bare_identifier)))) + name: (aesthetic_name))) + (mapping_element + (explicit_mapping + value: (mapping_value + (column_reference + (identifier + (bare_identifier)))) + name: (aesthetic_name))))) + (viz_clause + (draw_clause + (geom_type))))) + +================================================================================ +SQL source with Jinja var containing dict literal +================================================================================ + +SELECT * FROM {{ var('table', {'fallback': 'orders'}) }} +VISUALISE x AS x, y AS y +DRAW point + +-------------------------------------------------------------------------------- + +(query + (sql_portion + (sql_statement + (select_statement + (select_body + (from_clause + (table_ref + table: (jinja_template))))))) + (visualise_statement + (visualise_keyword) + (global_mapping + (mapping_list + (mapping_element + (explicit_mapping + value: (mapping_value + (column_reference + (identifier + (bare_identifier)))) + name: (aesthetic_name))) + (mapping_element + (explicit_mapping + value: (mapping_value + (column_reference + (identifier + (bare_identifier)))) + name: (aesthetic_name))))) + (viz_clause + (draw_clause + (geom_type))))) + +================================================================================ +Layer source with Jinja ref +================================================================================ + +VISUALISE +DRAW point MAPPING x AS x FROM {{ ref('fct_orders') }} + +-------------------------------------------------------------------------------- + +(query + (visualise_statement + (visualise_keyword) + (viz_clause + (draw_clause + (geom_type) + (mapping_clause + (mapping_list + (mapping_element + (explicit_mapping + value: (mapping_value + (column_reference + (identifier + (bare_identifier)))) + name: (aesthetic_name)))) + layer_source: (jinja_template))))))