From 1e9b0995d03e99d61f9b75ecd2812dc3bc49d3a8 Mon Sep 17 00:00:00 2001 From: Mosha Pasumansky Date: Wed, 12 Aug 2026 11:05:56 -0700 Subject: [PATCH] BigQuery: support `CREATE VECTOR INDEX` BigQuery can create a vector index for approximate nearest-neighbor search over an embedding column: CREATE [OR REPLACE] VECTOR INDEX [IF NOT EXISTS] ON () OPTIONS(index_type = 'IVF', distance_type = 'COSINE', ...) `VECTOR` is not a keyword, so `parse_create` previously failed with `Expected: an object type after CREATE, found: VECTOR`. ### Changes - Add `vector`, `or_replace` and `options` fields to `CreateIndex`. `VECTOR` is a modifier on `CREATE INDEX` (like `EXTERNAL` on `CREATE TABLE`), so it reuses the existing node rather than a new statement variant. `Display` renders `CREATE [OR REPLACE ]VECTOR INDEX ...` plus a trailing `OPTIONS(...)`. - Parse the form in `parse_create` via a small `parse_create_vector_index` helper; the `OPTIONS(...)` clause reuses `parse_options` / `SqlOption`, so it parses and renders the same way as `CREATE TABLE` / `CREATE VIEW` OPTIONS. - Plain `CREATE INDEX` is unchanged (all three fields default to false/empty). - New test `parse_bigquery_create_vector_index` in `tests/sqlparser_bigquery.rs` verifies the round-trip and covers `OR REPLACE`, `IF NOT EXISTS`, multi-part names and the OPTIONS-less form. Docs: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_vector_index_statement Co-authored-by: Claude Opus 4.8 --- src/ast/ddl.rs | 13 ++++++- src/ast/spans.rs | 4 ++ src/parser/mod.rs | 40 +++++++++++++++++++ tests/sqlparser_bigquery.rs | 77 +++++++++++++++++++++++++++++++++++++ tests/sqlparser_common.rs | 6 +++ tests/sqlparser_postgres.rs | 33 ++++++++++++++++ 6 files changed, 172 insertions(+), 1 deletion(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index a0e69ad8ac..0709f63ebe 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -2827,6 +2827,10 @@ pub struct CreateIndex { pub using: Option, /// columns included in the index pub columns: Vec, + /// whether this is a BigQuery `CREATE VECTOR INDEX` + pub vector: bool, + /// whether the statement is `CREATE OR REPLACE` + pub or_replace: bool, /// whether the index is unique pub unique: bool, /// whether the index is created concurrently @@ -2843,6 +2847,8 @@ pub struct CreateIndex { pub nulls_distinct: Option, /// WITH clause: pub with: Vec, + /// BigQuery `OPTIONS(...)` clause, e.g. on `CREATE VECTOR INDEX` + pub options: Vec, /// WHERE clause: pub predicate: Option, /// Index options: @@ -2860,8 +2866,10 @@ impl fmt::Display for CreateIndex { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, - "CREATE {unique}INDEX {concurrently}{async_}{if_not_exists}", + "CREATE {or_replace}{unique}{vector}INDEX {concurrently}{async_}{if_not_exists}", + or_replace = if self.or_replace { "OR REPLACE " } else { "" }, unique = if self.unique { "UNIQUE " } else { "" }, + vector = if self.vector { "VECTOR " } else { "" }, concurrently = if self.concurrently { "CONCURRENTLY " } else { @@ -2882,6 +2890,9 @@ impl fmt::Display for CreateIndex { write!(f, " USING {value} ")?; } write!(f, "({})", display_comma_separated(&self.columns))?; + if !self.options.is_empty() { + write!(f, " OPTIONS({})", display_comma_separated(&self.options))?; + } if !self.include.is_empty() { write!(f, " INCLUDE ({})", display_comma_separated(&self.include))?; } diff --git a/src/ast/spans.rs b/src/ast/spans.rs index a34fe66d98..ab5f105c8f 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -699,6 +699,8 @@ impl Spanned for CreateIndex { table_name, using: _, columns, + vector: _, // bool + or_replace: _, // bool unique: _, // bool concurrently: _, // bool r#async: _, // bool @@ -706,6 +708,7 @@ impl Spanned for CreateIndex { include, nulls_distinct: _, // bool with, + options, predicate, index_options: _, alter_options, @@ -718,6 +721,7 @@ impl Spanned for CreateIndex { .chain(columns.iter().map(|i| i.column.span())) .chain(include.iter().map(|i| i.span)) .chain(with.iter().map(|i| i.span())) + .chain(options.iter().map(|i| i.span())) .chain(predicate.iter().map(|i| i.span())) .chain(alter_options.iter().map(|i| i.span())), ) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b2b3f42bbf..530fd2534f 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5288,6 +5288,13 @@ impl<'a> Parser<'a> { self.parse_create_schema(or_replace) } else if self.parse_keyword(Keyword::WAREHOUSE) { self.parse_create_warehouse(or_replace).map(Into::into) + } else if matches!( + &self.peek_token_ref().token, + Token::Word(w) if w.keyword == Keyword::NoKeyword && w.value.eq_ignore_ascii_case("VECTOR") + ) { + // BigQuery `CREATE [OR REPLACE] VECTOR INDEX ...`; VECTOR is not a keyword. + self.next_token(); + self.parse_create_vector_index(or_replace).map(Into::into) } else if or_replace { self.expected_ref( "[EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION or SCHEMA or WAREHOUSE after CREATE OR REPLACE", @@ -8245,6 +8252,36 @@ impl<'a> Parser<'a> { Ok(Statement::Discard { object_type }) } + /// Parse a BigQuery `CREATE [OR REPLACE] VECTOR INDEX` statement (`VECTOR` already consumed). + fn parse_create_vector_index(&mut self, or_replace: bool) -> Result { + self.expect_keyword_is(Keyword::INDEX)?; + let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); + let name = self.parse_object_name(false)?; + self.expect_keyword_is(Keyword::ON)?; + let table_name = self.parse_object_name(false)?; + let columns = self.parse_parenthesized_index_column_list()?; + let options = self.parse_options(Keyword::OPTIONS)?; + Ok(CreateIndex { + name: Some(name), + table_name, + using: None, + columns, + vector: true, + or_replace, + unique: false, + concurrently: false, + r#async: false, + if_not_exists, + include: vec![], + nulls_distinct: None, + with: vec![], + options, + predicate: None, + index_options: vec![], + alter_options: vec![], + }) + } + /// Parse a `CREATE INDEX` statement. pub fn parse_create_index(&mut self, unique: bool) -> Result { let concurrently = self.parse_keyword(Keyword::CONCURRENTLY); @@ -8326,6 +8363,8 @@ impl<'a> Parser<'a> { table_name, using, columns, + vector: false, + or_replace: false, unique, concurrently, r#async, @@ -8333,6 +8372,7 @@ impl<'a> Parser<'a> { include, nulls_distinct, with, + options: vec![], predicate, index_options, alter_options, diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index f6d4483c24..e1c10d943e 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2950,3 +2950,80 @@ fn test_create_snapshot_table() { "CREATE SNAPSHOT TABLE IF NOT EXISTS dataset_id.table1 CLONE dataset_id.table2 FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) OPTIONS(expiration_timestamp = TIMESTAMP '2025-01-01 00:00:00 UTC')", ); } +#[test] +fn parse_bigquery_create_vector_index() { + // `CREATE VECTOR INDEX ... OPTIONS(...)` is a CreateIndex flagged `vector`. + let sql = + "CREATE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE', dimension = 4)"; + match bigquery().verified_stmt(sql) { + Statement::CreateIndex(CreateIndex { + name, + table_name, + using, + columns, + vector, + or_replace, + unique, + if_not_exists, + with, + options, + .. + }) => { + assert!(vector); + assert!(!or_replace); + assert!(!unique); + assert!(!if_not_exists); + assert_eq!(name.unwrap().to_string(), "emb"); + assert_eq!(table_name.to_string(), "t"); + assert_eq!(using, None); + assert!(with.is_empty()); + assert_eq!( + columns.iter().map(|c| c.to_string()).collect::>(), + vec!["embedding"] + ); + assert_eq!( + options, + vec![ + SqlOption::KeyValue { + key: Ident::new("distance_type"), + value: Expr::Value( + Value::SingleQuotedString("COSINE".to_string()).with_empty_span() + ), + }, + SqlOption::KeyValue { + key: Ident::new("dimension"), + value: Expr::value(number("4")), + }, + ] + ); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // OR REPLACE, IF NOT EXISTS, multi-part names, and the OPTIONS-less form all round-trip. + match bigquery().verified_stmt( + "CREATE OR REPLACE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE')", + ) { + Statement::CreateIndex(CreateIndex { + vector, or_replace, .. + }) => { + assert!(vector); + assert!(or_replace); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + match bigquery().verified_stmt( + "CREATE VECTOR INDEX IF NOT EXISTS mydataset.emb ON mydataset.t(embedding) OPTIONS(distance_type = 'EUCLIDEAN')", + ) { + Statement::CreateIndex(CreateIndex { + vector, + if_not_exists, + .. + }) => { + assert!(vector); + assert!(if_not_exists); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + bigquery().verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)"); +} diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 0800bc41fd..99e1304839 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -9728,6 +9728,9 @@ fn test_create_index_with_using_function() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq!("idx_name", name.to_string()); assert_eq!("test", table_name.to_string()); @@ -9785,6 +9788,9 @@ fn test_create_index_with_with_clause() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { pretty_assertions::assert_eq!("title_idx", name.to_string()); pretty_assertions::assert_eq!("films", table_name.to_string()); diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index a7128eafd8..9c12dc8af7 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2993,6 +2993,9 @@ fn parse_create_index() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3030,6 +3033,9 @@ fn parse_create_anonymous_index() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq!(None, name); assert_eq_vec(&["my_table"], &table_name); @@ -3150,6 +3156,9 @@ fn parse_create_indices_with_operator_classes() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3179,6 +3188,9 @@ fn parse_create_indices_with_operator_classes() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3263,6 +3275,9 @@ fn parse_create_bloom() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["bloomidx"], &name); assert_eq_vec(&["tbloom"], &table_name); @@ -3320,6 +3335,9 @@ fn parse_create_brin() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["brin_sensor_data_recorded_at"], &name); assert_eq_vec(&["sensor_data"], &table_name); @@ -3388,6 +3406,9 @@ fn parse_create_index_concurrently() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3425,6 +3446,9 @@ fn parse_create_index_with_predicate() { predicate: Some(_), index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3462,6 +3486,9 @@ fn parse_create_index_with_include() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3499,6 +3526,9 @@ fn parse_create_index_with_nulls_distinct() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3534,6 +3564,9 @@ fn parse_create_index_with_nulls_distinct() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name);