From a6798370261a9ea925c66f7df0036fb04890c411 Mon Sep 17 00:00:00 2001 From: wenshao Date: Wed, 21 Jun 2017 12:49:49 +0800 Subject: [PATCH] improved sql parser. --- .../ast/statement/SQLCharacterDataType.java | 13 + .../statement/SQLCreateTableStatement.java | 14 + .../sql/ast/statement/SQLSelectItem.java | 24 + .../ast/statement/SQLSelectQueryBlock.java | 18 + .../oracle/parser/OracleExprParser.java | 2 +- .../com/alibaba/druid/sql/parser/Lexer.java | 7 +- .../com/alibaba/druid/sql/parser/Token.java | 3 +- .../com/alibaba/druid/util/MySqlUtils.java | 22 + .../com/alibaba/druid/util/OracleUtils.java | 27 +- .../com/alibaba/druid/util/lang/Consumer.java | 23 + .../META-INF/druid/parser/mysql/keywords | 622 ++++++++++++++++++ .../META-INF/druid/parser/oracle/keywords | 110 ++++ .../select/OracleSelectTest66_not_eq.java | 74 +++ 13 files changed, 952 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/util/lang/Consumer.java create mode 100644 src/main/resources/META-INF/druid/parser/mysql/keywords create mode 100644 src/main/resources/META-INF/druid/parser/oracle/keywords create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest66_not_eq.java diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCharacterDataType.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCharacterDataType.java index 39a16b2ab3..9a2861221d 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCharacterDataType.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCharacterDataType.java @@ -17,6 +17,8 @@ package com.alibaba.druid.sql.ast.statement; import com.alibaba.druid.sql.ast.SQLCommentHint; import com.alibaba.druid.sql.ast.SQLDataTypeImpl; +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr; import com.alibaba.druid.sql.visitor.SQLASTVisitor; import java.util.List; @@ -78,6 +80,17 @@ public class SQLCharacterDataType extends SQLDataTypeImpl { this.hints = hints; } + public int getLength() { + if (this.arguments.size() == 1) { + SQLExpr arg = this.arguments.get(0); + if (arg instanceof SQLIntegerExpr) { + return ((SQLIntegerExpr) arg).getNumber().intValue(); + } + } + + return -1; + } + @Override protected void accept0(SQLASTVisitor visitor) { if (visitor.visit(this)) { diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java index 1cd6f4b081..c1f13e5900 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.List; import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLDataType; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.SQLStatementImpl; @@ -27,6 +28,7 @@ import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlTableIndex; import com.alibaba.druid.sql.visitor.SQLASTVisitor; import com.alibaba.druid.stat.TableStat; +import com.alibaba.druid.util.lang.Consumer; public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLStatement { @@ -213,4 +215,16 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS return null; } + + public void forEachColumn(Consumer columnConsumer) { + if (columnConsumer == null) { + return; + } + + for (SQLTableElement element : this.tableElementList) { + if (element instanceof SQLColumnDefinition) { + columnConsumer.accept((SQLColumnDefinition) element); + } + } + } } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectItem.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectItem.java index 7db00d3c22..bac2e6afaf 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectItem.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectItem.java @@ -157,4 +157,28 @@ public class SQLSelectItem extends SQLObjectImpl implements SQLReplaceable { return false; } + + public boolean match(String alias) { + if (alias == null) { + return false; + } + + String alias_normalized = SQLUtils.normalize(alias); + + if (alias_normalized.equalsIgnoreCase(this.alias)) { + return true; + } + + if (expr instanceof SQLIdentifierExpr) { + String ident = ((SQLIdentifierExpr) expr).getName(); + return alias_normalized.equalsIgnoreCase(SQLUtils.normalize(ident)); + } + + if (expr instanceof SQLPropertyExpr) { + String ident = ((SQLPropertyExpr) expr).getName(); + return alias_normalized.equalsIgnoreCase(SQLUtils.normalize(ident)); + } + + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java index 851fa06fd8..48aa96baea 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLSelectQueryBlock.java @@ -18,7 +18,9 @@ package com.alibaba.druid.sql.ast.statement; import java.util.ArrayList; import java.util.List; +import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.*; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.visitor.SQLASTVisitor; public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery, SQLReplaceable { @@ -421,4 +423,20 @@ public class SQLSelectQueryBlock extends SQLObjectImpl implements SQLSelectQuery } return false; } + + public SQLSelectItem findSelectItem(String ident) { + if (ident == null) { + return null; + } + + String ident_normalized = SQLUtils.normalize(ident); + + for (SQLSelectItem item : this.selectList) { + if (item.match(ident_normalized)) { + return item; + } + } + + return null; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java index d540e16310..98814ff80c 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleExprParser.java @@ -982,7 +982,7 @@ public class OracleExprParser extends SQLExprParser { rightExp = equalityRest(rightExp); expr = new SQLBinaryOpExpr(expr, SQLBinaryOperator.Equality, rightExp, getDbType()); - } else if (lexer.token() == Token.BANGEQ) { + } else if (lexer.token() == Token.BANGEQ || lexer.token() == Token.CARETEQ) { lexer.nextToken(); rightExp = shift(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java index 2b477b3b7c..e8bcc1cc54 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Lexer.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Lexer.java @@ -582,7 +582,12 @@ public class Lexer { break; case '^': scanChar(); - token = Token.CARET; + if (ch == '=') { + scanChar(); + token = Token.CARETEQ; + } else { + token = Token.CARET; + } break; case '%': scanChar(); diff --git a/src/main/java/com/alibaba/druid/sql/parser/Token.java b/src/main/java/com/alibaba/druid/sql/parser/Token.java index bf9990f025..9a34972b2d 100644 --- a/src/main/java/com/alibaba/druid/sql/parser/Token.java +++ b/src/main/java/com/alibaba/druid/sql/parser/Token.java @@ -334,7 +334,8 @@ public enum Token { SLASH("/"), AMP("&"), BAR("|"), - CARET("^"), + CARET("^"), + CARETEQ("^="), PERCENT("%"), LTLT("<<"), GTGT(">>"), diff --git a/src/main/java/com/alibaba/druid/util/MySqlUtils.java b/src/main/java/com/alibaba/druid/util/MySqlUtils.java index f4220b6025..297263a20e 100644 --- a/src/main/java/com/alibaba/druid/util/MySqlUtils.java +++ b/src/main/java/com/alibaba/druid/util/MySqlUtils.java @@ -29,6 +29,8 @@ import java.sql.Connection; import java.sql.Driver; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; +import java.util.HashSet; +import java.util.Set; public class MySqlUtils { static Class utilClass; @@ -104,4 +106,24 @@ public class MySqlUtils { } return null; } + + private static Set keywords; + + public static boolean isKeyword(String name) { + if (name == null) { + return false; + } + + String name_lower = name.toLowerCase(); + + Set words = keywords; + + if (words == null) { + words = new HashSet(); + Utils.loadFromFile("META-INF/druid/parser/mysql/keywords", words); + keywords = words; + } + + return words.contains(name_lower); + } } diff --git a/src/main/java/com/alibaba/druid/util/OracleUtils.java b/src/main/java/com/alibaba/druid/util/OracleUtils.java index a01541296a..5529cfd7da 100644 --- a/src/main/java/com/alibaba/druid/util/OracleUtils.java +++ b/src/main/java/com/alibaba/druid/util/OracleUtils.java @@ -15,10 +15,8 @@ */ package com.alibaba.druid.util; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; +import java.util.ArrayList; import java.util.HashSet; import java.util.Set; @@ -28,6 +26,7 @@ import javax.transaction.xa.XAException; import oracle.jdbc.OracleConnection; import oracle.jdbc.OracleResultSet; import oracle.jdbc.OracleStatement; +import oracle.jdbc.driver.DatabaseError; import oracle.jdbc.internal.OraclePreparedStatement; import oracle.jdbc.xa.client.OracleXAConnection; import oracle.sql.ROWID; @@ -203,4 +202,24 @@ public class OracleUtils { return tables.contains(table_lower); } + + private static Set keywords; + + public static boolean isKeyword(String name) { + if (name == null) { + return false; + } + + String name_lower = name.toLowerCase(); + + Set words = keywords; + + if (words == null) { + words = new HashSet(); + Utils.loadFromFile("META-INF/druid/parser/oracle/keywords", words); + keywords = words; + } + + return words.contains(name_lower); + } } diff --git a/src/main/java/com/alibaba/druid/util/lang/Consumer.java b/src/main/java/com/alibaba/druid/util/lang/Consumer.java new file mode 100644 index 0000000000..49e49f3800 --- /dev/null +++ b/src/main/java/com/alibaba/druid/util/lang/Consumer.java @@ -0,0 +1,23 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.util.lang; + +/** + * Created by wenshao on 20/06/2017. + */ +public interface Consumer { + void accept(T t); +} diff --git a/src/main/resources/META-INF/druid/parser/mysql/keywords b/src/main/resources/META-INF/druid/parser/mysql/keywords new file mode 100644 index 0000000000..b974301b79 --- /dev/null +++ b/src/main/resources/META-INF/druid/parser/mysql/keywords @@ -0,0 +1,622 @@ +accessible +account +action +add +after +against +aggregate +algorithm +all +alter +always +analyse +analyze +and +any +as +asc +ascii +asensitive +at +autoextend_size +auto_increment +avg +avg_row_length +backup +before +begin +between +bigint +binary +binlog +bit +blob +block +bool +boolean +both +btree +by +byte +cache +call +cascade +cascaded +case +catalog_name +chain +change +changed +channel +char +character +charset +check +checksum +cipher +class_origin +client +close +coalesce +code +collate +collation +column +columns +column_format +column_name +comment +commit +committed +compact +completion +compressed +compression +concurrent +condition +connection +consistent +constraint +constraint_catalog +constraint_name +constraint_schema +contains +context +continue +convert +cpu +create +cross +cube +current +current_date +current_time +current_timestamp +current_user +cursor +cursor_name +data +database +databases +datafile +date +datetime +day +day_hour +day_microsecond +day_minute +day_second +deallocate +dec +decimal +declare +default +default_auth +definer +delayed +delay_key_write +delete +desc +describe +des_key_file +deterministic +diagnostics +directory +disable +discard +disk +distinct +distinctrow +div +do +double +drop +dual +dumpfile +duplicate +dynamic +each +else +elseif +enable +enclosed +encryption +end +ends +engine +engines +enum +error +errors +escape +escaped +event +events +every +exchange +execute +exists +exit +expansion +expire +explain +export +extended +extent_size +false +fast +faults +fetch +fields +file +file_block_size +filter +first +fixed +float +float4 +float8 +flush +follows +for +force +foreign +format +found +from +full +fulltext +function +general +generated +geometry +geometrycollection +get +get_format +global +grant +grants +group +group_replication +handler +hash +having +help +high_priority +host +hosts +hour +hour_microsecond +hour_minute +hour_second +identified +if +ignore +ignore_server_ids +import +in +index +indexes +infile +initial_size +inner +inout +insensitive +insert +insert_method +install +instance +int +int1 +int2 +int3 +int4 +int8 +integer +interval +into +invoker +io +io_after_gtids +io_before_gtids +io_thread +ipc +is +isolation +issuer +iterate +join +json +key +keys +key_block_size +kill +language +last +leading +leave +leaves +left +less +level +like +limit +linear +lines +linestring +list +load +local +localtime +localtimestamp +lock +locks +logfile +logs +long +longblob +longtext +loop +low_priority +master +master_auto_position +master_bind +master_connect_retry +master_delay +master_heartbeat_period +master_host +master_log_file +master_log_pos +master_password +master_port +master_retry_count +master_server_id +master_ssl +master_ssl_ca +master_ssl_capath +master_ssl_cert +master_ssl_cipher +master_ssl_crl +master_ssl_crlpath +master_ssl_key +master_ssl_verify_server_cert +master_tls_version +master_user +match +maxvalue +max_connections_per_hour +max_queries_per_hour +max_rows +max_size +max_statement_time +max_updates_per_hour +max_user_connections +medium +mediumblob +mediumint +mediumtext +memory +merge +message_text +microsecond +middleint +migrate +minute +minute_microsecond +minute_second +min_rows +mod +mode +modifies +modify +month +multilinestring +multipoint +multipolygon +mutex +mysql_errno +name +names +national +natural +nchar +ndb +ndbcluster +never +new +next +no +nodegroup +nonblocking +none +not +no_wait +no_write_to_binlog +null +number +numeric +nvarchar +offset +old_password +on +one +only +open +optimize +optimizer_costs +option +optionally +options +or +order +out +outer +outfile +owner +pack_keys +page +parser +parse_gcol_expr +partial +partition +partitioning +partitions +password +phase +plugin +plugins +plugin_dir +point +polygon +port +precedes +precision +prepare +preserve +prev +primary +privileges +procedure +processlist +profile +profiles +proxy +purge +quarter +query +quick +range +read +reads +read_only +read_write +real +rebuild +recover +redofile +redo_buffer_size +redundant +references +regexp +relay +relaylog +relay_log_file +relay_log_pos +relay_thread +release +reload +remove +rename +reorganize +repair +repeat +repeatable +replace +replicate_do_db +replicate_do_table +replicate_ignore_db +replicate_ignore_table +replicate_rewrite_db +replicate_wild_do_table +replicate_wild_ignore_table +replication +require +reset +resignal +restore +restrict +resume +return +returned_sqlstate +returns +reverse +revoke +right +rlike +rollback +rollup +rotate +routine +row +rows +row_count +row_format +rtree +savepoint +schedule +schema +schemas +schema_name +second +second_microsecond +security +select +sensitive +separator +serial +serializable +server +session +set +share +show +shutdown +signal +signed +simple +slave +slow +smallint +snapshot +socket +some +soname +sounds +source +spatial +specific +sql +sqlexception +sqlstate +sqlwarning +sql_after_gtids +sql_after_mts_gaps +sql_before_gtids +sql_big_result +sql_buffer_result +sql_cache +sql_calc_found_rows +sql_no_cache +sql_small_result +sql_thread +sql_tsi_day +sql_tsi_hour +sql_tsi_minute +sql_tsi_month +sql_tsi_quarter +sql_tsi_second +sql_tsi_week +sql_tsi_year +ssl +stacked +start +starting +starts +stats_auto_recalc +stats_persistent +stats_sample_pages +status +stop +storage +stored +straight_join +string +subclass_origin +subject +subpartition +subpartitions +super +suspend +swaps +switches +table +tables +tablespace +table_checksum +table_name +temporary +temptable +terminated +text +than +then +time +timestamp +timestampadd +timestampdiff +tinyblob +tinyint +tinytext +to +trailing +transaction +trigger +triggers +true +truncate +type +types +uncommitted +undefined +undo +undofile +undo_buffer_size +unicode +uninstall +union +unique +unknown +unlock +unsigned +until +update +upgrade +usage +use +user +user_resources +use_frm +using +utc_date +utc_time +utc_timestamp +validation +value +values +varbinary +varchar +varcharacter +variables +varying +view +virtual +wait +warnings +week +weight_string +when +where +while +with +without +work +wrapper +write +x509 +xa +xid +xml +xor +year +year_month +zerofill diff --git a/src/main/resources/META-INF/druid/parser/oracle/keywords b/src/main/resources/META-INF/druid/parser/oracle/keywords new file mode 100644 index 0000000000..18a3798cd6 --- /dev/null +++ b/src/main/resources/META-INF/druid/parser/oracle/keywords @@ -0,0 +1,110 @@ +access +add +all +alter +and +any +as +asc +audit +between +by +char +check +cluster +column +column_value +comment +compress +connect +create +current +date +decimal +default +delete +desc +distinct +drop +else +exclusive +exists +file +float +for +from +grant +group +having +identified +immediate +in +increment +index +initial +insert +integer +intersect +into +is +level +like +lock +long +maxextents +minus +mlslabel +mode +modify +nested_table_id +noaudit +nocompress +not +nowait +null +number +of +offline +on +online +option +or +order +pctfree +prior +public +raw +rename +resource +revoke +row +rowid +rownum +rows +select +session +set +share +size +smallint +start +successful +synonym +sysdate +table +then +to +trigger +uid +union +unique +update +user +validate +values +varchar +varchar2 +view +whenever +where +with \ No newline at end of file diff --git a/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest66_not_eq.java b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest66_not_eq.java new file mode 100644 index 0000000000..90cc7b6299 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/oracle/select/OracleSelectTest66_not_eq.java @@ -0,0 +1,74 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.bvt.sql.oracle.select; + +import com.alibaba.druid.sql.OracleTest; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; +import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor; +import org.junit.Assert; + +import java.util.List; + +public class OracleSelectTest66_not_eq extends OracleTest { + + public void test_0() throws Exception { + String sql = // + "SELECT *\n" + + "FROM customers\n" + + "WHERE last_name ^= 'Anderson';"; // + + OracleStatementParser parser = new OracleStatementParser(sql); + List statementList = parser.parseStatementList(); + SQLStatement stmt = statementList.get(0); + print(statementList); + + Assert.assertEquals(1, statementList.size()); + + OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor(); + stmt.accept(visitor); + + System.out.println("Tables : " + visitor.getTables()); + System.out.println("fields : " + visitor.getColumns()); + System.out.println("coditions : " + visitor.getConditions()); + System.out.println("relationships : " + visitor.getRelationships()); + System.out.println("orderBy : " + visitor.getOrderByColumns()); + + Assert.assertEquals(1, visitor.getTables().size()); + + Assert.assertEquals(2, visitor.getColumns().size()); + + { + String text = SQLUtils.toOracleString(stmt); + + assertEquals("SELECT *\n" + + "FROM customers\n" + + "WHERE last_name != 'Anderson';", text); + } + + { + String text = SQLUtils.toOracleString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION); + + Assert.assertEquals("select *\n" + + "from customers\n" + + "where last_name != 'Anderson';", text); + } + // Assert.assertTrue(visitor.getColumns().contains(new TableStat.Column("acduser.vw_acd_info", "xzqh"))); + + // Assert.assertTrue(visitor.getOrderByColumns().contains(new TableStat.Column("employees", "last_name"))); + } +}