Alawn/Druid
1
0
Fork 0
mirror of https://github.com/alibaba/druid.git synced 2023-03-04 19:17:16 +00:00
Code Issues Releases Wiki Activity

Merge pull request #896 from wenshao/master

Browse Source 
improved sql parser for odps show tables
This commit is contained in:
高铁
2015-07-11 14:29:01 +08:00
parent ba2a4ce6d0 b36374f7bf
commit 682b856f3d
32 changed files with 763 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/alibaba/druid/sql/SQLUtils.java
+23 -3
View File
@@ -226,7 +226,10 @@ public class SQLUtils {
public static String format(String sql, String dbType, List<Object> parameters) {
try {
List<SQLStatement> statementList = toStatementList(sql, dbType);
SQLStatementParser parser = SQLParserUtils.createSQLStatementParser(sql, dbType);
parser.setKeepComments(true);
List<SQLStatement> statementList = parser.parseStatementList();
return toSQLString(statementList, dbType, parameters);
} catch (ParserException ex) {
@@ -247,10 +250,27 @@ public class SQLUtils {
}
for (int i = 0; i < statementList.size(); i++) {
SQLStatement stmt = statementList.get(i);
if (i > 0) {
out.append(";\n");
visitor.println(";");
}
List<String> comments = stmt.getBeforeCommentsDirect();
if (comments != null){
for(String comment : comments) {
visitor.println(comment);
}
}
stmt.accept(visitor);
if (i == statementList.size() - 1) {
Boolean semi = (Boolean) stmt.getAttribute("format.semi");
if (semi != null && semi.booleanValue()) {
visitor.println();
visitor.print(";");
}
}
statementList.get(i).accept(visitor);
}
return out.toString();
src/main/java/com/alibaba/druid/sql/ast/SQLObject.java
+13
View File
@@ -15,6 +15,7 @@
*/
package com.alibaba.druid.sql.ast;
import java.util.List;
import java.util.Map;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
@@ -34,6 +35,18 @@ public interface SQLObject {
void putAttribute(String name, Object value);
Map<String, Object> getAttributesDirect();
void addBeforeComment(String comment);
List<String> getBeforeCommentsDirect();
void addAfterComment(String comment);
List<String> getAfterCommentsDirect();
boolean hasBeforeComment();
boolean hasAfterComment()
void output(StringBuffer buf);
}
src/main/java/com/alibaba/druid/sql/ast/SQLObjectImpl.java
+67
View File
@@ -15,6 +15,7 @@
*/
package com.alibaba.druid.sql.ast;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -107,4 +108,70 @@ public abstract class SQLObjectImpl implements SQLObject {
public Map<String, Object> getAttributesDirect() {
return attributes;
}
@SuppressWarnings("unchecked")
public void addBeforeComment(String comment) {
if (attributes == null) {
attributes = new HashMap<String, Object>(1);
}
List<String> comments = (List<String>) attributes.get("format.before_comment");
if (comments == null) {
comments = new ArrayList<String>(2);
attributes.put("format.before_comment", comments);
}
comments.add(comment);
}
@SuppressWarnings("unchecked")
public List<String> getBeforeCommentsDirect() {
if (attributes == null) {
return null;
}
return (List<String>) attributes.get("format.before_comment");
}
@SuppressWarnings("unchecked")
public void addAfterComment(String comment) {
if (attributes == null) {
attributes = new HashMap<String, Object>(1);
}
List<String> comments = (List<String>) attributes.get("format.after_comment");
if (comments == null) {
comments = new ArrayList<String>(2);
attributes.put("format.after_comment", comments);
}
comments.add(comment);
}
@SuppressWarnings("unchecked")
public List<String> getAfterCommentsDirect() {
if (attributes == null) {
return null;
}
return (List<String>) attributes.get("format.after_comment");
}
public boolean hasBeforeComment() {
List<String> comments = getBeforeCommentsDirect();
if (comments == null) {
return false;
}
return !comments.isEmpty();
}
public boolean hasAfterComment() {
List<String> comments = getAfterCommentsDirect();
if (comments == null) {
return false;
}
return !comments.isEmpty();
}
}
src/main/java/com/alibaba/druid/sql/ast/statement/SQLShowTablesStatement.java
+44
View File
@@ -0,0 +1,44 @@
package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.SQLStatementImpl;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class SQLShowTablesStatement extends SQLStatementImpl {
protected SQLName database;
protected SQLExpr like;
public SQLName getDatabase() {
return database;
}
public void setDatabase(SQLName database) {
if (database != null) {
database.setParent(this);
}
this.database = database;
}
public SQLExpr getLike() {
return like;
}
public void setLike(SQLExpr like) {
if (like != null) {
like.setParent(this);
}
this.like = like;
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, database);
acceptChild(visitor, like);
}
}
}
src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlShowStatement.java
+1 -1
View File
@@ -15,6 +15,6 @@
*/
package com.alibaba.druid.sql.dialect.mysql.ast.statement;
public interface MySqlShowStatement {
public interface MySqlShowStatement extends MySqlStatement {
}
src/main/java/com/alibaba/druid/sql/dialect/mysql/ast/statement/MySqlShowTablesStatement.java
+12 -20
View File
@@ -16,15 +16,14 @@
package com.alibaba.druid.sql.dialect.mysql.ast.statement;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.statement.SQLShowTablesStatement;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class MySqlShowTablesStatement extends MySqlStatementImpl implements MySqlShowStatement {
public class MySqlShowTablesStatement extends SQLShowTablesStatement implements MySqlShowStatement {
private boolean full;
private SQLName database;
private SQLExpr like;
private SQLExpr where;
public boolean isFull() {
@@ -35,22 +34,6 @@ public class MySqlShowTablesStatement extends MySqlStatementImpl implements MySq
this.full = full;
}
public SQLName getDatabase() {
return database;
}
public void setDatabase(SQLName database) {
this.database = database;
}
public SQLExpr getLike() {
return like;
}
public void setLike(SQLExpr like) {
this.like = like;
}
public SQLExpr getWhere() {
return where;
}
@@ -67,4 +50,13 @@ public class MySqlShowTablesStatement extends MySqlStatementImpl implements MySq
}
visitor.endVisit(this);
}
@Override
protected void accept0(SQLASTVisitor visitor) {
if (visitor instanceof MySqlASTVisitor) {
accept0((MySqlASTVisitor) visitor);
} else {
throw new IllegalArgumentException("not support visitor type : " + visitor.getClass().getName());
}
}
}
src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlLexer.java
+13 -4
View File
@@ -109,6 +109,9 @@ public class MySqlLexer extends Lexer {
stringVal = subString(mark, bufPos);
token = Token.LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
@@ -448,13 +451,16 @@ public class MySqlLexer extends Lexer {
stringVal = subString(mark, bufPos);
token = Token.MULTI_LINE_COMMENT;
hasComment = true;
}
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
if (keepComments) {
commentVal = stringVal;
}
}
endOfComment = isEOF();
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
}
if (!isHint && !isAllowComment() && !isSafeComment(stringVal)) {
throw new NotAllowCommentException();
@@ -492,6 +498,9 @@ public class MySqlLexer extends Lexer {
stringVal = subString(mark, bufPos + 1);
token = Token.LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsExprParser.java
+12 -1
View File
@@ -48,6 +48,11 @@ public class OdpsExprParser extends SQLExprParser {
this.lexer.nextToken();
}
public OdpsExprParser(String sql, Lexer.CommentHandler commentHandler){
this(new OdpsLexer(sql, commentHandler));
this.lexer.nextToken();
}
protected SQLExpr parseAliasExpr(String alias) {
return new SQLCharExpr(alias);
}
@@ -97,7 +102,13 @@ public class OdpsExprParser extends SQLExprParser {
}
final String alias = as();
SQLSelectItem item = new SQLSelectItem(expr, alias);
if (lexer.hasComment() && lexer.isKeepComments()) {
item.addAfterComment(lexer.commentVal());
}
return new SQLSelectItem(expr, alias);
return item;
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsLexer.java
+24 -2
View File
@@ -52,10 +52,17 @@ public class OdpsLexer extends Lexer {
super.keywods = DEFAULT_ODPS_KEYWORDS;
}
public OdpsLexer(String input, CommentHandler commentHandler){
super(input, commentHandler);
super.keywods = DEFAULT_ODPS_KEYWORDS;
}
public void scanComment() {
if (ch != '/' && ch != '-') {
throw new IllegalStateException();
}
Token lastToken = this.token;
mark = pos;
bufPos = 0;
@@ -95,9 +102,16 @@ public class OdpsLexer extends Lexer {
stringVal = subString(mark + startHintSp, (bufPos - startHintSp) - 1);
token = Token.HINT;
} else {
stringVal = subString(mark, bufPos);
stringVal = subString(mark, bufPos + 1);
token = Token.MULTI_LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
}
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
}
if (token != Token.HINT && !isAllowComment()) {
@@ -138,10 +152,18 @@ public class OdpsLexer extends Lexer {
bufPos++;
}
stringVal = subString(mark + 1, bufPos);
stringVal = subString(mark, bufPos);
token = Token.LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
endOfComment = isEOF();
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
}
return;
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsSelectParser.java
+6 -2
View File
@@ -40,14 +40,18 @@ public class OdpsSelectParser extends SQLSelectParser {
return queryRest(select);
}
OdpsSelectQueryBlock queryBlock = new OdpsSelectQueryBlock();
if (lexer.hasComment() && lexer.isKeepComments()) {
queryBlock.addBeforeComment(lexer.commentVal());
}
accept(Token.SELECT);
if (lexer.token() == Token.COMMENT) {
lexer.nextToken();
}
OdpsSelectQueryBlock queryBlock = new OdpsSelectQueryBlock();
if (lexer.token() == Token.DISTINCT) {
queryBlock.setDistionOption(SQLSetQuantifier.DISTINCT);
lexer.nextToken();
src/main/java/com/alibaba/druid/sql/dialect/odps/parser/OdpsStatementParser.java
+30 -2
View File
@@ -26,6 +26,7 @@ import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSetStatement;
import com.alibaba.druid.sql.ast.statement.SQLShowTablesStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsert;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsInsertStatement;
@@ -41,9 +42,11 @@ import com.alibaba.druid.sql.parser.Token;
import com.alibaba.druid.util.JdbcConstants;
public class OdpsStatementParser extends SQLStatementParser {
public OdpsStatementParser(String sql){
super(new OdpsExprParser(sql));
super (new OdpsLexer(sql), JdbcConstants.ODPS);
this.exprParser = new OdpsExprParser(this.lexer);
this.lexer.setCommentHandler(commentCallBack);
this.lexer.nextToken();
}
public OdpsStatementParser(SQLExprParser exprParser){
@@ -107,6 +110,11 @@ public class OdpsStatementParser extends SQLStatementParser {
public OdpsInsert parseOdpsInsert() {
OdpsInsert insert = new OdpsInsert();
if (lexer.hasComment()) {
insert.addBeforeComment(lexer.commentVal());
}
SQLSelectParser selectParser = createSQLSelectParser();
accept(Token.INSERT);
@@ -172,6 +180,25 @@ public class OdpsStatementParser extends SQLStatementParser {
return stmt;
}
if (identifierEquals("TABLES")) {
lexer.nextToken();
SQLShowTablesStatement stmt = new SQLShowTablesStatement();
if (lexer.token() == Token.FROM) {
lexer.nextToken();
stmt.setDatabase(this.exprParser.name());
}
if (lexer.token() == Token.LIKE) {
lexer.nextToken();
stmt.setLike(this.exprParser.expr());
}
return stmt;
}
throw new ParserException("TODO " + lexer.token() + " " + lexer.stringVal());
}
@@ -211,4 +238,5 @@ public class OdpsStatementParser extends SQLStatementParser {
}
}
}
src/main/java/com/alibaba/druid/sql/dialect/odps/visitor/OdpsOutputVisitor.java
+25 -2
View File
@@ -153,6 +153,10 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
@Override
public boolean visit(OdpsInsert x) {
if (x.hasBeforeComment()) {
printComment(x.getBeforeCommentsDirect(), "\n");
println();
}
if (x.isOverwrite()) {
print("INSERT OVERWRITE TABLE ");
} else {
@@ -231,12 +235,22 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
protected void printSelectList(List<SQLSelectItem> selectList) {
incrementIndent();
for (int i = 0, size = selectList.size(); i < size; ++i) {
SQLSelectItem selectItem = selectList.get(i);
if (i != 0) {
print(", ");
printComment(selectList.get(i - 1).getAfterCommentsDirect(), "\n");
println();
}
selectList.get(i).accept(this);
selectItem.accept(this);
if (i == selectList.size() - 1 && selectItem.hasAfterComment()) {
print(' ');
printComment(selectItem.getAfterCommentsDirect(), "\n");
}
}
decrementIndent();
}
@@ -374,6 +388,11 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
@Override
public boolean visit(OdpsSelectQueryBlock x) {
if (isPrettyFormat() && x.hasBeforeComment()) {
printComment(x.getBeforeCommentsDirect(), "\n");
println();
}
print("SELECT ");
if (SQLSetQuantifier.ALL == x.getDistionOption()) {
@@ -397,6 +416,10 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
print("WHERE ");
x.getWhere().setParent(x);
x.getWhere().accept(this);
if (x.getWhere().hasAfterComment() && isPrettyFormat()) {
print(' ');
printComment(x.getWhere().getAfterCommentsDirect(), "\n");
}
}
if (x.getGroupBy() != null) {
src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleLexer.java
+6
View File
@@ -231,6 +231,9 @@ public class OracleLexer extends Lexer {
stringVal = subString(mark, bufPos);
token = Token.MULTI_LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
}
if (token != Token.HINT && !isAllowComment()) {
@@ -274,6 +277,9 @@ public class OracleLexer extends Lexer {
stringVal = subString(mark + 1, bufPos);
token = Token.LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
endOfComment = isEOF();
return;
}
src/main/java/com/alibaba/druid/sql/dialect/sqlserver/parser/SQLServerLexer.java
+6
View File
@@ -106,6 +106,9 @@ public class SQLServerLexer extends Lexer {
stringVal = subString(mark, bufPos);
token = Token.MULTI_LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
}
if (token != Token.HINT && !isAllowComment() && !isSafeComment(stringVal)) {
@@ -145,6 +148,9 @@ public class SQLServerLexer extends Lexer {
stringVal = subString(mark + 1, bufPos);
token = Token.LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
endOfComment = isEOF();
if (!isAllowComment() && (isEOF() || !isSafeComment(stringVal))) {
src/main/java/com/alibaba/druid/sql/parser/Lexer.java
+50 -3
View File
@@ -56,6 +56,8 @@ public class Lexer {
protected Keywords keywods = Keywords.DEFAULT_KEYWORDS;
protected String stringVal;
protected String commentVal;
protected boolean skipComment = true;
@@ -73,9 +75,24 @@ public class Lexer {
protected boolean hasComment = false;
protected boolean endOfComment = false;
protected boolean keepComments = false;
public Lexer(String input){
this(input, null);
}
public Lexer(String input, CommentHandler commentHandler){
this(input, true);
this.commentHandler = commentHandler;
}
public boolean isKeepComments() {
return keepComments;
}
public void setKeepComments(boolean keepComments) {
this.keepComments = keepComments;
}
public CommentHandler getCommentHandler() {
@@ -270,6 +287,7 @@ public class Lexer {
public final void nextToken() {
bufPos = 0;
hasComment = false;
for (;;) {
if (isWhitespace(ch)) {
@@ -759,6 +777,8 @@ public class Lexer {
}
private void scanMultiLineComment() {
Token lastToken = this.token;
scanChar();
scanChar();
mark = pos;
@@ -782,12 +802,25 @@ public class Lexer {
stringVal = subString(mark, bufPos);
token = Token.MULTI_LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
}
if (!isAllowComment() && !isSafeComment(stringVal)) {
throw new NotAllowCommentException();
}
}
private void scanSingleLineComment() {
Token lastToken = this.token;
scanChar();
scanChar();
mark = pos;
mark = pos - 1;
bufPos = 0;
for (;;) {
@@ -817,6 +850,17 @@ public class Lexer {
stringVal = subString(mark, bufPos);
token = Token.LINE_COMMENT;
hasComment = true;
if (keepComments) {
commentVal = stringVal;
}
if (commentHandler != null && commentHandler.handle(lastToken, stringVal)) {
return;
}
if (!isAllowComment() && !isSafeComment(stringVal)) {
throw new NotAllowCommentException();
}
}
public void scanIdentifier() {
@@ -972,6 +1016,10 @@ public class Lexer {
public final String stringVal() {
return stringVal;
}
public final String commentVal() {
return commentVal;
}
private boolean isOperator(char ch) {
switch (ch) {
@@ -1080,11 +1128,10 @@ public class Lexer {
}
public static interface CommentHandler {
boolean handle(Token lastToken, String comment);
}
public boolean isHasComment() {
public boolean hasComment() {
return hasComment;
}
src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
+10 -1
View File
@@ -1179,6 +1179,10 @@ public class SQLExprParser extends SQLParser {
public SQLExpr andRest(SQLExpr expr) {
for (;;) {
if (lexer.token() == Token.AND || lexer.token() == Token.AMPAMP) {
if (lexer.isKeepComments() && lexer.hasComment()) {
expr.addAfterComment(lexer.commentVal());
}
lexer.nextToken();
SQLExpr rightExp = relational();
@@ -1682,7 +1686,12 @@ public class SQLExprParser extends SQLParser {
item.setValue(new SQLIdentifierExpr(lexer.stringVal()));
lexer.nextToken();
} else {
item.setValue(expr());
if (lexer.token() == Token.ALL) {
item.setValue(new SQLIdentifierExpr(lexer.stringVal()));
lexer.nextToken();
} else {
item.setValue(expr());
}
}
return item;
src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java
+22 -4
View File
@@ -158,6 +158,12 @@ public class SQLSelectParser extends SQLParser {
return queryRest(select);
}
SQLSelectQueryBlock queryBlock = new SQLSelectQueryBlock();
if (lexer.hasComment() && lexer.isKeepComments()) {
queryBlock.addBeforeComment(lexer.commentVal());
}
accept(Token.SELECT);
@@ -165,8 +171,6 @@ public class SQLSelectParser extends SQLParser {
lexer.nextToken();
}
SQLSelectQueryBlock queryBlock = new SQLSelectQueryBlock();
if (lexer.token() == Token.DISTINCT) {
queryBlock.setDistionOption(SQLSetQuantifier.DISTINCT);
lexer.nextToken();
@@ -234,7 +238,15 @@ public class SQLSelectParser extends SQLParser {
if (lexer.token() == Token.WHERE) {
lexer.nextToken();
queryBlock.setWhere(expr());
SQLExpr where = expr();
if (lexer.hasComment() && lexer.isKeepComments() //
&& lexer.token() != Token.INSERT // odps multi-insert
) {
where.addAfterComment(lexer.commentVal());
}
queryBlock.setWhere(where);
}
}
@@ -326,7 +338,13 @@ public class SQLSelectParser extends SQLParser {
parseTableSourceQueryTableExpr(tableReference);
return parseTableSourceRest(tableReference);
SQLTableSource tableSrc = parseTableSourceRest(tableReference);
if (lexer.hasComment() && lexer.isKeepComments()) {
tableSrc.addAfterComment(lexer.commentVal());
}
return tableSrc;
}
private void parseTableSourceQueryTableExpr(SQLExprTableSource tableReference) {
src/main/java/com/alibaba/druid/sql/parser/SQLStatementParser.java
+66 -8
View File
@@ -88,12 +88,16 @@ import com.alibaba.druid.sql.dialect.odps.parser.OdpsSelectParser;
public class SQLStatementParser extends SQLParser {
protected SQLExprParser exprParser;
protected SQLExprParser exprParser;
protected boolean parseCompleteValues = true;
protected int parseValuesSize = 3;
protected boolean parseCompleteValues = true;
protected int parseValuesSize = 3;
protected List<String> comments;
protected CommentCallback commentCallBack = new CommentCallback();
public SQLStatementParser(String sql){
this(sql, null);
}
@@ -101,11 +105,23 @@ public class SQLStatementParser extends SQLParser {
public SQLStatementParser(String sql, String dbType){
this(new SQLExprParser(sql, dbType));
}
public SQLStatementParser(SQLExprParser exprParser){
super(exprParser.getLexer(), exprParser.getDbType());
this.exprParser = exprParser;
}
protected SQLStatementParser(Lexer lexer, String dbType){
super(lexer, dbType);
}
public boolean isKeepComments() {
return lexer.isKeepComments();
}
public void setKeepComments(boolean keepComments) {
this.lexer.setKeepComments(keepComments);
}
public SQLExprParser getExprParser() {
return exprParser;
@@ -122,13 +138,30 @@ public class SQLStatementParser extends SQLParser {
}
public void parseStatementList(List<SQLStatement> statementList, int max) {
List<String> comments = null;
if (this.comments != null) {
comments = this.comments;
this.comments = null;
}
int size_original = statementList.size();
for (;;) {
boolean isFirstStmt = statementList.size() == size_original + 1;
if (isFirstStmt && comments != null && lexer.isKeepComments()) {
SQLStatement firstStmt = statementList.get(size_original);
for (String comment : comments) {
firstStmt.addBeforeComment(comment);
}
comments = null;
}
if (max != -1) {
if (statementList.size() >= max) {
return;
}
}
if (lexer.token() == Token.EOF) {
return;
}
@@ -136,8 +169,15 @@ public class SQLStatementParser extends SQLParser {
return;
}
if (lexer.token() == (Token.SEMI)) {
if (lexer.token() == Token.SEMI) {
size_original = statementList.size();
this.comments = null;
lexer.nextToken();
comments = this.comments;
if (lexer.isKeepComments()) {
SQLStatement stmt = statementList.get(statementList.size() - 1);
stmt.getAttributes().put("format.semi", Boolean.TRUE);
}
continue;
}
@@ -354,7 +394,7 @@ public class SQLStatementParser extends SQLParser {
public SQLStatement parseShow() {
throw new ParserException("TODO " + lexer.token() + " " + lexer.stringVal());
}
public SQLUseStatement parseUse() {
accept(Token.USE);
SQLUseStatement stmt = new SQLUseStatement(getDbType());
@@ -1781,4 +1821,22 @@ public class SQLStatementParser extends SQLParser {
public void setParseValuesSize(int parseValuesSize) {
this.parseValuesSize = parseValuesSize;
}
class CommentCallback implements Lexer.CommentHandler {
@Override
public boolean handle(Token lastToken, String comment) {
addComment(comment);
return true;
}
}
void addComment(String comment) {
if (comments == null) {
comments = new ArrayList<String>();
}
comments.add(comment);
}
}
src/main/java/com/alibaba/druid/sql/visitor/ParameterizedOutputVisitorUtils.java
+1 -1
View File
@@ -66,7 +66,7 @@ public class ParameterizedOutputVisitorUtils {
stmt.accept(visitor);
}
if (visitor.getReplaceCount() == 0 && !parser.getLexer().isHasComment()) {
if (visitor.getReplaceCount() == 0 && !parser.getLexer().hasComment()) {
return sql;
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java
+42 -1
View File
@@ -136,6 +136,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSetStatement;
import com.alibaba.druid.sql.ast.statement.SQLShowTablesStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableElement;
import com.alibaba.druid.sql.ast.statement.SQLTruncateStatement;
@@ -349,6 +350,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Printab
visitBinaryLeft(item, x.getOperator());
if (relational) {
if (isPrettyFormat() && item.hasAfterComment()) {
print(' ');
printComment(item.getAfterCommentsDirect(), "\n");
}
println();
} else {
print(" ");
@@ -683,6 +688,10 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Printab
}
public boolean visit(SQLSelectQueryBlock x) {
if (isPrettyFormat() && x.hasBeforeComment()) {
printComment(x.getBeforeCommentsDirect(), "\n");
}
print("SELECT ");
if (SQLSetQuantifier.ALL == x.getDistionOption()) {
@@ -762,11 +771,16 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Printab
public boolean visit(SQLExprTableSource x) {
x.getExpr().accept(this);
if (x.getAlias() != null) {
print(' ');
print(x.getAlias());
}
if (isPrettyFormat() && x.hasAfterComment()) {
print(' ');
printComment(x.getAfterCommentsDirect(), "\n");
}
return false;
}
@@ -2166,4 +2180,31 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Printab
x.getTo().accept(this);
return false;
}
@Override
public boolean visit(SQLShowTablesStatement x) {
print("SHOW TABLES");
if (x.getDatabase() != null) {
print(" FROM ");
x.getDatabase().accept(this);
}
if (x.getLike() != null) {
print(" LIKE ");
x.getLike().accept(this);
}
return false;
}
protected void printComment(List<String> comments, String seperator) {
if (comments != null) {
for (int i = 0; i < comments.size(); ++i) {
if (i != 0) {
print(seperator);
}
String comment = comments.get(i);
print(comment);
}
}
}
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java
+5
View File
@@ -113,6 +113,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSetStatement;
import com.alibaba.druid.sql.ast.statement.SQLShowTablesStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTruncateStatement;
import com.alibaba.druid.sql.ast.statement.SQLUnionQuery;
@@ -569,4 +570,8 @@ public interface SQLASTVisitor {
boolean visit(SQLAlterTableRename x);
void endVisit(SQLShowTablesStatement x);
boolean visit(SQLShowTablesStatement x);
}
src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java
+11
View File
@@ -115,6 +115,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLSetStatement;
import com.alibaba.druid.sql.ast.statement.SQLShowTablesStatement;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTruncateStatement;
import com.alibaba.druid.sql.ast.statement.SQLUnionQuery;
@@ -1115,4 +1116,14 @@ public class SQLASTVisitorAdapter implements SQLASTVisitor {
return true;
}
@Override
public void endVisit(SQLShowTablesStatement x) {
}
@Override
public boolean visit(SQLShowTablesStatement x) {
return true;
}
}
src/test/java/com/alibaba/druid/bvt/sql/mysql/MySqlFormatTest.java
+2 -1
View File
@@ -29,6 +29,7 @@ public class MySqlFormatTest extends TestCase {
"\ta INT, \n" + //
"\tb CHAR(20), \n" + //
"\tINDEX(a)\n" + //
")", SQLUtils.format(text, JdbcUtils.MYSQL));
")" //
+ "\n;", SQLUtils.format(text, JdbcUtils.MYSQL));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsFormatCommentTest.java
+34
View File
@@ -0,0 +1,34 @@
package com.alibaba.druid.bvt.sql.odps;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import junit.framework.TestCase;
public class OdpsFormatCommentTest extends TestCase {
public void test_column_comment() throws Exception {
String sql = "select f1 -- aaa"
+ "\n from t1";
Assert.assertEquals("SELECT f1 -- aaa" //
+ "\nFROM t1", SQLUtils.formatOdps(sql));
}
public void test_column_2_comment() throws Exception {
String sql = "select f1 -- aaa"
+ "\n, f2 -- bbb"
+ "\n from t1";
Assert.assertEquals("SELECT f1, -- aaa" //
+ "\n\tf2 -- bbb" //
+ "\nFROM t1", SQLUtils.formatOdps(sql));
}
public void test_column_2_multi_comment() throws Exception {
String sql = "select f1 /*aa*/"
+ "\n, f2 -- bbb"
+ "\n from t1";
Assert.assertEquals("SELECT f1, /*aa*/" //
+ "\n\tf2 -- bbb" //
+ "\nFROM t1", SQLUtils.formatOdps(sql));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsFormatCommentTest2.java
+27
View File
@@ -0,0 +1,27 @@
package com.alibaba.druid.bvt.sql.odps;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import junit.framework.TestCase;
public class OdpsFormatCommentTest2 extends TestCase {
public void test_column_comment() throws Exception {
String sql = "--[Subject -]" //
+ "\n--[Author -高铁/035139]"//
+ "\n--[Created -2015-06-10 13:19:18]"//
+ "\n--[Update ]"//
+ "\nset odps.sql.mapper.split.size=2048;"//
+ "\nselect * from dual;";
Assert.assertEquals("--[Subject -]"
+ "\n--[Author -高铁/035139]"
+ "\n--[Created -2015-06-10 13:19:18]"
+ "\n--[Update ]"
+ "\nSET odps.sql.mapper.split.size = 2048;"
+ "\nSELECT *"
+ "\nFROM dual"
+ "\n;", SQLUtils.formatOdps(sql));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsFormatCommentTest3.java
+30
View File
@@ -0,0 +1,30 @@
package com.alibaba.druid.bvt.sql.odps;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import junit.framework.TestCase;
public class OdpsFormatCommentTest3 extends TestCase {
public void test_column_comment() throws Exception {
String sql = "-- 使用服务模式" //
+ "\nset odps.service.mode=all;" //
+ "\n-- 使用新引擎"//
+ "\nset odps.nvm.enabled=true;"//
+ "\nselect f1 -- aa"//
+ "\nfrom t0;"//
+ "\nselect f2 -- aa"//
+ "\nfrom t1;";//
Assert.assertEquals("-- 使用服务模式"//
+ "\nSET odps.service.mode = all;"//
+ "\n-- 使用新引擎"//
+ "\nSET odps.nvm.enabled = true;"//
+ "\nSELECT f1 -- aa"//
+ "\nFROM t0;"//
+ "\nSELECT f2 -- aa"//
+ "\nFROM t1"
+ "\n;", SQLUtils.formatOdps(sql));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsFormatCommentTest4.java
+40
View File
@@ -0,0 +1,40 @@
package com.alibaba.druid.bvt.sql.odps;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import junit.framework.TestCase;
public class OdpsFormatCommentTest4 extends TestCase {
public void test_column_comment() throws Exception {
String sql = "from (select * from xxx) a"
+ "\n -- comment_0" //
+ "\ninsert overwrite table b partition(ds='20150711')" //
+ "\nselect f0, f1, f2" //
+ "\nwhere name rlike 'xxxx'"
+ "\n -- comment_1" //
+ "\ninsert overwrite table c partition(ds='20150711')" //
+ "\nselect f0, f1, f2" //
+ "\nwhere name rlike 'kk';"
;//
Assert.assertEquals("FROM (" //
+ "\n\tSELECT *"//
+ "\n\tFROM xxx"//
+ "\n) a" //
+ "\n-- comment_0"//
+ "\nINSERT OVERWRITE TABLE b PARTITION (ds='20150711')"//
+ "\nSELECT f0, "//
+ "\n\tf1, "//
+ "\n\tf2"//
+ "\nWHERE name RLIKE 'xxxx'" //
+ "\n-- comment_1"//
+ "\nINSERT OVERWRITE TABLE c PARTITION (ds='20150711')"//
+ "\nSELECT f0, "//
+ "\n\tf1, "//
+ "\n\tf2"//
+ "\nWHERE name RLIKE 'kk'"
+ "\n;", SQLUtils.formatOdps(sql));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsFormatCommentTest5.java
+48
View File
@@ -0,0 +1,48 @@
package com.alibaba.druid.bvt.sql.odps;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import junit.framework.TestCase;
public class OdpsFormatCommentTest5 extends TestCase {
public void test_column_comment() throws Exception {
String sql = "select *" //
+ "\nfrom t -- xxxx"//
+ "\nwhere id > 0;"
;//
Assert.assertEquals("SELECT *" //
+ "\nFROM t -- xxxx" //
+ "\nWHERE id > 0"
+ "\n;", SQLUtils.formatOdps(sql));
}
public void test_column_comment_as() throws Exception {
String sql = "select *" //
+ "\nfrom xxxx a-- xxxx"//
+ "\nwhere id > 0;"
;//
Assert.assertEquals("SELECT *" //
+ "\nFROM xxxx a -- xxxx" //
+ "\nWHERE id > 0"
+ "\n;", SQLUtils.formatOdps(sql));
}
public void test_column_comment_subquery() throws Exception {
String sql = "select *" //
+ "\nfrom (" //
+ "\n-- comment_xxx" //
+ "\nselect * from t" //
+ "\n) a;"
;//
Assert.assertEquals("SELECT *" //
+ "\nFROM (" //
+ "\n\t-- comment_xxx" //
+ "\n\tSELECT *" //
+ "\n\tFROM t" //
+ "\n) a"
+ "\n;", SQLUtils.formatOdps(sql));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsFormatCommentTest6.java
+25
View File
@@ -0,0 +1,25 @@
package com.alibaba.druid.bvt.sql.odps;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import junit.framework.TestCase;
public class OdpsFormatCommentTest6 extends TestCase {
public void test_column_comment() throws Exception {
String sql = "select *" //
+ "\nfrom t "//
+ "\nwhere status = '20' -- comment xxx"
+ "\nand flag & 127 > 0 -- comment kkkkk"
+ "\n;"
;//
Assert.assertEquals("SELECT *"
+ "\nFROM t"
+ "\nWHERE status = '20' -- comment xxx"
+ "\n\tAND flag & 127 > 0 -- comment kkkkk"
+ "\n;", SQLUtils.formatOdps(sql));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsFormatTest.java
+26
View File
@@ -0,0 +1,26 @@
package com.alibaba.druid.bvt.sql.odps;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import junit.framework.TestCase;
public class OdpsFormatTest extends TestCase {
public void test_format() throws Exception {
String sql = "select * from t1; ;select * from t2;";
Assert.assertEquals("SELECT *" //
+ "\nFROM t1;" //
+ "\nSELECT *" //
+ "\nFROM t2"//
+ "\n;", SQLUtils.formatOdps(sql));
}
public void test_no_semi() throws Exception {
String sql = "select * from t1; ;select * from t2";
Assert.assertEquals("SELECT *" //
+ "\nFROM t1;" //
+ "\nSELECT *" //
+ "\nFROM t2", SQLUtils.formatOdps(sql));
}
}
src/test/java/com/alibaba/druid/bvt/sql/odps/OdpsShowTablesTest0.java
+40
View File
@@ -0,0 +1,40 @@
package com.alibaba.druid.bvt.sql.odps;
import junit.framework.TestCase;
import org.junit.Assert;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.odps.parser.OdpsStatementParser;
import com.alibaba.druid.sql.parser.Token;
public class OdpsShowTablesTest0 extends TestCase {
public void test_0() throws Exception {
String sql = "show tables";
OdpsStatementParser parser = new OdpsStatementParser(sql);
SQLStatement stmt = parser.parseStatementList().get(0);
parser.match(Token.EOF);
String output = SQLUtils.toOdpsString(stmt);
Assert.assertEquals("SHOW TABLES", output);
}
public void test_from() throws Exception {
String sql = "show tables from xx";
OdpsStatementParser parser = new OdpsStatementParser(sql);
SQLStatement stmt = parser.parseStatementList().get(0);
parser.match(Token.EOF);
String output = SQLUtils.toOdpsString(stmt);
Assert.assertEquals("SHOW TABLES FROM xx", output);
}
public void test_from_like() throws Exception {
String sql = "show tables from xx like '*'";
OdpsStatementParser parser = new OdpsStatementParser(sql);
SQLStatement stmt = parser.parseStatementList().get(0);
parser.match(Token.EOF);
String output = SQLUtils.toOdpsString(stmt);
Assert.assertEquals("SHOW TABLES FROM xx LIKE '*'", output);
}
}
src/test/java/com/alibaba/druid/bvt/sql/postgresql/PGFormatTest.java
+2 -1
View File
@@ -30,6 +30,7 @@ public class PGFormatTest extends TestCase {
"\n\tfooid int," + //
"\n\tfoosubid int," + //
"\n\tfooname text" + //
"\n)", formatedSql);
"\n)"
+ "\n;", formatedSql);
}
}