From ade87e4b8ec167768e76a7bdce284f978e6be066 Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 30 Nov 2017 17:05:19 +0800 Subject: [PATCH] improved hive sql parser support. --- .../statement/SQLCreateTableStatement.java | 12 +++++ .../hive/parser/HiveCreateTableParser.java | 23 ++++++++++ .../hive/visitor/HiveOutputVisitor.java | 31 +++++++++++++ .../visitor/OracleToMySqlOutputVisitor.java | 11 ----- .../ast/stmt/OracleCreateTableStatement.java | 46 ++----------------- .../parser/OracleCreateTableParser.java | 3 +- .../oracle/visitor/OracleASTVisitor.java | 4 -- .../visitor/OracleASTVisitorAdapter.java | 7 --- .../oracle/visitor/OracleOutputVisitor.java | 20 -------- .../visitor/OracleSchemaStatVisitor.java | 10 ---- .../postgresql/visitor/PGOutputVisitor.java | 23 ---------- .../sql/visitor/SQLASTOutputVisitor.java | 16 +++++++ .../druid/sql/visitor/SQLASTVisitor.java | 3 ++ .../sql/visitor/SQLASTVisitorAdapter.java | 8 ++++ .../druid/sql/visitor/SchemaStatVisitor.java | 5 ++ .../java/com/alibaba/druid/util/FnvHash.java | 1 + 16 files changed, 105 insertions(+), 118 deletions(-) diff --git a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java index 52f99638e5..63bc9a5a19 100644 --- a/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/ast/statement/SQLCreateTableStatement.java @@ -60,6 +60,7 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS protected boolean onCommitDeleteRows; // for hive & odps + protected SQLExternalRecordFormat rowFormat; protected final List partitionColumns = new ArrayList(2); protected final List clusteredBy = new ArrayList(); protected final List sortedBy = new ArrayList(); @@ -1094,4 +1095,15 @@ public class SQLCreateTableStatement extends SQLStatementImpl implements SQLDDLS } this.partitionColumns.add(column); } + + public SQLExternalRecordFormat getRowFormat() { + return rowFormat; + } + + public void setRowFormat(SQLExternalRecordFormat x) { + if (x != null) { + x.setParent(this); + } + this.rowFormat = x; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java index 9aff9dc5b0..e944ecd94c 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/parser/HiveCreateTableParser.java @@ -15,6 +15,7 @@ */ package com.alibaba.druid.sql.dialect.hive.parser; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; @@ -120,6 +121,11 @@ public class HiveCreateTableParser extends SQLCreateTableParser { } } + if (lexer.token() == Token.COMMENT) { + lexer.nextToken(); + SQLExpr comment = this.exprParser.expr(); + stmt.setComment(comment); + } if (lexer.token() == Token.PARTITIONED) { lexer.nextToken(); @@ -159,6 +165,23 @@ public class HiveCreateTableParser extends SQLCreateTableParser { accept(Token.RPAREN); } + if (lexer.token() == Token.ROW) { + lexer.nextToken(); + acceptIdentifier("FORMAT"); + + if (lexer.identifierEquals(FnvHash.Constants.DELIMITED)) { + lexer.nextToken(); + acceptIdentifier("FIELDS"); + acceptIdentifier("TERMINATED"); + accept(Token.BY); + SQLExternalRecordFormat format = new SQLExternalRecordFormat(); + format.setTerminatedBy(this.exprParser.expr()); + stmt.setRowFormat(format); + } else { + throw new ParserException("TODO " + lexer.info()); + } + } + if (lexer.identifierEquals(FnvHash.Constants.SORTED)) { lexer.nextToken(); accept(Token.BY); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java index 170bed8ff2..5d2ee3e63f 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/hive/visitor/HiveOutputVisitor.java @@ -15,6 +15,7 @@ */ package com.alibaba.druid.sql.dialect.hive.visitor; +import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.statement.*; import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement; @@ -69,6 +70,13 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis print(')'); } + SQLExpr comment = x.getComment(); + if (comment != null) { + println(); + print0(ucase ? "COMMENT " : "comment "); + comment.accept(this); + } + int partitionSize = x.getPartitionColumns().size(); if (partitionSize > 0) { println(); @@ -104,6 +112,13 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis print(')'); } + SQLExternalRecordFormat format = x.getRowFormat(); + if (format != null) { + println(); + print0(ucase ? "ROW FORMAT DELIMITED " : "row format delimited "); + visit(format); + } + List sortedBy = x.getSortedBy(); if (sortedBy.size() > 0) { println(); @@ -141,4 +156,20 @@ public class HiveOutputVisitor extends SQLASTOutputVisitor implements HiveASTVis public void endVisit(HiveCreateTableStatement x) { } + + public boolean visit(SQLExternalRecordFormat x) { + if (x.getDelimitedBy() != null) { + println(); + print0(ucase ? "LINES TERMINATED BY " : "lines terminated by "); + x.getDelimitedBy().accept(this); + } + + if (x.getTerminatedBy() != null) { + println(); + print0(ucase ? "FIELDS TERMINATED BY " : "fields terminated by "); + x.getTerminatedBy().accept(this); + } + + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/OracleToMySqlOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/OracleToMySqlOutputVisitor.java index ec3fa6a946..d49d3fe0bc 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/OracleToMySqlOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/mysql/visitor/OracleToMySqlOutputVisitor.java @@ -15,7 +15,6 @@ */ package com.alibaba.druid.sql.dialect.mysql.visitor; -import com.alibaba.druid.sql.ast.statement.SQLAlterProcedureStatement; import com.alibaba.druid.sql.dialect.oracle.ast.OracleDataTypeIntervalDay; import com.alibaba.druid.sql.dialect.oracle.ast.OracleDataTypeIntervalYear; import com.alibaba.druid.sql.dialect.oracle.ast.clause.*; @@ -926,16 +925,6 @@ public class OracleToMySqlOutputVisitor extends MySqlOutputVisitor implements Or } - @Override - public boolean visit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - return false; - } - - @Override - public void endVisit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - - } - @Override public boolean visit(OracleCreateTableStatement.OIDIndex x) { return false; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateTableStatement.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateTableStatement.java index fce5b7f3f0..419f3bf051 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateTableStatement.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/ast/stmt/OracleCreateTableStatement.java @@ -18,11 +18,9 @@ package com.alibaba.druid.sql.dialect.oracle.ast.stmt; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLName; import com.alibaba.druid.sql.ast.SQLObject; -import com.alibaba.druid.sql.ast.SQLPartitionBy; -import com.alibaba.druid.sql.ast.expr.SQLNumericLiteralExpr; +import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat; import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; import com.alibaba.druid.sql.dialect.oracle.ast.OracleSQLObject; -import com.alibaba.druid.sql.dialect.oracle.ast.OracleSQLObjectImpl; import com.alibaba.druid.sql.dialect.oracle.ast.OracleSegmentAttributes; import com.alibaba.druid.sql.dialect.oracle.ast.OracleSegmentAttributesImpl; import com.alibaba.druid.sql.dialect.oracle.ast.clause.OracleLobStorageClause; @@ -310,7 +308,7 @@ public class OracleCreateTableStatement extends SQLCreateTableStatement implemen private SQLName externalType; private SQLExpr externalDirectory; - private OracleExternalRecordFormat externalDirectoryRecordFormat; + private SQLExternalRecordFormat externalDirectoryRecordFormat; private List externalDirectoryLocation = new ArrayList(); private SQLExpr externalRejectLimit; @@ -350,11 +348,11 @@ public class OracleCreateTableStatement extends SQLCreateTableStatement implemen this.externalDirectory = externalDirectory; } - public OracleExternalRecordFormat getExternalDirectoryRecordFormat() { + public SQLExternalRecordFormat getExternalDirectoryRecordFormat() { return externalDirectoryRecordFormat; } - public void setExternalDirectoryRecordFormat(OracleExternalRecordFormat recordFormat) { + public void setExternalDirectoryRecordFormat(SQLExternalRecordFormat recordFormat) { if (recordFormat != null) { recordFormat.setParent(this); } @@ -377,42 +375,6 @@ public class OracleCreateTableStatement extends SQLCreateTableStatement implemen } } - public static class OracleExternalRecordFormat extends OracleSQLObjectImpl { - private SQLExpr delimitedBy; - private SQLExpr terminatedBy; - - @Override - public void accept0(OracleASTVisitor visitor) { - if (visitor.visit(this)) { - acceptChild(visitor, delimitedBy); - acceptChild(visitor, terminatedBy); - } - visitor.endVisit(this); - } - - public SQLExpr getDelimitedBy() { - return delimitedBy; - } - - public void setDelimitedBy(SQLExpr delimitedBy) { - if (delimitedBy != null) { - delimitedBy.setParent(this); - } - this.delimitedBy = delimitedBy; - } - - public SQLExpr getTerminatedBy() { - return terminatedBy; - } - - public void setTerminatedBy(SQLExpr terminatedBy) { - if (terminatedBy != null) { - terminatedBy.setParent(this); - } - this.terminatedBy = terminatedBy; - } - } - public static class OIDIndex extends OracleSegmentAttributesImpl implements OracleSQLObject{ private SQLName name; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleCreateTableParser.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleCreateTableParser.java index 92385a83ed..e32bd3447a 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleCreateTableParser.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/parser/OracleCreateTableParser.java @@ -22,6 +22,7 @@ import com.alibaba.druid.sql.ast.SQLPartitionByList; import com.alibaba.druid.sql.ast.SQLPartitionByRange; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLNumericLiteralExpr; +import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat; import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; import com.alibaba.druid.sql.ast.statement.SQLSelect; import com.alibaba.druid.sql.dialect.oracle.ast.clause.OracleLobStorageClause; @@ -266,7 +267,7 @@ public class OracleCreateTableParser extends SQLCreateTableParser { if (lexer.token() == Token.LPAREN) { lexer.nextToken(); - OracleCreateTableStatement.OracleExternalRecordFormat recordFormat = new OracleCreateTableStatement.OracleExternalRecordFormat(); + SQLExternalRecordFormat recordFormat = new SQLExternalRecordFormat(); if (lexer.identifierEquals("RECORDS")) { lexer.nextToken(); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitor.java index a735c69c65..9d63ff68cf 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitor.java @@ -398,10 +398,6 @@ public interface OracleASTVisitor extends SQLASTVisitor { void endVisit(OracleCreateTableStatement.Organization x); - boolean visit(OracleCreateTableStatement.OracleExternalRecordFormat x); - - void endVisit(OracleCreateTableStatement.OracleExternalRecordFormat x); - boolean visit(OracleCreateTableStatement.OIDIndex x); void endVisit(OracleCreateTableStatement.OIDIndex x); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitorAdapter.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitorAdapter.java index f2feb28e53..d77cdb53d6 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitorAdapter.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleASTVisitorAdapter.java @@ -990,13 +990,6 @@ public class OracleASTVisitorAdapter extends SQLASTVisitorAdapter implements Ora } - public boolean visit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - return true; - } - - public void endVisit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - - } public boolean visit(OracleCreateTableStatement.OIDIndex x) { return true; } diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java index c1249953d7..efd91ba632 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleOutputVisitor.java @@ -3020,26 +3020,6 @@ public class OracleOutputVisitor extends SQLASTOutputVisitor implements OracleAS } - public boolean visit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - if (x.getDelimitedBy() != null) { - println(); - print0(ucase ? "RECORDS DELIMITED BY " : "records delimited by "); - x.getDelimitedBy().accept(this); - } - - if (x.getTerminatedBy() != null) { - println(); - print0(ucase ? "FIELDS TERMINATED BY " : "fields terminated by "); - x.getTerminatedBy().accept(this); - } - - return false; - } - - public void endVisit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - - } - public boolean visit(OracleCreateTableStatement.OIDIndex x) { print0(ucase ? "OIDINDEX" : "oidindex"); diff --git a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleSchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleSchemaStatVisitor.java index bd664cc030..f73e0b2128 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleSchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/oracle/visitor/OracleSchemaStatVisitor.java @@ -1223,16 +1223,6 @@ public class OracleSchemaStatVisitor extends SchemaStatVisitor implements Oracle } - @Override - public boolean visit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - return false; - } - - @Override - public void endVisit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - - } - @Override public boolean visit(OracleCreateTableStatement.OIDIndex x) { return false; diff --git a/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java index bd237b8a65..e8301717aa 100644 --- a/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java @@ -46,7 +46,6 @@ import com.alibaba.druid.util.JdbcConstants; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; public class PGOutputVisitor extends SQLASTOutputVisitor implements PGASTVisitor, OracleASTVisitor { @@ -1890,28 +1889,6 @@ public class PGOutputVisitor extends SQLASTOutputVisitor implements PGASTVisitor } - @Override - public boolean visit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - if (x.getDelimitedBy() != null) { - println(); - print0(ucase ? "RECORDS DELIMITED BY " : "records delimited by "); - x.getDelimitedBy().accept(this); - } - - if (x.getTerminatedBy() != null) { - println(); - print0(ucase ? "FIELDS TERMINATED BY " : "fields terminated by "); - x.getTerminatedBy().accept(this); - } - - return false; - } - - @Override - public void endVisit(OracleCreateTableStatement.OracleExternalRecordFormat x) { - - } - @Override public boolean visit(OracleCreateTableStatement.OIDIndex x) { print0(ucase ? "OIDINDEX" : "oidindex"); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java index d8aa288294..c214b8602b 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java @@ -6025,4 +6025,20 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet } return false; } + + public boolean visit(SQLExternalRecordFormat x) { + if (x.getDelimitedBy() != null) { + println(); + print0(ucase ? "RECORDS DELIMITED BY " : "records delimited by "); + x.getDelimitedBy().accept(this); + } + + if (x.getTerminatedBy() != null) { + println(); + print0(ucase ? "FIELDS TERMINATED BY " : "fields terminated by "); + x.getTerminatedBy().accept(this); + } + + return false; + } } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java index 50e98ad78a..2dc3a7a7c3 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitor.java @@ -799,4 +799,7 @@ public interface SQLASTVisitor { boolean visit(SQLDropTypeStatement x); void endVisit(SQLDropTypeStatement x); + + boolean visit(SQLExternalRecordFormat x); + void endVisit(SQLExternalRecordFormat x); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java index 9e34197ce2..651cfc5af5 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SQLASTVisitorAdapter.java @@ -1866,6 +1866,14 @@ public class SQLASTVisitorAdapter implements SQLASTVisitor { } + public boolean visit(SQLExternalRecordFormat x) { + return true; + } + + public void endVisit(SQLExternalRecordFormat x) { + + } + public final boolean isEnabled(VisitorFeature feature) { return VisitorFeature.isEnabled(this.features, feature); } diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java index 8ad5d38486..3aabc2a0d1 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java @@ -2416,4 +2416,9 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter { public boolean visit(SQLDropTypeStatement x) { return false; } + + @Override + public boolean visit(SQLExternalRecordFormat x) { + return false; + } } diff --git a/src/main/java/com/alibaba/druid/util/FnvHash.java b/src/main/java/com/alibaba/druid/util/FnvHash.java index aa98fb6584..1e9d2e10e6 100644 --- a/src/main/java/com/alibaba/druid/util/FnvHash.java +++ b/src/main/java/com/alibaba/druid/util/FnvHash.java @@ -621,5 +621,6 @@ public final class FnvHash { long RELEASE = fnv1a_64_lower("RELEASE"); long MERGE = fnv1a_64_lower("MERGE"); long INHERITS = fnv1a_64_lower("INHERITS"); + long DELIMITED = fnv1a_64_lower("DELIMITED"); } }