improved hive sql parser

This commit is contained in:
高铁
2019-10-29 01:15:14 +08:00
parent 1e8cc71c50
commit 1a4aea6c6e
29 changed files with 2005 additions and 110 deletions
@@ -26,6 +26,7 @@ import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLTableElement;
import com.alibaba.druid.sql.dialect.hive.stmt.HiveCreateTableStatement;
import com.alibaba.druid.sql.parser.Lexer;
import com.alibaba.druid.sql.parser.ParserException;
import com.alibaba.druid.sql.parser.SQLCreateTableParser;
import com.alibaba.druid.sql.parser.SQLExprParser;
@@ -38,6 +39,10 @@ public class HiveCreateTableParser extends SQLCreateTableParser {
super(exprParser);
}
public HiveCreateTableParser(Lexer lexer) {
super(new HiveExprParser(lexer));
}
public SQLCreateTableStatement parseCreateTable(boolean acceptCreate) {
HiveCreateTableStatement stmt = newCreateStatement();
@@ -1,5 +1,5 @@
/*
* Copyright 1999-2018 Alibaba Group Holding Ltd.
* Copyright 1999-2017 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,24 +15,25 @@
*/
package com.alibaba.druid.sql.dialect.hive.parser;
import java.util.Arrays;
import com.alibaba.druid.sql.ast.SQLCurrentTimeExpr;
import com.alibaba.druid.sql.ast.SQLCurrentUserExpr;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.expr.SQLArrayExpr;
import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.druid.sql.parser.Lexer;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.SQLParserFeature;
import com.alibaba.druid.sql.parser.Token;
import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
import com.alibaba.druid.sql.ast.statement.SQLExternalRecordFormat;
import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlOutFileExpr;
import com.alibaba.druid.sql.parser.*;
import com.alibaba.druid.util.FnvHash;
import java.util.Arrays;
public class HiveExprParser extends SQLExprParser {
private final static String[] AGGREGATE_FUNCTIONS;
private final static long[] AGGREGATE_FUNCTIONS_CODES;
static {
String[] strings = { "AVG", "COUNT", "MAX", "MIN", "STDDEV", "SUM", "ROW_NUMBER",
"ROWNUMBER" };
"ROWNUMBER" };
AGGREGATE_FUNCTIONS_CODES = FnvHash.fnv1a_64_lower(strings, true);
AGGREGATE_FUNCTIONS = new String[AGGREGATE_FUNCTIONS_CODES.length];
@@ -60,19 +61,229 @@ public class HiveExprParser extends SQLExprParser {
}
public SQLExpr primaryRest(SQLExpr expr) {
if (lexer.token() == Token.COLONCOLON) {
lexer.nextToken();
String propertyName = lexer.stringVal();
lexer.nextToken();
expr = new SQLPropertyExpr(expr, propertyName);
} else if (lexer.token() == Token.LBRACKET) {
SQLArrayExpr array = new SQLArrayExpr();
array.setExpr(expr);
lexer.nextToken();
this.exprList(array.getValues(), array);
accept(Token.RBRACKET);
expr = array;
// if(lexer.token() == Token.COLON) {
// lexer.nextToken();
// expr = dotRest(expr);
// return expr;
// }
switch (lexer.token()) {
case LBRACKET:
SQLArrayExpr array = new SQLArrayExpr();
array.setExpr(expr);
lexer.nextToken();
this.exprList(array.getValues(), array);
accept(Token.RBRACKET);
return primaryRest(array);
case LITERAL_CHARS:
if (expr instanceof SQLCharExpr) {
String text2 = ((SQLCharExpr) expr).getText();
do {
String chars = lexer.stringVal();
text2 += chars;
lexer.nextToken();
} while (lexer.token() == Token.LITERAL_CHARS || lexer.token() == Token.LITERAL_ALIAS);
expr = new SQLCharExpr(text2);
}
break;
case IDENTIFIER:
if (lexer.identifierEquals(FnvHash.Constants.BD) && expr instanceof SQLNumericLiteralExpr) {
lexer.nextToken();
Number num = ((SQLNumericLiteralExpr) expr).getNumber();
expr = new SQLDecimalExpr(num.toString());
}
break;
default:
break;
}
return super.primaryRest(expr);
}
public SQLExpr primary() {
final Token tok = lexer.token();
switch (tok) {
case IDENTIFIER:
final long hash_lower = lexer.hash_lower();
if (hash_lower == FnvHash.Constants.OUTLINE) {
lexer.nextToken();
SQLExpr file = primary();
SQLExpr expr = new MySqlOutFileExpr(file);
return primaryRest(expr);
}
SQLCurrentTimeExpr currentTimeExpr = null;
if (hash_lower == FnvHash.Constants.CURRENT_TIMESTAMP) {
currentTimeExpr = new SQLCurrentTimeExpr(SQLCurrentTimeExpr.Type.CURRENT_TIMESTAMP);
} else if (hash_lower == FnvHash.Constants.CURRENT_DATE) {
currentTimeExpr = new SQLCurrentTimeExpr(SQLCurrentTimeExpr.Type.CURRENT_DATE);
} else if (hash_lower == FnvHash.Constants.CURRENT_USER && isEnabled(SQLParserFeature.EnableCurrentUserExpr)) {
lexer.nextToken();
return primaryRest(new SQLCurrentUserExpr());
}
if (currentTimeExpr != null) {
String methodName = lexer.stringVal();
lexer.nextToken();
if (lexer.token() == Token.LPAREN) {
lexer.nextToken();
if (lexer.token() == Token.LPAREN) {
lexer.nextToken();
} else {
return primaryRest(
methodRest(new SQLIdentifierExpr(methodName), false)
);
}
}
return primaryRest(currentTimeExpr);
}
default:
break;
}
return super.primary();
}
public SQLExternalRecordFormat parseRowFormat() {
lexer.nextToken();
acceptIdentifier("FORMAT");
if (lexer.identifierEquals(FnvHash.Constants.DELIMITED)) {
lexer.nextToken();
}
SQLExternalRecordFormat format = new SQLExternalRecordFormat();
if (lexer.identifierEquals(FnvHash.Constants.FIELDS)) {
lexer.nextToken();
acceptIdentifier("TERMINATED");
accept(Token.BY);
format.setTerminatedBy(this.expr());
} else if (lexer.identifierEquals("FIELD")) {
throw new ParserException("syntax error, expect FIELDS, " + lexer.info());
}
if (lexer.token() == Token.ESCAPE || lexer.identifierEquals(FnvHash.Constants.ESCAPED)) {
lexer.nextToken();
accept(Token.BY);
format.setEscapedBy(this.expr());
}
if (lexer.identifierEquals(FnvHash.Constants.LINES)) {
lexer.nextToken();
acceptIdentifier("TERMINATED");
accept(Token.BY);
format.setLinesTerminatedBy(this.expr());
}
if (lexer.identifierEquals(FnvHash.Constants.COLLECTION)) {
lexer.nextToken();
acceptIdentifier("ITEMS");
acceptIdentifier("TERMINATED");
accept(Token.BY);
format.setCollectionItemsTerminatedBy(this.expr());
}
if (lexer.identifierEquals(FnvHash.Constants.MAP)) {
lexer.nextToken();
acceptIdentifier("KEYS");
acceptIdentifier("TERMINATED");
accept(Token.BY);
format.setMapKeysTerminatedBy(this.expr());
}
if (lexer.identifierEquals(FnvHash.Constants.SERDE)) {
lexer.nextToken();
format.setSerde(this.expr());
}
return format;
}
protected SQLExpr parseAliasExpr(String alias) {
String chars = alias.substring(1, alias.length() - 1);
StringBuilder buf = null;
for (int i = 0; i < chars.length(); ++i) {
char ch = chars.charAt(i);
if (ch == '\\' && i < chars.length() - 1) {
char next = chars.charAt(i + 1);
if (next == '\\') {
if (buf == null) {
buf = new StringBuilder();
buf.append(chars.substring(0, i));
}
buf.append('\\');
++i;
} else if (next == '\"') {
if (buf == null) {
buf = new StringBuilder();
buf.append(chars.substring(0, i));
}
buf.append('"');
++i;
} else {
if (buf != null) {
buf.append(ch);
}
}
} else {
if (buf != null) {
buf.append(ch);
}
}
}
if (buf != null) {
chars = buf.toString();
}
return new SQLCharExpr(chars);
}
public SQLColumnDefinition parseColumnRest(SQLColumnDefinition column) {
if (lexer.identifierEquals(FnvHash.Constants.MAPPED)) {
lexer.nextToken();
accept(Token.BY);
this.parseAssignItem(column.getMappedBy(), column);
}
if (lexer.identifierEquals(FnvHash.Constants.COLPROPERTIES)) {
lexer.nextToken();
this.parseAssignItem(column.getColProperties(), column);
}
return super.parseColumnRest(column);
}
protected SQLExpr parseInterval() {
accept(Token.INTERVAL);
SQLExpr value = expr();
if (lexer.token() != Token.IDENTIFIER) {
throw new ParserException("Syntax error. " + lexer.info());
}
String unit = lexer.stringVal();
lexer.nextToken();
SQLIntervalExpr intervalExpr = new SQLIntervalExpr();
intervalExpr.setValue(value);
SQLIntervalUnit intervalUnit = SQLIntervalUnit.valueOf(unit.toUpperCase());
if (intervalUnit == SQLIntervalUnit.YEAR
&& lexer.token() == Token.TO) {
lexer.nextToken();
acceptIdentifier("MONTH");
intervalUnit = SQLIntervalUnit.YEAR_TO_MONTH;
}
intervalExpr.setUnit(intervalUnit);
return intervalExpr;
}
}
@@ -17,12 +17,18 @@ package com.alibaba.druid.sql.dialect.hive.parser;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLOrderingSpecification;
import com.alibaba.druid.sql.ast.expr.SQLSizeExpr;
import com.alibaba.druid.sql.ast.statement.SQLExprTableSource;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock;
import com.alibaba.druid.sql.ast.statement.SQLTableSampling;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.parser.Lexer;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.SQLSelectListCache;
import com.alibaba.druid.sql.parser.SQLSelectParser;
import com.alibaba.druid.sql.parser.Token;
import com.alibaba.druid.util.FnvHash;
public class HiveSelectParser extends SQLSelectParser {
@@ -70,4 +76,82 @@ public class HiveSelectParser extends SQLSelectParser {
}
}
protected SQLTableSource parseTableSourceRest(SQLTableSource tableSource) {
if (lexer.identifierEquals(FnvHash.Constants.TABLESAMPLE) && tableSource instanceof SQLExprTableSource) {
Lexer.SavePoint mark = lexer.mark();
lexer.nextToken();
if (lexer.token() == Token.LPAREN) {
lexer.nextToken();
SQLTableSampling sampling = new SQLTableSampling();
if (lexer.identifierEquals(FnvHash.Constants.BUCKET)) {
lexer.nextToken();
SQLExpr bucket = this.exprParser.primary();
sampling.setBucket(bucket);
if (lexer.token() == Token.OUT) {
lexer.nextToken();
accept(Token.OF);
SQLExpr outOf = this.exprParser.primary();
sampling.setOutOf(outOf);
}
if (lexer.token() == Token.ON) {
lexer.nextToken();
SQLExpr on = this.exprParser.expr();
sampling.setOn(on);
}
}
if (lexer.token() == Token.LITERAL_INT || lexer.token() == Token.LITERAL_FLOAT) {
SQLExpr val = this.exprParser.primary();
if (lexer.identifierEquals(FnvHash.Constants.ROWS)) {
lexer.nextToken();
sampling.setRows(val);
} else {
acceptIdentifier("PERCENT");
sampling.setPercent(val);
}
}
if (lexer.token() == Token.IDENTIFIER) {
String strVal = lexer.stringVal();
char first = strVal.charAt(0);
char last = strVal.charAt(strVal.length() - 1);
if (last >= 'a' && last <= 'z') {
last -= 32; // to upper
}
boolean match = false;
if ((first == '.' || (first >= '0' && first <= '9'))) {
switch (last) {
case 'B':
case 'K':
case 'M':
case 'G':
case 'T':
case 'P':
match = true;
break;
default:
break;
}
}
SQLSizeExpr size = new SQLSizeExpr(strVal.substring(0, strVal.length() - 2), last);
sampling.setByteLength(size);
lexer.nextToken();
}
final SQLExprTableSource table = (SQLExprTableSource) tableSource;
table.setSampling(sampling);
accept(Token.RPAREN);
} else {
lexer.reset(mark);
}
}
return super.parseTableSourceRest(tableSource);
}
}
@@ -81,4 +81,13 @@ public class MySqlSubPartitionByKey extends SQLSubPartitionBy implements MySqlOb
cloneTo(x);
return x;
}
public boolean isPartitionByColumn(long columnNameHashCode64) {
for (SQLName column : columns) {
if (column.nameHashCode64() == columnNameHashCode64) {
return true;
}
}
return false;
}
}