improved sql parser.

This commit is contained in:
wenshao
2017-10-02 02:15:21 +08:00
parent 630d7fbc7c
commit 27c10df625
18 changed files with 165 additions and 114 deletions
@@ -18,13 +18,17 @@ package com.alibaba.druid.sql.ast.expr;
import com.alibaba.druid.sql.ast.SQLExprImpl;
import com.alibaba.druid.sql.ast.SQLObject;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
import com.alibaba.druid.util.Utils;
import java.math.BigInteger;
import java.util.Collections;
import java.util.List;
public class SQLBinaryExpr extends SQLExprImpl implements SQLLiteralExpr, SQLValuableExpr {
private String value;
private String text;
private transient Number val;
public SQLBinaryExpr(){
@@ -32,15 +36,46 @@ public class SQLBinaryExpr extends SQLExprImpl implements SQLLiteralExpr, SQLVal
public SQLBinaryExpr(String value){
super();
this.value = value;
this.text = value;
}
public String getValue() {
return value;
public String getText() {
return text;
}
public Number getValue() {
if (text == null) {
return null;
}
if (val == null) {
long[] words = new long[text.length() / 64 + 1];
for (int i = text.length() - 1; i >= 0; --i) {
char ch = text.charAt(i);
if (ch == '1') {
int wordIndex = i >> 6;
words[wordIndex] |= (1L << (text.length() - 1 - i));
}
}
if (words.length == 1) {
val = words[0];
} else {
byte[] bytes = new byte[words.length * 8];
for (int i = 0; i < words.length; ++i) {
Utils.putLong(bytes, (words.length - 1 - i) * 8, words[i]);
}
val = new BigInteger(bytes);
}
}
return val;
}
public void setValue(String value) {
this.value = value;
this.text = value;
}
public void accept0(SQLASTVisitor visitor) {
@@ -51,7 +86,7 @@ public class SQLBinaryExpr extends SQLExprImpl implements SQLLiteralExpr, SQLVal
public void output(StringBuffer buf) {
buf.append("b'");
buf.append(value);
buf.append(text);
buf.append('\'');
}
@@ -59,12 +94,12 @@ public class SQLBinaryExpr extends SQLExprImpl implements SQLLiteralExpr, SQLVal
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((value == null) ? 0 : value.hashCode());
result = prime * result + ((text == null) ? 0 : text.hashCode());
return result;
}
public SQLBinaryExpr clone() {
return new SQLBinaryExpr(value);
return new SQLBinaryExpr(text);
}
@Override
@@ -84,11 +119,11 @@ public class SQLBinaryExpr extends SQLExprImpl implements SQLLiteralExpr, SQLVal
return false;
}
SQLBinaryExpr other = (SQLBinaryExpr) obj;
if (value == null) {
if (other.value != null) {
if (text == null) {
if (other.text != null) {
return false;
}
} else if (!value.equals(other.value)) {
} else if (!text.equals(other.text)) {
return false;
}
return true;
@@ -13,19 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.dialect.odps.ast;
package com.alibaba.druid.sql.ast.statement;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLName;
import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.ast.statement.SQLTableSourceImpl;
import com.alibaba.druid.sql.dialect.odps.visitor.OdpsASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class OdpsLateralViewTableSource extends SQLTableSourceImpl {
public class SQLLateralViewTableSource extends SQLTableSourceImpl {
private SQLTableSource tableSource;
@@ -35,10 +32,6 @@ public class OdpsLateralViewTableSource extends SQLTableSourceImpl {
@Override
protected void accept0(SQLASTVisitor visitor) {
accept0((OdpsASTVisitor) visitor);
}
protected void accept0(OdpsASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, tableSource);
acceptChild(visitor, method);
@@ -24,7 +24,7 @@ import com.alibaba.druid.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem;
import com.alibaba.druid.sql.ast.statement.SQLSelectQuery;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsLateralViewTableSource;
import com.alibaba.druid.sql.ast.statement.SQLLateralViewTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsSelectQueryBlock;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsValuesTableSource;
import com.alibaba.druid.sql.parser.SQLExprParser;
@@ -167,41 +167,4 @@ public class OdpsSelectParser extends SQLSelectParser {
return super.parseTableSource();
}
protected SQLTableSource parseTableSourceRest(SQLTableSource tableSource) {
tableSource = super.parseTableSourceRest(tableSource);
if ("LATERAL".equalsIgnoreCase(tableSource.getAlias()) && lexer.token() == Token.VIEW) {
return parseLateralView(tableSource);
}
if (lexer.identifierEquals("LATERAL")) {
lexer.nextToken();
return parseLateralView(tableSource);
}
return tableSource;
}
protected SQLTableSource parseLateralView(SQLTableSource tableSource) {
accept(Token.VIEW);
if ("LATERAL".equalsIgnoreCase(tableSource.getAlias())) {
tableSource.setAlias(null);
}
OdpsLateralViewTableSource lateralViewTabSrc = new OdpsLateralViewTableSource();
lateralViewTabSrc.setTableSource(tableSource);
SQLMethodInvokeExpr udtf = (SQLMethodInvokeExpr) this.exprParser.expr();
lateralViewTabSrc.setMethod(udtf);
String alias = as();
lateralViewTabSrc.setAlias(alias);
accept(Token.AS);
this.exprParser.names(lateralViewTabSrc.getColumns());
return parseTableSourceRest(lateralViewTabSrc);
}
}
@@ -15,6 +15,7 @@
*/
package com.alibaba.druid.sql.dialect.odps.visitor;
import com.alibaba.druid.sql.ast.statement.SQLLateralViewTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.*;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
@@ -103,10 +104,6 @@ public interface OdpsASTVisitor extends SQLASTVisitor {
void endVisit(OdpsGrantStmt x);
boolean visit(OdpsGrantStmt x);
void endVisit(OdpsLateralViewTableSource x);
boolean visit(OdpsLateralViewTableSource x);
void endVisit(OdpsValuesTableSource x);
@@ -231,16 +231,6 @@ public class OdpsASTVisitorAdapter extends SQLASTVisitorAdapter implements OdpsA
return true;
}
@Override
public void endVisit(OdpsLateralViewTableSource x) {
}
@Override
public boolean visit(OdpsLateralViewTableSource x) {
return true;
}
@Override
public void endVisit(OdpsValuesTableSource x) {
@@ -798,26 +798,6 @@ public class OdpsOutputVisitor extends SQLASTOutputVisitor implements OdpsASTVis
return false;
}
@Override
public void endVisit(OdpsLateralViewTableSource x) {
}
@Override
public boolean visit(OdpsLateralViewTableSource x) {
x.getTableSource().accept(this);
this.indentCount++;
println();
print0(ucase ? "LATERAL VIEW " : "lateral view ");
x.getMethod().accept(this);
print(' ');
print0(x.getAlias());
print0(ucase ? " AS " : " as ");
printAndAccept(x.getColumns(), ", ");
this.indentCount--;
return false;
}
public boolean visit(SQLCharExpr x) {
String text = x.getText();
@@ -23,8 +23,6 @@ import com.alibaba.druid.sql.visitor.SchemaStatVisitor;
import com.alibaba.druid.stat.TableStat;
import com.alibaba.druid.util.JdbcConstants;
import java.util.Map;
public class OdpsSchemaStatVisitor extends SchemaStatVisitor implements OdpsASTVisitor {
public OdpsSchemaStatVisitor() {
@@ -265,16 +263,6 @@ public class OdpsSchemaStatVisitor extends SchemaStatVisitor implements OdpsASTV
return super.visit((SQLGrantStatement) x);
}
@Override
public void endVisit(OdpsLateralViewTableSource x) {
}
@Override
public boolean visit(OdpsLateralViewTableSource x) {
return true;
}
@Override
public void endVisit(OdpsValuesTableSource x) {
@@ -557,11 +557,11 @@ public class PGOutputVisitor extends SQLASTOutputVisitor implements PGASTVisitor
public void endVisit(PGLineSegmentsExpr x) {
}
@Override
public boolean visit(SQLBinaryExpr x) {
print0(ucase ? "B'" : "b'");
print0(x.getValue());
print0(x.getText());
print('\'');
return false;
@@ -738,6 +738,16 @@ public class SQLSelectParser extends SQLParser {
return parseTableSourceRest(join);
}
if (tableSource.aliasHashCode64() == FnvHash.Constants.LATERAL
&& lexer.token() == Token.VIEW) {
return parseLateralView(tableSource);
}
if (lexer.identifierEquals(FnvHash.Constants.LATERAL)) {
lexer.nextToken();
return parseLateralView(tableSource);
}
return tableSource;
}
@@ -846,4 +856,25 @@ public class SQLSelectParser extends SQLParser {
queryBlock.setConnectBy(this.exprParser.expr());
}
}
protected SQLTableSource parseLateralView(SQLTableSource tableSource) {
accept(Token.VIEW);
if ("LATERAL".equalsIgnoreCase(tableSource.getAlias())) {
tableSource.setAlias(null);
}
SQLLateralViewTableSource lateralViewTabSrc = new SQLLateralViewTableSource();
lateralViewTabSrc.setTableSource(tableSource);
SQLMethodInvokeExpr udtf = (SQLMethodInvokeExpr) this.exprParser.expr();
lateralViewTabSrc.setMethod(udtf);
String alias = as();
lateralViewTabSrc.setAlias(alias);
accept(Token.AS);
this.exprParser.names(lateralViewTabSrc.getColumns());
return parseTableSourceRest(lateralViewTabSrc);
}
}
@@ -1609,11 +1609,11 @@ class SchemaResolveVisitorFactory {
}
} else if (ctxTable instanceof SQLSubqueryTableSource) {
tableSource = ctxTable.findTableSourceWithColumn(hash);
} else if (ctxTable instanceof OdpsLateralViewTableSource) {
} else if (ctxTable instanceof SQLLateralViewTableSource) {
tableSource = ctxTable.findTableSourceWithColumn(hash);
if (tableSource == null) {
tableSource = ((OdpsLateralViewTableSource) ctxTable).getTableSource();
tableSource = ((SQLLateralViewTableSource) ctxTable).getTableSource();
}
} else {
for (SchemaResolveVisitor.Context parentCtx = ctx;
@@ -3918,7 +3918,7 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
@Override
public boolean visit(SQLBinaryExpr x) {
print0("b'");
print0(x.getValue());
print0(x.getText());
print('\'');
return false;
@@ -5420,4 +5420,20 @@ public class SQLASTOutputVisitor extends SQLASTVisitorAdapter implements Paramet
return false;
}
///////////// for odps & hive
@Override
public boolean visit(SQLLateralViewTableSource x) {
x.getTableSource().accept(this);
this.indentCount++;
println();
print0(ucase ? "LATERAL VIEW " : "lateral view ");
x.getMethod().accept(this);
print(' ');
print0(x.getAlias());
print0(ucase ? " AS " : " as ");
printAndAccept(x.getColumns(), ", ");
this.indentCount--;
return false;
}
}
@@ -762,4 +762,8 @@ public interface SQLASTVisitor {
boolean visit(SQLIntervalExpr x);
void endVisit(SQLIntervalExpr x);
boolean visit(SQLLateralViewTableSource x);
void endVisit(SQLLateralViewTableSource x);
}
@@ -1749,6 +1749,16 @@ public class SQLASTVisitorAdapter implements SQLASTVisitor {
}
@Override
public boolean visit(SQLLateralViewTableSource x) {
return true;
}
@Override
public void endVisit(SQLLateralViewTableSource x) {
}
public final boolean isEnabled(VisitorFeature feature) {
return VisitorFeature.isEnabled(this.features, feature);
}
@@ -581,7 +581,7 @@ public class SQLEvalVisitorUtils {
}
public static boolean visit(SQLEvalVisitor visitor, SQLBinaryExpr x) {
String text = x.getValue();
String text = x.getText();
long[] words = new long[text.length() / 64 + 1];
for (int i = text.length()-1; i >= 0 ; --i) {
@@ -30,7 +30,7 @@ import com.alibaba.druid.sql.ast.expr.*;
import com.alibaba.druid.sql.ast.statement.*;
import com.alibaba.druid.sql.dialect.mysql.ast.expr.MySqlExpr;
import com.alibaba.druid.sql.dialect.mysql.visitor.MySqlASTVisitorAdapter;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsLateralViewTableSource;
import com.alibaba.druid.sql.ast.statement.SQLLateralViewTableSource;
import com.alibaba.druid.sql.dialect.odps.ast.OdpsValuesTableSource;
import com.alibaba.druid.sql.dialect.oracle.ast.expr.OracleDbLinkExpr;
import com.alibaba.druid.sql.dialect.oracle.ast.expr.OracleExpr;
@@ -44,7 +44,6 @@ import com.alibaba.druid.stat.TableStat.Mode;
import com.alibaba.druid.stat.TableStat.Relationship;
import com.alibaba.druid.util.FnvHash;
import com.alibaba.druid.util.JdbcConstants;
import com.alibaba.druid.util.StringUtils;
public class SchemaStatVisitor extends SQLASTVisitorAdapter {
@@ -1106,7 +1105,7 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter {
return false;
} else if (tableSource instanceof SQLUnionQueryTableSource) {
return false;
} else if (tableSource instanceof OdpsLateralViewTableSource) {
} else if (tableSource instanceof SQLLateralViewTableSource) {
return false;
} else {
if (x.getResolvedProcudure() != null) {
@@ -1237,7 +1236,7 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter {
}
} else if (tableSource instanceof SQLWithSubqueryClause.Entry
|| tableSource instanceof SQLSubqueryTableSource
|| tableSource instanceof OdpsLateralViewTableSource) {
|| tableSource instanceof SQLLateralViewTableSource) {
return false;
} else {
boolean skip = false;
@@ -508,5 +508,7 @@ public final class FnvHash {
long UNSUPPORTED = fnv1a_64_lower("UNSUPPORTED");
long USERENV = fnv1a_64_lower("USERENV");
long NUMTODSINTERVAL = fnv1a_64_lower("NUMTODSINTERVAL");
long LATERAL = fnv1a_64_lower("LATERAL");
}
}