bug fixed for oracle sql parser

This commit is contained in:
温少
2012-02-27 20:08:38 +08:00
parent 431c684425
commit 9cf9c0d467
17 changed files with 427 additions and 76 deletions
@@ -18,7 +18,7 @@ package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class SQLExprTableSource extends SQLTableSource {
public class SQLExprTableSource extends SQLTableSourceImpl {
private static final long serialVersionUID = 1L;
@@ -18,7 +18,7 @@ package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class SQLJoinTableSource extends SQLTableSource {
public class SQLJoinTableSource extends SQLTableSourceImpl {
private static final long serialVersionUID = 1L;
@@ -17,7 +17,7 @@ package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
public class SQLSubqueryTableSource extends SQLTableSource {
public class SQLSubqueryTableSource extends SQLTableSourceImpl {
private static final long serialVersionUID = 1L;
@@ -1,42 +1,10 @@
/*
* Copyright 1999-2011 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLObjectImpl;
import com.alibaba.druid.sql.ast.SQLObject;
public abstract class SQLTableSource extends SQLObjectImpl {
public interface SQLTableSource extends SQLObject {
private static final long serialVersionUID = 1L;
String getAlias();
protected String alias;
public SQLTableSource(){
}
public SQLTableSource(String alias){
this.alias = alias;
}
public String getAlias() {
return this.alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
void setAlias(String alias);
}
@@ -0,0 +1,42 @@
/*
* Copyright 1999-2011 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.druid.sql.ast.statement;
import com.alibaba.druid.sql.ast.SQLObjectImpl;
public abstract class SQLTableSourceImpl extends SQLObjectImpl implements SQLTableSource {
private static final long serialVersionUID = 1L;
protected String alias;
public SQLTableSourceImpl(){
}
public SQLTableSourceImpl(String alias){
this.alias = alias;
}
public String getAlias() {
return this.alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
}
@@ -1,6 +1,10 @@
package com.alibaba.druid.sql.dialect.oracle.ast.stmt;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.statement.SQLInsertStatement;
import com.alibaba.druid.sql.dialect.oracle.ast.OracleHint;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.OracleErrorLoggingClause;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.OracleReturningClause;
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleASTVisitor;
@@ -10,8 +14,17 @@ public class OracleInsertStatement extends SQLInsertStatement implements OracleS
private static final long serialVersionUID = 1L;
private OracleReturningClause returning;
private OracleReturningClause returning;
private OracleErrorLoggingClause errorLogging;
private List<OracleHint> hints = new ArrayList<OracleHint>();
public List<OracleHint> getHints() {
return hints;
}
public void setHints(List<OracleHint> hints) {
this.hints = hints;
}
public OracleReturningClause getReturning() {
return returning;
@@ -20,6 +20,7 @@ import java.util.List;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.FlashbackQueryClause;
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
@@ -29,6 +30,7 @@ public class OracleSelectJoin extends SQLJoinTableSource implements OracleSelect
private final List<SQLExpr> using = new ArrayList<SQLExpr>();
protected OracleSelectPivotBase pivot;
protected FlashbackQueryClause flashback;
public OracleSelectJoin(String alias){
super(alias);
@@ -38,6 +40,14 @@ public class OracleSelectJoin extends SQLJoinTableSource implements OracleSelect
}
public FlashbackQueryClause getFlashback() {
return flashback;
}
public void setFlashback(FlashbackQueryClause flashback) {
this.flashback = flashback;
}
public OracleSelectPivotBase getPivot() {
return pivot;
}
@@ -57,6 +67,7 @@ public class OracleSelectJoin extends SQLJoinTableSource implements OracleSelect
acceptChild(visitor, this.right);
acceptChild(visitor, this.condition);
acceptChild(visitor, this.using);
acceptChild(visitor, this.flashback);
}
visitor.endVisit(this);
@@ -17,6 +17,7 @@ package com.alibaba.druid.sql.dialect.oracle.ast.stmt;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.FlashbackQueryClause;
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleASTVisitor;
import com.alibaba.druid.sql.visitor.SQLASTVisitor;
@@ -26,10 +27,20 @@ public class OracleSelectSubqueryTableSource extends SQLSubqueryTableSource impl
protected OracleSelectPivotBase pivot;
protected FlashbackQueryClause flashback;
public OracleSelectSubqueryTableSource(){
}
public FlashbackQueryClause getFlashback() {
return flashback;
}
public void setFlashback(FlashbackQueryClause flashback) {
this.flashback = flashback;
}
public OracleSelectSubqueryTableSource(String alias){
super(alias);
}
@@ -58,13 +69,9 @@ public class OracleSelectSubqueryTableSource extends SQLSubqueryTableSource impl
protected void accept0(OracleASTVisitor visitor) {
if (visitor.visit(this)) {
acceptChild(visitor, this.select);
acceptChild(visitor, this.flashback);
}
visitor.endVisit(this);
}
public void output(StringBuffer buf) {
buf.append("(");
this.select.output(buf);
buf.append(")");
}
}
@@ -15,14 +15,16 @@
*/
package com.alibaba.druid.sql.dialect.oracle.ast.stmt;
public interface OracleSelectTableSource {
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.FlashbackQueryClause;
String getAlias();
void setAlias(String alias);
public interface OracleSelectTableSource extends SQLTableSource {
OracleSelectPivotBase getPivot();
void setPivot(OracleSelectPivotBase pivot);
FlashbackQueryClause getFlashback();
void setFlashback(FlashbackQueryClause flashback);
}
@@ -113,6 +113,7 @@ public class OracleLexer extends Lexer {
map.put("UNLIMITED", Token.UNLIMITED);
map.put("RETURNING", Token.RETURNING);
map.put("LIMIT", Token.LIMIT);
map.put("OF", Token.OF);
DEFAULT_ORACLE_KEYWORDS = new Keywords(map);
}
@@ -323,4 +324,5 @@ public class OracleLexer extends Lexer {
token = Token.LITERAL_INT;
}
}
}
@@ -28,6 +28,7 @@ import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.ast.statement.SQLUnionOperator;
import com.alibaba.druid.sql.ast.statement.SQLUnionQuery;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.CycleClause;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.FlashbackQueryClause;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.FlashbackQueryClause.AsOfFlashbackQueryClause;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.FlashbackQueryClause.VersionsFlashbackQueryClause;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.GroupingSetExpr;
@@ -246,6 +247,8 @@ public class OracleSelectParser extends SQLSelectParser {
lexer.nextToken();
}
this.createExprParser().parseHints(queryBlock.getHints());
parseSelectList(queryBlock);
parseFrom(queryBlock);
@@ -666,6 +669,10 @@ public class OracleSelectParser extends SQLSelectParser {
private void parseTableSourceQueryTableExpr(OracleSelectTableReference tableReference) {
tableReference.setExpr(this.createExprParser().expr());
{
FlashbackQueryClause clause = flashback();
tableReference.setFlashback(clause);
}
if (identifierEquals("SAMPLE")) {
lexer.nextToken();
@@ -733,27 +740,7 @@ public class OracleSelectParser extends SQLSelectParser {
tableReference.setPartition(partition);
}
if (lexer.token() == Token.AS) {
lexer.nextToken();
if (identifierEquals("OF")) {
lexer.nextToken();
AsOfFlashbackQueryClause clause = new AsOfFlashbackQueryClause();
if (identifierEquals("SCN")) {
clause.setType(AsOfFlashbackQueryClause.Type.SCN);
lexer.nextToken();
} else {
accept(Token.TIMESTAMP);
clause.setType(AsOfFlashbackQueryClause.Type.TIMESTAMP);
}
clause.setExpr(createExprParser().primary());
tableReference.setFlashback(clause);
} else {
throw new SQLParseException("TODO");
}
} else if (identifierEquals("VERSIONS")) {
if (identifierEquals("VERSIONS")) {
lexer.nextToken();
if (lexer.token() == Token.BETWEEN) {
@@ -784,8 +771,40 @@ public class OracleSelectParser extends SQLSelectParser {
}
protected SQLTableSource parseTableSourceRest(SQLTableSource tableSource) {
if ((tableSource.getAlias() == null) || (tableSource.getAlias().length() == 0)) {
private FlashbackQueryClause flashback() {
if (lexer.token() == Token.AS) {
lexer.nextToken();
}
if (lexer.token() == Token.OF) {
lexer.nextToken();
AsOfFlashbackQueryClause clause = new AsOfFlashbackQueryClause();
if (identifierEquals("SCN")) {
clause.setType(AsOfFlashbackQueryClause.Type.SCN);
lexer.nextToken();
} else {
accept(Token.TIMESTAMP);
clause.setType(AsOfFlashbackQueryClause.Type.TIMESTAMP);
}
clause.setExpr(createExprParser().primary());
return clause;
}
return null;
}
protected SQLTableSource parseTableSourceRest(OracleSelectTableSource tableSource) {
if (lexer.token() == Token.AS) {
lexer.nextToken();
if (lexer.token() == Token.OF) {
tableSource.setFlashback(flashback());
}
tableSource.setAlias(as());
} else if ((tableSource.getAlias() == null) || (tableSource.getAlias().length() == 0)) {
if (lexer.token() != Token.LEFT && lexer.token() != Token.RIGHT && lexer.token() != Token.FULL) {
tableSource.setAlias(as());
}
@@ -15,14 +15,17 @@
*/
package com.alibaba.druid.sql.dialect.oracle.parser;
import java.util.ArrayList;
import java.util.List;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.ast.statement.SQLInsertInto;
import com.alibaba.druid.sql.ast.statement.SQLSelect;
import com.alibaba.druid.sql.ast.statement.SQLSelectStatement;
import com.alibaba.druid.sql.ast.statement.SQLTableSource;
import com.alibaba.druid.sql.ast.statement.SQLUpdateSetItem;
import com.alibaba.druid.sql.dialect.oracle.ast.OracleHint;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.OracleErrorLoggingClause;
import com.alibaba.druid.sql.dialect.oracle.ast.clause.OracleReturningClause;
import com.alibaba.druid.sql.dialect.oracle.ast.stmt.OracleInsertStatement;
@@ -49,6 +52,16 @@ public class OracleStatementParser extends SQLStatementParser {
protected OracleExprParser createExprParser() {
return new OracleExprParser(lexer);
}
protected void parseInsert0_hinits(SQLInsertInto insertStatement) {
if (insertStatement instanceof OracleInsertStatement) {
OracleInsertStatement stmt = (OracleInsertStatement) insertStatement;
this.createExprParser().parseHints(stmt.getHints());
} else {
List<OracleHint> hints = new ArrayList<OracleHint>();
this.createExprParser().parseHints(hints);
}
}
@Override
public void parseStatementList(List<SQLStatement> statementList) throws ParserException {
@@ -70,6 +70,7 @@ public class CharTypes {
// identifierFlags['`'] = true;
identifierFlags['_'] = true;
identifierFlags['$'] = true;
identifierFlags['#'] = true;
}
public static boolean isIdentifierChar(char c) {
@@ -252,7 +252,13 @@ public class Lexer {
scanChar();
token = COLONEQ;
} else {
token = COLON;
if (isDigit(ch)) {
unscan();
scanVariable();
} else {
unscan();
token = COLON;
}
}
return;
case '.':
@@ -570,8 +570,8 @@ public class SQLExprParser extends SQLParser {
if (lexer.token() == Token.DOT) {
lexer.nextToken();
if (lexer.token() != Token.IDENTIFIER) {
throw new ParserException("error");
if (lexer.token() != Token.LITERAL_ALIAS && lexer.token() != Token.IDENTIFIER) {
throw new ParserException("error, " + lexer.token());
}
name = new SQLPropertyExpr(name, lexer.stringVal());
@@ -173,12 +173,18 @@ public class SQLStatementParser extends SQLParser {
parseInsert0(insertStatement, true);
}
protected void parseInsert0_hinits(SQLInsertInto insertStatement) {
}
protected void parseInsert0(SQLInsertInto insertStatement, boolean acceptSubQuery) {
accept(Token.INTO);
SQLName tableName = this.exprParser.name();
insertStatement.setTableName(tableName);
parseInsert0_hinits(insertStatement);
if (lexer.token() == Token.IDENTIFIER) {
insertStatement.setAlias(lexer.stringVal());
lexer.nextToken();
@@ -0,0 +1,261 @@
package com.alibaba.druid.sql;
import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.List;
import junit.framework.Assert;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
import com.alibaba.druid.sql.dialect.oracle.visitor.OracleSchemaStatVisitor;
import com.alibaba.fastjson.JSON;
public class TestTransform extends OracleTest {
private String jdbcUrl;
private String user;
private String password;
private DruidDataSource dataSource;
public void setUp() throws Exception {
jdbcUrl = "jdbc:oracle:thin:@10.20.149.18:1521:emdb";
user = "wardon";
password = "wardon";
dataSource = new DruidDataSource();
dataSource.setInitialSize(1);
dataSource.setUrl(jdbcUrl);
dataSource.setUsername(user);
dataSource.setPassword(password);
}
public void test_transform() throws Exception {
String sql = "SELECT SNAP_DATE, DBNAME, SQL_ID, PIECE, SQL_TEXT" + //
" , COMMAND_TYPE, LAST_SNAP_DATE, DB_PK, SQL_PARSE_RESULT " + //
" FROM db_day_sql_fulltext " + //
" ORDER BY sql_id, piece";
Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql);
Record r = null;
while (rs.next()) {
r = new Record();
Date d1 = rs.getDate(1);
String s2 = rs.getString(2);
String sqlId = rs.getString(3);
r.setSnapshotDate(d1);
r.setDbName(s2);
r.setSqlId(sqlId);
r.setPiece(rs.getInt(4));
r.setSqlText(rs.getString(5));
r.setCommandType(rs.getInt(6));
r.setLastSnapshotDate(rs.getDate(7));
r.setDbPk(rs.getLong(8));
System.out.println(r.getSqlText());
schemaStat(r);
}
rs.close();
stmt.close();
conn.close();
}
public void schemaStat(Record r) {
String sql = r.getSqlText();
OracleStatementParser parser = new OracleStatementParser(sql);
List<SQLStatement> statementList = parser.parseStatementList();
SQLStatement statemen = statementList.get(0);
Assert.assertEquals(1, statementList.size());
System.out.println(output(statementList));
OracleSchemaStatVisitor visitor = new OracleSchemaStatVisitor();
statemen.accept(visitor);
System.out.println("Tables : " + visitor.getTables());
System.out.println("fields : " + visitor.getColumns());
System.out.println("coditions : " + visitor.getConditions());
System.out.println("relationships : " + visitor.getRelationships());
System.out.println();
System.out.println();
System.out.println();
}
public void clearResult() throws Exception {
Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
stmt.execute("DELETE FROM db_day_sql_fulltext");
stmt.close();
conn.close();
}
public void f_test_migrate() throws Exception {
clearResult();
String sql = "SELECT SNAP_DATE, DBNAME, SQL_ID, PIECE, SQL_TEXT" + //
" , COMMAND_TYPE, LAST_SNAP_DATE, DB_PK, SQL_PARSE_RESULT " + //
" FROM db_day_sqltext " + //
" WHERE db_pk = 40 and snap_date = trunc(sysdate) " + //
" ORDER BY sql_id, piece";
Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql);
Record r = null;
String lastSqlId = null;
while (rs.next()) {
Date d1 = rs.getDate(1);
String s2 = rs.getString(2);
String sqlId = rs.getString(3);
if (lastSqlId == null || !lastSqlId.equals(sqlId)) {
if (r != null) {
System.out.println(r.getSqlText());
System.out.println();
insert(r);
}
r = new Record();
r.setSnapshotDate(d1);
r.setDbName(s2);
r.setSqlId(sqlId);
r.setPiece(rs.getInt(4));
r.setSqlText(rs.getString(5));
r.setCommandType(rs.getInt(6));
r.setLastSnapshotDate(rs.getDate(7));
r.setDbPk(rs.getLong(8));
} else {
r.setSqlText(r.getSqlText() + rs.getString(5));
}
lastSqlId = sqlId;
}
rs.close();
stmt.close();
conn.close();
}
public void insert(Record r) throws Exception {
String sql = "INSERT INTO db_day_sql_fulltext " + //
"(SNAP_DATE, DBNAME, SQL_ID, PIECE, SQL_TEXT" + //
", COMMAND_TYPE, LAST_SNAP_DATE, DB_PK, SQL_PARSE_RESULT)" + //
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";
Connection conn = dataSource.getConnection();
PreparedStatement stmt = conn.prepareStatement(sql);
stmt.setDate(1, r.getSnapshotDate());
stmt.setString(2, r.getDbName());
stmt.setString(3, r.getSqlId());
stmt.setInt(4, r.getPiece());
stmt.setString(5, r.getSqlText());
stmt.setInt(6, r.getCommandType());
stmt.setDate(7, r.getLastSnapshotDate());
stmt.setLong(8, r.getDbPk());
stmt.setString(9, r.getSqlText());
stmt.execute();
stmt.close();
conn.close();
}
public static class Record {
private Date snapshotDate;
private String dbName;
private String sqlId;
private String sqlText;
private Integer piece;
private Integer commandType;
private Date lastSnapshotDate;
private Long dbPk;
public Date getSnapshotDate() {
return snapshotDate;
}
public void setSnapshotDate(Date snapshotDate) {
this.snapshotDate = snapshotDate;
}
public String getDbName() {
return dbName;
}
public void setDbName(String dbName) {
this.dbName = dbName;
}
public String getSqlId() {
return sqlId;
}
public void setSqlId(String sqlId) {
this.sqlId = sqlId;
}
public String getSqlText() {
return sqlText;
}
public void setSqlText(String sqlText) {
this.sqlText = sqlText;
}
public Integer getPiece() {
return piece;
}
public void setPiece(Integer piece) {
this.piece = piece;
}
public Integer getCommandType() {
return commandType;
}
public void setCommandType(Integer commandType) {
this.commandType = commandType;
}
public Date getLastSnapshotDate() {
return lastSnapshotDate;
}
public void setLastSnapshotDate(Date lastSnapshotDate) {
this.lastSnapshotDate = lastSnapshotDate;
}
public Long getDbPk() {
return dbPk;
}
public void setDbPk(Long dbPk) {
this.dbPk = dbPk;
}
public String toString() {
return JSON.toJSONString(this);
}
}
}