From b88e34128e430e6107bb72cf1d9732abf98a17f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E9=93=81?= Date: Thu, 15 Jul 2021 13:19:07 +0800 Subject: [PATCH] improved odps support --- .../sql/repository/SchemaRepository.java | 74 ++++++++++++++- .../SchemaResolveVisitorFactory.java | 30 +++++- .../druid/sql/visitor/SchemaStatVisitor.java | 9 +- .../support/opds/udf/ExportInputTables.java | 92 +++++++++++++++++++ .../support/opds/udf/ExportOutputTables.java | 79 ++++++++++++++++ .../SchemaStatTest23_createView.java | 34 +++++++ 6 files changed, 309 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/alibaba/druid/support/opds/udf/ExportInputTables.java create mode 100644 src/main/java/com/alibaba/druid/support/opds/udf/ExportOutputTables.java create mode 100644 src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest23_createView.java diff --git a/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java b/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java index 2f27f0dd87..2c209d536d 100644 --- a/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java +++ b/src/main/java/com/alibaba/druid/sql/repository/SchemaRepository.java @@ -303,6 +303,11 @@ public class SchemaRepository { if (stmtList.size() == 1) { return schemaObject; } + } else if (stmt instanceof SQLCreateViewStatement) { + SchemaObject schemaObject = acceptView((SQLCreateViewStatement) stmt); + if (stmtList.size() == 1) { + return schemaObject; + } } else { accept(stmt); } @@ -574,6 +579,66 @@ public class SchemaRepository { return null; } + public SchemaObject findView(SQLName name) { + if (name instanceof SQLIdentifierExpr) { + return findView( + ((SQLIdentifierExpr) name).getName()); + } + + if (name instanceof SQLPropertyExpr) { + SQLPropertyExpr propertyExpr = (SQLPropertyExpr) name; + SQLExpr owner = propertyExpr.getOwner(); + String schema; + String catalog = null; + if (owner instanceof SQLIdentifierExpr) { + schema = ((SQLIdentifierExpr) owner).getName(); + } else if (owner instanceof SQLPropertyExpr){ + schema = ((SQLPropertyExpr) owner).getName(); + catalog = ((SQLPropertyExpr) owner).getOwnernName(); + } else { + return null; + } + + long tableHashCode64 = propertyExpr.nameHashCode64(); + + Schema schemaObj = findSchema(schema, false); + if (schemaObj != null) { + SchemaObject table = schemaObj.findView(tableHashCode64); + if (table != null) { + return table; + } + } + + String ddl = loadDDL(catalog, schema, propertyExpr.getName()); + + if (ddl == null) { + schemaObj = findSchema(schema, true); + } else { + List stmtList = SQLUtils.parseStatements(ddl, schemaDbType); + for (SQLStatement stmt : stmtList) { + accept(stmt); + } + + if (stmtList.size() == 1) { + SQLStatement stmt = stmtList.get(0); + if (stmt instanceof SQLCreateTableStatement) { + SQLCreateTableStatement createStmt = (SQLCreateTableStatement) stmt; + String schemaName = createStmt.getSchema(); + schemaObj = findSchema(schemaName, true); + } + } + } + + if (schemaObj == null) { + return null; + } + + return schemaObj.findView(tableHashCode64); + } + + return null; + } + private boolean renameTable(SQLName name, SQLName to) { Schema schema; if (name instanceof SQLPropertyExpr) { @@ -937,7 +1002,7 @@ public class SchemaRepository { return true; } - boolean acceptView(SQLCreateViewStatement x) { + SchemaObject acceptView(SQLCreateViewStatement x) { String schemaName = x.getSchema(); Schema schema = findSchema(schemaName, true); @@ -945,12 +1010,13 @@ public class SchemaRepository { String name = x.computeName(); SchemaObject view = schema.findTableOrView(name); if (view != null) { - return false; + return view; } SchemaObject object = new SchemaObject(schema, name, SchemaObjectType.View, x.clone()); - schema.objects.put(object.nameHashCode64(), object); - return true; + long nameHashCode64 = FnvHash.hashCode64(name); + schema.objects.put(nameHashCode64, object); + return object; } boolean acceptView(SQLAlterViewStatement x) { diff --git a/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java b/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java index a4209fa7c4..03d96a032f 100644 --- a/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java +++ b/src/main/java/com/alibaba/druid/sql/repository/SchemaResolveVisitorFactory.java @@ -44,6 +44,7 @@ import com.alibaba.druid.sql.dialect.sqlserver.ast.SQLServerSelectQueryBlock; import com.alibaba.druid.sql.dialect.sqlserver.ast.stmt.SQLServerInsertStatement; import com.alibaba.druid.sql.dialect.sqlserver.ast.stmt.SQLServerUpdateStatement; import com.alibaba.druid.sql.dialect.sqlserver.visitor.SQLServerASTVisitorAdapter; +import com.alibaba.druid.sql.visitor.SQLASTVisitor; import com.alibaba.druid.sql.visitor.SQLASTVisitorAdapter; import com.alibaba.druid.util.FnvHash; import com.alibaba.druid.util.PGUtils; @@ -1910,12 +1911,24 @@ class SchemaResolveVisitorFactory { } } } + return; + } + + SchemaObject view = repository.findView((SQLName) expr); + if (view != null) { + x.setSchemaObject(view); + return; } } + return; + } - } else if (expr instanceof SQLMethodInvokeExpr) { + if (expr instanceof SQLMethodInvokeExpr) { visitor.visit((SQLMethodInvokeExpr) expr); - } else if (expr instanceof SQLQueryExpr) { + return; + } + + if (expr instanceof SQLQueryExpr) { SQLSelect select = ((SQLQueryExpr) expr) .getSubQuery(); @@ -1933,9 +1946,10 @@ class SchemaResolveVisitorFactory { } } //if (queryBlock.findColumn()) - } else { - expr.accept(visitor); + return; } + + expr.accept(visitor); } static void resolve(SchemaResolveVisitor visitor, SQLAlterTableStatement x) { @@ -2224,6 +2238,14 @@ class SchemaResolveVisitorFactory { return true; } } + + SchemaRepository repo = visitor.getRepository(); + if (repo != null) { + SchemaObject view = repo.findView(x); + if (view != null && view.getStatement() instanceof SQLCreateViewStatement) { + x.setResolvedOwnerObject(view.getStatement()); + } + } } SQLDeclareItem declareItem = parentCtx.findDeclare(hash); diff --git a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java index dc670c0059..97d68390c5 100644 --- a/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java +++ b/src/main/java/com/alibaba/druid/sql/visitor/SchemaStatVisitor.java @@ -1924,6 +1924,12 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter { if (isSimpleExprTableSource(x)) { TableStat stat = getTableStatWithUnwrap(expr); if (stat == null) { + if (expr instanceof SQLIdentifierExpr + && ((SQLIdentifierExpr) expr).getResolvedOwnerObject() instanceof SQLCreateViewStatement) { + SQLCreateViewStatement createView = (SQLCreateViewStatement) ((SQLIdentifierExpr) expr).getResolvedOwnerObject(); + createView.getSubQuery().accept(this); + } + return false; } @@ -1969,7 +1975,8 @@ public class SchemaStatVisitor extends SQLASTVisitorAdapter { SQLObject resolvedOwnerObject = identifierExpr.getResolvedOwnerObject(); if (resolvedOwnerObject instanceof SQLSubqueryTableSource - || resolvedOwnerObject instanceof SQLWithSubqueryClause.Entry) { + || resolvedOwnerObject instanceof SQLWithSubqueryClause.Entry + || resolvedOwnerObject instanceof SQLCreateViewStatement) { return true; } diff --git a/src/main/java/com/alibaba/druid/support/opds/udf/ExportInputTables.java b/src/main/java/com/alibaba/druid/support/opds/udf/ExportInputTables.java new file mode 100644 index 0000000000..b713853ea8 --- /dev/null +++ b/src/main/java/com/alibaba/druid/support/opds/udf/ExportInputTables.java @@ -0,0 +1,92 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.support.opds.udf; + +import com.alibaba.druid.DbType; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.statement.*; +import com.alibaba.druid.sql.repository.SchemaRepository; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.stat.TableStat; +import com.aliyun.odps.udf.UDF; + +import java.util.List; +import java.util.Map; + +public class ExportInputTables extends UDF { + + public String evaluate(String sql) throws Throwable { + return evaluate(sql, null, false); + } + + public String evaluate(String sql, String dbTypeName) throws Throwable { + return evaluate(sql, dbTypeName, false); + } + + public String evaluate(String sql, String dbTypeName, boolean throwError) throws Throwable { + DbType dbType = dbTypeName == null ? null : DbType.valueOf(dbTypeName); + + Throwable error = null; + try { + SchemaRepository repo = new SchemaRepository(dbType); + List statementList = SQLUtils.parseStatements(sql, dbType); + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(repo, dbType); + + for (SQLStatement stmt : statementList) { + if (stmt instanceof SQLCreateViewStatement) { + repo.accept(stmt); + } else if (stmt instanceof SQLCreateTableStatement) { + repo.accept(stmt); + } else if (stmt instanceof SQLDropTableStatement) { + repo.accept(stmt); + } else if (stmt instanceof SQLDropViewStatement) { + repo.accept(stmt); + } + + stmt.accept(visitor); + } + + StringBuffer buf = new StringBuffer(); + + for (Map.Entry entry : visitor.getTables().entrySet()) { + TableStat.Name name = entry.getKey(); + + TableStat stat = entry.getValue(); + if (stat.getSelectCount() == 0) { + continue; + } + + if (buf.length() != 0) { + buf.append(','); + } + buf.append(name.toString()); + } + + return buf.toString(); + } catch (Exception ignored) { + error = ignored; + } catch (StackOverflowError ignored) { + error = ignored; + } + + if (throwError && error != null) { + throw error; + } + + return null; + } +} diff --git a/src/main/java/com/alibaba/druid/support/opds/udf/ExportOutputTables.java b/src/main/java/com/alibaba/druid/support/opds/udf/ExportOutputTables.java new file mode 100644 index 0000000000..8d064b89f0 --- /dev/null +++ b/src/main/java/com/alibaba/druid/support/opds/udf/ExportOutputTables.java @@ -0,0 +1,79 @@ +/* + * Copyright 1999-2017 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.druid.support.opds.udf; + +import com.alibaba.druid.DbType; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.stat.TableStat; +import com.aliyun.odps.udf.UDF; + +import java.util.List; +import java.util.Map; + +public class ExportOutputTables extends UDF { + + public String evaluate(String sql) throws Throwable { + return evaluate(sql, null, false); + } + + public String evaluate(String sql, String dbTypeName) throws Throwable { + return evaluate(sql, dbTypeName, false); + } + + public String evaluate(String sql, String dbTypeName, boolean throwError) throws Throwable { + DbType dbType = dbTypeName == null ? null : DbType.valueOf(dbTypeName); + + Throwable error = null; + try { + List statementList = SQLUtils.parseStatements(sql, dbType); + SchemaStatVisitor visitor = SQLUtils.createSchemaStatVisitor(dbType); + + for (SQLStatement stmt : statementList) { + stmt.accept(visitor); + } + + StringBuffer buf = new StringBuffer(); + + for (Map.Entry entry : visitor.getTables().entrySet()) { + TableStat.Name name = entry.getKey(); + + TableStat stat = entry.getValue(); + if (stat.getInsertCount() == 0) { + continue; + } + + if (buf.length() != 0) { + buf.append(','); + } + buf.append(name.toString()); + } + + return buf.toString(); + } catch (Exception ignored) { + // skip + } catch (StackOverflowError ignored) { + // skip + } + + if (throwError && error != null) { + throw error; + } + + return null; + } +} diff --git a/src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest23_createView.java b/src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest23_createView.java new file mode 100644 index 0000000000..9eac16dd35 --- /dev/null +++ b/src/test/java/com/alibaba/druid/bvt/sql/schemaStat/SchemaStatTest23_createView.java @@ -0,0 +1,34 @@ +package com.alibaba.druid.bvt.sql.schemaStat; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition; +import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.parser.SQLParserUtils; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.alibaba.druid.sql.repository.SchemaRepository; +import com.alibaba.druid.sql.visitor.SchemaStatVisitor; +import com.alibaba.druid.support.opds.udf.ExportInputTables; +import com.alibaba.druid.util.JdbcConstants; +import junit.framework.TestCase; + +public class SchemaStatTest23_createView extends TestCase { + + public void test_schemaStat() throws Throwable { + SchemaRepository repository = new SchemaRepository(JdbcConstants.ODPS); + + String sql = "DROP VIEW IF EXISTS v1;\n" + + "CREATE VIEW v1 \n" + + "AS \n" + + "SELECT * FROM s1.t1;\n" + + "select * from v1;"; +// sql = "SELECT * FROM ( SELECT * FROM DEPT) X WHERE X.dname='cs'"; + + String inputs = new ExportInputTables() + .evaluate(sql, "odps"); + assertEquals("s1.t1", inputs); + } +}