diff --git a/README.md b/README.md index 455a0db..e9c0d2d 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,7 @@ ### 本地部署文档详见[ms开发文档](./all/ms/deploy.md) ## 项目概览 - `mybatis-generator-extend-maven-plugin`模块是便于快速根据条件生成mybatis-generator-plugin配置文件的maven扩展插件，基本增 -删改查、按条件分页查询`dao` `servcie` `controller`层利用了词向量、卷积神经网络、使用人工智能技术进行代码语意分析(好吧，我吹不 -下去了)自动生成，自动生成70%代码，降低开发中重复性耗时工作,此插件可在其他项目中单独使用。 +删改查、按条件分页查询`dao` `servcie` `controller`层自动生成，自动生成70%代码，降低开发中重复性耗时工作,此插件可在其他项目中单独使用。 - 合理的拆分，拥有众多的可复用模块，开箱即用 - 高可用注册中心 - 高可用配置配置管理中心，通过`rabbitmq`支持配置刷新 @@ -25,7 +24,7 @@ - 支持docker容器部署 - 使用flume收集系统日志，便于日志分析 - 热点数据使用redis缓存 -- 多数据源支持 +- 多数据库,数据库分表支持 - 完善的项目使用文档 - 用心的开发者 ## 整体技术选型 diff --git a/all/docker/maven/shardbatis-2.0.0B.jar b/all/docker/maven/shardbatis-2.0.0B.jar new file mode 100644 index 0000000..9b5f2cb Binary files /dev/null and b/all/docker/maven/shardbatis-2.0.0B.jar differ diff --git a/all/docker/rabbitmq/rabbitmq-runsh.sh b/all/docker/rabbitmq/rabbitmq-runsh.sh index cafd704..c58a1bb 100644 --- a/all/docker/rabbitmq/rabbitmq-runsh.sh +++ b/all/docker/rabbitmq/rabbitmq-runsh.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -docker run -d --name rabbitmq -p 5671:5671 -p 5672:5672 -p 25672:25672 -p 4369:4369 rabbitmq \ No newline at end of file +docker run -d --name rabbitmq -p 5671:5671 -p 5672:5672 -p 25672:25672 -p 4369:4369 -p 15672:15672 -p 15671:15671 rabbitmq:3-management \ No newline at end of file diff --git a/all/quickStartDev/startKafka-kafka.sh b/all/quickStartDev/startKafka-kafka.sh new file mode 100644 index 0000000..1d42b22 --- /dev/null +++ b/all/quickStartDev/startKafka-kafka.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +cd /opt/tools/kafka_2.11-1.1.0 +bin/kafka-server-start.sh config/server.properties \ No newline at end of file diff --git a/all/quickStartDev/startKafka-zookeeper.sh b/all/quickStartDev/startKafka-zookeeper.sh new file mode 100644 index 0000000..fbb1e5e --- /dev/null +++ b/all/quickStartDev/startKafka-zookeeper.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +cd /opt/tools/kafka_2.11-1.1.0 +bin/zookeeper-server-start.sh config/zookeeper.properties + + diff --git a/boot-starter/boot-starter-ssm/README.md b/boot-starter/boot-starter-ssm/README.md index 3e537b4..20d860f 100644 --- a/boot-starter/boot-starter-ssm/README.md +++ b/boot-starter/boot-starter-ssm/README.md @@ -27,4 +27,8 @@ spring-boot提供了默认的启动时执行数据库脚本功能，指定`sprin 更多的数据源，像增加多个druid数据源一样增加`com.micro.fast.common.init.properties.SqlInitProperties`类的配置bean,然后满足上述 两个数据源时同样要求即可。 - +### 启用基于shardbatis的分表功能 +- 通过设置shardbatis.configPath(shardbatis的xml配置文件的classpath路径)属性来激活分表功能．使用`@IdShard`来在mapper接口参数中指定分表策略字段 +或者`@IdShard('filedName')`来指定参数对象中哪个字段是分表策略所需的字段. +- shardbatis.tableCount 指定分表的数量 +- 注意,使用默认的分表策略的时候mapper接口中不允许使用只有一个map传递参数. diff --git a/boot-starter/boot-starter-ssm/pom.xml b/boot-starter/boot-starter-ssm/pom.xml index 0263333..92caa09 100644 --- a/boot-starter/boot-starter-ssm/pom.xml +++ b/boot-starter/boot-starter-ssm/pom.xml @@ -55,6 +55,16 @@ com.github.jsqlparser jsqlparser + + + org.shardbatis + shardbatis + 2.0.0B + + + net.sf.jsqlparser + jsqlparser + mysql diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/config/SqlSessionFactoryConfig.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/config/SqlSessionFactoryConfig.java index b94495a..703d261 100644 --- a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/config/SqlSessionFactoryConfig.java +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/config/SqlSessionFactoryConfig.java @@ -1,6 +1,7 @@ package com.micro.fast.common.config; import com.github.pagehelper.PageHelper; +import com.google.code.shardbatis.plugin.ShardPlugin; import com.micro.fast.common.config.param.MybatisParamConfig; import org.apache.ibatis.plugin.Interceptor; import org.apache.ibatis.session.SqlSessionFactory; @@ -8,12 +9,12 @@ import org.mybatis.spring.SqlSessionFactoryBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver; -import org.springframework.core.io.support.ResourcePropertySource; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.EnableTransactionManagement; @@ -49,6 +50,13 @@ public class SqlSessionFactoryConfig implements TransactionManagementConfigurer{ @Autowired private MybatisParamConfig mybatisParamConfig; + /** + * 分表插件的配置 + */ + @Autowired(required = false) + @Qualifier("shardPlugin") + private ShardPlugin shardPlugin; + /** * mybatis连接工厂bean * @return sqlSessionFactory @@ -97,7 +105,12 @@ public class SqlSessionFactoryConfig implements TransactionManagementConfigurer{ } } pageHelper.setProperties(properties); - sqlSessionFactoryBean.setPlugins(new Interceptor[]{pageHelper}); + // 选择性启用shardbatisf分表插件 + if (shardPlugin == null){ + sqlSessionFactoryBean.setPlugins(new Interceptor[]{pageHelper}); + } else { + sqlSessionFactoryBean.setPlugins(new Interceptor[]{pageHelper,shardPlugin}); + } //设置详细的配置 org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); //开启下划线转驼峰命名 diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/IdShard.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/IdShard.java new file mode 100644 index 0000000..19b9f92 --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/IdShard.java @@ -0,0 +1,13 @@ +package com.micro.fast.common.shardbatis; + +import java.lang.annotation.*; + +/** + * 指定分表依据的参数的注解，分表的参数上必须有这个注解 + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.PARAMETER) +@Documented +public @interface IdShard{ + String value() default "id"; +} diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/IdShardUtils.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/IdShardUtils.java new file mode 100644 index 0000000..027a9de --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/IdShardUtils.java @@ -0,0 +1,158 @@ +package com.micro.fast.common.shardbatis; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.Map; + +/** + * 根据idshard注解判断哪个参数是分表的依据，以及根据反射获取分表的依据参数的值 + * + * @author lsy + */ +@Slf4j +public class IdShardUtils { + + public static final String PARAM = "param"; + + /** + * 工具类私有构造防止被实例化 + */ + private IdShardUtils() { + + } + + /** + * 根据statementId反射出方法，查找方法中带有IdShard朱姐标记的参数，获取其值用于分表策略 + * + * @param params + * @param statementId + * @return + */ + public static Object getShardIdValue(Object params, String statementId) throws Exception { + int lastIndexOfD = statementId.lastIndexOf("."); + String methodName = statementId.substring(lastIndexOfD + 1); + String className = statementId.substring(0, lastIndexOfD); + Class aClass = null; + try { + aClass = Class.forName(className); + Method[] methods = aClass.getMethods(); + for (Method method : methods) { + String name = method.getName(); + int parameterIndex = 0; + // 方法名相同就找到了语句 + if (name.equals(methodName)) { + Parameter[] parameters = method.getParameters(); + for (Parameter parameter : parameters) { + parameterIndex++; + // 参数是否包含IdShard.class注解 + Annotation[] annotations = parameter.getAnnotations(); + for (Annotation annotation : annotations) { + // 找到带有分表标记符号的参数 + if (annotation.annotationType() == IdShard.class) { + Class type = parameter.getType(); + // statement是不是只有一个参数，有一个的参数的话参数类型应该相同.但是这一个参数不能是map类型 + if (params.getClass() == type) { + return getValueByFieldName((IdShard) annotation,params); + } else { + // 如果有多个参数的话，mybatis会把参数封装成map类型,key是param+参数的顺序 + String key = PARAM + (parameterIndex + 1); + Map map = (HashMap)params; + Object o = map.get(key); + return getValueByFieldName((IdShard) annotation,o); + } + } + } + + } + } + } + } catch (ClassNotFoundException e) { + log.info("sql语句对应的类不存在:{}", e); + } + // 找不到就返回null + return null; + } + + /** + * 根据注解和mybatis的参数 + * @param idShard + * @param params + * @return + * @throws Exception + */ + private static Object getValueByFieldName(IdShard idShard, Object params) throws Exception { + // 如果是基础类型（表明是主键）就直接返回 + if (isBasicType(params)) { + return params; + } else { + // 如果不是基础类型，就使用IdShard的value属性指定对象的哪个属性是分表策略需要的值 + String shardFieldName = idShard.value(); + if (StringUtils.isBlank(shardFieldName)) { + throw new Exception("the shardFieldName was not annotated"); + } + Field field = null; + try { + // 根据字段名反射获取字段的类类型 + field = params.getClass().getDeclaredField(shardFieldName); + } catch (NoSuchFieldException e) { + field = params.getClass().getSuperclass().getDeclaredField(shardFieldName); + } + // 设置属性的访问权限 + field.setAccessible(true); + // 获取对象中属性的值 + return field.get(params); + } + } + + /** + * 　检查参数是不是基础类型 + * + * @param param + * @return + */ + private static boolean isBasicType(Object param) { + if (param == null) { + return false; + } + if (param instanceof String) { + return true; + } + if (param instanceof BigDecimal) { + return true; + } + if (param instanceof Integer) { + return true; + } + if (param instanceof Long) { + return true; + } + if (param instanceof Double) { + return true; + } + if (param instanceof Float) { + return true; + } + if (param instanceof Character) { + return true; + } + if (param instanceof Byte) { + return true; + } + if (param instanceof Short) { + return true; + } + if (param instanceof Boolean) { + return true; + } + return false; + } + + +} diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/config/ShardbatisConfig.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/config/ShardbatisConfig.java new file mode 100644 index 0000000..7b8d01c --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/config/ShardbatisConfig.java @@ -0,0 +1,32 @@ +package com.micro.fast.common.shardbatis.config; + +import com.google.code.shardbatis.plugin.ShardPlugin; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Properties; + +/** + * shardbatis的配置 + * @author lsy + */ +@Configuration +public class ShardbatisConfig { + + /** + * shardPlugin的配置类,当这个配置属性不存在的时候分表插件不生效 + * @param xmlClassPath shard + * @return + */ + @Bean(name = "shardPlugin") + @ConditionalOnProperty(prefix = "shardbatis",name = "configPath",matchIfMissing = false) + public ShardPlugin shardPlugin(@Value("${shardbatis.configPath}") String xmlClassPath){ + ShardPlugin shardPlugin = new ShardPlugin(); + Properties properties = new Properties(); + properties.setProperty(ShardPlugin.SHARDING_CONFIG,xmlClassPath); + shardPlugin.setProperties(properties); + return shardPlugin; + } +} diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/config/package-info.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/config/package-info.java new file mode 100644 index 0000000..850d1e6 --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/config/package-info.java @@ -0,0 +1,4 @@ +/** + * shardbatis的java config 包 + */ +package com.micro.fast.common.shardbatis.config; \ No newline at end of file diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/package-info.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/package-info.java new file mode 100644 index 0000000..8cffa3d --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/package-info.java @@ -0,0 +1,4 @@ +/** + * 利用shardbatis实现分表 + */ +package com.micro.fast.common.shardbatis; \ No newline at end of file diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/properties/package-info.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/properties/package-info.java new file mode 100644 index 0000000..5fa6208 --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/properties/package-info.java @@ -0,0 +1,4 @@ +/** + * shardbatis插件配置参数 + */ +package com.micro.fast.common.shardbatis.properties; \ No newline at end of file diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/shardstrategy/SsmDefaultShardStrategy.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/shardstrategy/SsmDefaultShardStrategy.java new file mode 100644 index 0000000..966c8e4 --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/shardstrategy/SsmDefaultShardStrategy.java @@ -0,0 +1,66 @@ +package com.micro.fast.common.shardbatis.shardstrategy; + +import com.google.code.shardbatis.strategy.ShardStrategy; +import com.micro.fast.common.config.SqlSessionFactoryConfig; +import com.micro.fast.common.shardbatis.IdShardUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.context.annotation.Configuration; + +/** + * 提供默认的分表策略 + * @author lsy + */ +@Slf4j +@Configuration +@AutoConfigureBefore(SqlSessionFactoryConfig.class) +public class SsmDefaultShardStrategy implements ShardStrategy{ + + + private static Integer tableCount; + + /** + * 分表策略的实现 + * @param baseTableName　逻辑表名 + * @param params mybatis语句的参数 + * @param statementId　语句的id，一般是类名.方法名 + * @return　 + */ + @Override + public String getTargetTableName(String baseTableName, Object params, String statementId) { + StringBuilder shardTableName = new StringBuilder(baseTableName); + shardTableName.append("_"); + try { + Object shardIdValue = IdShardUtils.getShardIdValue(params, statementId); + if (shardIdValue == null){ + shardTableName.append("000"); + }else { + String s = shardIdValue.toString(); + Integer integer = Integer.valueOf(s); + Integer index = integer % tableCount; + if (index<10){ + shardTableName.append("00"+index); + } else if (index<100){ + shardTableName.append("0"+index); + } else { + shardTableName.append(index); + } + } + } catch (Exception e) { + log.info("分表失败:{}",e); + throw new RuntimeException("分表失败"); + } + + return shardTableName.toString(); + } + + /** + * springBoot不允许直接给静态变量赋值,但是可以通过非静态方法给静态变量赋值 + * @param tableCount + */ + @Value("${shardbatis.tableCount}") + public void setTableCount(Integer tableCount){ + SsmDefaultShardStrategy.tableCount = tableCount; + } +} diff --git a/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/shardstrategy/package-info.java b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/shardstrategy/package-info.java new file mode 100644 index 0000000..fc2f9a2 --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/java/com/micro/fast/common/shardbatis/shardstrategy/package-info.java @@ -0,0 +1,4 @@ +/** + *提供默认的分表策略 + */ +package com.micro.fast.common.shardbatis.shardstrategy; \ No newline at end of file diff --git a/boot-starter/boot-starter-ssm/src/main/resources/shard_config.xml b/boot-starter/boot-starter-ssm/src/main/resources/shard_config.xml new file mode 100644 index 0000000..33e185e --- /dev/null +++ b/boot-starter/boot-starter-ssm/src/main/resources/shard_config.xml @@ -0,0 +1,19 @@ + + + + + + + + + xxx.dao.UserMapper.insertSelective + xxx.dao.UserMapper.selectByPrimaryKey + xxx.UserMapper.updateByPrimaryKeySelective + + + + + + diff --git a/pom.xml b/pom.xml index 329f148..1837c47 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,8 @@ true 1.5.7 2.2.1 + 2.0.0B + 0.7.0 @@ -318,6 +320,17 @@ gatling-charts-highcharts ${gatling.version} + + + org.shardbatis + shardbatis + ${shardbatis.version} + + + net.sf.jsqlparser + jsqlparser + ${net.sf.jsqlparser.jsqlparser.version} + @@ -405,11 +418,11 @@ registry.cn-hangzhou.aliyuncs.com - - - - - + + + + +