1.pom文件
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
2.ESClientSpringFactory.class
public class ESClientSpringFactory {
public static int CONNECT_TIMEOUT_MILLIS = 1000;
public static int SOCKET_TIMEOUT_MILLIS = 30000;
public static int CONNECTION_REQUEST_TIMEOUT_MILLIS = 500;
public static int MAX_CONN_PER_ROUTE = 10;
public static int MAX_CONN_TOTAL = 30;
public static String USERNAME = "";
public static String PASSWORD = "";
private static HttpHost[] HTTP_HOST;
private RestClientBuilder builder;
private RestClient restClient;
private RestHighLevelClient restHighLevelClient;
private static ESClientSpringFactory esClientSpringFactory = new ESClientSpringFactory();
private ESClientSpringFactory(){}
public static ESClientSpringFactory build(HttpHost[] httpHostArray, Integer maxConnectNum, Integer maxConnectPerRoute){
HTTP_HOST = httpHostArray;
MAX_CONN_TOTAL = maxConnectNum;
MAX_CONN_PER_ROUTE = maxConnectPerRoute;
return esClientSpringFactory;
}
public static ESClientSpringFactory build(HttpHost[] httpHostArray, Integer maxConnectNum, Integer maxConnectPerRoute, String username, String password){
HTTP_HOST = httpHostArray;
MAX_CONN_TOTAL = maxConnectNum;
MAX_CONN_PER_ROUTE = maxConnectPerRoute;
USERNAME = username;
PASSWORD = password;
return esClientSpringFactory;
}
public static ESClientSpringFactory build(HttpHost[] httpHostArray, Integer connectTimeOut, Integer socketTimeOut, Integer connectionRequestTime, Integer maxConnectNum, Integer maxConnectPerRoute){
HTTP_HOST = httpHostArray;
CONNECT_TIMEOUT_MILLIS = connectTimeOut;
SOCKET_TIMEOUT_MILLIS = socketTimeOut;
CONNECTION_REQUEST_TIMEOUT_MILLIS = connectionRequestTime;
MAX_CONN_TOTAL = maxConnectNum;
MAX_CONN_PER_ROUTE = maxConnectPerRoute;
return esClientSpringFactory;
}
public void init(){
builder = RestClient.builder(HTTP_HOST);
setConnectTimeOutConfig();
setMutiConnectConfig();
restClient = builder.build();
//restHighLevelClient = new RestHighLevelClient(builder);
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
//es账号密码
new UsernamePasswordCredentials(USERNAME, PASSWORD));
restHighLevelClient = new RestHighLevelClient(
builder.setHttpClientConfigCallback((httpClientBuilder) -> {
httpClientBuilder.disableAuthCaching();
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpClientBuilder;
})
.setRequestConfigCallback(builder -> {
builder.setConnectTimeout(CONNECT_TIMEOUT_MILLIS);
return builder;
})
);
log.info("init factory" + Arrays.toString(HTTP_HOST));
}
/**
* 配置连接时间延时
* */
public void setConnectTimeOutConfig(){
builder.setRequestConfigCallback(requestConfigBuilder -> {
requestConfigBuilder.setConnectTimeout(CONNECT_TIMEOUT_MILLIS);
requestConfigBuilder.setSocketTimeout(SOCKET_TIMEOUT_MILLIS);
requestConfigBuilder.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MILLIS);
return requestConfigBuilder;
});
}
public void setUserAndPassword(){
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
//es账号密码
new UsernamePasswordCredentials(USERNAME, PASSWORD));
}
/**
* 使用异步httpclient时设置并发连接数
* */
public void setMutiConnectConfig(){
builder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setMaxConnTotal(MAX_CONN_TOTAL);
httpClientBuilder.setMaxConnPerRoute(MAX_CONN_PER_ROUTE);
return httpClientBuilder;
});
}
public RestClient getClient(){
return restClient;
}
public RestHighLevelClient getRhlClient(){
return restHighLevelClient;
}
public void close() {
if (restClient != null) {
try {
restClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
log.info("close client");
}
}
3.ElasticsearchRestClient.class
@Configuration
@ComponentScan(basePackageClasses=ESClientSpringFactory.class)
@Data
@Slf4j
public class ElasticsearchRestClient {
@Value("${elasticsearch.client.connectNum}")
private Integer connectNum;
@Value("${elasticsearch.client.connectPerRoute}")
private Integer connectPerRoute;
@Value("${elasticsearch.hostlist}")
private String hostList;
@Value("${elasticsearch.username}")
private String username;
@Value("${elasticsearch.password}")
private String password;
@Bean
public HttpHost[] httpHost(){
//解析hostlist配置信息
String[] split = hostList.split(",");
//创建HttpHost数组，其中存放es主机和端口的配置信息
HttpHost[] httpHostArray = new HttpHost[split.length];
for(int i=0;i<split.length;i++){
String item = split[i];
httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
}
log.info("init HttpHost");
return httpHostArray;
}
@Bean(initMethod="init",destroyMethod="close")
public ESClientSpringFactory getFactory(){
log.info("ESClientSpringFactory 初始化");
return ESClientSpringFactory.
build(httpHost(), connectNum, connectPerRoute,username,password);
}
@Bean
@Scope("singleton")
public RestClient getRestClient(){
log.info("RestClient 初始化");
return getFactory().getClient();
}
@Bean(name = "restHighLevelClient")
@Scope("singleton")
public RestHighLevelClient getRHLClient(){
log.info("RestHighLevelClient 初始化");
return getFactory().getRhlClient();
}
}
4.EsAttribute.class
@Component
public class EsAttribute {
@Value("${elasticsearch.indexName}")
private String indexName;
@Bean
public String indexName(){
return indexName;
}
}
5.索引实体类
@Data
@Document(indexName = "#{@indexName}", shards = 5, replicas = 0)
public class EsDataInfo {
@Id
private String key;
/**
* 数据编号
**/
@Field(type = FieldType.Keyword)
private String dataCode;
/**
* 描述
**/
@Field(type = FieldType.Text, analyzer = "ik_max_word", searchAnalyzer = "ik_max_word")
private String description;
/**
* 分类
**/
@Field(type = FieldType.Keyword)
private String category;
/**
* 标签
*/
private List<String> tags = new ArrayList<>();
}
6.EsDataInfoRepository.class
@Repository
public interface EsDataInfoRepository extends ElasticsearchRepository<EsDataInfo, String> {
EsDataInfo findByDataCode(String id);
}
7.readme.txt
-- 使用方法
此包是项目中基于elasticsearch进行全文搜索 或 局部模糊搜索、统计的模块
1、 如果要使用此包 需在pom文件中加入
<dependency>
<groupId>com.***</groupId>
<artifactId>elastic-search</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
2、 在配置文件中加入
elasticsearch:
# ElasticSearch的链接地址
hostlist: ******:9200
username: elastic
password: ***
# 索引名称（可根据不同环境配置不同的索引，相当于MYSQL中的库名称）
indexName: test
client:
connectNum: 10
connectPerRoute: 50
3、启动类要添加类注解
@EnableElasticsearchRepositories(basePackages = "com.**.elastic.module.elasticsearch.mapper")
8.在实际项目中增加readme中的pom依赖，并增加配置相关文件
9.EsDataInfoService.class测试初始化和简单查询
@Slf4j
@Service
public class EsDataInfoService {
@Autowired
private ElasticsearchRestTemplate elasticsearchTemplate;
@Resource
private EsDataInfoRepository esDataInfoRepository;
public void initIndex() {
IndexOperations templateIndex = elasticsearchTemplate.indexOps(EsDataInfo.class);
// templateIndex.delete();
Document mapping = templateIndex.createMapping();
templateIndex.putMapping(mapping);
log.info("templateIndex:" + templateIndex);
log.info("putMapping:" + mapping);
}
public JsonResult initDatas() {
log.info("=================清除原始数据库开始================");
esDataInfoRepository.deleteAll();
log.info("=================清除原始数据库结束================");
mysql中的getMax();
log.info("==============开始创建索引文档==================数量为：====== " + max.getItems().size());
StringBuffer errorIds = new StringBuffer();
for (int i = 0; i < max.getItems().size(); i++) {
try {
initDataById(max.getItems().get(i).getId());
} catch (Exception e) {
log.error("initArticle error.", e);
errorIds.append(max.getItems().get(i).getId()).append(",");
log.info("!!!!!!!!!!!!!!!!!!!!创建第【" + (i+1) + "】条失败!!!!!!!!!!!!!!!!!!!!!");
continue;
}
log.info("==============创建第【" + (i+1) + "】条成功==================");
}
return new JsonResult(Constant.CODE_SUCCESS, "inits successful!", errorIds);
}
public JsonResult initData(String id) {
ExecutorService executorService = SingleThreadPool.getExecutorService();
// 开启线程同步单个数据
executorService.execute(() -> {
try {
this.initDataById(id);
Thread.sleep(1 * 1000);
} catch (Exception e) {
log.error("initData error.", e);
}
});
return new JsonResult(Constant.CODE_SUCCESS, "ID:"+id+"索引文档成功");
}
public JsonResult initDataById(String id) {
//构建数据
EsDataInfo es = new EsDataInfo();
//todo
esDataInfoRepository.save(es);
return new JsonResult(Constant.CODE_SUCCESS, "ID:"+id+"索引文档成功");
}
/**
* 删除 <br>
*
* @param id
**/
public JsonResult delete(String id) {
esDataInfoRepository.deleteById(id);
return new JsonResult(Constant.CODE_SUCCESS, "删除文档成功");
}
/**
* 查询
* @param dto
* @return
*/
public JsonPageResult queryByPage(DataQueryDTO dto) {
NativeSearchQueryBuilder searchQueryBuilder = new NativeSearchQueryBuilder();
//查询参数
if (getQueryBuilder(dto) != null) {
searchQueryBuilder = searchQueryBuilder.withQuery(getQueryBuilder(dto));
}
//分页参数
searchQueryBuilder = searchQueryBuilder.withPageable(PageRequest.of(dto.getPageNo() - 1, dto.getPageSize()));
//高亮字段
List<String> fieldList = new ArrayList<>();
if(dto.isHighlight()){
//高亮参数
HighlightBuilder.Field[] array = getHighlightBuilderField(dto, fieldList);
if (ObjectUtils.isNotEmpty(array)) {
searchQueryBuilder = searchQueryBuilder.withHighlightFields(array);
}
}
NativeSearchQuery searchQuery = searchQueryBuilder.build();
//查询总数
// log.info("build====>{}", searchQuery.getQuery());
long count = elasticsearchTemplate.count(searchQuery, EsDataInfo.class);
if (count == 0) {
return new JsonPageResult(new ArrayList<>(),0,dto.getPageNo(),dto.getPageSize());
}
//自定义查询结果封装
List<SeniorItemVO> itemVOS = new ArrayList<>();
SearchHits<EsDataInfo> search = elasticsearchTemplate.search(searchQuery, EsDataInfo.class);
for (SearchHit<EsDataInfo> searchHit : search.getSearchHits()) {
EsDataInfo info = searchHit.getContent();
if (ObjectUtils.isNotEmpty(fieldList) && ObjectUtils.isNotEmpty(searchHit.getHighlightFields())) {
Map<String, List<String>> highlightFields = searchHit.getHighlightFields();
for (String field : fieldList) {
//匹配到的字段设置内容高亮
List<String> strings = highlightFields.get(field);
if (ObjectUtils.isNotEmpty(strings)) {
StringBuffer fragmentBuffer = new StringBuffer("");
for (String fragment : strings) {
fragmentBuffer = fragmentBuffer.append(fragment);
}
EsUtil.setContentByFiled(info, field, fragmentBuffer.toString());
}
}
}
SeniorItemVO itemVO = new SeniorItemVO();
//todo把info中的值copy到返回类itemVO中
itemVOS.add(itemVO);
}
return new JsonPageResult(itemVOS,count,dto.getPageNo(),dto.getPageSize());
}
private BoolQueryBuilder getQueryBuilder(DataQueryDTO dto) {
/**
* 组合查询
* must(QueryBuilders) : AND
* mustNot(QueryBuilders): NOT
* should: : OR
*/
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (StringUtils.isNotEmpty(dto.getKeyword())) {
BoolQueryBuilder keywordQueryBuilder = QueryBuilders.boolQuery();
keywordQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("title", dto.getKeyword())).boost(50);
keywordQueryBuilder.should(QueryBuilders.matchPhraseQuery("regions.keyword", dto.getKeyword())).boost(30);
keywordQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery("description",dto.getKeyword())).boost(2);
keywordQueryBuilder.should(QueryBuilders.matchPhraseQuery("userName",dto.getKeyword())).boost(10);
boolQueryBuilder.must(keywordQueryBuilder);
}
//国家
if(StringUtils.isNotEmpty(dto.getCountry())){
BoolQueryBuilder countryQueryBuilder = QueryBuilders.boolQuery();
countryQueryBuilder.should(QueryBuilders.matchQuery("regions.keyword", dto.getCountry()));
boolQueryBuilder.must(countryQueryBuilder);
}
//城市
if(StringUtils.isNotEmpty(dto.getCity())){
BoolQueryBuilder cityQueryBuilder = QueryBuilders.boolQuery();
cityQueryBuilder.should(QueryBuilders.matchQuery("regions.keyword", dto.getCity()));
boolQueryBuilder.must(cityQueryBuilder);
}
//标签
if(StringUtils.isNotEmpty(dto.getTagsNames())){
BoolQueryBuilder tagQueryBuilder = QueryBuilders.boolQuery();
String[] split = dto.getTagsNames().split(",");
List<String> list = Stream.of(split).collect(Collectors.toList());
for (String s : list) {
tagQueryBuilder.should(QueryBuilders.matchQuery("tags.keyword", s));
}
boolQueryBuilder.must(tagQueryBuilder);
}
//分类
if(StringUtils.isNotEmpty(dto.getCategoryNames())){
BoolQueryBuilder categoryQueryBuilder = QueryBuilders.boolQuery();
String[] split = dto.getCategoryNames().split(",");
List<String> list = Stream.of(split).collect(Collectors.toList());
for (String s : list) {
categoryQueryBuilder.should(QueryBuilders.matchQuery("category", s));
}
boolQueryBuilder.must(categoryQueryBuilder);
}
//类型
if(StringUtils.isNotEmpty(dto.getTypeNames())){
BoolQueryBuilder categoryQueryBuilder = QueryBuilders.boolQuery();
String[] split = dto.getTypeNames().split(",");
List<String> list = Stream.of(split).collect(Collectors.toList());
for (String s : list) {
categoryQueryBuilder.should(QueryBuilders.matchQuery("types.keyword", s));
}
boolQueryBuilder.must(categoryQueryBuilder);
}
//分类
if(StringUtils.isNotEmpty(dto.getRegionNames())){
BoolQueryBuilder categoryQueryBuilder = QueryBuilders.boolQuery();
String[] split = dto.getRegionNames().split(",");
List<String> list = Stream.of(split).collect(Collectors.toList());
for (String s : list) {
categoryQueryBuilder.should(QueryBuilders.matchQuery("regions.keyword", s));
}
boolQueryBuilder.must(categoryQueryBuilder);
}
return boolQueryBuilder;
}
private HighlightBuilder.Field[] getHighlightBuilderField(DataQueryDTO dto, List<String> fieldList) {
List<String> highlightList = new ArrayList<>();
List<HighlightBuilder.Field> list = new ArrayList<>();
if (StringUtils.isNotEmpty(dto.getKeyword())) {
list.add(EsUtil.setHighlighter("title"));
fieldList.add("title");
highlightList.add("title");
list.add(EsUtil.setHighlighter("userName"));
fieldList.add("userName");
highlightList.add("userName");
list.add(EsUtil.setHighlighter("description"));
fieldList.add("description");
highlightList.add("description");
}
return list.toArray(new HighlightBuilder.Field[list.size()]);
}
/**
* 左侧汇总
* @param dto
* @return
*/
public JsonResult queryByCheckbox(DataQueryDTO dto) {
//查询汇总数据
FrontSeniorVO frontSeniorVO = new FrontSeniorVO();
NativeSearchQuery build = new NativeSearchQueryBuilder()
.withQuery(getQueryBuilder(dto))
.addAggregation(AggregationBuilders.terms("tagcount")
.field("tags.keyword"))
.addAggregation(AggregationBuilders.terms("typecount")
.field("types.keyword"))
.addAggregation(AggregationBuilders.terms("regionscount")
.field("regions.keyword"))
.addAggregation(AggregationBuilders.terms("categorycount")
.field("category"))
.build();
// log.info("build====>{}", build.getQuery());
// AggregatedPage<DataInfo> query = (AggregatedPage) esDataInfoRepository.search(build);
SearchHits<EsDataInfo> search = elasticsearchTemplate.search(build, EsDataInfo.class);
frontSeniorVO.setTagList(getGroupItemVOS(search,"tagcount"));
frontSeniorVO.setTypeList(getGroupItemVOS(search,"typecount"));
frontSeniorVO.setRegionList(getGroupItemVOS(search,"regionscount"));
frontSeniorVO.setCategoryList(getGroupItemVOS(search,"categorycount"));
return new JsonResult(frontSeniorVO);
}
private List<GroupItemVO> getGroupItemVOS(SearchHits<EsDataInfo> query, String type) {
List<GroupItemVO> groupItemVOList = new ArrayList<>();
Terms term1 = (Terms) query.getAggregations().getAsMap().get(type);
for (Terms.Bucket bucket : term1.getBuckets()) {
GroupItemVO vo = new GroupItemVO();
String name = bucket.getKey().toString();
vo.setCount(bucket.getDocCount()+"");
vo.setName(name);
groupItemVOList.add(vo);
}
return groupItemVOList;
}
}
10.使用到的工具类
public class EsUtil<T> {
/**
* 设置字段高亮
*
* @param searchSourceBuilder
* @param highlightFields 逗号分割
*/
public static void setHighlighter(SearchSourceBuilder searchSourceBuilder, String highlightFields) {
if (ObjectUtils.isNotEmpty(highlightFields)) {
String[] highlightFieldArray = highlightFields.split(",");
for (String highlightField : highlightFieldArray) {
HighlightBuilder highlightBuilder = new HighlightBuilder();
//设置前缀
highlightBuilder.preTags("<span style='color:red' >");
//设置后缀
highlightBuilder.postTags("</span>");
HighlightBuilder.Field highlightTitle = new HighlightBuilder.Field(highlightField);
//荧光笔类型
highlightTitle.highlighterType("unified");
// 设置高亮字段
highlightBuilder.field(highlightTitle);
searchSourceBuilder.highlighter(highlightBuilder);
}
}
}
public static HighlightBuilder.Field setHighlighter(String highlightField) {
HighlightBuilder.Field field = new HighlightBuilder.Field(highlightField)
.preTags("<span style='color:red' >")
//fragmentSize:最大高亮分片数 numOfFragments:从第一个分片获取高亮片段
.postTags("</span>").fragmentSize(100).numOfFragments(0);
return field;
}
/**
* 根据字段名设置
* @param o
* @param field
* @param content
*/
public static void setContentByFiled(Object o,String field,String content) {
try{
Class<?> oClass = o.getClass();
oClass.getDeclaredMethod("set" + firstCharToUpperCase(field), String.class).invoke(o, content);
}catch (Exception e){
e.printStackTrace();
}
}
/**
* 首字母变大写
*
* @param str
* @return
*/
public static String firstCharToUpperCase(String str) {
char firstChar = str.charAt(0);
if (firstChar >= 'a' && firstChar <= 'z') {
char[] arr = str.toCharArray();
arr[0] -= ('a' - 'A');
return new String(arr);
}
return str;
}
}
11.指定下版本，这步应该在前面
<es.version>7.4.0</es.version>
