日志解析，强推ELK！
日志解析，强推ELK！
日志解析，强推ELK！
（有时间我要搭一套 flag+1）
接上文，我这边接了个BI项目，就是帆软的BI项目，但是发现报表的错误日志里面实在是太多了，发现帆软的决策报表里有个查询的口子，于是写了个接口调用工具拉取帆软的报错信息
搭建restTemplate连接池
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.10</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
</dependency>
<dependency>
我用的boot版本是2.1.5
创建连接池
@Configuration
public class RestTemplateConfiguration {
/**
* 返回RestTemplate
* @param factory
* @return
*/
@Bean
public RestTemplate restTemplate(ClientHttpRequestFactory factory) {
RestTemplate restTemplate = new RestTemplate(factory);
return restTemplate;
}
/**
* ClientHttpRequestFactory接口的另一种实现方式（推荐使用），即：
* HttpComponentsClientHttpRequestFactory：底层使用Httpclient连接池的方式创建Http连接请求
* @return
*/
@Bean
public HttpComponentsClientHttpRequestFactory httpComponentsClientHttpRequestFactory() {
//Httpclient连接池，长连接保持30秒
PoolingHttpClientConnectionManager connectionManager =
new PoolingHttpClientConnectionManager(30, TimeUnit.SECONDS);
//设置总连接数
connectionManager.setMaxTotal(1000);
//设置同路由的并发数
connectionManager.setDefaultMaxPerRoute(1000);
//设置header
List<Header> headers = new ArrayList<Header>();
headers.add(new BasicHeader("User-Agent",
"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.04"));
headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
headers.add(new BasicHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3"));
headers.add(new BasicHeader("Connection", "keep-alive"));
headers.add(new BasicHeader("RequestType", "queue"));
//创建HttpClient
HttpClient httpClient = HttpClientBuilder.create()
.setConnectionManager(connectionManager)
.setDefaultHeaders(headers)
.setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)) //设置重试次数
.setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) //设置保持长连接
.build();
//创建HttpComponentsClientHttpRequestFactory实例
HttpComponentsClientHttpRequestFactory requestFactory =
new HttpComponentsClientHttpRequestFactory(httpClient);
//设置客户端和服务端建立连接的超时时间
requestFactory.setConnectTimeout(20000);
//设置客户端从服务端读取数据的超时时间
requestFactory.setReadTimeout(20000);
//设置从连接池获取连接的超时时间，不宜过长
requestFactory.setConnectionRequestTimeout(200);
//缓冲请求数据，默认为true。通过POST或者PUT大量发送数据时，建议将此更改为false，以免耗尽内存
requestFactory.setBufferRequestBody(false);
return requestFactory;
}
}
不用连接池更快，反正也是个小工具
模拟登录，抓起错误信息
public List<String> getCookie(){
HttpHeaders headers = new HttpHeaders();
MultiValueMap<String, String> map = new LinkedMultiValueMap<String, String>(); //map里面是请求体的内容
map.add("fr_username", PASSWORD);
map.add("fr_password",PASSWORD);
headers.setContentType(MediaType.parseMediaType("application/x-www-form-urlencoded; charset=UTF-8"));
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<MultiValueMap<String, String>>(map, headers);
ResponseEntity<String> response = restTemplate.postForEntity(LOGIN_URL, request, String.class); //地址替换为自己的
return response.getHeaders().get("Set-Cookie");
}
拿到cookie信息后，获取总量信息
/**
* 保存组Group
*/
@Test
public void getTotal(){
HttpHeaders headers = new HttpHeaders();
List<String> cookies = getCookie();
headers.put(HttpHeaders.COOKIE,cookies); //将cookie存入头部
MultiValueMap<String, String> map = new LinkedMultiValueMap<String, String>(); //请求体给予内容
map.add("pn", "1"); //应用名称
map.add("pageCount", "10"); //执行器名称
map.add("startTime", START_DATE); //排序方式
map.add("endTime", END_DATE); //注册方式 ： 0为
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<MultiValueMap<String, String>>(map, headers);
ResponseEntity<String> response = restTemplate.postForEntity(ERROR_INFO_URL, request, String.class);
if(!StringUtils.isEmpty(response.getBody())){
JSONObject jsonObject = JSONObject.parseObject(response.getBody());
int totalPage = Integer.parseInt(jsonObject.get("totalpages").toString());
List<ErrorEntity> errorEntityList = new ArrayList<>();
if(totalPage>1){
for(int i =1;i< totalPage;i++){
getErrorInfo(i,errorEntityList);
}
}else{
errorEntityList = JSONArray.parseArray(jsonObject.get("items").toString(),ErrorEntity.class);
}
EasyExcel.write(result +"180_FR_ERROR_" + START_DATE + ".xlsx", ErrorEntity.class).sheet(START_DATE).doWrite(errorEntityList);
}
}
获取分页信息后这里就是循环调用的过程了
/**
* 保存组Group
*/
public void getErrorInfo(Integer page,List<ErrorEntity> errorEntities){
HttpHeaders headers = new HttpHeaders();
List<String> cookies = getCookie();
headers.put(HttpHeaders.COOKIE,cookies); //将cookie存入头部
MultiValueMap<String, String> map = new LinkedMultiValueMap<String, String>(); //请求体给予内容
map.add("pn", page.toString()); //应用名称
map.add("pageCount", "10"); //执行器名称
map.add("startTime", START_DATE); //排序方式
map.add("endTime", END_DATE); //注册方式 ： 0为
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<MultiValueMap<String, String>>(map, headers);
ResponseEntity<String> response = restTemplate.postForEntity(ERROR_INFO_URL, request, String.class);
if(!StringUtils.isEmpty(response.getBody())){
JSONObject jsonObject = JSONObject.parseObject(response.getBody());
errorEntities.addAll(JSONArray.parseArray(jsonObject.get("items").toString(),ErrorEntity.class));
}
}
进行文件封装
1576674727210.jpg
