一、安装部署
1.1 挂载目录准备
创建es、logstash和kibana的三个挂载目录，然后将目录权限修改为1000
chown -R 1000:1000 xxxxx
1.2 es挂载目录
下一般需要挂载数据目录和配置文件。初次启动时，先不挂在es的目录，等es启动成功后，进入es容器，将es的配置目录和数据目录拷贝出来即可。
docker exec -it elasticsearch sh
tar -cvf config.tar config/
tar -cvf data.tar data/
exit
docker cp elasticsearch:/usr/share/elasticsearch/config.tar es挂载目录
docker cp elasticsearch:/usr/share/elasticsearch/data.tar es挂载目录
tar -vxf data.tar
tar -vxf config.tar
1.3 logstash挂载目录
config：logstash的服务配置logstash.yml
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://elk-es:9200" ]
pipeline：日志收集器的配置，可以有多个配置文件，配置方式也有很多种，这里以tcp方式为例（本例子的日志发送端已将日志全部格式化成json了）
input {
tcp {
port => 5044
codec => json_lines
}
}
output {
elasticsearch {
hosts => ["http://elk-es:9200"]
index => "log-%{+YYYY.MM.dd}"
#user => "elastic"
#password => "changeme"
}
}
1.4 kibana挂载目录
config：kibana的服务配置目录，kibana.yml
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elk-es:9200" ]
monitoring.ui.container.elasticsearch.enabled: true
1.5 启动脚本
注意初次启动时删除es的挂载目录，待拷贝完es的配置和数据目录后再修改。
docker-compose.yml
version: '3'
services:
elk-es:
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0
environment:
- discovery.type=single-node
volumes:
- /data/docker-space/elk/bind/es/data:/usr/share/elasticsearch/data
- /data/docker-space/elk/bind/es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
networks:
- elk
elk-logstash:
image: docker.elastic.co/logstash/logstash:7.10.0
volumes:
- /data/docker-space/elk/bind/logstash/pipeline:/usr/share/logstash/pipeline
- /data/docker-space/elk/bind/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
command: -f /usr/share/logstash/pipeline/logstash.conf
depends_on:
- elk-es
ports:
- "5044:5044"
networks:
- elk
elk-kibana:
image: docker.elastic.co/kibana/kibana:7.10.0
environment:
- ELASTICSEARCH_URL=http://elk-es:9200
volumes:
- /data/docker-space/elk/bind/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
depends_on:
- elk-es
ports:
- "5601:5601"
networks:
- elk
networks:
elk:
driver: bridge
1.6 启动
docker-compose up -d
二、spring-boot适配
2.1 添加依赖包
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
</dependency>
2.2 修改logback.xml
太过简单，此处省略，自行搜索
自此，技术上的探索已经完毕，服务启动后日志会通过logstash保存到es，然后可通过kibana查看。
但是，实际使用中，通常是需要更精确的使用，比如将日志按照需要结构化、添加接口拦截器，从而实现接口日志的采集功能等。
三、spring-boot日志结构化
3.1 自定义日志对象
@Data
public class LocalLogEntity {
@JsonIgnore
private final static Format dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").toFormat();
// 应用环境
private String appEnv;
// 应用名称
private String appName;
// 应用IP
private String appIp;
// 线程名称
private String threadName;
// 日志ID，可用于日志追踪
private String trackId;
// 日志名称
private ELogName logName = ELogName.NORMAL_LOG;;
// 日志时间
private String logTime;
// 日志级别
private String logLevel;
// 输出日志的类
private String logger;
// 输出日志的方法
private String loggerMethod;
// 输出日志的代码行数
private int loggerLine;
// 日志信息
private String message;
// 接口地址
private String apiUrl;
// 接口状态
private String apiStatus;
// 请求IP
private String requestIp;
// 路径参数
private String pathParams;
// 请求体参数
private String bodyParams;
// 用户ID
private String userId;
// 用户部门ID
private String deptId;
// 用户部门上级ID
private String parentId;
@JsonIgnore
public void setContext(Context context) {
if(context == null) {
return;
}
this.appEnv = context.getProperty("appEnv");
this.appName = context.getProperty("appName");
this.appIp = context.getProperty("appIp");
}
@JsonIgnore
public void setEvent(ILoggingEvent event) {
if(event == null) {
return;
}
this.logLevel = event.getLevel().levelStr;
this.threadName = event.getThreadName();
this.logTime = dateTimeFormatter.format(LocalDateTime.now());
this.trackId = event.getMDCPropertyMap().get(SecurityConstants.LOG_TRACK_ID);
this.message = event.getMessage();
// 有异常信息时，获取异常抛出的信息
if (event.getThrowableProxy() != null) {
StackTraceElementProxy[] traces = event.getThrowableProxy().getStackTraceElementProxyArray();
StackTraceElement element = traces[0].getStackTraceElement();
if ("某自定义exception".equals(element.getClassName())) {
element = event.getThrowableProxy().getStackTraceElementProxyArray()[1].getStackTraceElement();
this.logName = ELogName.BUSSINESS_ERROR;
} else {
this.logName = ELogName.SYSTEM_ERROR;
StringBuffer sb = new StringBuffer();
sb.append(String.format("%s: %s\r\n",event.getThrowableProxy().getClassName(),event.getThrowableProxy().getMessage()));
for(int i=0;i<traces.length;i++) {
sb.append(String.format("\t%s\r\n",traces[i].getSTEAsString()));
}
this.message = sb.toString();
}
this.logger = element.getClassName();
this.loggerMethod = element.getMethodName();
this.loggerLine = element.getLineNumber();
} else {
this.logger = event.getLoggerName();
this.loggerMethod = event.getCallerData()[0].getMethodName();
this.loggerLine = event.getCallerData()[0].getLineNumber();
try {
LocalBeanUtils.copyJsonToEntity(JSON.parseObject(event.getMessage()), this);
} catch (Exception e) {
}
}
}
/**
* 设置会话信息
*/
public void setSessionInfo() {
SessionInfo sessionInfo = SecurityUtils.getSessionInfo();
if(sessionInfo==null) {
return;
}
SessionUser user = sessionInfo.getUser();
if(user != null) {
this.userId = user.getUserId();
}
SessionDept dept = sessionInfo.getCurrentDept();
if(dept !=null) {
this.deptId = dept.getDeptId();
this.parentId = dept.getParentId();
}
}
}
3.2 自定义日志采集器
public class LocalLogJsonEncoder implements Encoder<ILoggingEvent> {
private Charset charset = Charset.forName("UTF-8");
private Context context;
private String appIp;
@Getter
@Setter
private Level level = Level.WARN;
@Override
public byte[] headerBytes() {
// System.out.println("headerBytes");
return new byte[0];
}
@Override
public byte[] footerBytes() {
// System.out.println("footerBytes");
return new byte[0];
}
@Override
public void setContext(Context context) {
this.context = context;
}
@Override
public Context getContext() {
return context;
}
@Override
public void addStatus(Status status) {
// System.out.println("addStatus");
// System.out.println(JSON.toJSONString(status));
}
@Override
public void addInfo(String msg) {
// System.out.println("addInfo2");
}
@Override
public void addInfo(String msg, Throwable ex) {
// System.out.println("addInfo2");
}
@Override
public void addWarn(String msg) {
// System.out.println("addWarn");
}
@Override
public void addWarn(String msg, Throwable ex) {
// System.out.println("addWarn2");
}
@Override
public void addError(String msg) {
// System.out.println("addError");
// System.out.println(msg);
}
@Override
public void addError(String msg, Throwable ex) {
// System.out.println("addError2");
}
@Override
public void start() {
this.appIp = IpUtils.getHostIp();
}
@Override
public void stop() {
// System.out.println("stop");
}
@Override
public boolean isStarted() {
// System.out.println("isStarted");
return false;
}
@Override
public byte[] encode(ILoggingEvent event) {
if (!event.getLevel().isGreaterOrEqual(this.level)) {
// 高于配置的日志级别才记录日志
return null;
}
LocalLogEntity entity;
try {
entity = JSON.parseObject(event.getMessage(), LocalLogEntity.class);
if (entity == null) {
entity = new LocalLogEntity();
}
} catch (Exception e) {
entity = new LocalLogEntity();
}
entity.setContext(this.context);
entity.setEvent(event);
entity.setAppIp(appIp);
return String.format("%s\r\n", JSON.toJSONString(entity)).getBytes(charset);
}
}
3.3 修改logback.xml
<?xmlversion="1.0"encoding="UTF-8"?>
<configurationscan="true"debug="false">
<springPropertyscope="context"name="log_level"source="logging.level"defaultValue="error"/>
<springPropertyscope="context"name="appEnv"source="spring.profiles.active"defaultValue="unknow"/>
<springPropertyscope="context"name="appName"source="spring.application.name"defaultValue="unknow"/>
<springPropertyscope="context"name="logstash_address"source="logging.logstash.address" defaultValue="unknow_logstash_address"/>
<!-- ConsoleAppender 控制台输出日志 -->
<appendername="STDOUT"class="ch.qos.logback.core.ConsoleAppender">
<!-- 对日志进行格式化 -->
......
</appender>
<!--输出到 logstash的 appender -->
<appendername="logstash"class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>${logstash_address}</destination>
<encoderclass="com.xxxxx.LocalLogJsonEncoder">
<level>WARN</level>
</encoder>
</appender>
<!-- 全部环境的日志输出配置 -->
<rootlevel="INFO">
<appender-refref="STDOUT"/>
</root>
<!-- elk日志输出配置 -->
<rootlevel="WARN">
<appender-refref="logstash"/>
</root>
</configuration>
这样es中存储的日志mapping就是自定义的日志对象，后续可用于自定义的业务统计需求和分析。
3.4 接口拦截
@Slf4j
@Configuration
@ConditionalOnProperty(value = "logging.local.enabled", havingValue = "true", matchIfMissing = true)
public class LogInterceptor implements HandlerInterceptor, WebMvcConfigurer {
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler)
throws Exception {
String traceId = request.getHeader(SecurityConstants.LOG_TRACK_ID);
if(LocalStringUtils.isEmpty(traceId)) {
traceId = IdUtils.getSnowId();
}
MDC.put(SecurityConstants.LOG_TRACK_ID, traceId);
String requestBody = "{}";
if (request instanceof LocalHttpServletRequestWrapper) {
LocalHttpServletRequestWrapper requestWrapper = (LocalHttpServletRequestWrapper) request;
byte[] bytes = requestWrapper.getBody();
requestBody = new String(bytes);
}
log.warn(LocalLogFormater.apiStart(IpUtils.getIpAddr(request), request.getRequestURI(),
request.getParameterMap(), requestBody));
return HandlerInterceptor.super.preHandle(request, response, handler);
}
@Override
public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler,
ModelAndView modelAndView) throws Exception {
HandlerInterceptor.super.postHandle(request, response, handler, modelAndView);
}
@Override
public void afterCompletion(HttpServletRequest httpRequest, HttpServletResponse response, Object handler,
Exception ex) throws Exception {
String url = httpRequest.getRequestURI().substring(httpRequest.getContextPath().length());// 请求的URL
String bussinessCode = response.getHeader(SecurityConstants.BUSSINESS_CODE);
if (LocalStringUtils.isEmpty(bussinessCode)) {
log.warn(LocalLogFormater.apiEnd(String.valueOf(response.getStatus()), url));
} else {
log.warn(LocalLogFormater.apiEnd(bussinessCode, url));
}
HandlerInterceptor.super.afterCompletion(httpRequest, response, handler, ex);
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(new LogInterceptor()).addPathPatterns("/**");
}
}
参数过滤器
@Slf4j
@Configuration
@ConditionalOnProperty(value = "log.local.enabled", havingValue = "true", matchIfMissing = true)
public class LogRequestBodyReadFilter implements Filter {
@Override
public void init(FilterConfig filterConfig) throws ServletException {
}
@Override
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)
throws IOException, ServletException {
if (servletRequest instanceof HttpServletRequest) {
HttpServletRequest request = (HttpServletRequest) servletRequest;
String contentType = servletRequest.getContentType();
if (contentType==null || !contentType.contains("multipart")) {
/*
* 只要不是文件上传接口，都转换成可以反复读取输入流的Wrapper
* 后续的日志拦截器就可以读取body，并且不影响接口读取参数了
*/
filterChain.doFilter(new LocalHttpServletRequestWrapper(request), servletResponse);
return;
}
}
filterChain.doFilter(servletRequest, servletResponse);
}
@Override
public void destroy() {
}
}
public class LocalHttpServletRequestWrapper extends HttpServletRequestWrapper {
private final byte[] body;
public LocalHttpServletRequestWrapper(HttpServletRequest request) throws IOException {
super(request);
request.setCharacterEncoding("UTF-8");
ServletInputStream inputStream = request.getInputStream();
DataInputStream dataInputStream = new DataInputStream(inputStream);
body = dataInputStream.readAllBytes();
}
public byte[] getBody(){
return this.body;
}
@Override
public BufferedReader getReader() throws IOException {
return new BufferedReader(new InputStreamReader(getInputStream()));
}
@Override
public ServletInputStream getInputStream() {
final ByteArrayInputStream basis = new ByteArrayInputStream(body);
return new ServletInputStream() {
@Override
public int read() {
return basis.read();
}
@Override
public boolean isFinished() {
return false;
}
@Override
public boolean isReady() {
return false;
}
@Override
public void setReadListener(ReadListener readListener) {
}
};
}
}
