基于Docker搭建ElastiFlow，接入网络设备的sflow和log

0、架构图
image.png
1、宿主机环境准备
  • 基础环境配置
# 关闭selinux
setenforce 0    # 临时关闭
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config    # 永久关闭

# 关闭并禁用防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
  • rsyslog配置
# rsyslog配置文件
[root@testhost mnt]# grep -v "^#\|^$" /etc/rsyslog.conf
$ModLoad imuxsock # provides support for local system logging (e.g. via logger command)
$ModLoad imjournal # provides access to the systemd journal
$ModLoad imudp
$UDPServerRun 514
$ModLoad imtcp
$InputTCPServerRun 514
$WorkDirectory /var/lib/rsyslog
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$IncludeConfig /etc/rsyslog.d/*.conf
$OmitLocalLogging on
$IMJournalStateFile imjournal.state
*.info;mail.none;authpriv.none;cron.none;local6.none                /var/log/messages
$template h3c,"/apps/logs/h3c_log/%FROMHOST-IP%.log"      # 交换机日志路径和文件名格式
local6.* ?h3c
authpriv.*                                              /var/log/secure
mail.*                                                  -/var/log/maillog
cron.*                                                  /var/log/cron
*.emerg                                                 :omusrmsg:*
uucp,news.crit                                          /var/log/spooler
local7.*                                                /var/log/boot.log

# 重启rsyslog服务
systemctl restart rsyslog.service

# 检查514端口是否正常监听
netstat -antupl |grep syslog

# 创建交换机日志存放目录
mkdir -p /apps/logs/h3c_log/
2、交换机配置（h3c为例）
<H3C>dis curr | inc info-center
 undo info-center logfile enable
 info-center loghost source Vlan-interface3
 info-center loghost 192.168.10.100 facility local6      # 192.168.10.100为rsyslog的地址

远程ssh登录或退出交换机的命令行触发产生日志后可在rsyslog服务器上看到对应的日志

ls -alF /apps/logs/h3c_log/
3、docker相关环境准备
# 安装docker
curl https://download.docker.com/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker.repo
yum install docker-ce -y

# 全局修改docker配置
vi /etc/docker/daemon.json
{
    "registry-mirrors": ["https://registry.docker-cn.com","http://hub-mirror.c.163.com"],      # 修改镜像仓库源
    "log-driver": "json-file",      # 指定日志驱动为json文件
    "log-opts": {
        "max-size" : "100m",      # 指定日志文件最大为100M
        "max-file": "10",       # 指定保存最多10个日志文件
        "compress": "true"      # 开启切割后的日志压缩
    }
}

# 启用docker服务
systemctl daemon-reload
systemctl start docker
systemctl enable docker

# 安装docker-compose
sudo curl -L "https://github.com/docker/compose/releases/download/1.28.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose

# 修改响应时间（解决docker-compose启动容器超时问题）
vi /etc/profile
export COMPOSE_HTTP_TIMEOUT=500
export DOCKER_CLIENT_TIMEOUT=500
# 使配置生效
source /etc/profile

# 新建目录
mkdir -p /apps/elastiflow    # 脚本及配置文件目录
mkdir /elastiflow_data && chown -R 1000:1000 /elastiflow_data    # 宿主机上es持久化数据目录
4、应用配置
  • filebeat配置文件：filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/networklogs/*.log      # 日志文件路径列表，可用通配符，不递归
  tags: ["h3c"]      # 标记tag，可用于分组
  include_lines: ['LOGIN','Failed','failed','error','ERROR','\bDOWN\b','\bdown\b','\bUP\b','\bup\b']      # 只输出匹配行
output.logstash:      # output到logstash
  hosts: ["127.0.0.1:5044"]
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  • logstash配置文件：logstash_networklog.conf
input {
  beats {
    port => 5044
  }
}
filter {
  if "huawei" in [tags] {
    grok{      # 解析文本构造
      match => {"message" => "%{SYSLOGTIMESTAMP:time} %{DATA:hostname} %{GREEDYDATA:info}"}
        }
  }
   else if "h3c" in [tags] {
    grok{
      match => {"message" => "%{SYSLOGTIMESTAMP:time} %{YEAR:year} %{DATA:hostname} %{GREEDYDATA:info}"}
        }
  }
mutate {      # 对字段做处理 重命名、删除、替换和修改字段
      remove_field => ["message","time","year","offset","tags","path","host","@version","[log]","[prospector]","[beat]","[input][type]","[source]"]
    }
}
output{
stdout {codec => rubydebug}
elasticsearch {
    index => "networklogs-%{+YYYY.MM.dd}"
    hosts => ["127.0.0.1:9200"]
    sniffing => false
    }
}
  • logstash管道配置文件：logstash_pipelines.yml
- pipeline.id: main
  path.config: "/etc/logstash/conf.d/*.conf"      # 加载networklog.conf配置
- pipeline.id: elastiflow
  path.config: "/etc/logstash/elastiflow/conf.d/*.conf"      # 加载elastiflow配置（sflow使用）
  • docker-compose.yml
version: '3'

services:
  elastiflow-elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.1
    container_name: elastiflow-elasticsearch
    restart: 'no'
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: host
    volumes:
      - /elastiflow_data:/usr/share/elasticsearch/data       # 持久化数据目录
    environment:
      ES_JAVA_OPTS: '-Xms8g -Xmx8g'
      cluster.name: elastiflow
      bootstrap.memory_lock: 'true'
      network.host: 0.0.0.0
      http.port: 9200
      discovery.type: 'single-node'
      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000
      action.destructive_requires_name: 'true'

  elastiflow-kibana:
    image: docker.elastic.co/kibana/kibana:7.8.1
    container_name: elastiflow-kibana
    restart: 'no'
    depends_on:
      - elastiflow-elasticsearch
    network_mode: host
    environment:
      SERVER_HOST: 0.0.0.0
      SERVER_PORT: 5601
      SERVER_MAXPAYLOADBYTES: 8388608
      ELASTICSEARCH_HOSTS: "http://127.0.0.1:9200"
      ELASTICSEARCH_REQUESTTIMEOUT: 132000
      ELASTICSEARCH_SHARDTIMEOUT: 120000
      KIBANA_DEFAULTAPPID: "dashboard/653cf1e0-2fd2-11e7-99ed-49759aed30f5"
      KIBANA_AUTOCOMPLETETIMEOUT: 3000
      KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000
      LOGGING_DEST: stdout
      LOGGING_QUIET: 'false'
      I18N_LOCALE: zh-CN

  elastiflow-logstash:
    image: robcowart/elastiflow-logstash:4.0.1
    container_name: elastiflow-logstash
    restart: 'no'
    depends_on:
      - elastiflow-elasticsearch
    network_mode: host
    volumes:
      - /apps/elastiflow/logstash_networklog.conf:/etc/logstash/conf.d/networklog.conf
      - /apps/elastiflow/logstash_pipelines.yml:/usr/share/logstash/config/pipelines.yml
    environment:
      # JVM Heap size - this MUST be at least 3GB (4GB preferred)
      LS_JAVA_OPTS: '-Xms4g -Xmx4g'

      # ElastiFlow global configuration
      ELASTIFLOW_AGENT_ID: elastiflow
      ELASTIFLOW_GEOIP_CACHE_SIZE: 16384
      ELASTIFLOW_GEOIP_LOOKUP: 'true'
      ELASTIFLOW_ASN_LOOKUP: 'true'
      ELASTIFLOW_OUI_LOOKUP: 'false'
      ELASTIFLOW_POPULATE_LOGS: 'true'
      ELASTIFLOW_KEEP_ORIG_DATA: 'true'
      ELASTIFLOW_DEFAULT_APPID_SRCTYPE: '__UNKNOWN'

      # Name resolution option
      ELASTIFLOW_RESOLVE_IP2HOST: 'false'
      ELASTIFLOW_NAMESERVER: '127.0.0.1'
      ELASTIFLOW_DNS_HIT_CACHE_SIZE: 25000
      ELASTIFLOW_DNS_HIT_CACHE_TTL: 900
      ELASTIFLOW_DNS_FAILED_CACHE_SIZE: 75000
      ELASTIFLOW_DNS_FAILED_CACHE_TTL: 3600
      ELASTIFLOW_ES_HOST: '127.0.0.1:9200'
      #ELASTIFLOW_ES_USER: 'elastic'
      #ELASTIFLOW_ES_PASSWD: 'changeme'
      ELASTIFLOW_NETFLOW_IPV4_PORT: 2055
      ELASTIFLOW_NETFLOW_UDP_WORKERS: 2
      ELASTIFLOW_NETFLOW_UDP_QUEUE_SIZE: 4096
      ELASTIFLOW_NETFLOW_UDP_RCV_BUFF: 2000000
      ELASTIFLOW_SFLOW_IPV4_PORT: 6343
      ELASTIFLOW_SFLOW_UDP_WORKERS: 2
      ELASTIFLOW_SFLOW_UDP_QUEUE_SIZE: 4096
      ELASTIFLOW_SFLOW_UDP_RCV_BUFF: 2000000
      ELASTIFLOW_IPFIX_UDP_IPV4_PORT: 4739
      ELASTIFLOW_IPFIX_UDP_WORKERS: 2
      ELASTIFLOW_IPFIX_UDP_QUEUE_SIZE: 4096
      ELASTIFLOW_IPFIX_UDP_RCV_BUFF: 2000000

  elastiflow-filebeat:
    user: root
    container_name: elastiflow-filebeat
    restart: 'no'
    image: docker.elastic.co/beats/filebeat:7.8.1
    depends_on:
      - elastiflow-logstash
    network_mode: host
    volumes:
#      - /var/run/docker.sock:/var/run/docker.sock
#      - /var/log/remote_log:/usr/share/filebeat/remote_log  #use your log file location to update this line
      - /apps/logs/h3c_log/:/var/log/networklogs/
      - /apps/elastiflow/filebeat.yml:/usr/share/filebeat/filebeat.yml
5、启动容器
docker-compose up -d
6、验证和测试
  • docker验证
docker ps -a    # docker运行情况
journalctl -u docker.service     # docker 引擎日志
docker logs -f xxxxx     # docker日志
  • 相关端口
netstat -antupl | grep 9200      # es端口
netstat -antupl | grep 6343      # sflow端口
netstat -antupl | grep 5044      # logstash端口
netstat -antupl | grep 5601      # kibana端口
  • filebeat测试
docker exec -it elastiflow-filebeat /bin/bash
filebeat test config    # 测试配置文件
filebeat test output    # 测试连接logstash 5044端口是否正常
7、页面配置-sflow

http://x.x.x.x:5601打开kibana页面
Management -> Stack Management -> Kibana Saved Objects 导入模板文件elastiflow.kibana.7.8.x.json（该文件从github中获取 https://github.com/robcowart/elastiflow/tree/master/kibana）
效果图如下

image.png

8、页面配置-交换机log

浏览器打开http://x.x.x.x:5601
打开管理页面：Home --> Management --> Stack Management

image.png

image.png

image.png

image.png

image.png

image.png

image.png

9、es管理

查看es集群健康状态curl -XGET 'http://127.0.0.1:9200/_cluster/health?pretty'

[root@testhost]# curl -XGET 'http://127.0.0.1:9200/_cluster/health?pretty'
{
  "cluster_name" : "elastiflow",
  "status" : "yellow",    # 集群状态
  "timed_out" : false,
  "number_of_nodes" : 1,    # 集群中节点数量
  "number_of_data_nodes" : 1,    # 集群中存放数据的节点总数
  "active_primary_shards" : 17,    # 集群中全部索引的主分片总数
  "active_shards" : 17,    # 集群中全部索引的所有分片（包括主分片和副本分片）总数
  "relocating_shards" : 0,    # 当下正在多个节点间移动的分片数量
  "initializing_shards" : 0,    # 新创建的分片数量
  "unassigned_shards" : 11,    # 集群中定义的，却未能发现的分片数量（此项非0的话集群状态为yellow）
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 60.71428571428571
}

说明：单点部署的 Elasticsearch，默认分片的副本数为 1，而相同的分片不能在同一个节点上，所以就出现上面 unsigned shards 非0的问题，最简单的解决方案是增加es的节点。
参考链接：
https://github.com/robcowart/elastiflow

最后编辑于 ：
©著作权归作者所有,转载或内容合作请联系作者
平台声明：文章内容（如有图片或视频亦包括在内）由作者上传并发布，文章内容仅代表作者本人观点，简书系信息发布平台，仅提供信息存储服务。

推荐阅读更多精彩内容