ElasticFlow搭建，接入网络设备sflow

CentOS 7 中部署安装（非docker）

1、jdk环境和其他工具准备

yum install java-openjdk-devel java-openjdk    # jdk
java -version
yum install git unzip net-tools lrzsz

2、下载elk安装包并安装

# 创建安装包存放目录
mkdir -p /home/apps/elk
# 下载安装包
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.8.1-x86_64.rpm
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.8.1-x86_64.rpm
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.8.1.rpm
# 安装
rpm -ivh elasticsearch-7.8.1-x86_64.rpm kibana-7.8.1-x86_64.rpm logstash-7.8.1.rpm
# 加入systemd管理，使其开机自启
systemctl daemon-reload
systemctl enable elasticsearch.service 
systemctl enable kibana.service 
systemctl enable logstash.service

3、修改配置文件

vim /etc/elasticsearch/elasticsearch.yml

检查配置文件关键配置如下

[root@testhost elk]# grep -v "^$\|^#" /etc/elasticsearch/elasticsearch.yml
cluster.name: elastiflow
node.name: elastiflow
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: true
network.host: 0.0.0.0
http.port: 9200
action.destructive_requires_name: true
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
discovery.type: single-node

修改es的jvm配置
vim /etc/elasticsearch/jvm.options
检查配置文件关键配置如下

-Xms4g
-Xmx4g

修改kibana配置文件vim /etc/kibana/kibana.yml

server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:9200"]

4、启动服务

顺序启动

systemctl restart elasticsearch.service
systemctl status elasticsearch.service 
systemctl restart kibana.service
systemctl status kibana.service

5、安装logstash plugin

/usr/share/logstash/bin/logstash-plugin install logstash-codec-sflow
/usr/share/logstash/bin/logstash-plugin install logstash-codec-netflow
/usr/share/logstash/bin/logstash-plugin install logstash-input-udp
/usr/share/logstash/bin/logstash-plugin install logstash-input-tcp
/usr/share/logstash/bin/logstash-plugin install logstash-filter-dns
/usr/share/logstash/bin/logstash-plugin install logstash-filter-geoip
/usr/share/logstash/bin/logstash-plugin install logstash-filter-translate

6、logstash中的elastiflow模块下载部署

git clone https://github.com/robcowart/elastiflow.git
wget https://github.com/robcowart/elastiflow/archive/master.zip
unzip master.zip
cp -a /home/apps/elk/elastiflow/logstash/elastiflow/. /etc/logstash/elastiflow/
cp -a /home/apps/elk/elastiflow/logstash.service.d/. /etc/systemd/system/logstash.service.d/
sz /home/apps/elk/elastiflow-master/kibana/elastiflow.kibana.7.8.x.ndjson    # 把kibana模板保存到本地，后面步骤需要在kibana页面中导入此模板文件

修改logstash jvm配置文件vim /etc/logstash/jvm.options（此项很重要，需要把jvm堆改大，否则logstash启动会报OOM）

[root@testhost elk]# grep -v "^$\|^#" /etc/logstash/jvm.options 
-Xms4g
-Xmx4g
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djruby.compile.invokedynamic=true
-Djruby.jit.threshold=0
-Djruby.regexp.interruptible=true
-XX:+HeapDumpOnOutOfMemoryError
-Djava.security.egd=file:/dev/urandom
-Dlog4j2.isThreadContextMapInheritable=true

修改管道配置文件vim /etc/logstash/pipelines.yml

# This file is where you define your pipelines. You can define multiple.
# For more information on multiple pipelines, see the documentation:
#   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html

#- pipeline.id: main
#  path.config: "/etc/logstash/conf.d/*.conf"

- pipeline.id: elastiflow
  path.config: "/etc/logstash/elastiflow/conf.d/*.conf"

执行logstash系统脚本/usr/share/logstash/bin/system-install
启动logstash并设置开机自启

systemctl daemon-reload
systemctl enable logstash
systemctl start logstash.service

7、其他测试项

关闭防火墙和selinux

systemctl stop firewalld.service
systemctl disable firewalld.service

# 临时禁用selinux
setenforce 0

测试es

[root@testhost ~]# curl http://0.0.0.0:9200
{
  "name" : "elastiflow",
  "cluster_name" : "elastiflow",
  "cluster_uuid" : "a_PBznEvT0iULfVEC1MdGA",
  "version" : {
    "number" : "7.8.1",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "b5ca9c58fb664ca8bf9e4057fc229b3396bf3a89",
    "build_date" : "2020-07-21T16:40:44.668009Z",
    "build_snapshot" : false,
    "lucene_version" : "8.5.1",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

检查端口

[root@testhost ~]# netstat -antupl |grep java
tcp6       0      0 :::9200                 :::*                    LISTEN      3924/java           
tcp6       0      0 :::9300                 :::*                    LISTEN      3924/java           
tcp6       0      0 127.0.0.1:9600          :::*                    LISTEN      4181/java           
tcp6       0      0 :::4739                 :::*                    LISTEN      4181/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42400         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42410         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42404         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42406         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:42412         127.0.0.1:9200          ESTABLISHED 4181/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42412         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42398         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42394         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42408         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42402         ESTABLISHED 3924/java           
tcp6       0      0 127.0.0.1:9200          127.0.0.1:42396         ESTABLISHED 3924/java           
udp        0      0 0.0.0.0:2055            0.0.0.0:*                           4181/java           
udp        0      0 0.0.0.0:4739            0.0.0.0:*                           4181/java           
udp        0      0 0.0.0.0:6343            0.0.0.0:*                           4181/java      # elastiflow监听端口

[root@testhost ~]# netstat -antp |grep 5601
tcp        0      0 0.0.0.0:5601            0.0.0.0:*               LISTEN      3289/node      # kibana web端口

检查日志

tailf /var/log/logstash/logstash-plain.log

elasticflow各协议的端口查看

[root@testhost ~]# grep -i sflow /etc/systemd/system/logstash.service.d/elastiflow.conf    # 只查看sflow相关端口和配置
# sFlow - IPv4
Environment="ELASTIFLOW_SFLOW_IPV4_HOST=0.0.0.0"
Environment="ELASTIFLOW_SFLOW_IPV4_PORT=6343"
# sFlow - IPv6
Environment="ELASTIFLOW_SFLOW_IPV6_HOST=[::]"
Environment="ELASTIFLOW_SFLOW_IPV6_PORT=56343"
# sFlow - UDP input options
Environment="ELASTIFLOW_SFLOW_UDP_WORKERS=4"
Environment="ELASTIFLOW_SFLOW_UDP_QUEUE_SIZE=4096"
Environment="ELASTIFLOW_SFLOW_UDP_RCV_BUFF=33554432"

8、kibana配置

浏览器打开http://127.0.0.1:5601

image.png

二、交换机配置（H3C）

# 全局sflow配置
sflow agent ip 192.168.10.35
sflow source ip 192.168.10.35
sflow collector 1 ip 192.168.10.10 description "Test Collector"
sflow sampling-rate 1000

# 接口下应用sflow配置
sflow flow collector 1

# 查看sflow状态
<H3C>dis sflow
sFlow datagram version: 5
Global information:
Agent IP: 192.168.10.35(CLI)
Source address: 192.168.10.35 
Collector information:
ID    IP              Port  Aging      Size VPN-instance Description
1     192.168.10.10   6343  N/A        1400              "Test Collector"       
Port counter sampling information:
Interface      Instance   CID   Interval(s)
Port flow sampling information:
Interface      Instance   FID   MaxHLen Rate       Mode      Status
GE1/0/24       1          1     128     1000       Random    Active

参考链接：
https://cloud.tencent.com/developer/article/1648854
https://blog.51cto.com/coolsky/3190806

最后编辑于：2021.11.02 14:12:31

©著作权归作者所有,转载或内容合作请联系作者
【社区内容提示】社区部分内容疑似由AI辅助生成，浏览时请结合常识与多方信息审慎甄别。
平台声明：文章内容（如有图片或视频亦包括在内）由作者上传并发布，文章内容仅代表作者本人观点，简书系信息发布平台，仅提供信息存储服务。

ElasticFlow搭建，接入网络设备sflow