Fluentd 以及logtail使用日记

一/通过sh设置环境和安装

#!/bin/bash

# 增加文件描述符的最大数量
echo "Increasing the maximum number of file descriptors..."

# 检查当前的文件描述符限制
current_limit=$(ulimit -n)
echo "Current file descriptor limit: $current_limit"

# 更新 /etc/security/limits.conf
{
    echo "root soft nofile 65536"
    echo "root hard nofile 65536"
    echo "* soft nofile 65536"
    echo "* hard nofile 65536"
} >> /etc/security/limits.conf

echo "Updated /etc/security/limits.conf to increase file descriptor limits."

# 优化网络内核参数
echo "Optimizing network kernel parameters..."

# 更新 /etc/sysctl.conf
{
    echo "net.core.somaxconn = 1024"
    echo "net.core.netdev_max_backlog = 5000"
    echo "net.core.rmem_max = 16777216"
    echo "net.core.wmem_max = 16777216"
    echo "net.ipv4.tcp_wmem = 4096 12582912 16777216"
    echo "net.ipv4.tcp_rmem = 4096 12582912 16777216"
    echo "net.ipv4.tcp_max_syn_backlog = 8096"
    echo "net.ipv4.tcp_slow_start_after_idle = 0"
    echo "net.ipv4.tcp_tw_reuse = 1"
    echo "net.ipv4.ip_local_port_range = 10240 65535"
    echo "net.ipv4.ip_local_reserved_ports = 24224"
} >> /etc/sysctl.conf

echo "Updated /etc/sysctl.conf with network kernel parameters."

# 应用 sysctl 配置
sysctl -p

# 检查和设置符号链接保护
echo "Checking and setting symbolic link protection..."

# 检查 fs.protected_hardlinks 和 fs.protected_symlinks
protected_hardlinks=$(sysctl fs.protected_hardlinks | awk '{print $3}')
protected_symlinks=$(sysctl fs.protected_symlinks | awk '{print $3}')

if [ "$protected_hardlinks" -ne 1 ]; then
    echo "fs.protected_hardlinks is not set to 1. Setting it now..."
    echo "fs.protected_hardlinks = 1" >> /etc/sysctl.conf
fi

if [ "$protected_symlinks" -ne 1 ]; then
    echo "fs.protected_symlinks is not set to 1. Setting it now..."
    echo "fs.protected_symlinks = 1" >> /etc/sysctl.conf
fi

# 应用 sysctl 配置
sysctl -p

echo "All configurations have been applied. Please restart your machine for the changes to take full effect."


# 检查系统版本
OS_VERSION=$(lsb_release -is)
OS_CODENAME=$(lsb_release -cs)

echo "Detected OS: $OS_VERSION $OS_CODENAME"

# 根据系统版本选择安装脚本
case "$OS_VERSION" in
    Ubuntu)
        case "$OS_CODENAME" in
            focal)
                INSTALL_SCRIPT="https://calyptia-fluentd.s3.us-east-2.amazonaws.com/calyptia-fluentd-1-ubuntu-focal.sh"
                ;;
            bionic)
                INSTALL_SCRIPT="https://calyptia-fluentd.s3.us-east-2.amazonaws.com/calyptia-fluentd-1-ubuntu-bionic.sh"
                ;;
            xenial)
                INSTALL_SCRIPT="https://calyptia-fluentd.s3.us-east-2.amazonaws.com/calyptia-fluentd-1-ubuntu-xenial.sh"
                ;;
            *)
                echo "Unsupported Ubuntu version: $OS_CODENAME"
                exit 1
                ;;
        esac
        ;;
    Debian)
        case "$OS_CODENAME" in
            buster)
                INSTALL_SCRIPT="https://calyptia-fluentd.s3.us-east-2.amazonaws.com/calyptia-fluentd-1-debian-buster.sh"
                ;;
            *)
                echo "Unsupported Debian version: $OS_CODENAME"
                exit 1
                ;;
        esac
        ;;
    *)
        echo "Unsupported OS: $OS_VERSION"
        exit 1
        ;;
esac

# 执行安装脚本
echo "Installing Calyptia Fluentd from $INSTALL_SCRIPT..."
curl -fsSL "$INSTALL_SCRIPT" | sh


# 检查和设置符号链接保护
echo "Checking and setting symbolic link protection..."
{
    echo "fs.protected_hardlinks = 1"
    echo "fs.protected_symlinks = 1"
} >> /etc/sysctl.conf

# 应用 sysctl 配置
sysctl -p

# 启动 Calyptia Fluentd 服务
echo "Starting Calyptia Fluentd service..."
sudo systemctl start calyptia-fluentd.service

# 发布示例日志
echo "Publishing example log..."
curl -X POST -d 'json={"json":"message"}' http://localhost:8888/debug.test

# 查看日志
echo "Last log entry:"
sudo tail -n 1 /var/log/calyptia-fluentd/calyptia-fluentd.log

echo "Installation and configuration completed."

2/修改配置文件：

cd /etc/calyptia-fluentd/
vi calyptia-fluentd.conf

####
## Source descriptions:
####

<source>
  @type tail
  @id input_pay_out_log
  path /app/pay-staging/pay_out.log
  pos_file /var/log/calyptia-fluentd/pay_out.log.pos
  tag pay_out.log
  format none  # 根据日志格式选择合适的解析方式
</source>

####
## Output descriptions:
####

<match pay_out.log>
  @type aliyun_sls
  @id output_aliyun_sls
  project pay_test  # 替换为您的阿里云项目名称
  logstore test-acquirng-pay  # 替换为您的日志库名称
  access_key_id LTAI5tSNftH7uPMSv79dZSD8  # 替换为您的阿里云 Access Key ID
  access_key_secret 5BieZeRlxzqVnPbvzNXTIGNGOA9fIV  # 替换为您的阿里云 Access Key Secret
  region us-west-1  # 替换为您的阿里云区域，例如 cn-hangzhou
  log_tag pay_out.log
</match>

3/安装sls插件

sudo apt update
sudo apt install -y git curl
git clone https://github.com/rbenv/rbenv.git ~/.rbenv
echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
echo 'eval "$(rbenv init -)"' >> ~/.bashrc
exec $SHELL
git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
rbenv install 3.0.0
rbenv global 3.0.0
ruby -v

装不上，放弃了，直接使用kafka模式
测试配置文件是否正常

 /opt/calyptia-fluentd/bin/fluentd --dry-run -c /etc/calyptia-fluentd/calyptia-fluentd.conf

sudo systemctl status calyptia-fluentd.service

kafka模式调试成功！

二/logtail安装
安装Logtail（Linux系统）

wget http://logtail-release-${region_id}.oss-${region_id}.aliyuncs.com/linux64/logtail.sh -O logtail.sh; chmod +x logtail.sh; ./logtail.sh install ${region_id}-acceleration

美国硅谷为：us-west-1，替换后如下

wget http://logtail-release-us-west-1.oss-us-west-1.aliyuncs.com/linux64/logtail.sh -O logtail.sh; chmod +x logtail.sh; ./logtail.sh install us-west-1-acceleration

启动Logtail

sudo /etc/init.d/ilogtaild start

停止Logtail

sudo /etc/init.d/ilogtaild stop

查看Logtail状态
使用以下命令查看Logtail状态。

sudo /etc/init.d/ilogtaild status

如果系统返回如下信息，表示已安装Logtail。

ilogtail is running

如果Logtail状态显示未运行，请卸载后重新安装。

查看Logtail版本
Logtail会将版本信息记录在/usr/local/ilogtail/app_info.json文件中的logtail_version字段。使用以下命令查看Logtail版本信息。

cat /usr/local/ilogtail/app_info.json

返回结果

{
  "UUID" : "0DF18E97-0F2D-486F-B77F-*********",
  "hostname" : "david*******",
  "instance_id" : "F4FAFADA-F1D7-11E7-846C-00163E30349E_*********_1515129548",
  "ip" : "**********",
  "logtail_version" : "0.16.30",
  "os" : "Linux; 2.6.32-220.23.2.ali1113.el5.x86_64; #1 SMP Thu Jul 4 20:09:15 CST 2013; x86_64",
  "update_time" : "2020-01-05 13:19:08"
}

卸载Logtail

wget http://logtail-release-${region_id}.oss-${region_id}.aliyuncs.com/linux64/logtail.sh -O logtail.sh; chmod +x logtail.sh; ./logtail.sh uninstall

修改配置文件

/usr/local/ilogtail/ilogtail_config.json

文件示例

 
$cat /usr/local/ilogtail/ilogtail_config.json
{
    "config_server_address" : "http://logtail.cn-hangzhou-intranet.log.aliyuncs.com",
    "data_server_list" :
    [
        {
            "cluster" : "cn-hangzhou",
            "endpoint" : "cn-hangzhou-intranet.log.aliyuncs.com"
        }
    ],
    "cpu_usage_limit" : 0.4,
    "mem_usage_limit" : 100,
    "max_bytes_per_sec" : 2097152,
    "process_thread_count" : 1,
    "send_request_concurrency" : 4,
    "streamlog_open" : false
}

配置用户标识

touch /etc/ilogtail/users/1*************9
查看
ls /etc/ilogtail/users/

配置机器组标识

echo "amazon-pay-prod" > /etc/ilogtail/1*************9

检查用户自定义标识是否写入成功，可使用以下命令。

cat /etc/ilogtail/1*************9

新增、删除、修改user_defined_id文件后，默认情况下，1分钟内生效。如果需要立即生效，请执行以下命令重启Logtail。

/etc/init.d/ilogtaild stop
/etc/init.d/ilogtaild start

logtail的AppInfo记录文件（app_info.json）路径为/usr/local/ilogtail/app_info.json。
Logtail运行日志（ilogtail.LOG）文件路径为/usr/local/ilogtail/ilogtail.LOG。

©著作权归作者所有,转载或内容合作请联系作者
平台声明：文章内容（如有图片或视频亦包括在内）由作者上传并发布，文章内容仅代表作者本人观点，简书系信息发布平台，仅提供信息存储服务。

推荐阅读更多精彩内容