Centos7安装Canal

1、创建用户并授权

mysql> CREATE USER canal@'%' IDENTIFIED BY '123456';
Query OK, 0 rows affected (0.10 sec)

mysql> GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';
Query OK, 0 rows affected (0.03 sec)

mysql> FLUSH PRIVILEGES; 
Query OK, 0 rows affected (0.03 sec)

2、下载

https://github.com/alibaba/canal/releases

3、解压

[root@localhost canal]# tar -zxvf canal.deployer-1.1.5.tar.gz
[root@localhost canal]# ll
总用量 4
drwxr-xr-x 2 root root   76 8月  28 11:11 bin
drwxr-xr-x 5 root root  123 8月  28 11:11 conf
drwxr-xr-x 2 root root 4096 8月  28 11:12 lib
drwxrwxrwx 2 root root    6 4月  19 16:15 logs
drwxrwxrwx 2 root root  177 4月  19 16:15 plugin

4、配置

[root@localhost canal]# vim conf/example/instance.properties 

# enable gtid use true/false
canal.instance.gtidon=false
canal.instance.master.address=192.168.171.128:3306
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal

#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#canal.instance.standby.gtid=

# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=123456
canal.instance.defaultDatabaseName=canaldb
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUg
unvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==

# table regex
canal.instance.filter.regex=canaldb\\..*
# table black regex
canal.instance.filter.black.regex=mysql\\.slave_.*
# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field
1/field2)
#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/na
me/contact/ch
# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2
:field1/field2)
#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_compa
ny:id/name/contact/ch

# mq config
canal.mq.topic=example
# dynamic topic route by schema or table regex
#canal.mq.dynamicTopic=mytest1.user,mytest2\\..*,.*\\..*
canal.mq.partition=0
# hash partition config
#canal.mq.partitionsNum=3
#canal.mq.partitionHash=test.table:id^name,.*\\..*
#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
#################################################

5、启动

# ./bin/startup.sh  启动
# ./bin/stop.sh 停止
[root@localhost canal]# ./bin/startup.sh

1、安装canal-adapter

[root@localhost canal-adapter]# tar -zxvf canal.adapter-1.1.5.tar.gz

[root@localhost canal-adapter]# ll
总用量 8
drwxr-xr-x 2 root root   76 8月  28 16:51 bin
drwxrwxrwx 8 root root  143 4月  19 15:48 conf
drwxr-xr-x 2 root root 4096 8月  28 16:51 lib
drwxrwxrwx 2 root root    6 4月  19 16:16 logs
drwxrwxrwx 2 root root 4096 4月  19 16:16 plugin

2、配置

# 修改配置文件：conf/application.yml
[root@localhost canal-adapter]# vim conf/application.yml
server:
  port: 8081
spring:
  jackson:
    date-format: yyyy-MM-dd HH:mm:ss
    time-zone: GMT+8
    default-property-inclusion: non_null

canal.conf:
  mode: tcp #tcp kafka rocketMQ rabbitMQ
  flatMessage: true
  zookeeperHosts:
  syncBatchSize: 1000
  retries: 0
  timeout:
  accessKey:
  secretKey:
  consumerProperties:
    # canal tcp consumer
    canal.tcp.server.host: 192.168.171.129:11111
    canal.tcp.zookeeper.hosts:
    canal.tcp.batch.size: 500
    canal.tcp.username:
    canal.tcp.password:
    # kafka consumer
    kafka.bootstrap.servers: 127.0.0.1:9092
    kafka.enable.auto.commit: false
    kafka.auto.commit.interval.ms: 1000
    kafka.auto.offset.reset: latest
    kafka.request.timeout.ms: 40000
    kafka.session.timeout.ms: 30000
    kafka.isolation.level: read_committed
    kafka.max.poll.records: 1000
    # rocketMQ consumer
    rocketmq.namespace:
    rocketmq.namesrv.addr: 127.0.0.1:9876
    rocketmq.batch.size: 1000
    rocketmq.enable.message.trace: false
    rocketmq.customized.trace.topic:
    rocketmq.access.channel:
    rocketmq.subscribe.filter:
    # rabbitMQ consumer
    rabbitmq.host:
    rabbitmq.virtual.host:
    rabbitmq.username:
    rabbitmq.password:
    rabbitmq.resource.ownerId:

  srcDataSources:
    defaultDS:
      # 配置主库的数据源
      url: jdbc:mysql://192.168.171.128:3306/canaldb?useUnicode=true&useSSL=false
      username: root
      password: 123456
  canalAdapters:
  - instance: example # canal instance Name or mq topic name
    groups:
    - groupId: g1
      outerAdapters:
      -name: logger
      - name: es7
        key: exampleKey
        hosts: 192.168.171.128:9300, 192.168.171.129:9300, 192.168.171.130:9300# 127.0.0.1:9200 for rest mode
        properties:
          mode: transport
          cluster.name: my-elasticsearch
#      - name: rdb
#        key: mysql1
#        properties:
#          jdbc.driverClassName: com.mysql.jdbc.Driver
#          jdbc.url: jdbc:mysql://127.0.0.1:3306/mytest2?useUnicode=true
#          jdbc.username: root
#          jdbc.password: 121212
#      - name: rdb
#        key: oracle1
#        properties:
#          jdbc.driverClassName: oracle.jdbc.OracleDriver
#          jdbc.url: jdbc:oracle:thin:@localhost:49161:XE
#          jdbc.username: mytest
#          jdbc.password: m121212
#      - name: rdb
#        key: postgres1
#        properties:
#          jdbc.driverClassName: org.postgresql.Driver
#          jdbc.url: jdbc:postgresql://localhost:5432/postgres
#          jdbc.username: postgres
#          jdbc.password: 121212
#          threads: 1
#          commitSize: 3000
#      - name: hbase
#        properties:
#          hbase.zookeeper.quorum: 127.0.0.1
#          hbase.zookeeper.property.clientPort: 2181
#          zookeeper.znode.parent: /hbase
#      - name: es
#        hosts: 127.0.0.1:9300 # 127.0.0.1:9200 for rest mode
#        properties:
#          mode: transport # or rest
#          # security.auth: test:123456 #  only used for rest mode
#          cluster.name: elasticsearch

3、ES配置

[root@localhost es7]# ll
总用量 12
-rwxrwxrwx 1 root root 449 4月  19 15:48 biz_order.yml
-rwxrwxrwx 1 root root 838 4月  19 15:48 customer.yml
-rwxrwxrwx 1 root root 354 4月  19 15:48 mytest_user.yml

[root@localhost es]# cp customer.yml ./canal.yml
[root@localhost es6]# vim canal.yml
dataSourceKey: defaultDS
outerAdapterKey: exampleKey
destination: example
groupId: g1
esMapping:
  _index: book
  _type: _doc
  _id: _id
  upsert: true
#  pk: id
  sql: "select t.id as _id, t.name, t.author, t.publishtime, t.price, t.publishgroup from tb_book t"
#  objFields:
#    _labels: array:;
  etlCondition: "where a.c_time>='{0}'"
  commitBatch: 3000

4、创建ES索引

put http://192.168.171.129:9200/book/_mapping
{
    "properties": {
        "id": {
            "type": "long"
        },
        "name": {
            "type": "text"
        },
        "author": {
            "type": "text"
        },
        "publishtime": {
            "type": "date"
        },
        "price": {
            "type": "float"
        },
        "publishgroup": {
            "type": "text"
        }
    }
}

5、启动

[root@localhost canal-adapter]# ./bin/startup.sh