-
制作Flink的Parcel包和csd文件
-
将Parcel包和manifest.json文件部署到httpd服务中
[root@node01 ~]# mkdir -p /var/www/html/cloudera-repos/flink-parcel/ [root@node01 ~]# cd /var/www/html/cloudera-repos/flink-parcel/ [root@node01 flink-parcel]# cp -R /root/github/cloudera/flink-parcel/FLINK-1.9.2-BIN-SCALA_2.12_build/* ./ [root@node01 flink-parcel]# ll total 240424 -rw-r--r-- 1 root root 246182815 Apr 17 13:33 FLINK-1.9.2-BIN-SCALA_2.12-el7.parcel -rw-r--r-- 1 root root 41 Apr 17 13:33 FLINK-1.9.2-BIN-SCALA_2.12-el7.parcel.sha -rw-r--r-- 1 root root 578 Apr 17 13:33 manifest.json浏览器访问是否部署成功
image -
将生成的csd文件,复制到cloudera-manager-server服务所在节点的/opt/cloudera/csd目录下
[root@node01 ~]# cd /opt/cloudera/csd/ [root@node01 csd]# cp ~/github/cloudera/flink-parcel/FLINK_ON_YARN-1.9.2.jar ./ [root@node01 csd]# cp ~/github/cloudera/flink-parcel/FLINK-1.9.2.jar ./ [root@node01 csd]# ll | grep FLINK -rw-r--r-- 1 root root 7737 Apr 17 13:37 FLINK-1.9.2.jar -rw-r--r-- 1 root root 7799 Apr 17 13:37 FLINK_ON_YARN-1.9.2.jar -
重启cloudera-scm-server服务
[root@node01 ~]# systemctl restart cloudera-scm-server -
配置远程Parcel存储库
image
image -
重启Cloudera Management Service
image -
下载分发激活
下载
image分配
image激活
image
image -
部署Flink
image
image
imageimage
image
-
遇到的问题
Error found before invoking supervisord: 'getpwnam(): name not found: flink'image解决办法:
在 Flink-yarn 服务所在的节点添加 flink 用户和角色：
[root@node01 ~]# groupadd flink [root@node01 ~]# useradd flink -g flink -
集群测试
[root@node01 ~]# /opt/cloudera/parcels/FLINK/lib/flink/bin/flink run -m yarn-cluster -yn 4 -yjm 1024 -ytm 1024 /opt/cloudera/parcels/FLINK/lib/flink/examples/streaming/WordCount.jar --input hdfs://mycluster/test/input/word --output hdfs://mycluster/test/output/wordcount-result ··· 20/05/07 19:56:57 INFO configuration.GlobalConfiguration: Loading configuration property: yarn.tags, flink 20/05/07 19:56:57 INFO rest.RestClusterClient: Submitting job af205be6ccc6f7336f85b28271d98f88 (detached: false). 20/05/07 19:57:10 INFO cli.CliFrontend: Program execution finished Program execution finished Job with JobID af205be6ccc6f7336f85b28271d98f88 has finished. Job Runtime: 10336 ms 20/05/07 19:57:10 INFO rest.RestClient: Shutting down rest endpoint. 20/05/07 19:57:10 INFO rest.RestClient: Rest endpoint shutdown complete. 20/05/07 19:57:10 INFO leaderretrieval.ZooKeeperLeaderRetrievalService: Stopping ZooKeeperLeaderRetrievalService /leader/rest_server_lock. 20/05/07 19:57:10 INFO leaderretrieval.ZooKeeperLeaderRetrievalService: Stopping ZooKeeperLeaderRetrievalService /leader/dispatcher_lock. 20/05/07 19:57:10 INFO imps.CuratorFrameworkImpl: backgroundOperationsLoop exiting 20/05/07 19:57:10 INFO zookeeper.ZooKeeper: Session: 0x471e8f2d12b091e closed 20/05/07 19:57:10 INFO zookeeper.ClientCnxn: EventThread shut down for session: 0x471e8f2d12b091e查看结果
[root@node01 ~]# hdfs dfs -cat /test/output/wordcount-result (hello,1) (flink,1) (hello,2) (spark,1) (hello,3) (hive,1) (hadoop,1) (kafka,1) (spark,2) (flink,2)如出现以下现象:
Retrying connect to server: 0.0.0.0/0.0.0.0:8032. Already tried 3 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)解决办法:
在 CDH 的 yarn 服务中添加 Gateway 服务。
