【采集项目-（4)业务数据采集】

1）在hadoop102的/opt/module/目录下创建 db _log 文件夹

[atguigu@hadoop102 module]$ mkdir db_log/

2）把g mall2020-mock–db-2021-11-14.jar和application.properties 上传到hadoop102的/opt/module/db_log 路径上。
3)创建数据库，建表

# 创建gmall数据库并初始化
create database gmall charset utf8
# 在gmall数据库执行脚本
source /opt/module/db_log/gmall.sql

4）编写生成日志脚本

#!/bin/bash
# 初始化日志
# 生成指定日期日志
DB_HOME='/opt/module/db_log'
DT="$1"
INIT="$2"
sed -i "/mock.clear=/s/.*/mock.clear=0/" $DB_HOME/application.properties
sed -i "/mock.clear.user=/s/.*/mock.clear.user=0/" $DB_HOME/application.properties

if [ "$DT" ]
then 
  sed -i "/mock.date=/s/.*/mock.date=$DT/" $DB_HOME/application.properties
else
  DT=$(cat $DB_HOME/application.properties | grep 'mock.date' | awk -F '=' '{print $2}')
fi
if [ "$INIT" = 'init' ]
then
  sed -i "/mock.clear=/s/.*/mock.clear=1/" $DB_HOME/application.properties
  sed -i "/mock.clear.user=/s/.*/mock.clear.user=1/" $DB_HOME/application.properties
fi

echo "正在生成$DT的数据"
cd $DB_HOME
java -jar gmall2020-mock-db-2021-11-14.jar 1&gt;/dev/null 2&gt;&amp;1

5）生成数据

# 生成数据
[atguigu@hadoop102 db_log]$ chmod +x db_mock.sh 
[atguigu@hadoop102 db_log]$ ll
总用量 15292
-rw-r--r--. 1 atguigu atguigu     1790 9月  18 09:10 application.properties
-rwxr-xr-x. 1 atguigu atguigu      749 9月  18 11:05 db_mock.sh
-rw-r--r--. 1 atguigu atguigu 15648406 2月   8 2022 gmall2020-mock-db-2021-11-14.jar
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-10 init
正在生成2020-06-10的数据
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-11
正在生成2020-06-11的数据
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-12
正在生成2020-06-12的数据
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-13
正在生成2020-06-13的数据
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-14
正在生成2020-06-14的数据
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-15
正在生成2020-06-15的数据
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-16
正在生成2020-06-16的数据
[atguigu@hadoop102 db_log]$ ./db_mock.sh 2020-06-17
正在生成2020-06-17的数据
[atguigu@hadoop102 db_log]$

官网地址：http://maxwells-daemon.io/
1）下载安装包
（1）地址：https://github.com/zendesk/maxwell/releases/download/v1.29.2/maxwell-1.29.2.tar.gz
注：Maxwell-1.30.0及以上版本不再支持JDK1.8。
（2）将安装包上传到h adoop102节点的/opt/software 目录
2）将安装包解压至/opt/mod ule

[atguigu@hadoop102 maxwell]$ tar -zxvf maxwell-1.29.2.tar.gz -C /opt/module/

[atguigu@hadoop102 module]$ mv maxwell-1.29.2/ maxwell

[mysqld]

#数据库id
server-id = 1
#启动binlog，该参数的值会作为binlog的文件名
log-bin=mysql-bin
#binlog类型，maxwell要求为row类型
binlog_format=row
#启用binlog的数据库，需根据实际情况作出修改
binlog-do-db=gmall

[atguigu@hadoop102 ~]$ sudo systemctl restart mysqld

msyql> CREATE DATABASE maxwell;

mysql> set global validate_password_policy=0;
mysql> set global validate_password_length=4;

mysql> CREATE USER 'maxwell'@'%' IDENTIFIED BY 'maxwell';
mysql> GRANT ALL ON maxwell.* TO 'maxwell'@'%';
mysql> GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'maxwell'@'%';

[atguigu@hadoop102 maxwell]$ cd /opt/module/maxwell
[atguigu@hadoop102 maxwell]$ cp config.properties.example config.properties

[atguigu@hadoop102 maxwell]$ vim config.properties

#Maxwell数据发送目的地，可选配置有stdout|file|kafka|kinesis|pubsub|sqs|rabbitmq|redis
producer=kafka
#目标Kafka集群地址
kafka.bootstrap.servers=hadoop102:9092,hadoop103:9092
#目标Kafka topic，可静态配置，例如:maxwell，也可动态配置，例如：%{database}_%{table}
kafka_topic=maxwell

#MySQL相关配置
host=hadoop102
user=maxwell
password=maxwell
jdbc_options=useSSL=false&amp;serverTimezone=Asia/Shanghai

[atguigu@hadoop102 ~]$ /opt/module/maxwell/bin/maxwell --config /opt/module/maxwell/config.properties --daemon

[atguigu@hadoop102 ~]$ ps -ef | grep maxwell | grep -v grep | grep maxwell | awk '{print $2}' | xargs kill -9

[atguigu@hadoop102 bin]$ vim mxw.sh

#!/bin/bash

MAXWELL_HOME=/opt/module/maxwell
status_maxwell(){
    result=`ps -ef | grep com.zendesk.maxwell.Maxwell | grep -v grep | wc -l`
    return $result
}
start_maxwell(){
    status_maxwell
    if [[ $? -lt 1 ]]; then
        echo "启动Maxwell"
        $MAXWELL_HOME/bin/maxwell --config $MAXWELL_HOME/config.properties --daemon
    else
        echo "Maxwell正在运行"
    fi
}
stop_maxwell(){
    status_maxwell
    if [[ $? -gt 0 ]]; then
        echo "停止Maxwell"
        ps -ef | grep com.zendesk.maxwell.Maxwell | grep -v grep | awk '{print $2}' | xargs kill -9
    else
        echo "Maxwell未在运行"
    fi
}
case $1 in
    start )
        start_maxwell
    ;;
    stop )
        stop_maxwell
    ;;
    restart )
       stop_maxwell
       start_maxwell
    ;;
esac

[atguigu@hadoop102 kafka]$ bin/kafka-console-consumer.sh --bootstrap-server hadoop102:9092 --topic maxwell

[root@hadoop102 db_log]# ./db_mock.sh 2020-06-18
正在生成2020-06-18的数据

[atguigu@hadoop102 maxwell]$ /opt/module/maxwell/bin/maxwell-bootstrap --database gmall --table spu_info --config /opt/module/maxwell/config.properties

{
    "database": "fooDB",
    "table": "barTable",
    "type": "bootstrap-start",
    "ts": 1450557744,
    "data": {}
}
{
    "database": "fooDB",
    "table": "barTable",
    "type": "bootstrap-insert",
    "ts": 1450557744,
    "data": {
        "txt": "hello"
    }
}
{
    "database": "fooDB",
    "table": "barTable",
    "type": "bootstrap-insert",
    "ts": 1450557744,
    "data": {
        "txt": "bootstrap!"
    }
}
{
    "database": "fooDB",
    "table": "barTable",
    "type": "bootstrap-complete",
    "ts": 1450557744,
    "data": {}
}