k8s部署canal-1.1.6版本实现MySQL数据库数据同步

1、版本说明

注意：

2、MySQL数据库说明

3、k8s部署yml文件说明

当前yml文件为两个MySQL数据库之间的同步,如果是MySQL和ES或者其他数据库之间的数据同步,此文档仅供参考；如果仅同步某个库下的某几张表数据,可以参考官方文档（https://github.com/alibaba/canal/wiki/Sync-RDB）,修改对应配置,这里不再多做说明

（1）请先部署canal deployer、canal adapter对应的configMap文件

③1、Canal deployer部署文件

---
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
app: canal-deployer
  name: canal-deployer
  namespace: canal-mysql
spec:
  replicas: 1
  minReadySeconds: 10
  selector:
matchLabels:
  app: canal-deployer
  template:
metadata:
  name: canal-deployer
  labels:
    app: canal-deployer
spec:
  volumes:
    - name: conf
      configMap:
        defaultMode: 493
        name: canal-deployer-configmap
  containers:
  - name: canal-deployer
    image: canal/canal-server:latest
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /home/admin/canal-server/conf/canal.properties
      name: conf
      subPath: canal.properties
    - mountPath: /home/admin/canal-server/conf/example/instance.properties
      name: conf
      subPath: instance.properties
    - mountPath: /home/admin/app.sh
      name: conf
      subPath: app.sh
    ports:
    - containerPort: 11111
      protocol: TCP
    - containerPort: 11112
      protocol: TCP

---
kind: Service
apiVersion: v1
metadata:
  name: canal-deployer
  namespace: canal-mysql
  labels:
app: canal-deployer
spec:
  ports:
  - port: 11111
name: deployer
  - port: 11112
name: metrics
  clusterIP: None
  selector:
app: canal-deployer

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: canal-deployer-configmap
  namespace: canal-mysql
data:
  app.sh: |-
#!/bin/bash
set -e

source /etc/profile
export JAVA_HOME=/usr/java/latest
export PATH=$JAVA_HOME/bin:$PATH
touch /tmp/start.log
host=◆hostname -i◆

# waitterm
#   wait TERM/INT signal.
#   see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
waitterm() {
        local PID
        # any process to block
        tail -f /dev/null 
        PID="$!"
        # setup trap, could do nothing, or just kill the blocker
        trap "kill -TERM ${PID}" TERM INT
        # wait for signal, ignore wait exit code
        wait "${PID}" || true
        # clear trap
        trap - TERM INT
        # wait blocker, ignore blocker exit code
        wait "${PID}" 2>/dev/null || true
}

# waittermpid "${PIDFILE}".
#   monitor process by pidfile  wait TERM/INT signal.
#   if the process disappeared, return 1, means exit with ERROR.
#   if TERM or INT signal received, return 0, means OK to exit.
waittermpid() {
        local PIDFILE PID do_run error
        PIDFILE="${1?}"
        do_run=true
        error=0
        trap "do_run=false" TERM INT
        while "${do_run}" ; do
                PID="$(cat "${PIDFILE}")"
                if ! ps -p "${PID}" >/dev/null 2>1 ; then
                        do_run=false
                        error=1
                else
                        sleep 1
                fi
        done
        trap - TERM INT
        return "${error}"
}

function checkStart() {
    local name=$1
    local cmd=$2
    local timeout=$3
    cost=5
    while [ $timeout -gt 0 ]; do
        ST=◆eval $cmd◆
        if [ "$ST" == "0" ]; then
            sleep 1
            let timeout=timeout-1
            let cost=cost◆1
        elif [ "$ST" == "" ]; then
            sleep 1
            let timeout=timeout-1
            let cost=cost◆1
        else
            break
        fi
    done
    echo "start $name successful"
}

function start_canal() {
    echo "start canal ..."
    managerAddress=◆perl -le 'print $ENV{"canal.admin.manager"}'◆
    if [ ! -z "$managerAddress" ] ; then
        # canal_local.properties mode
        adminPort=◆perl -le 'print $ENV{"canal.admin.port"}'◆
        if [ -z "$adminPort" ] ; then
            adminPort=11110
        fi

        su root -c 'cd /home/admin/canal-server/bin/  sh restart.sh local 1>>/tmp/start.log 2>1'
        sleep 5
        #check start
        checkStart "canal" "nc 12⑦0.0.1 $adminPort -w 1 -z | wc -l" 30
    else
        metricsPort=◆perl -le 'print $ENV{"canal.metrics.pull.port"}'◆
        if [ -z "$metricsPort" ] ; then
            metricsPort=11112
        fi

        destination=◆perl -le 'print $ENV{"canal.destinations"}'◆
        if [[ "$destination" =~ ',' ]]; then
            echo "multi destination:$destination is not support"
            exit 1;
        else
            if [ "$destination" != "" ]  [ "$destination" != "example" ] ; then
                if [ -d /home/admin/canal-server/conf/example ]; then
                    mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination
                fi
            fi 
        fi

        su root -c 'cd /home/admin/canal-server/bin/  sh restart.sh 1>>/tmp/start.log 2>1'
        sleep 5
        #check start
        checkStart "canal" "nc 12⑦0.0.1 $metricsPort -w 1 -z | wc -l" 30
    fi  
}

function stop_canal() {
    echo "stop canal"
    su root -c 'cd /home/admin/canal-server/bin/  sh stop.sh 1>>/tmp/start.log 2>1'
    echo "stop canal successful ..."
}

function start_exporter() {
    su root -c 'cd /home/admin/node_exporter  ./node_exporter 1>>/tmp/start.log 2>1 '
}

function stop_exporter() {
    su root -c 'killall node_exporter'
}

echo "==> START ..."

start_exporter
start_canal

echo "==> START SUCCESSFUL ..."

tail -f /dev/null 
# wait TERM signal
waitterm

echo "==> STOP"

stop_canal
stop_exporter

echo "==> STOP SUCCESSFUL ..."
  canal.properties: >-
canal.ip =
canal.register.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
# 修改点: 此处值修改为部署的zookeeper服务的地址
canal.zkServers = zookeeper:2181
canal.withoutNetty = false
canal.serverMode = tcp
canal.destinations = example
canal.auto.scan = true

canal.instance.detecting.enable = false
canal.instance.detecting.sql = select 1
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false

canal.instance.transaction.size =  1024
canal.instance.fallbackIntervalInSeconds = 60

canal.instance.network.receiveBufferSize = 16384
canal.instance.network.sendBufferSize = 16384
canal.instance.network.soTimeout = 30

canal.instance.filter.druid.ddl = true
canal.instance.filter.query.dcl = false
canal.instance.filter.query.dml = false
canal.instance.filter.query.ddl = false
canal.instance.filter.table.error = false
canal.instance.filter.rows = false
canal.instance.filter.transaction.entry = false
canal.instance.filter.dml.insert = false
canal.instance.filter.dml.update = false
canal.instance.filter.dml.delete = false

canal.conf.dir = ../conf
canal.instance.binlog.format = ROW,STATEMENT,MIXED
canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
canal.instance.get.ddl.isolation = false
canal.instance.parser.parallel = true
canal.instance.parser.parallelThreadSize = 16

canal.instance.tsdb.enable = false
canal.instance.tsdb.dir = ${canal.conf.dir}/tsdb
canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername = canal
canal.instance.tsdb.dbPassword = canal
canal.instance.tsdb.snapshot.interval = 24
canal.instance.tsdb.snapshot.expire = 360
canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml

canal.instance.global.mode = spring
canal.instance.global.lazy = false
canal.instance.global.spring.xml = classpath:spring/default-instance.xml
  instance.properties: |-
canal.instance.tsdb.enable = true
canal.instance.gtidon = false
# 修改点：将此处值修改为源库的地址
canal.instance.master.address = 192.16⑧91.131:3306
canal.instance.master.journal.name =
canal.instance.master.position =
canal.instance.master.timestamp=
canal.instance.master.gtid=
# 修改点: 将此处的用户名、密码修改为源库对应的用户名、密码
canal.instance.dbUsername = root
canal.instance.dbPassword = root@123456
canal.instance.connectionCharset = UTF-8
canal.instance.enableDruid = false
canal.instance.filter.regex=.*\\..*
# 过滤控制台打印与配置表无关的日志信息
canal.instance.filter.black.regex=mysql\\.slave_.*
canal.mq.topic = example
canal.mq.partition = 0

说明：

③2、Canal adapter部署文件

---
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
app: canal-adapter
  name: canal-adapter
  namespace: canal-mysql
spec:
  replicas: 1
  minReadySeconds: 10
  selector:
matchLabels:
  app: canal-adapter
  template:
metadata:
  name: canal-adapter
  labels:
    app: canal-adapter
spec:
  volumes:
  - name: conf
    defaultMode: 420
    configMap:
      name: canal-adapter-configmap
  containers:
  - name: canal-adapter
    image: funnyzak/canal-adapter:1.1.6
    imagePullPolicy: IfNotPresent
    volumeMounts:
      - mountPath: /opt/canal/canal-adapter/conf/application.yml
        name: conf
        subPath: application.yml
      - mountPath: /opt/canal/canal-adapter/conf/bootstrap.yml
        name: conf
        subPath: bootstrap.yml
      - mountPath: /opt/canal/canal-adapter/conf/rdb/monitoralter.yml
        name: conf
        subPath: monitoralter.yml
    ports:
    - containerPort: 8081
      protocol: TCP
 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: canal-adapter-configmap
  namespace: canal-mysql
data:
  application.yml: |-
server:
  port: 8081
logging:
  level:
    org.springframework: WARN
    com.alibaba.otter.canal.client.adapter.rdb: WARN
spring:
  jackson:
    date-format: yyyy-MM-dd HH:mm:ss
    time-zone: GMT◆8
    default-property-inclusion: non_null
canal.conf:
  mode: tcp
  flatMessage: true
  # 修改点：zookeeper服务对应的地址
  zookeeperHosts: zookeeper:2181
  syncBatchSize: 1000
  retries: 0
  timeout:
  accessKey:
  secretKey:
  consumerProperties:
    # 修改点：由于采用tcp方式,所以这里配置就是tcp方式下canal deployer对应的地址
    canal.tcp.server.host: canal-deployer:11111
    canal.tcp.zookeeper.hosts:
    canal.tcp.batch.size: 500
    canal.tcp.username:
    canal.tcp.password:
  # 修改点：源库数据源配置信息
  srcDataSources:
    monitorAlterDS:
      url: jdbc:mysql://192.16⑧91.131:3306/monitor_alert?useUnicode=truecharacterEncoding=utf8autoReconnect=trueuseSSL=false
      username: root
      password: root@123456
  # 修改点：目标库数据源配置信息
  canalAdapters:
  - instance: example
    groups:
    - groupId: g1
      outerAdapters:
      - name: logger
      - name: rdb
        key: mysql1
        properties:
          jdbc.driverClassName: com.mysql.jdbc.Driver
          jdbc.url: jdbc:mysql://192.16⑧91.135:3306/monitor_alert?useUnicode=truecharacterEncoding=utf8autoReconnect=trueuseSSL=false
          jdbc.username: root
          jdbc.password: admin@4321
  bootstrap.yml:
  # 修改点：镜像方式同步配置文件
  monitoralter.yml: |-
dataSourceKey: monitorAlterDS
destination: example
groupId: g1
outerAdapterKey: mysql1
concurrent: true
dbMapping:
  mirrorDb: true
  database: monitor_alert

## 例如：需要再同步product数据库下的表数据
# Deployment文件修改如下：
# 1、canal adapter的Deployment文件添加挂载的product数据库信息
containers:
- name: canal-adapter
  image: funnyzak/canal-adapter:1.1.6
  imagePullPolicy: IfNotPresent
  volumeMounts:
- mountPath: /opt/canal/canal-adapter/conf/application.yml
  name: conf
  subPath: application.yml
- mountPath: /opt/canal/canal-adapter/conf/bootstrap.yml
  name: conf
  subPath: bootstrap.yml
- mountPath: /opt/canal/canal-adapter/conf/rdb/monitoralter.yml
  name: conf
  subPath: monitoralter.yml
- mountPath: /opt/canal/canal-adapter/conf/rdb/product.yml
  name: conf
  subPath: product.yml

# configMap修改如下：
# 1、application.yml文件添加源库、目标库的地址信息
# 源库信息
srcDataSources:
  monitorAlterDS:
url: jdbc:mysql://192.16⑧91.131:3306/monitor_alert?useUnicode=truecharacterEncoding=utf8autoReconnect=trueuseSSL=false
username: root
password: root@123456
  productDS:
url: jdbc:mysql://192.16⑧91.131:3306/product?useUnicode=truecharacterEncoding=utf8autoReconnect=trueuseSSL=false
username: root
password: root@123456
# 目标库信息
canalAdapters:
- instance: example
  groups:
  - groupId: g1
outerAdapters:
- name: logger
- name: rdb
  key: mysql1
  properties:
    jdbc.driverClassName: com.mysql.jdbc.Driver
    jdbc.url: jdbc:mysql://192.16⑧91.135:3306/monitor_alert?useUnicode=truecharacterEncoding=utf8autoReconnect=trueuseSSL=false
    jdbc.username: root
    jdbc.password: admin@4321
 - name: rdb
   key: mysql2
   properties:
     jdbc.driverClassName: com.mysql.jdbc.Driver
     jdbc.url: jdbc:mysql://192.16⑧91.135:3306/product?useUnicode=truecharacterEncoding=utf8autoReconnect=trueuseSSL=false
     jdbc.username: root
     jdbc.password: admin@4321
# 2、添加对应的挂载配置文件product.yml
product.yml: |-
  dataSourceKey: productDS
  destination: example
  groupId: g1
  outerAdapterKey: mysql2
  concurrent: true
  dbMapping:
mirrorDb: true
database: product           

③3、Zookeeper部署文件

---
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
app: zk-deployment
  name: zookeeper
  namespace: canal-mysql
spec:
  replicas: 2
  selector:
matchLabels:
  app: zk
  template:
metadata:
  name: zk
  labels:
    app: zk
spec:
  volumes:
  - name: localtime
    hostPath:
      path: /usr/share/zoneinfo/Asia/Shanghai
  containers:
  - name: zookeeper
    image: zookeeper:③⑥2
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - name: localtime
      mountPath: /etc/localtime
      readOnly: true

---
kind: Service
apiVersion: v1
metadata:
  name: zookeeper
  namespace: canal-mysql
  labels:
app: zk
spec:
  ports:
  - port: 2181
name: client
  clusterIP: None
  selector:
app: zk

4、部署成功图

④1、Canal Deployer部署成功图

④2、Canal Adapter部署成功图

以上就是土嘎嘎小编大虾米为大家整理的相关主题介绍,如果您觉得小编更新的文章只要能对粉丝们有用，就是我们最大的鼓励和动力,不要忘记讲本站分享给您身边的朋友哦！！