Flink通过Native Kubernetes部署运行

在 Flink on Kubernetes 环境中，Job 部署方式分为Native Kubernetes和Kubernetes Operator2种，本文以Native Kubernetes appplication模式部署运行介绍。

一、环境准备，创建K8S账户以及授权

# 创建namespace
kubectl create ns flink
# 创建serviceaccount
kubectl create serviceaccount flink-service-account -n flink
# 用户授权
kubectl create clusterrolebinding flink-role-binding-flink --clusterrole=edit --serviceaccount=flink:flink-service-account

二、部署flink historyserver

1. flink-historyserver-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: flink-historyserver-configmap
  namespace: flink
data:
  # file format
  flink-conf.yaml: |
    execution.target: kubernetes-session
    rest.address: localhost
    rest.bind-address: localhost
    jobmanager.bind-host: localhost
    jobmanager.rpc.address: localhost
    jobmanager.rpc.port: 6123
    jobmanager.memory.process.size: 1600m
    jobmanager.execution.failover-strategy: region
    taskmanager.bind-host: localhost
    taskmanager.host: localhost
    taskmanager.memory.process.size: 1728m
    taskmanager.numberOfTaskSlots: 1
    parallelism.default: 1
    state.backend: filesystem
    fs.allowed-fallback-filesystems: s3
    s3.endpoint: http://127.0.0.1:9680
    s3.access-key: sk
    s3.secret-key: sk
    s3.path.style.access: true
    state.checkpoints.dir: s3://flink/flink-checkpoints
    # HistoryServer
    jobmanager.archive.fs.dir: s3://flink/completed-jobs/
    historyserver.web.address: 0.0.0.0
    historyserver.web.port: 8082
    historyserver.archive.fs.dir: s3://flink/completed-jobs/
    historyserver.archive.fs.refresh-interval: 10000
    historyserver.archive.clean-expired-jobs：false
    historyserver.archive.retained-jobs：-1
  log4j.properties: |
    # Allows this configuration to be modified at runtime. The file will be checked every 30 seconds.
    monitorInterval=30
    # This affects logging for both user code and Flink
    rootLogger.level = INFO
    rootLogger.appenderRef.file.ref = MainAppender
    # Uncomment this if you want to _only_ change Flink's logging
    #logger.flink.name = org.apache.flink
    #logger.flink.level = INFO
    # The following lines keep the log level of common libraries/connectors on
    # log level INFO. The root logger does not override this. You have to manually
    # change the log levels here.
    logger.akka.name = akka
    logger.akka.level = INFO
    logger.kafka.name= org.apache.kafka
    logger.kafka.level = INFO
    logger.hadoop.name = org.apache.hadoop
    logger.hadoop.level = INFO
    logger.zookeeper.name = org.apache.zookeeper
    logger.zookeeper.level = INFO
    logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3
    logger.shaded_zookeeper.level = INFO
    # Log all infos in the given file
    appender.main.name = MainAppender
    appender.main.type = RollingFile
    appender.main.append = true
    appender.main.fileName = ${sys:log.file}
    appender.main.filePattern = ${sys:log.file}.%i
    appender.main.layout.type = PatternLayout
    appender.main.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
    appender.main.policies.type = Policies
    appender.main.policies.size.type = SizeBasedTriggeringPolicy
    appender.main.policies.size.size = 100MB
    appender.main.policies.startup.type = OnStartupTriggeringPolicy
    appender.main.strategy.type = DefaultRolloverStrategy
    appender.main.strategy.max = ${env:MAX_LOG_FILE_NUMBER:-10}
    # Suppress the irrelevant (wrong) warnings from the Netty channel handler
    logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline
    logger.netty.level = OFF
  log4j-console.properties: |
    # This affects logging for both user code and Flink
    rootLogger.level = INFO
    rootLogger.appenderRef.console.ref = ConsoleAppender
    rootLogger.appenderRef.rolling.ref = RollingFileAppender
     
    # Uncomment this if you want to _only_ change Flink's logging
    #logger.flink.name = org.apache.flink
    #logger.flink.level = INFO
     
    # The following lines keep the log level of common libraries/connectors on
    # log level INFO. The root logger does not override this. You have to manually
    # change the log levels here.
    logger.akka.name = akka
    logger.akka.le