配置文件不是原始的 修改过，可以根据自己的配置文件来

jobmanager配置

flink-conf.yaml

################################################################################

#  Licensed to the Apache Software Foundation (ASF) under one

#  or more contributor license agreements.  See the NOTICE file

#  distributed with this work for additional information

#  regarding copyright ownership.  The ASF licenses this file

#  to you under the Apache License, Version 2.0 (the

#  "License"); you may not use this file except in compliance

#  with the License.  You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

#  Unless required by applicable law or agreed to in writing, software

#  distributed under the License is distributed on an "AS IS" BASIS,

#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

#  See the License for the specific language governing permissions and

# limitations under the License.

################################################################################

#==============================================================================

# Common

#==============================================================================

# The external address of the host on which the JobManager runs and can be

# reached by the TaskManagers and any clients which want to connect. This setting

# is only used in Standalone mode and may be overwritten on the JobManager side

# by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.

# In high availability mode, if you use the bin/start-cluster.sh script and setup

# the conf/masters file, this will be taken care of automatically. Yarn/Mesos

# automatically configure the host name based on the hostname of the node where the

# JobManager runs.

jobmanager.rpc.address: jobmanager

# The RPC port where the JobManager is reachable.

jobmanager.rpc.port: 6123

# The total process memory size for the JobManager.

#

# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.

jobmanager.memory.process.size: 2g

# The total process memory size for the TaskManager.

#

# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.

taskmanager.memory.process.size: 3g

taskmanager.memory.jvm-metaspace.size: 512m

# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.

# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.

#

# taskmanager.memory.flink.size: 1280m

# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.

taskmanager.numberOfTaskSlots: 4

# The parallelism used for programs that did not specify and other parallelism.

parallelism.default: 1

# The default file system scheme and authority.

#

# By default file paths without scheme are interpreted relative to the local

# root file system 'file:///'. Use this to override the default and interpret

# relative paths relative to a different file system,

# for example 'hdfs://mynamenode:12345'

#

# fs.default-scheme

#==============================================================================

# High Availability

#==============================================================================

# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.

#

# high-availability: zookeeper

# The path where metadata for master recovery is persisted. While ZooKeeper stores

# the small ground truth for checkpoint and leader election, this location stores

# the larger objects, like persisted dataflow graphs.

#

# Must be a durable file system that is accessible from all nodes

# (like HDFS, S3, Ceph, nfs, ...)

#

# high-availability.storageDir: hdfs:///flink/ha/

# The list of ZooKeeper quorum peers that coordinate the high-availability

# setup. This must be a list of the form:

# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)

#

# high-availability.zookeeper.quorum: localhost:2181

# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes

# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)

# The default value is "open" and it can be changed to "creator" if ZK security is enabled

#

# high-availability.zookeeper.client.acl: open

#==============================================================================

# Fault tolerance and checkpointing

#==============================================================================

# The backend that will be used to store operator state checkpoints if

# checkpointing is enabled.

#

# Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the

# <class-name-of-factory>.

#

# state.backend: filesystem

# Directory for checkpoints filesystem, when using any of the default bundled

# state backends.

#

# state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints

# Default target directory for savepoints, optional.

#

#state.savepoints.dir: file:/tmp/savepoint

# Flag to enable/disable incremental checkpoints for backends that

# support incremental checkpoints (like the RocksDB state backend).

#

# state.backend.incremental: false

# The failover strategy, i.e., how the job computation recovers from task failures.

# Only restart tasks that may have been affected by the task failure, which typically includes

# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.

jobmanager.execution.failover-strategy: region

#==============================================================================

# Rest & web frontend

#==============================================================================

# The port to which the REST client connects to. If rest.bind-port has

# not been specified, then the server will bind to this port as well.

#

#rest.port: 8081

# The address to which the REST client will connect to

#

#rest.address: 0.0.0.0

# Port range for the REST and web server to bind to.

#

#rest.bind-port: 8080-8090

# The address that the REST & web server binds to

#

#rest.bind-address: 0.0.0.0

# Flag to specify whether job submission is enabled from the web-based

# runtime monitor. Uncomment to disable.

#web.submit.enable: false

#==============================================================================

# Advanced

#==============================================================================

# Override the directories for temporary files. If not specified, the

# system-specific Java temporary directory (java.io.tmpdir property) is taken.

#

# For framework setups on Yarn or Mesos, Flink will automatically pick up the

# containers' temp directories without any need for configuration.

#

# Add a delimited list for multiple directories, using the system directory

# delimiter (colon ':' on unix) or a comma, e.g.:

#     /data1/tmp:/data2/tmp:/data3/tmp

#

# Note: Each directory entry is read from and written to by a different I/O

# thread. You can include the same directory multiple times in order to create

# multiple I/O threads against that directory. This is for example relevant for

# high-throughput RAIDs.

#

# io.tmp.dirs: /tmp

# The classloading resolve order. Possible values are 'child-first' (Flink's default)

# and 'parent-first' (Java's default).

#

# Child first classloading allows users to use different dependency/library

# versions in their application than those in the classpath. Switching back

# to 'parent-first' may help with debugging dependency issues.

#

# classloader.resolve-order: child-first

# The amount of memory going to the network stack. These numbers usually need

# no tuning. Adjusting them may be necessary in case of an "Insufficient number

# of network buffers" error. The default min is 64MB, the default max is 1GB.

#

# taskmanager.memory.network.fraction: 0.1

# taskmanager.memory.network.min: 64mb

# taskmanager.memory.network.max: 1gb

#==============================================================================

# Flink Cluster Security Configuration

#==============================================================================

# Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -

# may be enabled in four steps:

# 1. configure the local krb5.conf file

# 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)

# 3. make the credentials available to various JAAS login contexts

# 4. configure the connector to use JAAS/SASL

# The below configure how Kerberos credentials are provided. A keytab will be used instead of

# a ticket cache if the keytab path and principal are set.

# security.kerberos.login.use-ticket-cache: true

# security.kerberos.login.keytab: /path/to/kerberos/keytab

# security.kerberos.login.principal: flink-user

# The configuration below defines which JAAS login contexts

# security.kerberos.login.contexts: Client,KafkaClient

#==============================================================================

# ZK Security Configuration

#==============================================================================

# Below configurations are applicable if ZK ensemble is configured for security

# Override below configuration to provide custom ZK service name if configured

# zookeeper.sasl.service-name: zookeeper

# The configuration below must match one of the values set in "security.kerberos.login.contexts"

# zookeeper.sasl.login-context-name: Client

#==============================================================================

# HistoryServer

#==============================================================================

# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)

# Directory to upload completed jobs to. Add this directory to the list of

# monitored directories of the HistoryServer as well (see below).

#jobmanager.archive.fs.dir: hdfs:///completed-jobs/

# The address under which the web-based HistoryServer listens.

#historyserver.web.address: 0.0.0.0

# The port under which the web-based HistoryServer listens.

#historyserver.web.port: 8082

# Comma separated list of directories to monitor for completed jobs.

#historyserver.archive.fs.dir: hdfs:///completed-jobs/

# Interval in milliseconds for refreshing the monitored directories.

#historyserver.archive.fs.refresh-interval: 10000

blob.server.port: 6124

query.server.port: 6125

jobmanager.web.tmpdir: /tmp/flink-web

jobmanager.web.upload.dir: /tmp/flink-web

#任务失败后，重启时，每两次连续尝试之间间隔1 min，任务重启失败30次，则任务最终失败，配置默认全局生效。

restart-strategy: fixed-delay

restart-strategy.fixed-delay.attempts: 30

restart-strategy.fixed-delay.delay: 1 min

taskmanager

flink-conf.yaml 配置

################################################################################

#  Licensed to the Apache Software Foundation (ASF) under one

#  or more contributor license agreements.  See the NOTICE file

#  distributed with this work for additional information

#  regarding copyright ownership.  The ASF licenses this file

#  to you under the Apache License, Version 2.0 (the

#  "License"); you may not use this file except in compliance

#  with the License.  You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

#  Unless required by applicable law or agreed to in writing, software

#  distributed under the License is distributed on an "AS IS" BASIS,

#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

#  See the License for the specific language governing permissions and

# limitations under the License.

################################################################################

#==============================================================================

# Common

#==============================================================================

# The external address of the host on which the JobManager runs and can be

# reached by the TaskManagers and any clients which want to connect. This setting

# is only used in Standalone mode and may be overwritten on the JobManager side

# by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.

# In high availability mode, if you use the bin/start-cluster.sh script and setup

# the conf/masters file, this will be taken care of automatically. Yarn/Mesos

# automatically configure the host name based on the hostname of the node where the

# JobManager runs.

jobmanager.rpc.address: jobmanager

# The RPC port where the JobManager is reachable.

jobmanager.rpc.port: 6123

# The total process memory size for the JobManager.

#

# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.

jobmanager.memory.process.size: 2g

# The total process memory size for the TaskManager.

#

# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.

taskmanager.memory.process.size: 3g

taskmanager.memory.jvm-metaspace.size: 512m

# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.

# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.

#

# taskmanager.memory.flink.size: 1280m

# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.

taskmanager.numberOfTaskSlots: 4

# The parallelism used for programs that did not specify and other parallelism.

parallelism.default: 1

# The default file system scheme and authority.

#

# By default file paths without scheme are interpreted relative to the local

# root file system 'file:///'. Use this to override the default and interpret

# relative paths relative to a different file system,

# for example 'hdfs://mynamenode:12345'

#

# fs.default-scheme

#==============================================================================

# High Availability

#==============================================================================

# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.

#

# high-availability: zookeeper

# The path where metadata for master recovery is persisted. While ZooKeeper stores

# the small ground truth for checkpoint and leader election, this location stores

# the larger objects, like persisted dataflow graphs.

#

# Must be a durable file system that is accessible from all nodes

# (like HDFS, S3, Ceph, nfs, ...)

#

# high-availability.storageDir: hdfs:///flink/ha/

# The list of ZooKeeper quorum peers that coordinate the high-availability

# setup. This must be a list of the form:

# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)

#

# high-availability.zookeeper.quorum: localhost:2181

# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes

# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)

# The default value is "open" and it can be changed to "creator" if ZK security is enabled

#

# high-availability.zookeeper.client.acl: open

#==============================================================================

# Fault tolerance and checkpointing

#==============================================================================

# The backend that will be used to store operator state checkpoints if

# checkpointing is enabled.

#

# Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the

# <class-name-of-factory>.

#

# state.backend: filesystem

# Directory for checkpoints filesystem, when using any of the default bundled

# state backends.

#

# state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints

# Default target directory for savepoints, optional.

#

#state.savepoints.dir: file:/tmp/savepoint

# Flag to enable/disable incremental checkpoints for backends that

# support incremental checkpoints (like the RocksDB state backend).

#

# state.backend.incremental: false

# The failover strategy, i.e., how the job computation recovers from task failures.

# Only restart tasks that may have been affected by the task failure, which typically includes

# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.

jobmanager.execution.failover-strategy: region

#==============================================================================

# Rest & web frontend

#==============================================================================

# The port to which the REST client connects to. If rest.bind-port has

# not been specified, then the server will bind to this port as well.

#

#rest.port: 8081

# The address to which the REST client will connect to

#

#rest.address: 0.0.0.0

# Port range for the REST and web server to bind to.

#

#rest.bind-port: 8080-8090

# The address that the REST & web server binds to

#

#rest.bind-address: 0.0.0.0

# Flag to specify whether job submission is enabled from the web-based

# runtime monitor. Uncomment to disable.

#web.submit.enable: false

#==============================================================================

# Advanced

#==============================================================================

# Override the directories for temporary files. If not specified, the

# system-specific Java temporary directory (java.io.tmpdir property) is taken.

#

# For framework setups on Yarn or Mesos, Flink will automatically pick up the

# containers' temp directories without any need for configuration.

#

# Add a delimited list for multiple directories, using the system directory

# delimiter (colon ':' on unix) or a comma, e.g.:

#     /data1/tmp:/data2/tmp:/data3/tmp

#

# Note: Each directory entry is read from and written to by a different I/O

# thread. You can include the same directory multiple times in order to create

# multiple I/O threads against that directory. This is for example relevant for

# high-throughput RAIDs.

#

# io.tmp.dirs: /tmp

# The classloading resolve order. Possible values are 'child-first' (Flink's default)

# and 'parent-first' (Java's default).

#

# Child first classloading allows users to use different dependency/library

# versions in their application than those in the classpath. Switching back

# to 'parent-first' may help with debugging dependency issues.

#

# classloader.resolve-order: child-first

# The amount of memory going to the network stack. These numbers usually need

# no tuning. Adjusting them may be necessary in case of an "Insufficient number

# of network buffers" error. The default min is 64MB, the default max is 1GB.

#

# taskmanager.memory.network.fraction: 0.1

# taskmanager.memory.network.min: 64mb

# taskmanager.memory.network.max: 1gb

#==============================================================================

# Flink Cluster Security Configuration

#==============================================================================

# Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -

# may be enabled in four steps:

# 1. configure the local krb5.conf file

# 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)

# 3. make the credentials available to various JAAS login contexts

# 4. configure the connector to use JAAS/SASL

# The below configure how Kerberos credentials are provided. A keytab will be used instead of

# a ticket cache if the keytab path and principal are set.

# security.kerberos.login.use-ticket-cache: true

# security.kerberos.login.keytab: /path/to/kerberos/keytab

# security.kerberos.login.principal: flink-user

# The configuration below defines which JAAS login contexts

# security.kerberos.login.contexts: Client,KafkaClient

#==============================================================================

# ZK Security Configuration

#==============================================================================

# Below configurations are applicable if ZK ensemble is configured for security

# Override below configuration to provide custom ZK service name if configured

# zookeeper.sasl.service-name: zookeeper

# The configuration below must match one of the values set in "security.kerberos.login.contexts"

# zookeeper.sasl.login-context-name: Client

#==============================================================================

# HistoryServer

#==============================================================================

# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)

# Directory to upload completed jobs to. Add this directory to the list of

# monitored directories of the HistoryServer as well (see below).

#jobmanager.archive.fs.dir: hdfs:///completed-jobs/

# The address under which the web-based HistoryServer listens.

#historyserver.web.address: 0.0.0.0

# The port under which the web-based HistoryServer listens.

#historyserver.web.port: 8082

# Comma separated list of directories to monitor for completed jobs.

#historyserver.archive.fs.dir: hdfs:///completed-jobs/

# Interval in milliseconds for refreshing the monitored directories.

#historyserver.archive.fs.refresh-interval: 10000

blob.server.port: 6124

query.server.port: 6125

#flink重启后，不会丢失jar

jobmanager.web.tmpdir: /tmp/flink-web

jobmanager.web.upload.dir: /tmp/flink-web

#任务失败后，重启时，每两次连续尝试之间间隔1 min，任务重启失败30次，则任务最终失败，配置默认全局生效。

restart-strategy: fixed-delay

restart-strategy.fixed-delay.attempts: 30

restart-strategy.fixed-delay.delay: 1 min

docker-compose.yml

version: '3'

services:

  jobmanager:

    image: flink:1.13.0

    privileged: true

    expose:

    - "6123"

    ports:

    - "8081:8081"

    command: jobmanager

    environment:

    - JOB_MANAGER_RPC_ADDRESS=jobmanager

    - TZ=Asia/Shanghai

    volumes:

    - /data/flink/conf/job/flink-conf.yaml:/opt/flink/conf/flink-conf.yaml

    restart: unless-stopped

  taskmanager:

    image: flink:1.13.0

    privileged: true

    expose:

    - "6121"

    - "6122"

    depends_on:

    - jobmanager

    command: taskmanager

    environment:

    - JOB_MANAGER_RPC_ADDRESS=jobmanager

    - TZ=Asia/Shanghai

    volumes:

    - /data/flink/conf/task/flink-conf.yaml:/opt/flink/conf/flink-conf.yaml

    restart: unless-stopped

访问http://ip:8081/

这种方法部署有个问题没有解决就是

这里看不到日志，但是 logs日志可以看到

Linux使用docker安装flink的更多相关文章

  1. Linux通过docker安装运行酷Q--用QQ骰子君进行跑团

    Linux通过docker安装运行酷Q 文:铁乐与猫 需求:和小伙伴周末进行愉快的TRPG跑团,需要在QQ讨论组上加了qq小号后,将qq小号用酷Q配合投骰的应用变成骰子君. 限制:我个人的云计算服务器 ...

  2. Docker安装flink及避坑指南

    Docker安装flink 导航 无处不在的大数据 安装flink 拉取flink镜像 编写docker-compose.yml 生成启动 查看安装效果 常见坑及解决方案 问题1 问题2 参考   本 ...

  3. 【Linux】【自学笔记】Linux下面docker安装mysql

    写在前面: 捣腾继续,之前把一个SpringBoot的程序安装在docker上面,参考链接:https://www.cnblogs.com/aki-stones/p/2019-11-01-note.h ...

  4. Linux下docker安装教程

    目前最新版本的docker19.03支持nvidia显卡与容器的无缝对接,从而摆脱了对nvidia-docker的依赖.因此毫不犹豫安装19.03版本的docker,安装教程可参考官方教程Centos ...

  5. linux上Docker安装gogs私服亲测(详解)

    一.前言 有网友问我为什么要使用私服,可能大部分人都不是太懂,网上那么多存储仓库而且好用方便,但是你想过没有如果企业中的项目,放在人家的仓库上这个安全性不是太好,所以说一般企业都会有自己的私服.本章教 ...

  6. Linux、Docker安装Nginx

    Docker安装Nginx #docker images nginx #docker search nginx #docker pull nginx #docker run -it -p 8084:8 ...

  7. 使用 Linux Centos Docker 安装 2Bizbix

    在 Docker 安装 2Bizbix 安装 Centos 7 安装 mysql5.5 镜像 映射好数据库的配置文件和数据库目录 在 Windows 安装 2Bizbox 安装 jboss/base- ...

  8. Linux下Docker安装

    1 在 CentOS 6.4 上安装 docker   docker当前官方只支持Ubuntu,所以在 CentOS 安装Docker比较麻烦(Issue #172).   docker官方文档说要求 ...

  9. oracle linux 6 docker 安装

    docker对安装系统的内核版本有严格的要求,本文针对oracle linux 6.5进行讲解,其它系统参见: https://docs.docker.com/v1.5/installation/ 下 ...

随机推荐

  1. AtCoder Beginner Contest 200

    前言:果然自己连\(ABC\)都打不好了吗. 没看清题目,卡了巨久,排名一直跌,笔记本键盘坏了,心态崩了. 冷静. ------------------------------------------ ...

  2. Codeforces 870F - Path（数论+分类讨论+正难则反）

    Codeforces 题目传送门 & 洛谷题目传送门 首先考虑 \(d(u,v)\) 是个什么东西,分情况讨论: \(u\not\perp v\),\(d(u,v)=1\) \(u\perp ...

  3. 一次forEach 中 await 的使用

    forEach 和 await/async 的问题 最近在刷面试提的时候看见这样一道题 const list = [1, 2, 3] const square = num => { return ...

  4. DirectX12 3D 游戏开发与实战第九章内容（下）

    仅供个人学习使用,请勿转载.谢谢! 9.纹理贴图 学习目标 学习如何将局部纹理映射到网格三角形中 探究如何创建和启用纹理 学会如何通过纹理过滤来创建更加平滑的图像 探索如何使用寻址模式来进行多次贴图 ...

  5. 【豆科基因组】绿豆Mungbean, Vigna radiata基因组2014NC

    目录 来源 一.简介 二.结果 基因组组装 重复序列和转座子 基因组特征和基因注释 绿豆的驯化 豆科基因组复制历史 基于转录组分析的豇豆属形成 绿豆育种基因组资源 三.讨论 四.方法 材料 组装 SN ...

  6. linux中conda升级R到4.0？

    目录 前言 问题 曲线救国 前言 虽然我的win版本R已经用4了,但之前在Linux环境一直没用R4.0,因为Linux涉及的东西太多,担心不稳定,牵一发而动全身. 但现在有好些R包必须要用更新到R4 ...

  7. [R]在dplyr基础上编写函数-(2)substitute和quote

    关于这两个函数,官方是这么定义的: substitute returns the parse tree for the (unevaluated) expression expr, substitut ...

  8. 3 - 简单了解一下springboot中的yml语法 和 使用yml赋值

    1.简单了解yml语法 2.使用yml给实体类赋值 准备工作:导入依赖 <!-- 这个jar包就是为了实体类中使用@ConfigurationProperties(prefix = " ...

  9. 日常Java 2021/10/5

    java 异常处理 Throwable中包括Error 和Exception,Exception包括IOException和RuntimeException 抛出异常 1.异常运算条件 Arithme ...

  10. h5移动端设备像素比dpr介绍

    首先介绍一下概念 devicePixelRatio其实指的是window.devicePixelRatio window.devicePixelRatio是设备上物理像素和设备独立像素(device- ...