org.apache.flink.yarn

Class YarnResourceManager

java.lang.Object

org.apache.flink.runtime.rpc.RpcEndpoint

org.apache.flink.runtime.rpc.FencedRpcEndpoint<org.apache.flink.runtime.resourcemanager.ResourceManagerId>

org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

org.apache.flink.yarn.YarnResourceManager

All Implemented Interfaces:

org.apache.flink.runtime.leaderelection.LeaderContender, org.apache.flink.runtime.resourcemanager.ResourceManagerGateway, org.apache.flink.runtime.rpc.FencedRpcGateway<org.apache.flink.runtime.resourcemanager.ResourceManagerId>, org.apache.flink.runtime.rpc.RpcGateway, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

public class YarnResourceManager
extends org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>
implements org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

The yarn implementation of the resource manager. Used when the system is started via the resource framework YARN.

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

org.apache.flink.runtime.rpc.RpcEndpoint.MainThreadExecutor

Field Summary

Fields inherited from class org.apache.flink.runtime.resourcemanager.ResourceManager

RESOURCE_MANAGER_NAME

Fields inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

log, rpcServer

Constructor Summary

Constructors

Constructor and Description

YarnResourceManager(org.apache.flink.runtime.rpc.RpcService rpcService, String resourceManagerEndpointId, org.apache.flink.runtime.clusterframework.types.ResourceID resourceId, org.apache.flink.configuration.Configuration flinkConfig, Map<String,String> env, org.apache.flink.runtime.highavailability.HighAvailabilityServices highAvailabilityServices, org.apache.flink.runtime.heartbeat.HeartbeatServices heartbeatServices, org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager slotManager, org.apache.flink.runtime.metrics.MetricRegistry metricRegistry, org.apache.flink.runtime.resourcemanager.JobLeaderIdService jobLeaderIdService, org.apache.flink.runtime.entrypoint.ClusterInformation clusterInformation, org.apache.flink.runtime.rpc.FatalErrorHandler fatalErrorHandler, String webInterfaceUrl, org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup jobManagerMetricGroup)

Method Summary

Modifier and Type	Method and Description
protected org.apache.hadoop.yarn.client.api.NMClient	createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration)
protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest>	createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration, int yarnHeartbeatIntervalMillis, String webInterfaceUrl)
float	getProgress()
protected void	initialize()
protected void	internalDeregisterApplication(org.apache.flink.runtime.clusterframework.ApplicationStatus finalStatus, String diagnostics)
void	onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)
void	onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)
void	onError(Throwable error)
void	onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list)
void	onShutdownRequest()
CompletableFuture<Void>	postStop()
Collection<org.apache.flink.runtime.clusterframework.types.ResourceProfile>	startNewWorker(org.apache.flink.runtime.clusterframework.types.ResourceProfile resourceProfile)
boolean	stopWorker(YarnWorkerNode workerNode)
protected YarnWorkerNode	workerStarted(org.apache.flink.runtime.clusterframework.types.ResourceID resourceID)

Methods inherited from class org.apache.flink.runtime.resourcemanager.ResourceManager

cancelSlotRequest, clearStateAsync, closeJobManagerConnection, closeTaskManagerConnection, createSlotsPerWorker, deregisterApplication, disconnectJobManager, disconnectTaskManager, getNumberOfRegisteredTaskManagers, getNumberRequiredTaskManagerSlots, grantLeadership, handleError, heartbeatFromJobManager, heartbeatFromTaskManager, jobLeaderLostLeadership, notifySlotAvailable, onFatalError, prepareLeadershipAsync, registerInfoMessageListener, registerJobManager, registerTaskExecutor, releaseResource, removeJob, requestResourceOverview, requestSlot, requestTaskManagerFileUpload, requestTaskManagerInfo, requestTaskManagerInfo, requestTaskManagerMetricQueryServicePaths, revokeLeadership, sendInfoMessage, sendSlotReport, start, unRegisterInfoMessageListener

Methods inherited from class org.apache.flink.runtime.rpc.FencedRpcEndpoint

callAsyncWithoutFencing, getFencingToken, getMainThreadExecutor, getUnfencedMainThreadExecutor, runAsyncWithoutFencing, setFencingToken

Methods inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

callAsync, getAddress, getEndpointId, getHostname, getRpcService, getSelfGateway, getTerminationFuture, runAsync, scheduleRunAsync, scheduleRunAsync, shutDown, stop, validateRunsInMainThread

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.flink.runtime.rpc.FencedRpcGateway

getFencingToken

Methods inherited from interface org.apache.flink.runtime.rpc.RpcGateway

getAddress, getHostname

Methods inherited from interface org.apache.flink.runtime.leaderelection.LeaderContender

getAddress

Constructor Detail

YarnResourceManager

public YarnResourceManager(org.apache.flink.runtime.rpc.RpcService rpcService,
                           String resourceManagerEndpointId,
                           org.apache.flink.runtime.clusterframework.types.ResourceID resourceId,
                           org.apache.flink.configuration.Configuration flinkConfig,
                           Map<String,String> env,
                           org.apache.flink.runtime.highavailability.HighAvailabilityServices highAvailabilityServices,
                           org.apache.flink.runtime.heartbeat.HeartbeatServices heartbeatServices,
                           org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager slotManager,
                           org.apache.flink.runtime.metrics.MetricRegistry metricRegistry,
                           org.apache.flink.runtime.resourcemanager.JobLeaderIdService jobLeaderIdService,
                           org.apache.flink.runtime.entrypoint.ClusterInformation clusterInformation,
                           org.apache.flink.runtime.rpc.FatalErrorHandler fatalErrorHandler,
                           @Nullable
                           String webInterfaceUrl,
                           org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup jobManagerMetricGroup)

Method Detail

createAndStartResourceManagerClient

protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration,
                                                                                                                                                                     int yarnHeartbeatIntervalMillis,
                                                                                                                                                                     @Nullable
                                                                                                                                                                     String webInterfaceUrl)
                                                                                                                                                              throws Exception

Throws:

Exception

createAndStartNodeManagerClient

protected org.apache.hadoop.yarn.client.api.NMClient createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration)

initialize

protected void initialize()
                   throws org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException

Specified by:

initialize in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

Throws:

org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException

postStop

public CompletableFuture<Void> postStop()

Overrides:

postStop in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

internalDeregisterApplication

protected void internalDeregisterApplication(org.apache.flink.runtime.clusterframework.ApplicationStatus finalStatus,
                                             @Nullable
                                             String diagnostics)

Specified by:

internalDeregisterApplication in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

startNewWorker

public Collection<org.apache.flink.runtime.clusterframework.types.ResourceProfile> startNewWorker(org.apache.flink.runtime.clusterframework.types.ResourceProfile resourceProfile)

Specified by:

startNewWorker in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

stopWorker

public boolean stopWorker(YarnWorkerNode workerNode)

Specified by:

stopWorker in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

workerStarted

protected YarnWorkerNode workerStarted(org.apache.flink.runtime.clusterframework.types.ResourceID resourceID)

Specified by:

workerStarted in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

getProgress

public float getProgress()

Specified by:

getProgress in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainersCompleted

public void onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)

Specified by:

onContainersCompleted in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainersAllocated

public void onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)

Specified by:

onContainersAllocated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onShutdownRequest

public void onShutdownRequest()

Specified by:

onShutdownRequest in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onNodesUpdated

public void onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list)

Specified by:

onNodesUpdated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onError

public void onError(Throwable error)

Specified by:

onError in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler