public class FederationClientInterceptor extends AbstractClientRequestInterceptor
AbstractRequestInterceptorClient
class and provides an
implementation for federation of YARN RM and scaling an application across
multiple YARN SubClusters. All the federation specific implementation is
encapsulated in this class. This is always the last intercepter in the chain.user
Constructor and Description |
---|
FederationClientInterceptor() |
Modifier and Type | Method and Description |
---|---|
org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse |
cancelDelegationToken(org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse |
deleteReservation(org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse |
failApplicationAttempt(org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse |
forceKillApplication(org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest request)
The YARN Router will forward to the respective YARN RM in which the AM is
running.
|
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse |
getApplicationAttemptReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse |
getApplicationAttempts(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse |
getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest request)
The YARN Router will forward to the respective YARN RM in which the AM is
running.
|
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse |
getApplications(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse |
getAttributesToNodes(org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest request) |
protected org.apache.hadoop.yarn.api.ApplicationClientProtocol |
getClientRMProxyForSubCluster(org.apache.hadoop.yarn.server.federation.store.records.SubClusterId subClusterId) |
org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse |
getClusterMetrics(org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse |
getClusterNodeAttributes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse |
getClusterNodeLabels(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse |
getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse |
getContainerReport(org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse |
getContainers(org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse |
getDelegationToken(org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse |
getLabelsToNodes(org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse |
getNewApplication(org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest request)
YARN Router forwards every getNewApplication requests to any RM.
|
org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse |
getNewReservation(org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse |
getNodesToAttributes(org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse |
getNodeToLabels(org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse |
getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse |
getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse |
getResourceProfile(org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse |
getResourceProfiles(org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse |
getResourceTypeInfo(org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest request) |
void |
init(String userName)
Initializes the
ClientRequestInterceptor . |
org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse |
listReservations(org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse |
moveApplicationAcrossQueues(org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse |
renewDelegationToken(org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest request) |
void |
setNextInterceptor(ClientRequestInterceptor next)
Sets the
ClientRequestInterceptor in the chain. |
void |
shutdown()
Disposes the
ClientRequestInterceptor . |
org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse |
signalToContainer(org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse |
submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest request)
Today, in YARN there are no checks of any applicationId submitted.
|
org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse |
submitReservation(org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse |
updateApplicationPriority(org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse |
updateApplicationTimeouts(org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest request) |
org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse |
updateReservation(org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest request) |
getConf, getNextInterceptor, setConf
public void init(String userName)
AbstractClientRequestInterceptor
ClientRequestInterceptor
.init
in interface ClientRequestInterceptor
init
in class AbstractClientRequestInterceptor
userName
- the name of the clientpublic void setNextInterceptor(ClientRequestInterceptor next)
AbstractClientRequestInterceptor
ClientRequestInterceptor
in the chain.setNextInterceptor
in interface ClientRequestInterceptor
setNextInterceptor
in class AbstractClientRequestInterceptor
next
- the ClientRequestInterceptor to set in the pipelineprotected org.apache.hadoop.yarn.api.ApplicationClientProtocol getClientRMProxyForSubCluster(org.apache.hadoop.yarn.server.federation.store.records.SubClusterId subClusterId) throws org.apache.hadoop.yarn.exceptions.YarnException
org.apache.hadoop.yarn.exceptions.YarnException
public org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse getNewApplication(org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
ClientRMService
.
Router: the Client will timeout and resubmit.
ResourceManager: the Router will timeout and contacts another RM.
StateStore: not in the execution.org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
ClientRMService
.
In case of Router failure:
Scenario 1 – Crash before submission to the ResourceManager
The Client submits an application to the Router. • The Router selects one
SubCluster to forward the request. • The Router inserts a tuple into State
Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
crashes. • The Client timeouts and resubmits the application. • The Router
selects one SubCluster to forward the request. • The Router inserts a tuple
into State Store with the selected SubCluster (e.g. SC2) and the appId. •
Because the tuple is already inserted in the State Store, it returns the
previous selected SubCluster (e.g. SC1). • The Router submits the request
to the selected SubCluster (e.g. SC1).
Scenario 2 – Crash after submission to the ResourceManager
• The Client submits an application to the Router. • The Router selects one
SubCluster to forward the request. • The Router inserts a tuple into State
Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
submits the request to the selected SubCluster. • The Router crashes. • The
Client timeouts and resubmit the application. • The Router selects one
SubCluster to forward the request. • The Router inserts a tuple into State
Store with the selected SubCluster (e.g. SC2) and the appId. • The State
Store replies with the selected SubCluster (e.g. SC1). • The Router submits
the request to the selected SubCluster (e.g. SC1). When a client re-submits
the same application to the same RM, it does not raise an exception and
replies with operation successful message.
In case of Client failure: identical behavior as ClientRMService
.
In case of ResourceManager failure:
The Client submits an application to the Router. • The Router selects one
SubCluster to forward the request. • The Router inserts a tuple into State
Store with the selected SubCluster (e.g. SC1) and the appId. • The Router
submits the request to the selected SubCluster. • The entire SubCluster is
down – all the RMs in HA or the master RM is not reachable. • The Router
times out. • The Router selects a new SubCluster to forward the request. •
The Router update a tuple into State Store with the selected SubCluster
(e.g. SC2) and the appId. • The State Store replies with OK answer. • The
Router submits the request to the selected SubCluster (e.g. SC2).org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse forceKillApplication(org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
ClientRMService
.
Router: the Client will timeout and resubmit the request.
ResourceManager: the Router will timeout and the call will fail.
State Store: the Router will timeout and it will retry depending on the
FederationFacade settings - if the failure happened before the select
operation.org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
ClientRMService
.
Router: the Client will timeout and resubmit the request.
ResourceManager: the Router will timeout and the call will fail.
State Store: the Router will timeout and it will retry depending on the
FederationFacade settings - if the failure happened before the select
operation.org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse getApplications(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse getClusterMetrics(org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse getNewReservation(org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse submitReservation(org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse listReservations(org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse updateReservation(org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse deleteReservation(org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse getNodeToLabels(org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse getLabelsToNodes(org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse getClusterNodeLabels(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse getApplicationAttemptReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse getApplicationAttempts(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse getContainerReport(org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse getContainers(org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse getDelegationToken(org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse renewDelegationToken(org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse cancelDelegationToken(org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse failApplicationAttempt(org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse updateApplicationPriority(org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse signalToContainer(org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse updateApplicationTimeouts(org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse getResourceProfiles(org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse getResourceProfile(org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse getResourceTypeInfo(org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public void shutdown()
AbstractClientRequestInterceptor
ClientRequestInterceptor
.shutdown
in interface ClientRequestInterceptor
shutdown
in class AbstractClientRequestInterceptor
public org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse getAttributesToNodes(org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse getClusterNodeAttributes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
public org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse getNodesToAttributes(org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest request) throws org.apache.hadoop.yarn.exceptions.YarnException, IOException
org.apache.hadoop.yarn.exceptions.YarnException
IOException
Copyright © 2008–2022 Apache Software Foundation. All rights reserved.