Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Class SchedulerApplicationAttempt

java.lang.Object
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt
All Implemented Interfaces:
SchedulableEntity
Direct Known Subclasses:
FiCaSchedulerApp, FSAppAttempt
@Private @Unstable public class SchedulerApplicationAttempt extends Object implements SchedulableEntity
Represents an application attempt from the viewpoint of the scheduler. Each running app attempt in the RM corresponds to one instance of this class.

  • Field Details

    • lastMemoryAggregateAllocationUpdateTime

      protected long lastMemoryAggregateAllocationUpdateTime

    • appSchedulingInfo

      protected final AppSchedulingInfo appSchedulingInfo

    • attemptId

      protected org.apache.hadoop.yarn.api.records.ApplicationAttemptId attemptId

    • liveContainers

      protected Map<org.apache.hadoop.yarn.api.records.ContainerId,RMContainer> liveContainers

    • reservedContainers

      protected final Map<org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey,Map<org.apache.hadoop.yarn.api.records.NodeId,RMContainer>> reservedContainers

    • attemptResourceUsage

      protected ResourceUsage attemptResourceUsage

    • attemptOpportunisticResourceUsage

      protected ResourceUsage attemptOpportunisticResourceUsage
      Resource usage of opportunistic containers.

    • attemptResourceUsageAllocatedRemotely

      protected ResourceUsage attemptResourceUsageAllocatedRemotely
      Scheduled by a remote scheduler.

    • newlyAllocatedContainers

      protected List<RMContainer> newlyAllocatedContainers

    • tempContainerToKill

      protected List<RMContainer> tempContainerToKill

    • newlyPromotedContainers

      protected Map<org.apache.hadoop.yarn.api.records.ContainerId,RMContainer> newlyPromotedContainers

    • newlyDemotedContainers

      protected Map<org.apache.hadoop.yarn.api.records.ContainerId,RMContainer> newlyDemotedContainers

    • newlyDecreasedContainers

      protected Map<org.apache.hadoop.yarn.api.records.ContainerId,RMContainer> newlyDecreasedContainers

    • newlyIncreasedContainers

      protected Map<org.apache.hadoop.yarn.api.records.ContainerId,RMContainer> newlyIncreasedContainers

    • updatedNMTokens

      protected Set<org.apache.hadoop.yarn.api.records.NMToken> updatedNMTokens

    • updateContainerErrors

      protected List<org.apache.hadoop.yarn.api.records.UpdateContainerError> updateContainerErrors

    • lastScheduledContainer

      protected Map<org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey,Long> lastScheduledContainer

    • queue

      protected volatile Queue queue

    • isStopped

      protected volatile boolean isStopped

    • appAMNodePartitionName

      protected String appAMNodePartitionName

    • rmContext

      protected final RMContext rmContext

    • readLock

      protected ReentrantReadWriteLock.ReadLock readLock

    • writeLock

      protected ReentrantReadWriteLock.WriteLock writeLock

  • Constructor Details

    • SchedulerApplicationAttempt

      public SchedulerApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, String user, Queue queue, AbstractUsersManager abstractUsersManager, RMContext rmContext)

  • Method Details

    • setOpportunisticContainerContext

      public void setOpportunisticContainerContext(org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext oppContext)

    • getOpportunisticContainerContext

      public org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext getOpportunisticContainerContext()

    • getLiveContainers

      public Collection<RMContainer> getLiveContainers()
      Get the live containers of the application.
      Returns:
      live containers of the application

    • getAppSchedulingInfo

      public AppSchedulingInfo getAppSchedulingInfo()

    • getUpdateContext

      public ContainerUpdateContext getUpdateContext()

    • isPending

      public boolean isPending()
      Is this application pending?
      Returns:
      true if it is else false.

    • getApplicationAttemptId

      public org.apache.hadoop.yarn.api.records.ApplicationAttemptId getApplicationAttemptId()
      Get ApplicationAttemptId of the application master.
      Returns:
      ApplicationAttemptId of the application master

    • getApplicationId

      public org.apache.hadoop.yarn.api.records.ApplicationId getApplicationId()

    • getUser

      public String getUser()

    • getPendingRelease

      public Set<org.apache.hadoop.yarn.api.records.ContainerId> getPendingRelease()

    • getNewContainerId

      public long getNewContainerId()

    • getSchedulerKeys

      public Collection<org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey> getSchedulerKeys()

    • getPendingAsk

      public PendingAsk getPendingAsk(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey, String resourceName)

    • getOutstandingAsksCount

      public int getOutstandingAsksCount(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • getOutstandingAsksCount

      public int getOutstandingAsksCount(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey, String resourceName)

    • getQueueName

      public String getQueueName()

    • getAMResource

      public org.apache.hadoop.yarn.api.records.Resource getAMResource()

    • getAMResource

      public org.apache.hadoop.yarn.api.records.Resource getAMResource(String label)

    • setAMResource

      public void setAMResource(org.apache.hadoop.yarn.api.records.Resource amResource)

    • setAMResource

      public void setAMResource(String label, org.apache.hadoop.yarn.api.records.Resource amResource)

    • isAmRunning

      public boolean isAmRunning()

    • setAmRunning

      public void setAmRunning(boolean bool)

    • getUnmanagedAM

      public boolean getUnmanagedAM()

    • getRMContainer

      public RMContainer getRMContainer(org.apache.hadoop.yarn.api.records.ContainerId id)

    • addRMContainer

      public void addRMContainer(org.apache.hadoop.yarn.api.records.ContainerId id, RMContainer rmContainer)

    • removeRMContainer

      public boolean removeRMContainer(org.apache.hadoop.yarn.api.records.ContainerId containerId)
      Removes an RM container from the map of live containers related to this application attempt.
      Parameters:
      containerId - The container ID of the RMContainer to remove
      Returns:
      true if the container is in the map

    • resetReReservations

      protected void resetReReservations(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • addReReservation

      protected void addReReservation(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • getReReservations

      public int getReReservations(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • getCurrentReservation

      @Stable @Private public org.apache.hadoop.yarn.api.records.Resource getCurrentReservation()
      Get total current reservations. Used only by unit tests
      Returns:
      total current reservations

    • getQueue

      public Queue getQueue()

    • updateResourceRequests

      public boolean updateResourceRequests(List<org.apache.hadoop.yarn.api.records.ResourceRequest> requests)

    • updateSchedulingRequests

      public boolean updateSchedulingRequests(List<org.apache.hadoop.yarn.api.records.SchedulingRequest> requests)

    • recoverResourceRequestsForContainer

      public void recoverResourceRequestsForContainer(ContainerRequest containerRequest)

    • stop

      public void stop(RMAppAttemptState rmAppAttemptFinalState)

    • isStopped

      public boolean isStopped()

    • getReservedContainers

      public List<RMContainer> getReservedContainers()
      Get the list of reserved containers
      Returns:
      All of the reserved containers.

    • reserveIncreasedContainer

      public boolean reserveIncreasedContainer(SchedulerNode node, org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey, RMContainer rmContainer, org.apache.hadoop.yarn.api.records.Resource reservedResource)

    • reserve

      public RMContainer reserve(SchedulerNode node, org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey, RMContainer rmContainer, org.apache.hadoop.yarn.api.records.Container container)

    • setHeadroom

      public void setHeadroom(org.apache.hadoop.yarn.api.records.Resource globalLimit)

    • getHeadroom

      public org.apache.hadoop.yarn.api.records.Resource getHeadroom()
      Get available headroom in terms of resources for the application's user.
      Returns:
      available resource headroom

    • getNumReservedContainers

      public int getNumReservedContainers(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • containerLaunchedOnNode

      public void containerLaunchedOnNode(org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.yarn.api.records.NodeId nodeId)

    • showRequests

      public void showRequests()

    • getCurrentConsumption

      public org.apache.hadoop.yarn.api.records.Resource getCurrentConsumption()

    • updateNMTokens

      public void updateNMTokens(Collection<org.apache.hadoop.yarn.api.records.Container> containers)

    • pullPreviousAttemptContainers

      public List<org.apache.hadoop.yarn.api.records.Container> pullPreviousAttemptContainers()
      Called when AM heartbeats. These containers were recovered by the RM after the AM had registered. They are reported to the AM in the AllocateResponse#containersFromPreviousAttempts.
      Returns:
      Container List.

    • pullNewlyAllocatedContainers

      public List<org.apache.hadoop.yarn.api.records.Container> pullNewlyAllocatedContainers()

    • addToNewlyDemotedContainers

      public void addToNewlyDemotedContainers(org.apache.hadoop.yarn.api.records.ContainerId containerId, RMContainer rmContainer)

    • addToNewlyDecreasedContainers

      public void addToNewlyDecreasedContainers(org.apache.hadoop.yarn.api.records.ContainerId containerId, RMContainer rmContainer)

    • addToUpdateContainerErrors

      protected void addToUpdateContainerErrors(org.apache.hadoop.yarn.api.records.UpdateContainerError error)

    • addToNewlyAllocatedContainers

      @VisibleForTesting public void addToNewlyAllocatedContainers(SchedulerNode node, RMContainer rmContainer)

    • pullNewlyPromotedContainers

      public List<org.apache.hadoop.yarn.api.records.Container> pullNewlyPromotedContainers()

    • pullNewlyDemotedContainers

      public List<org.apache.hadoop.yarn.api.records.Container> pullNewlyDemotedContainers()

    • pullNewlyIncreasedContainers

      public List<org.apache.hadoop.yarn.api.records.Container> pullNewlyIncreasedContainers()

    • pullNewlyDecreasedContainers

      public List<org.apache.hadoop.yarn.api.records.Container> pullNewlyDecreasedContainers()

    • pullUpdateContainerErrors

      public List<org.apache.hadoop.yarn.api.records.UpdateContainerError> pullUpdateContainerErrors()

    • pullUpdatedNMTokens

      public List<org.apache.hadoop.yarn.api.records.NMToken> pullUpdatedNMTokens()

    • isWaitingForAMContainer

      public boolean isWaitingForAMContainer()

    • updateBlacklist

      public void updateBlacklist(List<String> blacklistAdditions, List<String> blacklistRemovals)

    • isPlaceBlacklisted

      public boolean isPlaceBlacklisted(String resourceName)

    • addMissedNonPartitionedRequestSchedulingOpportunity

      public int addMissedNonPartitionedRequestSchedulingOpportunity(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • resetMissedNonPartitionedRequestSchedulingOpportunity

      public void resetMissedNonPartitionedRequestSchedulingOpportunity(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • addSchedulingOpportunity

      public void addSchedulingOpportunity(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • subtractSchedulingOpportunity

      public void subtractSchedulingOpportunity(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)

    • getSchedulingOpportunities

      public int getSchedulingOpportunities(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)
      Return the number of times the application has been given an opportunity to schedule a task at the given priority since the last time it successfully did so.
      Parameters:
      schedulerKey - Scheduler Key
      Returns:
      number of scheduling opportunities

    • resetSchedulingOpportunities

      public void resetSchedulingOpportunities(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey)
      Should be called when an application has successfully scheduled a container, or when the scheduling locality threshold is relaxed. Reset various internal counters which affect delay scheduling
      Parameters:
      schedulerKey - The priority of the container scheduled.

    • resetSchedulingOpportunities

      public void resetSchedulingOpportunities(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerKey, long currentTimeMs)

    • getResourceUsageReport

      public org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport getResourceUsageReport()

    • getLiveContainersMap

      @VisibleForTesting public Map<org.apache.hadoop.yarn.api.records.ContainerId,RMContainer> getLiveContainersMap()

    • getLastScheduledContainer

      public Map<org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey,Long> getLastScheduledContainer()

    • transferStateFromPreviousAttempt

      public void transferStateFromPreviousAttempt(SchedulerApplicationAttempt appAttempt)

    • move

      public void move(Queue newQueue)

    • recoverContainer

      public boolean recoverContainer(SchedulerNode node, RMContainer rmContainer)

    • incNumAllocatedContainers

      public void incNumAllocatedContainers(NodeType containerType, NodeType requestType)

    • setApplicationHeadroomForMetrics

      public void setApplicationHeadroomForMetrics(org.apache.hadoop.yarn.api.records.Resource headroom)

    • recordContainerRequestTime

      public void recordContainerRequestTime(long value)

    • recordContainerAllocationTime

      public void recordContainerAllocationTime(long value)

    • getBlacklistedNodes

      public Set<String> getBlacklistedNodes()

    • hasPendingResourceRequest

      @Private public boolean hasPendingResourceRequest(String nodePartition, SchedulingMode schedulingMode)

    • getAppAttemptResourceUsage

      @VisibleForTesting public ResourceUsage getAppAttemptResourceUsage()

    • getPriority

      public org.apache.hadoop.yarn.api.records.Priority getPriority()
      Description copied from interface: SchedulableEntity
      Get the priority of the application.
      Specified by:
      getPriority in interface SchedulableEntity
      Returns:
      priority of the application.

    • setPriority

      public void setPriority(org.apache.hadoop.yarn.api.records.Priority appPriority)

    • getId

      public String getId()
      Description copied from interface: SchedulableEntity
      Id - each entity must have a unique id.
      Specified by:
      getId in interface SchedulableEntity
      Returns:
      id.

    • compareInputOrderTo

      public int compareInputOrderTo(SchedulableEntity other)
      Description copied from interface: SchedulableEntity
      Compare the passed SchedulableEntity to this one for input order. Input order is implementation defined and should reflect the correct ordering for first-in first-out processing.
      Specified by:
      compareInputOrderTo in interface SchedulableEntity
      Parameters:
      other - SchedulableEntity.
      Returns:
      correct ordering.

    • getSchedulingResourceUsage

      public ResourceUsage getSchedulingResourceUsage()
      Description copied from interface: SchedulableEntity
      View of Resources wanted and consumed by the entity.
      Specified by:
      getSchedulingResourceUsage in interface SchedulableEntity
      Returns:
      ResourceUsage.

    • setAppAMNodePartitionName

      public void setAppAMNodePartitionName(String partitionName)

    • getAppAMNodePartitionName

      public String getAppAMNodePartitionName()

    • updateAMContainerDiagnostics

      public void updateAMContainerDiagnostics(SchedulerApplicationAttempt.AMState state, String diagnosticMessage)

    • getPendingAppDiagnosticMessage

      protected void getPendingAppDiagnosticMessage(StringBuilder diagnosticMessage)

    • getActivedAppDiagnosticMessage

      protected void getActivedAppDiagnosticMessage(StringBuilder diagnosticMessage)

    • getWriteLock

      public ReentrantReadWriteLock.WriteLock getWriteLock()

    • isRecovering

      public boolean isRecovering()
      Description copied from interface: SchedulableEntity
      Whether application was running before RM restart.
      Specified by:
      isRecovering in interface SchedulableEntity
      Returns:
      true, application was running before RM restart; otherwise false.

    • setAttemptRecovering

      protected void setAttemptRecovering(boolean isRecovering)

    • getAppPlacementAllocator

      public <N extends SchedulerNode> AppPlacementAllocator<N> getAppPlacementAllocator(org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey schedulerRequestKey)

    • incUnconfirmedRes

      public void incUnconfirmedRes(org.apache.hadoop.yarn.api.records.Resource res)

    • decUnconfirmedRes

      public void decUnconfirmedRes(org.apache.hadoop.yarn.api.records.Resource res)

    • hashCode

      public int hashCode()
      Overrides:
      hashCode in class Object

    • equals

      public boolean equals(Object o)
      Overrides:
      equals in class Object

    • getApplicationSchedulingEnvs

      public Map<String,String> getApplicationSchedulingEnvs()

    • getPartition

      public String getPartition()
      Description copied from interface: SchedulableEntity
      Get partition corresponding to this entity.
      Specified by:
      getPartition in interface SchedulableEntity
      Returns:
      partition node label.

    • getStartTime

      public long getStartTime()
      Description copied from interface: SchedulableEntity
      Start time of the job.
      Specified by:
      getStartTime in interface SchedulableEntity
      Returns:
      start time