Package org.apache.hadoop.yarn.server.resourcemanager.scheduler

Class QueueMetrics

java.lang.Object
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics
All Implemented Interfaces:
org.apache.hadoop.metrics2.MetricsSource
Direct Known Subclasses:
CSQueueMetrics, FSQueueMetrics, PartitionQueueMetrics
@Private @Metrics(context="yarn") public class QueueMetrics extends Object implements org.apache.hadoop.metrics2.MetricsSource

  • Field Details

    • QUEUE_INFO

      protected static final org.apache.hadoop.metrics2.MetricsInfo QUEUE_INFO

    • USER_INFO

      protected static final org.apache.hadoop.metrics2.MetricsInfo USER_INFO

    • PARTITION_INFO

      protected static final org.apache.hadoop.metrics2.MetricsInfo PARTITION_INFO

    • registry

      protected final org.apache.hadoop.metrics2.lib.MetricsRegistry registry

    • queueName

      protected final String queueName

    • metricsSystem

      protected final org.apache.hadoop.metrics2.MetricsSystem metricsSystem

    • users

      protected final Map<String,QueueMetrics> users

    • conf

      protected final org.apache.hadoop.conf.Configuration conf

    • P_RECORD_INFO

      protected static final org.apache.hadoop.metrics2.MetricsInfo P_RECORD_INFO

    • DEFAULT_PARTITION

      public static final String DEFAULT_PARTITION
      See Also:

    • DEFAULT_PARTITION_JMX_STR

      public static final String DEFAULT_PARTITION_JMX_STR
      See Also:

    • METRIC_NAME_DELIMITER

      public static final String METRIC_NAME_DELIMITER
      See Also:

    • storedPartitionMetrics

      protected Set<String> storedPartitionMetrics

  • Constructor Details

    • QueueMetrics

      public QueueMetrics(org.apache.hadoop.metrics2.MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, org.apache.hadoop.conf.Configuration conf)

  • Method Details

    • tag

      protected QueueMetrics tag(org.apache.hadoop.metrics2.MetricsInfo info, String value)

    • sourceName

      protected static StringBuilder sourceName(String queueName)

    • forQueue

      public static QueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, org.apache.hadoop.conf.Configuration conf)

    • clearQueueMetrics

      @Private public static void clearQueueMetrics()
      Helper method to clear cache.

    • getQueueMetrics

      public static Map<String,QueueMetrics> getQueueMetrics()
      Returns the metrics cache to help prevent re-registrations.
      Returns:
      A string to QueueMetrics map.

    • forQueue

      public static QueueMetrics forQueue(org.apache.hadoop.metrics2.MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, org.apache.hadoop.conf.Configuration conf)

    • getUserMetrics

      public QueueMetrics getUserMetrics(String userName)

    • getPartitionQueueMetrics

      public QueueMetrics getPartitionQueueMetrics(String partition)
      Partition * Queue Metrics Computes Metrics at Partition (Node Label) * Queue Level. Sample JMX O/P Structure: PartitionQueueMetrics (labelX) QueueMetrics (A) metrics QueueMetrics (A1) metrics QueueMetrics (A2) metrics QueueMetrics (B) metrics
      Parameters:
      partition - Node Partition
      Returns:
      QueueMetrics

    • isConfigurationValidationSet

      public static boolean isConfigurationValidationSet(org.apache.hadoop.conf.Configuration conf)
      Check whether we are in a configuration validation mode. INTERNAL ONLY.
      Parameters:
      conf - the configuration to check
      Returns:
      true if

    • setConfigurationValidation

      public static void setConfigurationValidation(org.apache.hadoop.conf.Configuration conf, boolean value)
      Set configuration validation mode. INTERNAL ONLY.
      Parameters:
      conf - the configuration to update
      value - the value for the validation mode

    • getMetrics

      public void getMetrics(org.apache.hadoop.metrics2.MetricsCollector collector, boolean all)
      Specified by:
      getMetrics in interface org.apache.hadoop.metrics2.MetricsSource

    • submitApp

      public void submitApp(String user, boolean unmanagedAM)

    • submitAppAttempt

      public void submitAppAttempt(String user, boolean unmanagedAM)

    • runAppAttempt

      public void runAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, boolean unmanagedAM)

    • finishAppAttempt

      public void finishAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationId appId, boolean isPending, String user, boolean unmanagedAM)

    • finishApp

      public void finishApp(String user, RMAppState rmAppFinalState, boolean unmanagedAM)

    • moveAppFrom

      public void moveAppFrom(AppSchedulingInfo app, boolean unmanagedAM)

    • moveAppTo

      public void moveAppTo(AppSchedulingInfo app, boolean unmanagedAM)

    • setAvailableResourcesToQueue

      public void setAvailableResourcesToQueue(String partition, org.apache.hadoop.yarn.api.records.Resource limit)
      Set available resources. To be called by scheduler periodically as resources become available.
      Parameters:
      partition - Node Partition
      limit - resource limit

    • setAvailableResources

      public void setAvailableResources(org.apache.hadoop.yarn.api.records.Resource limit)
      Set Available resources with support for resource vectors.
      Parameters:
      limit - Resource.

    • setAvailableResourcesToQueue

      public void setAvailableResourcesToQueue(org.apache.hadoop.yarn.api.records.Resource limit)
      Set available resources. To be called by scheduler periodically as resources become available.
      Parameters:
      limit - resource limit

    • setAvailableResourcesToUser

      public void setAvailableResourcesToUser(String partition, String user, org.apache.hadoop.yarn.api.records.Resource limit)
      Set available resources. To be called by scheduler periodically as resources become available.
      Parameters:
      partition - Node Partition
      user - Name of the user.
      limit - resource limit

    • incrPendingResources

      public void incrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)
      Increment pending resource metrics
      Parameters:
      partition - Node Partition
      user - Name of the user.
      containers - containers count.
      res - the TOTAL delta of resources note this is different from the other APIs which use per container resource

    • internalIncrPendingResources

      public void internalIncrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)

    • createQueueMetricsForCustomResources

      protected void createQueueMetricsForCustomResources()

    • registerCustomResources

      protected void registerCustomResources()

    • decrPendingResources

      public void decrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)

    • internalDecrPendingResources

      protected void internalDecrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)

    • incrNodeTypeAggregations

      public void incrNodeTypeAggregations(String user, NodeType type)

    • allocateResources

      public void allocateResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res, boolean decrPending)

    • internalAllocateResources

      public void internalAllocateResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res, boolean decrPending)

    • allocateResources

      public void allocateResources(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)
      Allocate Resource for container size change.
      Parameters:
      partition - Node Partition
      user - Name of the user
      res - Resource.

    • releaseResources

      public void releaseResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)

    • internalReleaseResources

      public void internalReleaseResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)

    • preemptContainer

      public void preemptContainer()

    • updatePreemptedMemoryMBSeconds

      public void updatePreemptedMemoryMBSeconds(long mbSeconds)

    • updatePreemptedVcoreSeconds

      public void updatePreemptedVcoreSeconds(long vcoreSeconds)

    • updatePreemptedResources

      public void updatePreemptedResources(org.apache.hadoop.yarn.api.records.Resource res)

    • updatePreemptedForCustomResources

      public void updatePreemptedForCustomResources(org.apache.hadoop.yarn.api.records.Resource res)

    • updatePreemptedSecondsForCustomResources

      public void updatePreemptedSecondsForCustomResources(org.apache.hadoop.yarn.api.records.Resource res, long seconds)

    • reserveResource

      public void reserveResource(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)

    • internalReserveResources

      protected void internalReserveResources(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)

    • incrReserveResources

      public void incrReserveResources(org.apache.hadoop.yarn.api.records.Resource res)

    • unreserveResource

      public void unreserveResource(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)

    • internalUnReserveResources

      protected void internalUnReserveResources(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)

    • decrReserveResource

      public void decrReserveResource(org.apache.hadoop.yarn.api.records.Resource res)

    • incrActiveUsers

      public void incrActiveUsers()

    • decrActiveUsers

      public void decrActiveUsers()

    • activateApp

      public void activateApp(String user)

    • deactivateApp

      public void deactivateApp(String user)

    • addAppAttemptFirstContainerAllocationDelay

      public void addAppAttemptFirstContainerAllocationDelay(long latency)

    • getAppsSubmitted

      public int getAppsSubmitted()

    • getUnmanagedAppsSubmitted

      public int getUnmanagedAppsSubmitted()

    • getAppsRunning

      public int getAppsRunning()

    • getUnmanagedAppsRunning

      public int getUnmanagedAppsRunning()

    • getAppsPending

      public int getAppsPending()

    • getUnmanagedAppsPending

      public int getUnmanagedAppsPending()

    • getAppsCompleted

      public int getAppsCompleted()

    • getUnmanagedAppsCompleted

      public int getUnmanagedAppsCompleted()

    • getAppsKilled

      public int getAppsKilled()

    • getAppsFailed

      public int getAppsFailed()

    • getUnmanagedAppsFailed

      public int getUnmanagedAppsFailed()

    • getAllocatedResources

      public org.apache.hadoop.yarn.api.records.Resource getAllocatedResources()

    • getAvailableResources

      public org.apache.hadoop.yarn.api.records.Resource getAvailableResources()

    • getPendingResources

      public org.apache.hadoop.yarn.api.records.Resource getPendingResources()

    • getReservedResources

      public org.apache.hadoop.yarn.api.records.Resource getReservedResources()

    • getAggregatedPreemptedSecondsResources

      @VisibleForTesting public org.apache.hadoop.yarn.metrics.CustomResourceMetricValue getAggregatedPreemptedSecondsResources()
      Handle this specially as this has a long value and it could be truncated when casted into an int parameter of Resource.newInstance (vCores).
      Returns:
      QueueMetricsCustomResource

    • getAggregateMemoryMBSecondsPreempted

      @VisibleForTesting public org.apache.hadoop.metrics2.lib.MutableCounterLong getAggregateMemoryMBSecondsPreempted()

    • getAggregateVcoreSecondsPreempted

      @VisibleForTesting public org.apache.hadoop.metrics2.lib.MutableCounterLong getAggregateVcoreSecondsPreempted()

    • getAggregateMemoryMBPreempted

      @VisibleForTesting public long getAggregateMemoryMBPreempted()

    • getAggregateVcoresPreempted

      @VisibleForTesting public long getAggregateVcoresPreempted()

    • getAllocatedMB

      public long getAllocatedMB()

    • getAllocatedVirtualCores

      public int getAllocatedVirtualCores()

    • getAllocatedContainers

      public int getAllocatedContainers()

    • getAvailableMB

      public long getAvailableMB()

    • getAvailableVirtualCores

      public int getAvailableVirtualCores()

    • getPendingMB

      public long getPendingMB()

    • getPendingVirtualCores

      public int getPendingVirtualCores()

    • getPendingContainers

      public int getPendingContainers()

    • getReservedMB

      public long getReservedMB()

    • getReservedVirtualCores

      public int getReservedVirtualCores()

    • getReservedContainers

      public int getReservedContainers()

    • getActiveUsers

      public int getActiveUsers()

    • getActiveApps

      public int getActiveApps()

    • getMetricsSystem

      public org.apache.hadoop.metrics2.MetricsSystem getMetricsSystem()

    • getAggregateAllocatedContainers

      public long getAggregateAllocatedContainers()

    • getAggregateNodeLocalContainersAllocated

      public long getAggregateNodeLocalContainersAllocated()

    • getAggregateRackLocalContainersAllocated

      public long getAggregateRackLocalContainersAllocated()

    • getAggregateOffSwitchContainersAllocated

      public long getAggregateOffSwitchContainersAllocated()

    • getAggegatedReleasedContainers

      public long getAggegatedReleasedContainers()

    • getAggregatePreemptedContainers

      public long getAggregatePreemptedContainers()

    • fillInValuesFromAvailableResources

      public void fillInValuesFromAvailableResources(org.apache.hadoop.yarn.api.records.Resource fromResource, org.apache.hadoop.yarn.api.records.Resource targetResource)
      Fills in Resource values from available metrics values of custom resources to @code{targetResource}, only if the corresponding value of @code{targetResource} is zero. If @code{fromResource} has a value less than the available metrics value for a particular resource, it will be set to the @code{targetResource} instead.
      Parameters:
      fromResource - The resource to compare available resource values with.
      targetResource - The resource to save the values into.

    • getQueueMetricsForCustomResources

      @VisibleForTesting public QueueMetricsForCustomResources getQueueMetricsForCustomResources()

    • setQueueMetricsForCustomResources

      protected void setQueueMetricsForCustomResources(QueueMetricsForCustomResources metrics)

    • setParent

      public void setParent(QueueMetrics parent)

    • getParentQueue

      public Queue getParentQueue()

    • registerPartitionMetricsCreation

      protected void registerPartitionMetricsCreation(String metricName)

    • setParentQueue

      public void setParentQueue(Queue parentQueue)