Linux: Thread scheduling question

[cpillonel]

I am running RH ES 4, kernel 2.6.9. I am using the POSIX threads.

I have one "real-time" thread with prio 93 and policy SCHED_RR. This thread creates and resumes another thread with prio 91 and policy SCHED_RR. This second thread does a lot of computing things in while loops
and I have noticed that the first thread doesn't get any CPU time until the second one has terminated.
Why does the first thread doesn't get any CPU time since it has a greater priority and is runnable? There is no preemption when a thread is rotating in a while loop?
I thought that with policy SCHED_RR after a while there is automatically a yield, this is not correct?
Is the only solution to put some sched_yield() in my while loops? But it is difficult to know how many and where to put them...

Thank you for any help.

Cédric

[wje_lf]

We're not ignoring you. :) The site's been down for almost a day, and yesterday was St. Valentine's day and that's kept me busy. Give me a couple of hours and I'll have some sort of reply ready!

[wje_lf]

The following program sets things up as you describe. The major thread has priority 93. It sets up the minor thread as priority 91. Both threads then get to work.

#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <sched.h>
#include <time.h>
#include <unistd.h>

/*--------------------------------------------------------------------------*/

#define DESIRED_WORKLOAD_SECONDS  2
#define TIME_ARRAY_ELEMENTS      10

#define  DESIRED_MAJOR_PRIORITY  93
#define  DESIRED_MINOR_PRIORITY  91

int       requested_major_priority;
int       requested_minor_priority;
int       observed_max_priority;
int       observed_min_priority;

long long iterations_before_checking_time;

time_t    major_work_array[TIME_ARRAY_ELEMENTS];
time_t    minor_work_array[TIME_ARRAY_ELEMENTS];

/*--------------------------------------------------------------------------*/

void calculate_workload(void);

void check_error(char *activity,
                 int   status
                );

void do_work(time_t *time_array);

void *major_code(void *something);

void *minor_code(void *something);

/*--------------------------------------------------------------------------*/

void calculate_workload(void)
{
  volatile long long counter;

           long long limit;

           time_t    start;

  /*
   * We want something that will last between DESIRED_WORKLOAD_SECONDS and
   * twice that.
   */

  printf("calculating workload\n"
         "could take up to %d seconds\n"
         "trying limit =",
         4*DESIRED_WORKLOAD_SECONDS
        );

  fflush(stdout);

  for(limit=1;
      ;
      limit<<=1
     )
  {
    printf(" %lld",
           limit
          );

    fflush(stdout);

    start=time(NULL);

    for(counter=0;
        counter<limit;
        counter++
       )
    {
      /* Empty block. */
    }

    if(time(NULL)-start>=DESIRED_WORKLOAD_SECONDS)
    {
      break;   /* <--------- */
    }
  }

  iterations_before_checking_time=limit;

  printf("\ndone calculating workload\n");

} /* calculate_workload() */

/*--------------------------------------------------------------------------*/

void check_error(char *activity,
                 int   status
                )
{
  if(status)
  {
    fprintf(stderr,
            "error %d %s\n",
            status,
            activity
           );

    exit(1);
  }

} /* check_error() */

/*--------------------------------------------------------------------------*/

void do_work(time_t *time_array)
{
           int       slice_number;

  volatile long long counter;

  for(slice_number=0;
      slice_number<TIME_ARRAY_ELEMENTS;
      slice_number++
     )
  {
    for(counter=0;
        counter<iterations_before_checking_time;
        counter++
       )
    {
      /* Empty block. */
    }

    time_array[slice_number]=time(NULL);
  }

} /* do_work() */

/*--------------------------------------------------------------------------*/

void *major_code(void *something)
{
  pthread_t          minor_thread_id;

  pthread_attr_t     thread_attributes;

  struct sched_param thread_parameters;

  check_error("pthread_attr_init()",
              pthread_attr_init(&thread_attributes)
             );

  check_error("setting SCHED_RR for minor thread",
              pthread_attr_setschedpolicy(&thread_attributes,
                                          SCHED_RR
                                         )
             );

  thread_parameters.sched_priority=requested_minor_priority;

  check_error("setting scheduling parameters for minor thread",
              pthread_attr_setschedparam(&thread_attributes,
                                         &thread_parameters
                                        )
             );

  check_error("setting explicit (not inherited) scheduling for minor thread",
              pthread_attr_setinheritsched(&thread_attributes,
                                           PTHREAD_EXPLICIT_SCHED
                                          )
             );

  check_error("creating minor thread (maybe because you\'re not root?)",
              pthread_create(&minor_thread_id,
                             &thread_attributes,
                             minor_code,
                             NULL
                            )
             );

  do_work(major_work_array);

  check_error("joining minor thread",
              pthread_join(minor_thread_id,
                           NULL
                          )
             );

  return NULL;

} /* major_code() */

/*--------------------------------------------------------------------------*/

void *minor_code(void *something)
{
  do_work(minor_work_array);

  return NULL;

} /* minor_code() */

/*--------------------------------------------------------------------------*/

int main(void)
{
  int                 jndex;

  pthread_t           major_thread_id;

  pthread_attr_t      thread_attributes;

  struct sched_param  thread_parameters;

  struct tm          *the_local_time;

#if defined (_POSIX_THREAD_PRIORITY_SCHEDULING) && !defined (sun)
#else
  fprintf(stderr,
          "priority scheduling not supported\n"
         );

  exit(1);
#endif

  calculate_workload();

  observed_min_priority=sched_get_priority_min(SCHED_RR);
  observed_max_priority=sched_get_priority_max(SCHED_RR);

  if((observed_min_priority>=observed_max_priority) &&
     (DESIRED_MAJOR_PRIORITY!=DESIRED_MINOR_PRIORITY)
    )
  {
    fprintf(stderr,
            "observed minimum priority %d >= observed maximum priority %d\n",
            observed_min_priority,
            observed_max_priority
           );

    exit(1);
  }

  requested_major_priority=DESIRED_MAJOR_PRIORITY;
  requested_minor_priority=DESIRED_MINOR_PRIORITY;

  if(requested_major_priority==requested_minor_priority)
  {
    if(requested_major_priority>observed_max_priority)
    {
      requested_major_priority=observed_max_priority;
      requested_minor_priority=observed_max_priority;
    }

    if(requested_major_priority<observed_min_priority)
    {
      requested_major_priority=observed_min_priority;
      requested_minor_priority=observed_min_priority;
    }
  }
  else
  if(requested_major_priority<requested_minor_priority)
  {
    if(requested_minor_priority>observed_max_priority)
    {
      requested_minor_priority=observed_max_priority;

      if(requested_major_priority>=requested_minor_priority)
      {
        requested_major_priority=requested_minor_priority-1;
      }
    }

    if(requested_major_priority<observed_min_priority)
    {
      requested_major_priority=observed_min_priority;

      if(requested_minor_priority<=requested_major_priority)
      {
        requested_minor_priority=requested_major_priority+1;
      }
    }
  }
  else   /* requested_major_priority>requested_minor_priority */
  {
    if(requested_major_priority>observed_max_priority)
    {
      requested_major_priority=observed_max_priority;

      if(requested_minor_priority>=requested_major_priority)
      {
        requested_minor_priority=requested_major_priority-1;
      }
    }

    if(requested_minor_priority<observed_min_priority)
    {
      requested_minor_priority=observed_min_priority;

      if(requested_major_priority<=requested_minor_priority)
      {
        requested_major_priority=requested_minor_priority+1;
      }
    }
  }

  if((requested_major_priority!=DESIRED_MAJOR_PRIORITY)
     ||
     (requested_minor_priority!=DESIRED_MINOR_PRIORITY)
    )
  {
    printf("The desired major and minor priorities were %d and %d.\n"
           "We adjusted them to %d and %d,\n"
           "because the minimum priority is %d\n"
           "and the maximum priority is %d\n",
           DESIRED_MAJOR_PRIORITY,
           DESIRED_MINOR_PRIORITY,
           requested_major_priority,
           requested_minor_priority,
           observed_min_priority,
           observed_max_priority
          );
  }

  check_error("pthread_attr_init()",
              pthread_attr_init(&thread_attributes)
             );

  check_error("setting SCHED_RR for major thread",
              pthread_attr_setschedpolicy(&thread_attributes,
                                          SCHED_RR
                                         )
             );

  thread_parameters.sched_priority=requested_major_priority;

  check_error("setting scheduling parameters for major thread",
              pthread_attr_setschedparam(&thread_attributes,
                                         &thread_parameters
                                        )
             );

  check_error("setting explicit (not inherited) scheduling for major thread",
              pthread_attr_setinheritsched(&thread_attributes,
                                           PTHREAD_EXPLICIT_SCHED
                                          )
             );

  printf("about to begin the work, which could take up to %d seconds\n",
         4*DESIRED_WORKLOAD_SECONDS*TIME_ARRAY_ELEMENTS
        );

  check_error("creating major thread (maybe because you\'re not root?)",
              pthread_create(&major_thread_id,
                             &thread_attributes,
                             major_code,
                             NULL
                            )
             );

  check_error("joining major thread",
              pthread_join(major_thread_id,
                           NULL
                          )
             );

  printf("major thread:\n");

  for(jndex=0;
      jndex<TIME_ARRAY_ELEMENTS;
      jndex++
     )
  {
    the_local_time=localtime(major_work_array+jndex);

    printf("%02d:%02d:%02d\n",
           the_local_time->tm_hour,
           the_local_time->tm_min,
           the_local_time->tm_sec
          );
  }

  printf("minor thread:\n");

  for(jndex=0;
      jndex<TIME_ARRAY_ELEMENTS;
      jndex++
     )
  {
    the_local_time=localtime(minor_work_array+jndex);

    printf("%02d:%02d:%02d\n",
           the_local_time->tm_hour,
           the_local_time->tm_min,
           the_local_time->tm_sec
          );
  }

  return 0;

} /* main() */

When I run the program as root, I consistently get indications that the major thread does its computing first. While it's computing, of course my mouse pointer cannot move! The output looks something like this:

calculating workload
could take up to 8 seconds
trying limit = 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152 4194304 8388608 16777216 33554432 67108864
done calculating workload
about to begin the work, which could take up to 80 seconds
major thread:
17:08:23
17:08:25
17:08:27
17:08:29
17:08:31
17:08:33
17:08:35
17:08:37
17:08:39
17:08:41
minor thread:
17:08:43
17:08:45
17:08:47
17:08:49
17:08:51
17:08:53
17:08:55
17:08:57
17:08:59
17:09:01

When I reverse the priorities by altering the numerical values in lines 13 and 14 of the source, and then recompile, and then run as root, the results are consistently the reverse.

Do you get the same results when you run this program?