InterruptAttachEvent in SMP

Here is the senario

We had a resource manager for am I/O card which worked great under the
non-SMP build. But after switching to the SMP build (with processor
affinity set to the first processor), we started running into missed
interrupts (rather huge delays). The card has a free running timer that
causes an interrupt (IRQ 5). We use this to generate a pulse which is used
for timing using the InterruptAttachEvent.
The interrupt fires every 0.5 milliseconds. The resource manager runs at
max priority (63r)

I wrote a very simple version of the resource manager which does nothing but
a “dispatch_block”. On recieving an pulse, it does an InterruptLock(), then
it logs the time taken using ClockCycles() and does an InterruptUnLock()
followed by and InterruptUnmask and goes back into the wait state. Again
works great with non-SMP build (timetaken = 0.47 ~ 0.53 ms). But with the
SMP build I get times like (0.36 ~ 1.52 ms !!! ). I must say that the
spikes in response time occur only when I am doing something else too (eg.
moving the mouse around like crazy or opeing or closing a window).

Please advice me as to why this might be happening and how to avoid running
into this issue.

Thanks
Dennis…

Attached is the code

// Includes

#include <unistd.h>
#include <devctl.h>
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include <sys/iofunc.h>
#include <sys/dispatch.h>
#include <sys/neutrino.h>
#include <stg.h>
#include <stg_int.h>
#include <errno.h>
#include <inttypes.h>
#include <sys/iomsg.h>
#include <sys/syspage.h>
#include <semaphore.h>
#include <fcntl.h>
#include <math.h>
#include “…/testcode/plotter.h”
#include <sys/netmgr.h>
#include <sys/neutrino.h>
#include <time.h>

#define STG_PROCESSOR 0x01


static uint64_t processor_cycles_per_sec;
double time_taken;
double latency;
uint64_t interrupt_occured;
uint64_t service_started;
uint64_t service_finished;
uint64_t interrupt_done;

double ticks_to_ms(uint64_t tme)
{
return (((double)(tme))/processor_cycles_per_sec);
}


int stg_pulse_handler(message_context_t * ctp, int code, unsigned flags,
void *handle)
{
service_started = ClockCycles();
// do nothing
return 1;
}

int stg_enable_interrupt(int coid,int code)
{
// struct sigevent event;
int intr;
struct sigevent event;

SIGEV_PULSE_INIT(&event,coid,getprio(0),code,0);

// attach signal event to IRQ
intr = InterruptAttachEvent(STG_IRQ,&event,0);
// start interrupts, but don’t reset watchdog
out8(STG_BASE + CNTRL1,1<<7 | CNTRL1_IEN_T0 | CNTRL1_NOT_SLAVE);
return intr;
}

static resmgr_connect_funcs_t connect_funcs;
static resmgr_io_funcs_t io_funcs;
static iofunc_attr_t attr;
static intrspin_t spin_lock;



int main(int argc, char **argv)
{
dispatch_t *dpp;
resmgr_attr_t resmgr_attr;
dispatch_context_t *ctp;
resmgr_context_t *ctp_r;
pthread_attr_t pthread_attr;
int pulse_code;
int coid;
int priority;
int freq = 2000;
struct sched_param sched_params;
static void finish(int sig);
int i;
double *timing_data;
double *latency;
int interrupt_id;

// Keep it simple. Assign processor affinity at the very begining
ThreadCtl(_NTO_TCTL_IO,0);


if (ThreadCtl(_NTO_TCTL_RUNMASK,(void*) STG_PROCESSOR)==-1)
{
printf(“Error setting processor affinity”);
exit(-1);
}

// limit the frequency at which the resource manager can operate
if (argc == 2)
{
freq = atoi(argv[1]);
if (freq < 100)
freq = 100;
if (freq > 20000)
freq = 20000;
}

// set up thread attributes

pthread_attr_init(&pthread_attr);
pthread_attr_setinheritsched(&pthread_attr, PTHREAD_EXPLICIT_SCHED);
priority = sched_get_priority_max(SCHED_RR);
sched_params.sched_priority = priority;
pthread_attr_setschedpolicy(&pthread_attr, SCHED_RR);

// initialize logging data

timing_data = create_array(10freq,1);
latency = create_array(10
freq,1);

// initialize spin_lock
memset(&spin_lock,0,sizeof(spin_lock));

// initialize dispatch
if ((dpp = dispatch_create()) == NULL)
{
perror(“resmgr>Unable to dispatch_create\n”);
exit(EXIT_FAILURE);
}

// Actual Code
if (sched_setparam(0, &sched_params) == -1)
{
perror(“sched_getparam”);
exit(-1);
}

// initialize the stg card and make sure it’s installed
if (stg_init(freq))
exit(EXIT_FAILURE);

processor_cycles_per_sec = (double) (SYSPAGE_ENTRY(qtime)->cycles_per_sec);

// bind backup functions into the outcall tables

memset(&resmgr_attr,0,sizeof(resmgr_attr));
resmgr_attr.nparts_max = 1;
resmgr_attr.msg_max_size = 2048;

iofunc_func_init(_RESMGR_CONNECT_NFUNCS, &connect_funcs, _RESMGR_IO_NFUNCS,
&io_funcs);

iofunc_attr_init(&attr, S_IFNAM | 0666, 0, 0);

/* we also want to catch pulses triggered by stg hw interrupts */
if ((pulse_code = pulse_attach(dpp, MSG_FLAG_ALLOC_PULSE, 0,
&stg_pulse_handler , NULL)) == -1)
{
perror(“resmgr: pulse attach\n”);
exit(EXIT_FAILURE);
}

/* establish a name in the pathname space */
if (resmgr_attach(dpp, &resmgr_attr, “/dev/stg”, _FTYPE_ANY,
_RESMGR_FLAG_DIR, &connect_funcs, &io_funcs, &attr) == -1)
{
perror(“resmgr>Unable to resmgr_attach\n”);
exit(EXIT_FAILURE);
}

if ((coid = message_connect(dpp, MSG_FLAG_SIDE_CHANNEL)) == -1)
{
perror(“resmgr: message_connect:”);
exit(EXIT_FAILURE);
}

// All init should have been done at this point let the interrupt handler
become
// active

if ((interrupt_id = stg_enable_interrupt(coid, pulse_code)) == -1)
{
perror(“resmgr: stg_enable_interrupt”);
exit(EXIT_FAILURE);
}

ctp = dispatch_context_alloc(dpp);

// handle first event, to syncronize with interrupt generator
if ((ctp = dispatch_block(ctp)) == NULL)
{
printf(“DISPATCH BLOCK ERROR\n”);
exit(-1);
}

if (InterruptUnmask(STG_IRQ, interrupt_id) == -1)
{
perror(“resmgr interruptunmask”);
return -1;
}

// start the default state to be the No_controller state

for (i=0;i<10*freq+11;i++)
{

if ((ctp = dispatch_block(ctp)) == NULL)
{
perror(“resmgr>Unable to dispatch_block\n”);
exit(EXIT_FAILURE);
}

InterruptLock(&spin_lock);

interrupt_done = ClockCycles();

// handle events
dispatch_handler(ctp);
service_started = ClockCycles();

if (InterruptUnmask(STG_IRQ, interrupt_id) == -1)
{
perror(“resmgr interruptunmask”);
return -1;
}

if (i>10)
{

set_array_element(timing_data,i-11,0,ticks_to_ms(interrupt_done-interrupt_oc
cured)*1000);

set_array_element(latency,i-11,0,ticks_to_ms(service_started-interrupt_done)
*1000);
}
interrupt_occured = interrupt_done;
InterruptUnlock(&spin_lock);
}

write_mat_file(“timing_data.m”,“timing”,timing_data,CREATE);
write_mat_file(“timing_data.m”,“latency”,latency,ADD);
remove_array(timing_data);
remove_array(latency);
return EXIT_SUCCESS;
}