ve+0xf6 > #4 0xffffffff803936c8 at xpt_run_devq+0x148 > #5 0xffffffff80392b0c at xpt_action_default+0x35c > #6 0xffffffff803c3544 at dastart+0x314 > #7 0xffffffff80394f73 at xpt_run_allocq+0x83 > #8 0xffffffff803c5152 at dastrategy+0x82 > #9 0xffffffff80b438d4 at g_disk_start+0x314 > #10 0xffffffff80b46b55 at g_io_request+0x1d5 > #11 0xffffffff80b46b55 at g_io_request+0x1d5 > #12 0xffffffff821a34dc at vdev_geom_io_start+0x23c > #13 0xffffffff82347fb4 at zio_vdev_io_start+0x204 > #14 0xffffffff8234204f at zio_nowait+0x12f > #15 0xffffffff822a0ce7 at vdev_mirror_io_start+0x177 > #16 0xffffffff82347fb4 at zio_vdev_io_start+0x204 > #17 0xffffffff8234204f at zio_nowait+0x12f > Uptime: 10h16m26s > > > The drive it complains about in the first line is one that has SMART > errors: > > > connector 1E box 1 bay 17 HP MB010000JWAYK > 7PH8LU3G HPD5 S.M.A.R.T. predictive > failure. > > It is _not_ da122 though=E2=80=A6 The drive with a SMART error (da85) is = not in > use so basically is just sitting there. > > > > The code where I=E2=80=99m trying to figure out what=E2=80=99s going on i= s this part in > sys/dev/ciss/ciss.c: > > static void > ciss_complete(struct ciss_softc *sc, cr_qhead_t *qh) > { > struct ciss_request *cr; > > debug_called(2); > > /* > > > * Loop taking requests off the completed queue and performing > > > * completion processing on them. > > > */ > for (;;) { > int got_nonbusy =3D 0; > > if ((cr =3D ciss_dequeue_complete(sc, qh)) =3D=3D NULL) > break; > ciss_unmap_request(cr); > > if ((cr->cr_flags & CISS_REQ_BUSY) =3D=3D 0) { > cr->nonbusy_request_counter++; > ciss_printf(sc, > "WARNING: completing non-busy request #%d [onq=3D%d= , > length=3D%u, ccphys=3D%u, tag=3D%d, flags=3D0x%x (%s), sg_tag=3D0x%x, > cr_complete=3D%p]\n", > cr->nonbusy_request_counter, > cr->cr_onq, cr->cr_length, cr->cr_ccphys, cr->cr_ta= g, > cr->cr_flags, cr_flags2str(cr->cr_flags), > cr->cr_sg_tag, cr->cr_complete > ); > got_nonbusy =3D 1; > } > > cr->cr_flags &=3D ~CISS_REQ_BUSY; > > /* > > > * If the request has a callback, invoke it. > > > */ > if (cr->cr_complete !=3D NULL) { > cr->cr_complete(cr); > > /* pen: Attempt to make sure ciss_cam_complete() doesn=E2=80=99t = get > called multiple times */ > > if (cr->cr_complete =3D=3D ciss_cam_complete && got_nonbusy) { > ciss_printf(sc, "WARNING: clearing cr_complete due to > ciss_cam_complete & nonbusy\n"); > cr->cr_complete =3D NULL; > } > continue; > } > > /* > > > * If someone is sleeping on this request, wake them up. > > > */ > if (cr->cr_flags & CISS_REQ_SLEEP) { > cr->cr_flags &=3D ~CISS_REQ_SLEEP; > wakeup(cr); > continue; > } > > /* > > > * If someone is polling this request for completion, signal. > > > */ > if (cr->cr_flags & CISS_REQ_POLL) { > cr->cr_flags &=3D ~CISS_REQ_POLL; > continue; > } > > /* > > > * Give up and throw the request back on the free queue. This > > > * should never happen; resources will probably be lost. > > > */ > ciss_printf(sc, "WARNING: completed command with no submitter\n")= ; > ciss_enqueue_free(cr); > } > } > > > - Any suggestions? (Probably not but I thought I=E2=80=99d ask anyway :-) > > - Peter > > > > --0000000000007b16e4061911469c Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable
Hey Peter,

My apologie= s for my tardy response. The CISS driver is a bit older, and most of my exp= erience is with newer drivers...

=
On Thu, May 9, 2024 at 11:59=E2=80=AF= AM Peter Eriksson <pen@lysator.liu= .se> wrote:
Hi,

I=E2=80=99m trying to fix a bug in the cciss driver that has been there =E2= =80=9Cforever=E2=80=9D when using it with an HP H241 SAS HBA card.
The driver works fine when all (SAS, spinning rust) drives are behaving wel= l, but when some of them are starting to go bad it often goes into spin and= either hangs the kernel or panics. I=E2=80=99ve been trying to add instrum= entation to it in order to pin-point the problem and have been attempting s= ome workarounds (like clearing cr_complete since without that hack sometime= the driver get many many non-busy repeated requests with the same =E2=80= =9Ctag=E2=80=9D and then It panics with:

> ciss1: WARNING: completing non-busy request
> ciss1: WARNING: completing non-busy request
> ciss1: WARNING: completing non-busy request

<= /div>
This means that we're not tracking the state of the requests = correctly.
=C2=A0
> panic: cam_periph_done_panic: already done with ccb 0xfffff88abefc4000=

And I think this is the panic when we = detect that.
=C2=A0
> cpuid =3D 0
> time =3D 1714852260
> KDB: stack backtrace:
> #0 0xffffffff80c541b5 at kdb_backtrace+0x65
> #1 0xffffffff80c06b31 at vpanic+0x151
> #2 0xffffffff80c069d3 at panic+0x43
> #3 0xffffffff8039017c at cam_periph_done_panic+0x1c
> #4 0xffffffff80397193 at xpt_done_process+0x313
> #5 0xffffffff803990a5 at xpt_done_td+0xf5
> #6 0xffffffff80bc333e at fork_exit+0x7e
> #7 0xffffffff8108a44e at fork_trampoline+0xe

but it=E2=80=99s still there=E2=80=A6

I think I could use some input/suggestions from someone more knowledgeable = with CAM and device drivers in general=E2=80=A6

An example (this is with extra instrumentation added so not quite a stock k= ernel driver):

ciss1: *** Drive failure imminent, Port=3D1E Box=3D1 Bay=3D17 reason=3D0x37= [ts=3D4460, class=3D4, subclass=3D0, detail=3D1, tag=3D0x2]
(da122:ciss1:32:78:0): WRITE(16). CDB: 8a 00 00 00 00 04 7f 17 51 98 00 00 = 00 10 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): WRITE(16). CDB: 8a 00 00 00 00 04 7f 17 52 18 00 00 = 00 10 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 65 c8 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 63 c8 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 61 c8 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 5f c8 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 5f a8 00 00 0= 0 20 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 5f 38 00 00 0= 0 70 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 5d 38 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 5b 38 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 59 38 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 58 48 00 00 0= 0 f0 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 7f 17 56 48 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): WRITE(16). CDB: 8a 00 00 00 00 04 88 d8 48 d0 00 00 = 00 10 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): WRITE(16). CDB: 8a 00 00 00 00 04 88 d8 46 e0 00 00 = 01 f0 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 88 d8 8f d0 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
(da122:ciss1:32:78:0): READ(16). CDB: 88 00 00 00 00 04 88 d8 8d d0 00 00 0= 2 00 00 00
(da122:ciss1:32:78:0): CAM status: CCB request completed with an error
(da122:ciss1:32:78:0): Retrying command, 3 more tries remain
ciss1: WARNING: completing non-busy request #1 [onq=3D0, length=3D262144, c= cphys=3D1791048448, tag=3D72, flags=3D0x50 (----I-C), sg_tag=3D0x3, cr_comp= lete=3D0xffffffff806059a0]
ciss1: WARNING: clearing cr_complete due to ciss_cam_complete & nonbusy=
ciss1: WARNING: completing non-busy request #1 [onq=3D0, length=3D262144, c= cphys=3D1791318368, tag=3D313, flags=3D0x50 (----I-C), sg_tag=3D0x3, cr_com= plete=3D0xffffffff806059a0]
ciss1: WARNING: clearing cr_complete due to ciss_cam_complete & nonbusy=
ciss1: WARNING: completing non-busy request #1 [onq=3D0, length=3D8192, ccp= hys=3D1791226528, tag=3D231, flags=3D0x50 (----I-C), sg_tag=3D0x3, cr_compl= ete=3D0xffffffff806059a0]
ciss1: WARNING: clearing cr_complete due to ciss_cam_complete & nonbusy=
ciss1: WARNING: completing non-busy request #1 [onq=3D0, length=3D262144, c= cphys=3D1791313888, tag=3D309, flags=3D0x50 (----I-C), sg_tag=3D0x5, cr_com= plete=3D0xffffffff806059a0]
ciss1: WARNING: clearing cr_complete due to ciss_cam_complete & nonbusy=
ciss1: WARNING: completing non-busy request #1 [onq=3D0, length=3D8192, ccp= hys=3D1791206368, tag=3D213, flags=3D0x50 (----I-C), sg_tag=3D0x3, cr_compl= ete=3D0xffffffff806059a0]
ciss1: WARNING: clearing cr_complete due to ciss_cam_complete & nonbusy=
panic: camq_remove: Attempt to remove out-of-bounds index -1 from queue 0xf= ffff80107fa9838 of size 1

So these seem= to line up with the commands that encountered an error. The errors handlin= g code of most CAM drivers is the hardest to write and validate because mis= behaving disks, while not rare, are generally not used in validation.=C2=A0= All my good ones that were dying have since stopped working completely, fo= r example.

So in this case, you'll need to loo= k at the state tracking code for the ciss requests, and make sure that in t= he cases of errors the right transitions happen. I've fixed bugs in mpr= /mps that amounted to marking a request as busy again when it errored out i= n some of the weirder error scenarios (I don't recall the exact cases t= hough). You might look at mpr/mps for the evolution of this code there. IIR= C, the same author did both ciss and mps and many of the techniques were re= tained between the two. You might get good hints for bugs to fix by looking= at how things evolved there.

Basically, you'r= e going to have to find out if these complaints are due to the state tracki= ng being messed up, but the request really should be doing the thing that i= t's doing when the complaint is lodged (in which case you need to fix t= he point where the state becomes incorrect). Or if it's caught a real b= ug and the state of the request is proper, but we're doing something li= ke completing something twice, or failing to allocate a new request or one = of the many other silly bugs this tracking code usually prevents.

Warner
=C2=A0
cpuid =3D 7
time =3D 1715266713
KDB: stack backtrace:
#0 0xffffffff80c542d5 at kdb_backtrace+0x65
#1 0xffffffff80c06c51 at vpanic+0x151
#2 0xffffffff80c06af3 at panic+0x43
#3 0xffffffff80390676 at camq_remove+0xf6
#4 0xffffffff803936c8 at xpt_run_devq+0x148
#5 0xffffffff80392b0c at xpt_action_default+0x35c
#6 0xffffffff803c3544 at dastart+0x314
#7 0xffffffff80394f73 at xpt_run_allocq+0x83
#8 0xffffffff803c5152 at dastrategy+0x82
#9 0xffffffff80b438d4 at g_disk_start+0x314
#10 0xffffffff80b46b55 at g_io_request+0x1d5
#11 0xffffffff80b46b55 at g_io_request+0x1d5
#12 0xffffffff821a34dc at vdev_geom_io_start+0x23c
#13 0xffffffff82347fb4 at zio_vdev_io_start+0x204
#14 0xffffffff8234204f at zio_nowait+0x12f
#15 0xffffffff822a0ce7 at vdev_mirror_io_start+0x177
#16 0xffffffff82347fb4 at zio_vdev_io_start+0x204
#17 0xffffffff8234204f at zio_nowait+0x12f
Uptime: 10h16m26s


The drive it complains about in the first line is one that has SMART errors= :

>=C2=A0 =C2=A0 =C2=A0 =C2=A0 connector 1E box=C2=A0 1 bay 17=C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0HP=C2=A0 =C2=A0 =C2=A0 = MB010000JWAYK=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0= =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 7PH8LU3G=C2= =A0 =C2=A0 =C2=A0HPD5 S.M.A.R.T. predictive failure.

It is _not_ da122 though=E2=80=A6 The drive with a SMART error (da85) is no= t in use so basically is just sitting there.



The code where I=E2=80=99m trying to figure out what=E2=80=99s going on is = this part in sys/dev/ciss/ciss.c:

static void
ciss_complete(struct ciss_softc *sc, cr_qhead_t *qh)
{
=C2=A0 =C2=A0 struct ciss_request *cr;

=C2=A0 =C2=A0 debug_called(2);

=C2=A0 =C2=A0 /*=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0* Loop taking requests off the completed queue and perf= orming=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0= =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0* completion processing on them.=C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0*/
=C2=A0 =C2=A0 for (;;) {
=C2=A0 =C2=A0 =C2=A0 int got_nonbusy =3D 0;

=C2=A0 =C2=A0 =C2=A0 =C2=A0 if ((cr =3D ciss_dequeue_complete(sc, qh)) =3D= =3D NULL)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 ciss_unmap_request(cr);

=C2=A0 =C2=A0 =C2=A0 =C2=A0 if ((cr->cr_flags & CISS_REQ_BUSY) =3D= =3D 0) {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cr->nonbusy_request_counter++;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ciss_printf(sc,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 "WARNING: completing non-busy request #%d [onq=3D%d, length=3D%u, = ccphys=3D%u, tag=3D%d, flags=3D0x%x (%s), sg_tag=3D0x%x, cr_complete=3D%p]\= n",
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 cr->nonbusy_request_counter,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 cr->cr_onq, cr->cr_length, cr->cr_ccphys, cr->cr_tag,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 cr->cr_flags, cr_flags2str(cr->cr_flags),
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 cr->cr_sg_tag, cr->cr_complete
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 );
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 got_nonbusy =3D 1;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 }

=C2=A0 =C2=A0 =C2=A0 =C2=A0 cr->cr_flags &=3D ~CISS_REQ_BUSY;

=C2=A0 =C2=A0 =C2=A0 =C2=A0 /*=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0* If the request has a callback, invoke i= t.=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0*/
=C2=A0 =C2=A0 =C2=A0 =C2=A0 if (cr->cr_complete !=3D NULL) {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cr->cr_complete(cr);

=C2=A0 =C2=A0 =C2=A0 =C2=A0 /* pen: Attempt to make sure ciss_cam_complete(= ) doesn=E2=80=99t get called multiple times */

=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if (cr->cr_complete =3D=3D ciss_cam_c= omplete && got_nonbusy) {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ciss_printf(sc, "WARNING: cl= earing cr_complete due to ciss_cam_complete & nonbusy\n");
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cr->cr_complete =3D NULL;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 }
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 continue;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 }=C2=A0 =C2=A0 =C2=A0

=C2=A0 =C2=A0 =C2=A0 =C2=A0 /*=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0* If someone is sleeping on this request,= wake them up.=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0*/
=C2=A0 =C2=A0 =C2=A0 =C2=A0 if (cr->cr_flags & CISS_REQ_SLEEP) {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cr->cr_flags &=3D ~CISS_REQ_SLEEP= ;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 wakeup(cr);
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 continue;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 }

=C2=A0 =C2=A0 =C2=A0 =C2=A0 /*=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0* If someone is polling this request for = completion, signal.=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0*/
=C2=A0 =C2=A0 =C2=A0 =C2=A0 if (cr->cr_flags & CISS_REQ_POLL) {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cr->cr_flags &=3D ~CISS_REQ_POLL;=
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 continue;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 }

=C2=A0 =C2=A0 =C2=A0 =C2=A0 /*=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0* Give up and throw the request back on t= he free queue.=C2=A0 This=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0* should never happen; resources will pro= bably be lost.=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0*/
=C2=A0 =C2=A0 =C2=A0 =C2=A0 ciss_printf(sc, "WARNING: completed comman= d with no submitter\n");
=C2=A0 =C2=A0 =C2=A0 =C2=A0 ciss_enqueue_free(cr);
=C2=A0 =C2=A0 }
}


- Any suggestions? (Probably not but I thought I=E2=80=99d ask anyway :-)
- Peter



--0000000000007b16e4061911469c--