fc9da4e612e998fdcc8c548589f523b5d6454878
[mirror/winof/.git] / core / bus / kernel / bus_pnp.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  *\r
5  * This software is available to you under the OpenIB.org BSD license\r
6  * below:\r
7  *\r
8  *     Redistribution and use in source and binary forms, with or\r
9  *     without modification, are permitted provided that the following\r
10  *     conditions are met:\r
11  *\r
12  *      - Redistributions of source code must retain the above\r
13  *        copyright notice, this list of conditions and the following\r
14  *        disclaimer.\r
15  *\r
16  *      - Redistributions in binary form must reproduce the above\r
17  *        copyright notice, this list of conditions and the following\r
18  *        disclaimer in the documentation and/or other materials\r
19  *        provided with the distribution.\r
20  *\r
21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
22  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
23  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
24  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
25  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
26  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
27  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
28  * SOFTWARE.\r
29  *\r
30  * $Id$\r
31  */\r
32 \r
33 \r
34 \r
35 /*\r
36  * Implemenation of all PnP functionality for FDO (power policy owners).\r
37  */\r
38 \r
39 #include "bus_pnp.h"\r
40 #include "al_ca.h"\r
41 #include "al_init.h"\r
42 #include "al_dev.h"\r
43 #include "al_debug.h"\r
44 #include "bus_port_mgr.h"\r
45 #include "bus_iou_mgr.h"\r
46 #include "complib/cl_memory.h"\r
47 #include <initguid.h>\r
48 #include "iba/ib_ci_ifc.h"\r
49 #include "iba/ib_cm_ifc.h"\r
50 #include "al_cm_cep.h"\r
51 #include "al_mgr.h"\r
52 \r
53 \r
54 /* Interface names are generated by IoRegisterDeviceInterface. */\r
55 static UNICODE_STRING   al_ifc_name;\r
56 static UNICODE_STRING   ci_ifc_name;\r
57 static UNICODE_STRING   cm_ifc_name;\r
58 \r
59 KEVENT                                  g_ControlEvent;\r
60 ULONG                                   g_bfi_InstanceCount;\r
61 bus_filter_t                    g_bus_filters[MAX_BUS_FILTERS];\r
62 \r
63 extern PDEVICE_OBJECT   g_ControlDeviceObject;\r
64 extern UNICODE_STRING   g_CDO_dev_name, g_CDO_dos_name;\r
65 \r
66 \r
67 static NTSTATUS\r
68 fdo_start(\r
69         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
70         IN                                      IRP* const                              p_irp, \r
71                 OUT                             cl_irp_action_t* const  p_action );\r
72 \r
73 static void\r
74 fdo_release_resources(\r
75         IN                                      DEVICE_OBJECT* const    p_dev_obj );\r
76 \r
77 static NTSTATUS\r
78 fdo_query_remove(\r
79         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
80         IN                                      IRP* const                              p_irp, \r
81                 OUT                             cl_irp_action_t* const  p_action );\r
82 \r
83 static NTSTATUS\r
84 fdo_query_capabilities(\r
85         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
86         IN                                      IRP* const                              p_irp, \r
87                 OUT                             cl_irp_action_t* const  p_action );\r
88 \r
89 static NTSTATUS\r
90 fdo_query_bus_relations(\r
91         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
92         IN                                      IRP* const                              p_irp, \r
93                 OUT                             cl_irp_action_t* const  p_action );\r
94 \r
95 static NTSTATUS\r
96 __query_al_ifc(\r
97         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
98         IN                              IO_STACK_LOCATION* const        p_io_stack );\r
99 \r
100 static NTSTATUS\r
101 __get_relations(\r
102         IN              const   net64_t                                         ca_guid,\r
103         IN                              IRP* const                                      p_irp );\r
104 \r
105 static NTSTATUS\r
106 __query_ci_ifc(\r
107         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
108         IN                              IO_STACK_LOCATION* const        p_io_stack );\r
109 \r
110 static NTSTATUS\r
111 __query_cm_ifc(\r
112         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
113         IN                              IO_STACK_LOCATION* const        p_io_stack );\r
114 \r
115 static NTSTATUS\r
116 fdo_query_interface(\r
117         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
118         IN                              IRP* const                                      p_irp, \r
119                 OUT                     cl_irp_action_t* const          p_action );\r
120 \r
121 static NTSTATUS\r
122 __fdo_query_power(\r
123         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
124         IN                              IRP* const                                      p_irp,\r
125                 OUT                     cl_irp_action_t* const          p_action );\r
126 \r
127 static NTSTATUS\r
128 __fdo_set_power(\r
129         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
130         IN                              IRP* const                                      p_irp,\r
131                 OUT                     cl_irp_action_t* const          p_action );\r
132 \r
133 static void\r
134 __set_ifc(\r
135                 OUT                     ib_al_ifc_t* const                      p_ifc );\r
136 \r
137 \r
138 \r
139 \r
140 /* Global virtual function pointer tables shared between all instances of FDO. */\r
141 static const cl_vfptr_pnp_po_t          vfptr_fdo_pnp = {\r
142         "IB Bus",\r
143         fdo_start,\r
144         cl_irp_skip,\r
145         cl_irp_skip,\r
146         cl_do_sync_pnp,\r
147         fdo_query_remove,\r
148         fdo_release_resources,\r
149         cl_do_remove,\r
150         cl_do_sync_pnp,\r
151         cl_irp_skip,\r
152         fdo_query_capabilities,\r
153         cl_irp_skip,\r
154         cl_irp_skip,\r
155         cl_do_sync_pnp,\r
156         fdo_query_bus_relations,\r
157         cl_irp_ignore,\r
158         cl_irp_skip,\r
159         cl_irp_ignore,\r
160         cl_irp_ignore,\r
161         cl_irp_ignore,\r
162         cl_irp_ignore,\r
163         cl_irp_ignore,\r
164         fdo_query_interface,    /* QueryInterface */\r
165         cl_irp_ignore,\r
166         cl_irp_ignore,\r
167         cl_irp_ignore,\r
168         cl_irp_ignore,\r
169         __fdo_query_power,              /* QueryPower */\r
170         __fdo_set_power,                /* SetPower */\r
171         cl_irp_ignore,                  /* PowerSequence */\r
172         cl_irp_ignore                   /* WaitWake */\r
173 };\r
174 \r
175 \r
176 NTSTATUS\r
177 bus_add_device(\r
178         IN                              DRIVER_OBJECT                           *p_driver_obj,\r
179         IN                              DEVICE_OBJECT                           *p_pdo )\r
180 {\r
181         NTSTATUS                status;\r
182         DEVICE_OBJECT   *p_dev_obj, *p_next_do;\r
183         bus_fdo_ext_t   *p_ext=NULL;\r
184         bus_filter_t    *p_bfi;\r
185         int                             ic;\r
186 \r
187         BUS_ENTER( BUS_DBG_PNP );\r
188 \r
189         /* allocate a Bus Filter Instance */\r
190         p_bfi = alloc_bfi( p_driver_obj, &ic );\r
191         if ( !p_bfi )\r
192         {\r
193                 BUS_TRACE_EXIT( BUS_DBG_PNP,\r
194                         ("%s() Err - Exceeded MAX_BUS_FILTERS(%d)\n",MAX_BUS_FILTERS));\r
195                 return STATUS_UNSUCCESSFUL;\r
196         }\r
197 \r
198         /* Create the FDO device object to attach to the stack. */\r
199 \r
200         /* if 1st Bus Filter Instance, then create device names for user ioctl */\r
201         if ( ic == 1  && !g_ControlDeviceObject)\r
202         {\r
203                 status = IoCreateDevice( p_driver_obj, sizeof(bus_fdo_ext_t),\r
204                                                                  &g_CDO_dev_name, FILE_DEVICE_BUS_EXTENDER,\r
205                                                                  FILE_DEVICE_SECURE_OPEN, FALSE, &p_dev_obj );\r
206                 if( !NT_SUCCESS(status) )\r
207                 {\r
208                         BUS_PRINT( BUS_DBG_ERROR, \r
209                                 ("Failed to create ControlDeviceObject, status %x.\n",status) );\r
210                         goto err1;\r
211                 }\r
212         }\r
213         else {\r
214                 status = IoCreateDevice( p_driver_obj, sizeof(bus_fdo_ext_t),\r
215                                                                  NULL, FILE_DEVICE_BUS_EXTENDER,\r
216                                                                  FILE_DEVICE_SECURE_OPEN, FALSE, &p_dev_obj );\r
217                 if( !NT_SUCCESS(status) )\r
218                 {\r
219                         BUS_PRINT( BUS_DBG_ERROR, \r
220                                 ("Failed to create bus root FDO device.\n") );\r
221                         goto err1;\r
222                 }\r
223         }\r
224 \r
225         if ( ic == 1 ) {\r
226                 /* enable user-mode access to IB stack */\r
227                 BUS_PRINT( BUS_DBG_PNP, ("Remove-n-reCreate dos_name symlink\n") );\r
228                 IoDeleteSymbolicLink( &g_CDO_dos_name );\r
229                 status = IoCreateSymbolicLink( &g_CDO_dos_name, &g_CDO_dev_name );\r
230                 if( !NT_SUCCESS(status) )\r
231                 {\r
232                         BUS_PRINT( BUS_DBG_ERROR,\r
233                                 ("Failed to create symlink for dos name.\n") );\r
234                         goto err2;\r
235                 }\r
236                 BUS_TRACE( BUS_DBG_PNP, ("Created dos_name symlink\n") );\r
237         }\r
238 \r
239         p_ext = p_dev_obj->DeviceExtension;\r
240         p_ext->n_al_ifc_ref = 0;\r
241         p_ext->n_ci_ifc_ref = 0;\r
242 \r
243         p_next_do = IoAttachDeviceToDeviceStack( p_dev_obj, p_pdo );\r
244         if( !p_next_do )\r
245         {\r
246                 BUS_PRINT( BUS_DBG_ERROR, ("IoAttachToDeviceStack failed.\n") );\r
247                 status = STATUS_NO_SUCH_DEVICE;\r
248                 goto err2;\r
249         }\r
250 \r
251         cl_init_pnp_po_ext( p_dev_obj, p_next_do, p_pdo, bus_globals.dbg_lvl,\r
252                                                 &vfptr_fdo_pnp, NULL );\r
253 \r
254         p_bfi->p_bus_ext = p_ext;\r
255         p_ext->bus_filter = p_bfi;\r
256 \r
257         /*\r
258          * if not 1st Bus Filter Instance, then finished...\r
259          */\r
260         if ( ic > 1 )\r
261                 goto adxit;\r
262 \r
263         /* Register the upper interface (the one used by clients). */\r
264         status = IoRegisterDeviceInterface( p_pdo, &GUID_IB_AL_INTERFACE, NULL,\r
265                                                                                 &al_ifc_name );\r
266         if( !NT_SUCCESS( status ) )\r
267         {\r
268                 BUS_PRINT( BUS_DBG_ERROR, \r
269                         ("IoRegisterDeviceInterface for upper interface returned %08x\n",\r
270                         status) );\r
271                 status = STATUS_NO_SUCH_DEVICE;\r
272                 goto err3;\r
273         }\r
274 \r
275         /* Register the lower (CI) interface (the one used by HCA VPDs). */\r
276         status = IoRegisterDeviceInterface( p_pdo, &GUID_IB_CI_INTERFACE, NULL,\r
277                                                                                 &ci_ifc_name );\r
278         if( !NT_SUCCESS( status ) )\r
279         {\r
280                 BUS_PRINT( BUS_DBG_ERROR, \r
281                         ("IoRegisterDeviceInterface for lower interface returned %08x\n",\r
282                         status) );\r
283                 status = STATUS_NO_SUCH_DEVICE;\r
284                 goto err3;\r
285         }\r
286 \r
287         status = IoRegisterDeviceInterface( p_pdo, &GUID_INFINIBAND_INTERFACE_CM, NULL,\r
288                                                                                 &cm_ifc_name );\r
289         if( !NT_SUCCESS( status ) )\r
290         {\r
291                 BUS_PRINT( BUS_DBG_ERROR, \r
292                         ("IoRegisterDeviceInterface for cm interface returned %08x\n",\r
293                         status) );\r
294                 status = STATUS_NO_SUCH_DEVICE;\r
295                 goto err3;\r
296         }\r
297 \r
298 adxit:\r
299         BUS_PRINT( BUS_DBG_PNP, ("%s exit status 0\n", p_bfi->whoami) );\r
300 \r
301         BUS_EXIT( BUS_DBG_PNP );\r
302         return STATUS_SUCCESS;\r
303 \r
304 err3:\r
305         IoDetachDevice( p_ext->cl_ext.p_next_do );\r
306 err2:\r
307         IoDeleteDevice( p_dev_obj );\r
308 err1:\r
309         BUS_PRINT( BUS_DBG_PNP, ("%s exit status 0x%x\n", p_bfi->whoami,status) );\r
310         ic = free_bfi(p_bfi);\r
311         /* if last Bus filter, then cleanup */\r
312         if ( ic == 0 )\r
313         {\r
314                 IoDeleteSymbolicLink( &g_CDO_dos_name );\r
315                 RtlFreeUnicodeString( &al_ifc_name );\r
316         }\r
317         BUS_EXIT( BUS_DBG_PNP );\r
318         return status;\r
319 }\r
320 \r
321 \r
322 /* Forwards the request to the HCA's FDO. */\r
323 static NTSTATUS\r
324 __get_ifc(\r
325         IN                              DEVICE_OBJECT* const            pDevObj,\r
326         IN              const   GUID* const                                     pGuid,\r
327         IN                              USHORT                                          size,\r
328         IN                              USHORT                                          Version,\r
329         IN OUT                  PVOID                                           InterfaceSpecificData,\r
330                 OUT                     PINTERFACE                                      pHcaIfc )\r
331 {\r
332         NTSTATUS                        status;\r
333         IRP                                     *pIrp;\r
334         IO_STATUS_BLOCK         ioStatus;\r
335         IO_STACK_LOCATION       *pIoStack;\r
336         DEVICE_OBJECT           *pDev;\r
337         KEVENT                          event;\r
338 \r
339         BUS_ENTER( BUS_DBG_PNP );\r
340 \r
341         CL_ASSERT( KeGetCurrentIrql() < DISPATCH_LEVEL );\r
342 \r
343         pDev = IoGetAttachedDeviceReference( pDevObj );\r
344 \r
345         KeInitializeEvent( &event, NotificationEvent, FALSE );\r
346 \r
347         /* Build the IRP for the HCA. */\r
348         pIrp = IoBuildSynchronousFsdRequest( IRP_MJ_PNP, pDev,\r
349                 NULL, 0, NULL, &event, &ioStatus );\r
350         if( !pIrp )\r
351         {\r
352                 ObDereferenceObject( pDev );\r
353                 BUS_PRINT( BUS_DBG_PNP, \r
354                         ("IoBuildSynchronousFsdRequest failed.\n"));\r
355                 return STATUS_INSUFFICIENT_RESOURCES;\r
356         }\r
357 \r
358         /* Copy the request query parameters. */\r
359         pIoStack = IoGetNextIrpStackLocation( pIrp );\r
360         pIoStack->MinorFunction = IRP_MN_QUERY_INTERFACE;\r
361         pIoStack->Parameters.QueryInterface.Size = size;\r
362         pIoStack->Parameters.QueryInterface.Version = Version;\r
363         pIoStack->Parameters.QueryInterface.InterfaceType = pGuid;\r
364         pIoStack->Parameters.QueryInterface.Interface = (INTERFACE*)pHcaIfc;\r
365         pIoStack->Parameters.QueryInterface.InterfaceSpecificData = InterfaceSpecificData;\r
366 \r
367         pIrp->IoStatus.Status = STATUS_NOT_SUPPORTED;\r
368 \r
369         /* Send the IRP. */\r
370         status = IoCallDriver( pDev, pIrp );\r
371         if( status == STATUS_PENDING )\r
372         {\r
373                 KeWaitForSingleObject( &event, Executive, KernelMode,\r
374                         FALSE, NULL );\r
375 \r
376                 status = ioStatus.Status;\r
377         }\r
378         ObDereferenceObject( pDev );\r
379 \r
380         BUS_EXIT( BUS_DBG_PNP );\r
381         return status;\r
382 }\r
383 \r
384 static NTSTATUS\r
385 fdo_start(\r
386         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
387         IN                                      IRP* const                              p_irp, \r
388                 OUT                             cl_irp_action_t* const  p_action )\r
389 {\r
390         NTSTATUS                status;\r
391         bus_fdo_ext_t   *p_ext;\r
392         ib_api_status_t ib_status;\r
393         bus_filter_t    *p_bfi;\r
394         boolean_t               AL_init_here = FALSE;\r
395 \r
396         BUS_ENTER( BUS_DBG_PNP );\r
397 \r
398         p_ext = p_dev_obj->DeviceExtension;\r
399         p_bfi = p_ext->bus_filter;\r
400 \r
401         /* Handled on the way up. */\r
402         status = cl_do_sync_pnp( p_dev_obj, p_irp, p_action );\r
403         if( !NT_SUCCESS( status ) )\r
404         {\r
405                 BUS_TRACE_EXIT( BUS_DBG_ERROR, \r
406                         ("Lower drivers failed IRP_MN_START_DEVICE.\n") );\r
407                 return status;\r
408         }\r
409 \r
410         lock_control_event();\r
411         if ( !gp_async_proc_mgr ) {\r
412                 /* Initialize AL */\r
413                 ib_status = al_initialize();\r
414                 if( ib_status != IB_SUCCESS )\r
415                 {\r
416                         al_cleanup();\r
417                         BUS_TRACE_EXIT( BUS_DBG_ERROR, ("al_initialize returned %s.\n",\r
418                                                         ib_get_err_str(ib_status)) );\r
419                         unlock_control_event();\r
420                         return STATUS_UNSUCCESSFUL;\r
421                 }\r
422                 AL_init_here = TRUE;\r
423                 BUS_TRACE( BUS_DBG_PNP, ("AL initialized\n"));\r
424         }\r
425         unlock_control_event();\r
426 \r
427         /* Initialize the port manager. */\r
428         ib_status = create_port_mgr( p_ext->bus_filter, &p_ext->p_port_mgr );\r
429         if( ib_status != IB_SUCCESS )\r
430         {\r
431                 BUS_TRACE_EXIT( BUS_DBG_ERROR, ("create_port_mgr returned %s.\n",\r
432                         ib_get_err_str(ib_status)) );\r
433                 return STATUS_UNSUCCESSFUL;\r
434         }\r
435 \r
436         /* Initialize the IOU manager. */\r
437         ib_status = create_iou_mgr( p_ext->bus_filter, &p_ext->p_iou_mgr );\r
438         if( ib_status != IB_SUCCESS )\r
439         {\r
440                 BUS_TRACE_EXIT( BUS_DBG_ERROR, ("create_iou_mgr returned %s.\n",\r
441                         ib_get_err_str(ib_status)) );\r
442                 return STATUS_UNSUCCESSFUL;\r
443         }\r
444 \r
445         if ( AL_init_here ) {\r
446                 status = IoSetDeviceInterfaceState( &al_ifc_name, TRUE );\r
447                 ASSERT( NT_SUCCESS( status ) );\r
448 \r
449                 status = IoSetDeviceInterfaceState( &ci_ifc_name, TRUE );\r
450                 ASSERT( NT_SUCCESS( status ) );\r
451 \r
452                 status = IoSetDeviceInterfaceState( &cm_ifc_name, TRUE );\r
453                 ASSERT( NT_SUCCESS( status ) );\r
454         }\r
455 \r
456         /* get HCA verbs interface */\r
457         status = __get_ifc( p_dev_obj, &GUID_RDMA_INTERFACE_VERBS,\r
458                                                 sizeof(RDMA_INTERFACE_VERBS), \r
459                                                 VerbsVersion(VERBS_MAJOR_VER, VERBS_MINOR_VER), \r
460                                                 p_ext, (PINTERFACE)&p_ext->hca_ifc );\r
461         if( !NT_SUCCESS( status ) ) \r
462         {\r
463                 BUS_TRACE_EXIT(BUS_DBG_PNP,\r
464                         ("Getting MLX4 BUS interface failed: status=0x%x\n", status));\r
465                 return STATUS_UNSUCCESSFUL;\r
466         }\r
467         p_ext->hca_ifc_taken = TRUE;\r
468 \r
469         /* bind BFI to HCA by CA GUID. Have to be before ib_register_ca */\r
470         p_bfi->ca_guid = p_ext->hca_ifc.Verbs.guid;\r
471 \r
472         /* register HCA */\r
473         ib_status = ib_register_ca( &p_ext->hca_ifc.Verbs, p_ext->cl_ext.p_pdo );\r
474         if( ib_status != IB_SUCCESS )\r
475         {\r
476                 BUS_TRACE_EXIT( BUS_DBG_ERROR, ("ib_register_ca returned %s.\n",\r
477                         ib_get_err_str(ib_status)) );\r
478                 return STATUS_UNSUCCESSFUL;\r
479         }\r
480 \r
481         BUS_TRACE_EXIT(BUS_DBG_PNP, ("%s bound to CA guid %I64x\n",\r
482                                                                         p_bfi->whoami,p_bfi->ca_guid));\r
483         return status;\r
484 }\r
485 \r
486 \r
487 static NTSTATUS\r
488 fdo_query_remove(\r
489         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
490         IN                                      IRP* const                              p_irp, \r
491                 OUT                             cl_irp_action_t* const  p_action )\r
492 {\r
493         bus_fdo_ext_t   *p_ext;\r
494 \r
495         BUS_ENTER( BUS_DBG_PNP );\r
496 \r
497         p_ext = p_dev_obj->DeviceExtension;\r
498 \r
499         CL_ASSERT(p_ext->bus_filter);\r
500         BUS_PRINT( BUS_DBG_PNP,\r
501                 ("IRP_MN_QUERY_REMOVE_DEVICE %s @ FDO %p refs(CI %d AL %d)\n"\r
502                 "   %s CA %I64x\n", \r
503                 p_ext->cl_ext.vfptr_pnp_po->identity, p_ext, p_ext->n_ci_ifc_ref,\r
504                 p_ext->n_al_ifc_ref,\r
505                 p_ext->bus_filter->whoami, p_ext->bus_filter->ca_guid) );\r
506 \r
507         if( p_ext->n_ci_ifc_ref )\r
508         {\r
509                 /*\r
510                  * Our interface is still being held by someone.\r
511                  * Rollback the PnP state that was changed in the cl_ext handler.\r
512                  */\r
513                 cl_rollback_pnp_state( &p_ext->cl_ext );\r
514 \r
515                 /* Fail the query. */\r
516                 *p_action = IrpComplete;\r
517                 BUS_TRACE_EXIT( BUS_DBG_PNP, \r
518                         ("Failing IRP_MN_QUERY_REMOVE_DEVICE:\n"\r
519                         "\tLowerInterface has %d references\n", \r
520                         p_ext->n_ci_ifc_ref ) );\r
521                 return STATUS_UNSUCCESSFUL;\r
522         }\r
523 \r
524         /* remove port & iou managers */        \r
525         CL_ASSERT( p_ext->bus_filter );\r
526 \r
527         //TODO: Fail outstanding I/O operations.\r
528 \r
529         *p_action = IrpSkip;\r
530         /* The FDO driver must set the status even when passing down. */\r
531         p_irp->IoStatus.Status = STATUS_SUCCESS;\r
532 \r
533         BUS_EXIT( BUS_DBG_PNP );\r
534         return STATUS_SUCCESS;\r
535 }\r
536 \r
537 \r
538 /*\r
539  * This function gets called after releasing the remove lock and waiting\r
540  * for all other threads to release the lock.  No more modifications will\r
541  * occur to the PDO pointer vectors.\r
542  */\r
543 static void\r
544 fdo_release_resources(\r
545         IN                                      DEVICE_OBJECT* const    p_dev_obj )\r
546 {\r
547         bus_fdo_ext_t   *p_ext;\r
548         NTSTATUS                status;\r
549         bus_filter_t    *p_bfi;\r
550         int                             ic;\r
551         ib_api_status_t ib_status;\r
552 \r
553         BUS_ENTER( BUS_DBG_PNP );\r
554 \r
555         p_ext = p_dev_obj->DeviceExtension;\r
556         ic = get_bfi_count();\r
557 \r
558         p_bfi = p_ext->bus_filter;\r
559         CL_ASSERT( p_bfi );\r
560 \r
561         //TODO: Fail outstanding I/O operations.\r
562 \r
563         ib_status = ib_deregister_ca( p_ext->hca_ifc.Verbs.guid );\r
564         if( ib_status != IB_SUCCESS ) {\r
565                 BUS_PRINT( BUS_DBG_ERROR, ("ib_deregister_ca returned %s.\n",\r
566                         ib_get_err_str(ib_status)) );\r
567         }\r
568 \r
569         if ( p_ext->p_port_mgr && p_bfi->p_port_mgr ) {\r
570                 cl_obj_destroy( &p_ext->p_port_mgr->obj );\r
571                 p_ext->p_port_mgr = NULL;\r
572         }\r
573 \r
574         if ( p_ext->p_iou_mgr && p_bfi->p_iou_mgr ) {\r
575                 cl_obj_destroy( &p_ext->p_iou_mgr->obj );\r
576                 p_ext->p_iou_mgr = NULL;\r
577         }\r
578 \r
579         /* if not last HCA then release IFC reference, otherwise release IFC after\r
580          * IBAL has shutdown; keep the HCA present until IBAL is terminated.\r
581          */\r
582         if ( ic > 1 && p_ext->hca_ifc_taken ) {\r
583                 p_ext->hca_ifc.InterfaceHeader.InterfaceDereference(\r
584                         p_ext->hca_ifc.InterfaceHeader.Context);\r
585                 p_ext->hca_ifc_taken = FALSE;\r
586         }\r
587 \r
588         BUS_TRACE( BUS_DBG_PNP, ("Releasing BusFilter %s\n", p_bfi->whoami ));\r
589         if (p_bfi) {\r
590                 p_ext->bus_filter = NULL;\r
591                 p_bfi->p_bus_ext = NULL;\r
592         }\r
593 \r
594         ic = free_bfi( p_bfi );\r
595 \r
596         /* if not last Buf Filter Instance, then exit, otherwise cleanup/shutdown */\r
597         if ( ic > 0 ) {\r
598                 BUS_TRACE( BUS_DBG_PNP, ("%d remaining BusFilters\n", ic ));\r
599                 return;\r
600         }\r
601 \r
602         IoDeleteSymbolicLink( &g_CDO_dos_name );\r
603 \r
604         /* Disable any exported interfaces. */\r
605         status = IoSetDeviceInterfaceState( &al_ifc_name, FALSE );\r
606         ASSERT( NT_SUCCESS( status ) );\r
607         status = IoSetDeviceInterfaceState( &ci_ifc_name, FALSE );\r
608         ASSERT( NT_SUCCESS( status ) );\r
609         status = IoSetDeviceInterfaceState( &cm_ifc_name, FALSE );\r
610         ASSERT( NT_SUCCESS( status ) );\r
611 \r
612         /* Release the memory allocated for the interface symbolic names. */\r
613         RtlFreeUnicodeString( &cm_ifc_name );\r
614         RtlFreeUnicodeString( &ci_ifc_name );\r
615         RtlFreeUnicodeString( &al_ifc_name );\r
616 \r
617         al_cleanup();\r
618         cl_thread_suspend(50);  /* allow time for AL's async procs to run to exit */\r
619 \r
620         CL_ASSERT( !gp_async_proc_mgr && !gp_async_pnp_mgr && !gp_al_mgr );\r
621 \r
622         /* AL needs the HCA to stick around until AL cleanup has completed.\r
623          * Now that it's done, let the HCA fade away.\r
624          */\r
625         if ( p_ext->hca_ifc_taken ) {\r
626                 p_ext->hca_ifc.InterfaceHeader.InterfaceDereference(\r
627                         p_ext->hca_ifc.InterfaceHeader.Context);\r
628                 p_ext->hca_ifc_taken = FALSE;\r
629         }\r
630 \r
631         BUS_EXIT( BUS_DBG_PNP );\r
632 }\r
633 \r
634 \r
635 static NTSTATUS\r
636 fdo_query_capabilities(\r
637         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
638         IN                                      IRP* const                              p_irp, \r
639                 OUT                             cl_irp_action_t* const  p_action )\r
640 {\r
641         NTSTATUS                        status;\r
642         bus_fdo_ext_t           *p_ext;\r
643         IO_STACK_LOCATION       *p_io_stack;\r
644 \r
645         BUS_ENTER( BUS_DBG_PNP );\r
646 \r
647         p_ext = p_dev_obj->DeviceExtension;\r
648 \r
649         /* Process on the way up. */\r
650         status = cl_do_sync_pnp( p_dev_obj, p_irp, p_action );\r
651 \r
652         if( !NT_SUCCESS( status ) )\r
653         {\r
654                 BUS_TRACE_EXIT( BUS_DBG_ERROR, \r
655                         ("cl_do_sync_pnp returned %08x.\n", status) );\r
656                 return status;\r
657         }\r
658 \r
659         p_io_stack = IoGetCurrentIrpStackLocation( p_irp );\r
660 \r
661         /*\r
662          * Store the device power maping into our extension since we're\r
663          * the power policy owner.  The mapping is used when handling\r
664          * IRP_MN_SET_POWER IRPs.\r
665          */\r
666         cl_memcpy( p_ext->po_state, \r
667                 p_io_stack->Parameters.DeviceCapabilities.Capabilities->DeviceState,\r
668                 sizeof( p_ext->po_state ) );\r
669 \r
670         BUS_EXIT( BUS_DBG_PNP );\r
671         return status;\r
672 }\r
673 \r
674 \r
675 static NTSTATUS\r
676 fdo_query_bus_relations(\r
677         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
678         IN                                      IRP* const                              p_irp, \r
679                 OUT                             cl_irp_action_t* const  p_action )\r
680 {\r
681         NTSTATUS                        status = STATUS_SUCCESS; /*default to success*/\r
682         bus_fdo_ext_t           *p_ext;\r
683         bus_filter_t            *p_bfi;\r
684         int                                     waitLoop = 0;\r
685 \r
686         BUS_ENTER( BUS_DBG_PNP );\r
687 \r
688         p_ext = p_dev_obj->DeviceExtension;\r
689 \r
690         if ( !p_ext->bus_filter )\r
691         {\r
692                 /* BFI has already been released */\r
693                 *p_action = IrpComplete;\r
694                 BUS_TRACE_EXIT( BUS_DBG_PNP, ("NULL BFI\n") );\r
695                 return STATUS_SUCCESS;\r
696         }\r
697 \r
698         p_bfi = p_ext->bus_filter;\r
699         CL_ASSERT( p_bfi->magic == BFI_MAGIC );\r
700 \r
701         while ( p_bfi->ca_guid == 0ULL )\r
702         {\r
703                 /* HCA not yet bound to a BFI slot (no PNP ADD event seen), no bus\r
704                  * relations yet.\r
705                  */\r
706                 BUS_PRINT(BUS_DBG_ERROR, ("%s ca_guid %I64x\n",p_bfi->whoami,\r
707                                                                 p_bfi->ca_guid));\r
708                 cl_thread_suspend( 100 );       /* suspend for 100 ms */\r
709                 waitLoop++;\r
710                 if(waitLoop>50) break;\r
711         }\r
712         if ( p_bfi->ca_guid != 0ULL )\r
713         {\r
714                 status = port_mgr_get_bus_relations( p_bfi->ca_guid, p_irp );\r
715                 if( status == STATUS_SUCCESS || \r
716                         status == STATUS_NO_SUCH_DEVICE )\r
717                 {\r
718                         status = iou_mgr_get_bus_relations( p_bfi->ca_guid, p_irp );\r
719                 }\r
720                 if( status == STATUS_NO_SUCH_DEVICE )\r
721                         status = STATUS_SUCCESS;\r
722         }\r
723 \r
724         switch( status )\r
725         {\r
726         case STATUS_NO_SUCH_DEVICE:\r
727                 *p_action = IrpSkip;\r
728                 status = STATUS_SUCCESS;\r
729                 break;\r
730 \r
731         case STATUS_SUCCESS:\r
732                 *p_action = IrpPassDown;\r
733                 break;\r
734 \r
735         default:\r
736                 *p_action = IrpComplete;\r
737                 break;\r
738         }\r
739 \r
740         BUS_EXIT( BUS_DBG_PNP );\r
741         return status;\r
742 }\r
743 \r
744 \r
745 void\r
746 al_ref_ifc(\r
747         IN                              DEVICE_OBJECT*                          p_dev_obj )\r
748 {\r
749         bus_fdo_ext_t   *p_ext;\r
750 \r
751         BUS_ENTER( BUS_DBG_PNP );\r
752 \r
753         p_ext = p_dev_obj->DeviceExtension;\r
754 \r
755         CL_ASSERT( p_ext->n_al_ifc_ref >= 0 );\r
756 \r
757         cl_atomic_inc( &p_ext->n_al_ifc_ref );\r
758         ObReferenceObject( p_dev_obj );\r
759 \r
760         BUS_EXIT( BUS_DBG_PNP );\r
761 }\r
762 \r
763 \r
764 void\r
765 al_deref_ifc(\r
766         IN                              DEVICE_OBJECT*                          p_dev_obj )\r
767 {\r
768         bus_fdo_ext_t   *p_ext;\r
769 \r
770         BUS_ENTER( BUS_DBG_PNP );\r
771 \r
772         p_ext = p_dev_obj->DeviceExtension;\r
773 \r
774         CL_ASSERT( p_ext->n_al_ifc_ref > 0 );\r
775         cl_atomic_dec( &p_ext->n_al_ifc_ref );\r
776         ObDereferenceObject( p_dev_obj );\r
777 \r
778         BUS_EXIT( BUS_DBG_PNP );\r
779 }\r
780 \r
781 \r
782 void\r
783 al_ref_ci_ifc(\r
784         IN                              DEVICE_OBJECT*                          p_dev_obj )\r
785 {\r
786         bus_fdo_ext_t   *p_ext;\r
787 \r
788         BUS_ENTER( BUS_DBG_PNP );\r
789 \r
790         p_ext = p_dev_obj->DeviceExtension;\r
791 \r
792         CL_ASSERT( p_ext->n_ci_ifc_ref >= 0 );\r
793         cl_atomic_inc( &p_ext->n_ci_ifc_ref );\r
794         ObReferenceObject( p_dev_obj );\r
795 \r
796         BUS_EXIT( BUS_DBG_PNP );\r
797 }\r
798 \r
799 \r
800 void\r
801 al_deref_ci_ifc(\r
802         IN                              DEVICE_OBJECT*                          p_dev_obj )\r
803 {\r
804         bus_fdo_ext_t   *p_ext;\r
805 \r
806         BUS_ENTER( BUS_DBG_PNP );\r
807 \r
808         p_ext = p_dev_obj->DeviceExtension;\r
809 \r
810         CL_ASSERT( p_ext->n_ci_ifc_ref > 0 );\r
811 \r
812         cl_atomic_dec( &p_ext->n_ci_ifc_ref );\r
813         ObDereferenceObject( p_dev_obj );\r
814 \r
815 \r
816         BUS_EXIT( BUS_DBG_PNP );\r
817 }\r
818 \r
819 \r
820 static NTSTATUS\r
821 __query_al_ifc(\r
822         IN                                      DEVICE_OBJECT* const            p_dev_obj,\r
823         IN                                      IO_STACK_LOCATION* const        p_io_stack )\r
824 {\r
825         ib_al_ifc_t             *p_ifc;\r
826 \r
827         BUS_ENTER( BUS_DBG_PNP );\r
828 \r
829         if( p_io_stack->Parameters.QueryInterface.Version != \r
830                 AL_INTERFACE_VERSION )\r
831         {\r
832                 BUS_TRACE_EXIT( BUS_DBG_PNP, ("Incorrect interface version (%d)\n",\r
833                         p_io_stack->Parameters.QueryInterface.Version ) );\r
834                 return STATUS_NOT_SUPPORTED;\r
835         }\r
836 \r
837         if( p_io_stack->Parameters.QueryInterface.Size < sizeof(ib_al_ifc_t) )\r
838         {\r
839                 BUS_TRACE_EXIT( BUS_DBG_PNP, \r
840                         ("Buffer too small (%d given, %d required).\n",\r
841                         p_io_stack->Parameters.QueryInterface.Size, sizeof(ib_al_ifc_t)) );\r
842                 return STATUS_BUFFER_TOO_SMALL;\r
843         }\r
844 \r
845         // Copy the interface.\r
846         p_ifc = (ib_al_ifc_t*)p_io_stack->Parameters.QueryInterface.Interface;\r
847 \r
848         p_ifc->wdm.Size = sizeof(ib_al_ifc_t);\r
849         p_ifc->wdm.Version = AL_INTERFACE_VERSION;\r
850         p_ifc->wdm.Context = p_dev_obj;\r
851         p_ifc->wdm.InterfaceReference = al_ref_ifc;\r
852         p_ifc->wdm.InterfaceDereference = al_deref_ifc;\r
853 \r
854         __set_ifc( p_ifc );\r
855 \r
856         // take the reference before returning.\r
857         al_ref_ifc( p_dev_obj );\r
858         BUS_EXIT( BUS_DBG_PNP );\r
859         return STATUS_SUCCESS;\r
860 }\r
861 \r
862 static void\r
863 __set_ifc(\r
864                 OUT                     ib_al_ifc_t* const                      p_ifc )\r
865 {\r
866         BUS_ENTER( BUS_DBG_PNP );\r
867 \r
868         p_ifc->wdm.Size = sizeof(ib_al_ifc_t);\r
869         p_ifc->wdm.InterfaceReference = al_ref_ifc;\r
870         p_ifc->wdm.InterfaceDereference = al_deref_ifc;\r
871 \r
872         p_ifc->sync_destroy = ib_sync_destroy;\r
873         p_ifc->open_ca = ib_open_ca;\r
874         p_ifc->query_ca = ib_query_ca;\r
875         p_ifc->get_dev = get_ca_dev;\r
876         p_ifc->close_ca = ib_close_ca;\r
877         p_ifc->alloc_pd = ib_alloc_pd;\r
878         p_ifc->dealloc_pd = ib_dealloc_pd;\r
879         p_ifc->create_av = ib_create_av;\r
880         p_ifc->query_av = ib_query_av;\r
881         p_ifc->modify_av = ib_modify_av;\r
882         p_ifc->destroy_av = ib_destroy_av;\r
883         p_ifc->create_qp = ib_create_qp;\r
884         p_ifc->get_spl_qp = ib_get_spl_qp;\r
885         p_ifc->query_qp = ib_query_qp;\r
886         p_ifc->modify_qp = ib_modify_qp;\r
887         p_ifc->destroy_qp = ib_destroy_qp;\r
888         p_ifc->create_cq = ib_create_cq;\r
889         p_ifc->modify_cq = ib_modify_cq;\r
890         p_ifc->query_cq = ib_query_cq;\r
891         p_ifc->destroy_cq = ib_destroy_cq;\r
892         p_ifc->reg_mem = ib_reg_mem;\r
893         p_ifc->reg_phys = ib_reg_phys;\r
894         p_ifc->query_mr = ib_query_mr;\r
895         p_ifc->rereg_mem = ib_rereg_mem;\r
896         p_ifc->reg_shmid = ib_reg_shmid;\r
897         p_ifc->dereg_mr = ib_dereg_mr;\r
898         p_ifc->create_mw = ib_create_mw;\r
899         p_ifc->query_mw = ib_query_mw;\r
900         p_ifc->bind_mw = ib_bind_mw;\r
901         p_ifc->destroy_mw = ib_destroy_mw;\r
902         p_ifc->post_send = ib_post_send;\r
903         p_ifc->post_recv = ib_post_recv;\r
904         p_ifc->send_mad = ib_send_mad;\r
905         p_ifc->cancel_mad = ib_cancel_mad;\r
906         p_ifc->poll_cq = ib_poll_cq;\r
907         p_ifc->rearm_cq = ib_rearm_cq;\r
908         p_ifc->join_mcast = ib_join_mcast;\r
909         p_ifc->leave_mcast = ib_leave_mcast;\r
910         p_ifc->local_mad = ib_local_mad;\r
911         p_ifc->cm_listen = ib_cm_listen;\r
912         p_ifc->cm_cancel = ib_cm_cancel;\r
913         p_ifc->cm_req = ib_cm_req;\r
914         p_ifc->cm_rep = ib_cm_rep;\r
915         p_ifc->cm_rtu = ib_cm_rtu;\r
916         p_ifc->cm_rej = ib_cm_rej;\r
917         p_ifc->cm_mra = ib_cm_mra;\r
918         p_ifc->cm_lap = ib_cm_lap;\r
919         p_ifc->cm_apr = ib_cm_apr;\r
920         p_ifc->force_apm = ib_force_apm;\r
921         p_ifc->cm_dreq = ib_cm_dreq;\r
922         p_ifc->cm_drep = ib_cm_drep;\r
923         p_ifc->cm_handoff = ib_cm_handoff;\r
924         p_ifc->create_ioc = ib_create_ioc;\r
925         p_ifc->destroy_ioc = ib_destroy_ioc;\r
926         p_ifc->reg_ioc = ib_reg_ioc;\r
927         p_ifc->add_svc_entry = ib_add_svc_entry;\r
928         p_ifc->remove_svc_entry = ib_remove_svc_entry;\r
929         p_ifc->get_ca_guids = ib_get_ca_guids;\r
930         p_ifc->get_ca_by_gid = ib_get_ca_by_gid;\r
931         p_ifc->get_port_by_gid = ib_get_port_by_gid;\r
932         p_ifc->create_mad_pool = ib_create_mad_pool;\r
933         p_ifc->destroy_mad_pool = ib_destroy_mad_pool;\r
934         p_ifc->reg_mad_pool = ib_reg_mad_pool;\r
935         p_ifc->dereg_mad_pool = ib_dereg_mad_pool;\r
936         p_ifc->get_mad = ib_get_mad;\r
937         p_ifc->put_mad = ib_put_mad;\r
938         p_ifc->init_dgrm_svc = ib_init_dgrm_svc;\r
939         p_ifc->reg_mad_svc = ib_reg_mad_svc;\r
940         p_ifc->reg_svc = ib_reg_svc;\r
941         p_ifc->dereg_svc = ib_dereg_svc;\r
942         p_ifc->query = ib_query;\r
943         p_ifc->cancel_query = ib_cancel_query;\r
944         p_ifc->reg_pnp = ib_reg_pnp;\r
945         p_ifc->dereg_pnp = ib_dereg_pnp;\r
946         p_ifc->subscribe = ib_subscribe;\r
947         p_ifc->unsubscribe = ib_unsubscribe;\r
948         p_ifc->reject_ioc = ib_reject_ioc;\r
949         p_ifc->ci_call = ib_ci_call;\r
950         p_ifc->open_al = ib_open_al;\r
951         p_ifc->close_al = ib_close_al;\r
952         p_ifc->get_err_str = ib_get_err_str;\r
953         p_ifc->get_wc_status_str = ib_get_wc_status_str;\r
954         p_ifc->create_mlnx_fmr = mlnx_create_fmr;\r
955         p_ifc->map_phys_mlnx_fmr = mlnx_map_phys_fmr;\r
956         p_ifc->unmap_mlnx_fmr = mlnx_unmap_fmr;\r
957         p_ifc->destroy_mlnx_fmr = mlnx_destroy_fmr;\r
958         p_ifc->create_mlnx_fmr_pool = mlnx_create_fmr_pool;\r
959         p_ifc->destroy_mlnx_fmr_pool = mlnx_destroy_fmr_pool;\r
960         p_ifc->map_phys_mlnx_fmr_pool = mlnx_map_phys_fmr_pool;\r
961         p_ifc->unmap_mlnx_fmr_pool = mlnx_unmap_fmr_pool;\r
962         p_ifc->flush_mlnx_fmr_pool = mlnx_flush_fmr_pool;\r
963         p_ifc->create_srq = ib_create_srq;\r
964         p_ifc->modify_srq = ib_modify_srq;\r
965         p_ifc->query_srq = ib_query_srq;\r
966         p_ifc->destroy_srq = ib_destroy_srq;\r
967         p_ifc->post_srq_recv = ib_post_srq_recv;\r
968 \r
969         BUS_EXIT( BUS_DBG_PNP );\r
970 }\r
971 \r
972 \r
973 static NTSTATUS\r
974 __get_relations(\r
975         IN              const   net64_t                                         ca_guid,\r
976         IN                              IRP* const                                      p_irp )\r
977 {\r
978         UNUSED_PARAM( ca_guid );\r
979         UNUSED_PARAM( p_irp );\r
980 \r
981         BUS_ENTER( BUS_DBG_PNP );\r
982 \r
983         /*\r
984          * Now that ibbus is in the same device stack as the HCA driver, skip\r
985          * returning relations here as ibbus has already done the deed.\r
986          * This interface remains to minimize changes to HCA drivers for now.\r
987          */\r
988 \r
989         BUS_EXIT( BUS_DBG_PNP );\r
990         return STATUS_SUCCESS;\r
991 }\r
992 \r
993 \r
994 static NTSTATUS\r
995 __query_ci_ifc(\r
996         IN                                      DEVICE_OBJECT* const            p_dev_obj,\r
997         IN                                      IO_STACK_LOCATION* const        p_io_stack )\r
998 {\r
999         ib_ci_ifc_t             *p_ifc;\r
1000 \r
1001         BUS_ENTER( BUS_DBG_PNP );\r
1002 \r
1003         if( p_io_stack->Parameters.QueryInterface.Version != \r
1004                 IB_CI_INTERFACE_VERSION )\r
1005         {\r
1006                 BUS_TRACE_EXIT( BUS_DBG_PNP, ("Incorrect interface version (%d)\n",\r
1007                         p_io_stack->Parameters.QueryInterface.Version ) );\r
1008                 return STATUS_NOT_SUPPORTED;\r
1009         }\r
1010 \r
1011         if( p_io_stack->Parameters.QueryInterface.Size < sizeof(ib_ci_ifc_t) )\r
1012         {\r
1013                 BUS_TRACE_EXIT( BUS_DBG_PNP, \r
1014                         ("Buffer too small (%d given, %d required).\n",\r
1015                         p_io_stack->Parameters.QueryInterface.Size, sizeof(ib_ci_ifc_t)) );\r
1016                 return STATUS_BUFFER_TOO_SMALL;\r
1017         }\r
1018 \r
1019         /* Copy the interface. */\r
1020         p_ifc = (ib_ci_ifc_t*)p_io_stack->Parameters.QueryInterface.Interface;\r
1021 \r
1022         p_ifc->wdm.Size = sizeof(ib_ci_ifc_t);\r
1023         p_ifc->wdm.Version = IB_CI_INTERFACE_VERSION;\r
1024         p_ifc->wdm.Context = p_dev_obj;\r
1025         p_ifc->wdm.InterfaceReference = al_ref_ci_ifc;\r
1026         p_ifc->wdm.InterfaceDereference = al_deref_ci_ifc;\r
1027 \r
1028         /* Set the entry points. */\r
1029         p_ifc->register_ca = ib_register_ca;\r
1030         p_ifc->deregister_ca = ib_deregister_ca;\r
1031         p_ifc->get_relations = __get_relations;\r
1032         p_ifc->get_err_str = ib_get_err_str;\r
1033 \r
1034         /* take the reference before returning. */\r
1035         al_ref_ci_ifc( p_dev_obj );\r
1036         BUS_EXIT( BUS_DBG_PNP );\r
1037         return STATUS_SUCCESS;\r
1038 }\r
1039 \r
1040 \r
1041 static NTSTATUS\r
1042 __query_cm_ifc(\r
1043         IN                                      DEVICE_OBJECT* const            p_dev_obj,\r
1044         IN                                      IO_STACK_LOCATION* const        p_io_stack )\r
1045 {\r
1046         INFINIBAND_INTERFACE_CM *p_ifc;\r
1047 \r
1048         BUS_ENTER( BUS_DBG_PNP );\r
1049 \r
1050         if( p_io_stack->Parameters.QueryInterface.Version != IbaCmVersion(1, 0) )\r
1051         {\r
1052                 BUS_TRACE_EXIT( BUS_DBG_PNP, ("Incorrect interface version (%d)\n",\r
1053                         p_io_stack->Parameters.QueryInterface.Version ) );\r
1054                 return STATUS_NOT_SUPPORTED;\r
1055         }\r
1056 \r
1057         if( p_io_stack->Parameters.QueryInterface.Size < sizeof(INFINIBAND_INTERFACE_CM) )\r
1058         {\r
1059                 BUS_TRACE_EXIT( BUS_DBG_PNP, \r
1060                         ("Buffer too small (%d given, %d required).\n",\r
1061                         p_io_stack->Parameters.QueryInterface.Size, sizeof(INFINIBAND_INTERFACE_CM)) );\r
1062                 return STATUS_BUFFER_TOO_SMALL;\r
1063         }\r
1064 \r
1065         /* Copy the interface. */\r
1066         p_ifc = (INFINIBAND_INTERFACE_CM*)p_io_stack->Parameters.QueryInterface.Interface;\r
1067 \r
1068         p_ifc->InterfaceHeader.Size = sizeof(INFINIBAND_INTERFACE_CM);\r
1069         p_ifc->InterfaceHeader.Version = IbaCmVersion(1, 0);\r
1070         p_ifc->InterfaceHeader.Context = p_dev_obj;\r
1071         p_ifc->InterfaceHeader.InterfaceReference = al_ref_ifc;\r
1072         p_ifc->InterfaceHeader.InterfaceDereference = al_deref_ifc;\r
1073         cm_get_interface(&p_ifc->CM);\r
1074 \r
1075         /* take the reference before returning. */\r
1076         al_ref_ifc( p_dev_obj );\r
1077         BUS_EXIT( BUS_DBG_PNP );\r
1078         return STATUS_SUCCESS;\r
1079 }\r
1080 \r
1081 \r
1082 static NTSTATUS\r
1083 fdo_query_interface(\r
1084         IN                                      DEVICE_OBJECT* const    p_dev_obj,\r
1085         IN                                      IRP* const                              p_irp, \r
1086                 OUT                             cl_irp_action_t* const  p_action )\r
1087 {\r
1088         NTSTATUS                        status;\r
1089         IO_STACK_LOCATION       *p_io_stack;\r
1090 \r
1091         BUS_ENTER( BUS_DBG_PNP );\r
1092 \r
1093         PAGED_CODE();\r
1094 \r
1095         p_io_stack = IoGetCurrentIrpStackLocation( p_irp );\r
1096         \r
1097         /* Compare requested GUID with our supported interface GUIDs. */\r
1098         if( IsEqualGUID( p_io_stack->Parameters.QueryInterface.InterfaceType,\r
1099                 &GUID_IB_AL_INTERFACE ) )\r
1100         {\r
1101                 status = __query_al_ifc( p_dev_obj, p_io_stack );\r
1102         }\r
1103         else if( IsEqualGUID( p_io_stack->Parameters.QueryInterface.InterfaceType,\r
1104                 &GUID_IB_CI_INTERFACE ) )\r
1105         {\r
1106                 status = __query_ci_ifc( p_dev_obj, p_io_stack );\r
1107         }\r
1108         else if( IsEqualGUID( p_io_stack->Parameters.QueryInterface.InterfaceType,\r
1109                 &GUID_INFINIBAND_INTERFACE_CM ) )\r
1110         {\r
1111                 status = __query_cm_ifc( p_dev_obj, p_io_stack );\r
1112         }\r
1113         else\r
1114         {\r
1115                 status = p_irp->IoStatus.Status;\r
1116         }\r
1117 \r
1118         if( NT_SUCCESS( status ) )\r
1119                 *p_action = IrpSkip;\r
1120         else if( status == STATUS_BUFFER_TOO_SMALL )\r
1121                 *p_action = IrpComplete;\r
1122         else\r
1123                 *p_action = IrpIgnore;\r
1124 \r
1125         BUS_EXIT( BUS_DBG_PNP );\r
1126         return status;\r
1127 }\r
1128 \r
1129 \r
1130 static NTSTATUS\r
1131 __fdo_query_power(\r
1132         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
1133         IN                              IRP* const                                      p_irp,\r
1134                 OUT                     cl_irp_action_t* const          p_action )\r
1135 {\r
1136         NTSTATUS                        status = STATUS_SUCCESS;\r
1137         IO_STACK_LOCATION       *p_io_stack;\r
1138 \r
1139         BUS_ENTER( BUS_DBG_POWER );\r
1140 \r
1141         UNUSED_PARAM( p_dev_obj );\r
1142 \r
1143         p_io_stack = IoGetCurrentIrpStackLocation( p_irp );\r
1144 \r
1145         switch( p_io_stack->Parameters.Power.Type )\r
1146         {\r
1147         case SystemPowerState:\r
1148                 /* Fail any requests to hibernate or sleep the system. */\r
1149                 switch( p_io_stack->Parameters.Power.State.SystemState )\r
1150                 {\r
1151                         case PowerSystemHibernate:\r
1152                         case PowerSystemSleeping1:      // STANDBY support\r
1153                         case PowerSystemWorking:\r
1154                         case PowerSystemShutdown:\r
1155                                 break;\r
1156 \r
1157                         default:\r
1158                                 status = STATUS_NOT_SUPPORTED;\r
1159                 }\r
1160                 break;\r
1161 \r
1162         case DevicePowerState:\r
1163                 /* Fail any query for low power states. */\r
1164                 switch( p_io_stack->Parameters.Power.State.DeviceState )\r
1165                 {\r
1166                 case PowerDeviceD0:\r
1167                 case PowerDeviceD3:\r
1168                         /* We only support fully powered or off power states. */\r
1169                         break;\r
1170 \r
1171                 default:\r
1172                         status = STATUS_NOT_SUPPORTED;\r
1173                 }\r
1174                 break;\r
1175         }\r
1176 \r
1177         if( status == STATUS_NOT_SUPPORTED )\r
1178                 *p_action = IrpComplete;\r
1179         else\r
1180                 *p_action = IrpSkip;\r
1181 \r
1182         BUS_EXIT( BUS_DBG_POWER );\r
1183         return status;\r
1184 }\r
1185 \r
1186 \r
1187 static void\r
1188 __request_power_completion(\r
1189         IN                              DEVICE_OBJECT                           *p_dev_obj,\r
1190         IN                              UCHAR                                           minor_function,\r
1191         IN                              POWER_STATE                                     power_state,\r
1192         IN                              void                                            *context,\r
1193         IN                              IO_STATUS_BLOCK                         *p_io_status )\r
1194 {\r
1195         IRP                                     *p_irp;\r
1196         cl_pnp_po_ext_t         *p_ext;\r
1197 \r
1198         BUS_ENTER( BUS_DBG_PNP );\r
1199 \r
1200         UNUSED_PARAM( minor_function );\r
1201         UNUSED_PARAM( power_state );\r
1202 \r
1203         p_irp = (IRP*)context;\r
1204         p_ext = p_dev_obj->DeviceExtension;\r
1205 \r
1206         /* Propagate the device IRP status to the system IRP status. */\r
1207         p_irp->IoStatus.Status = p_io_status->Status;\r
1208 \r
1209         /* Continue Power IRP processing. */\r
1210         PoStartNextPowerIrp( p_irp );\r
1211         IoCompleteRequest( p_irp, IO_NO_INCREMENT );\r
1212         IoReleaseRemoveLock( &p_ext->remove_lock, p_irp );\r
1213         BUS_EXIT( BUS_DBG_PNP );\r
1214 }\r
1215 \r
1216 \r
1217 /*NOTE: Completion routines must NEVER be pageable. */\r
1218 static NTSTATUS\r
1219 __set_power_completion(\r
1220         IN                              DEVICE_OBJECT                           *p_dev_obj,\r
1221         IN                              IRP                                                     *p_irp,\r
1222         IN                              void                                            *context )\r
1223 {\r
1224         NTSTATUS                        status;\r
1225         POWER_STATE                     state;\r
1226         bus_fdo_ext_t           *p_ext;\r
1227         IO_STACK_LOCATION       *p_io_stack;\r
1228 \r
1229         BUS_ENTER( BUS_DBG_PNP );\r
1230 \r
1231         UNUSED_PARAM( context );\r
1232 \r
1233         p_ext = p_dev_obj->DeviceExtension;\r
1234         p_io_stack = IoGetCurrentIrpStackLocation( p_irp );\r
1235 \r
1236         if( !NT_SUCCESS( p_irp->IoStatus.Status ) )\r
1237         {\r
1238                 PoStartNextPowerIrp( p_irp );\r
1239                 IoReleaseRemoveLock( &p_ext->cl_ext.remove_lock, p_irp );\r
1240                 BUS_TRACE_EXIT( BUS_DBG_ERROR, \r
1241                         ("IRP_MN_SET_POWER for system failed by lower driver with %08x.\n",\r
1242                         p_irp->IoStatus.Status) );\r
1243                 return STATUS_SUCCESS;\r
1244         }\r
1245 \r
1246         state.DeviceState = \r
1247                 p_ext->po_state[p_io_stack->Parameters.Power.State.SystemState];\r
1248 \r
1249         /*\r
1250          * Send a device power IRP to our devnode.  Using our device object will\r
1251          * only work on win2k and other NT based systems.\r
1252          */\r
1253         status = PoRequestPowerIrp( p_dev_obj, IRP_MN_SET_POWER, state,\r
1254                 __request_power_completion, p_irp, NULL );\r
1255 \r
1256         if( status != STATUS_PENDING )\r
1257         {\r
1258                 PoStartNextPowerIrp( p_irp );\r
1259                 /* Propagate the failure. */\r
1260                 p_irp->IoStatus.Status = status;\r
1261                 IoCompleteRequest( p_irp, IO_NO_INCREMENT );\r
1262                 IoReleaseRemoveLock( &p_ext->cl_ext.remove_lock, p_irp );\r
1263                 BUS_TRACE( BUS_DBG_ERROR,\r
1264                         ("PoRequestPowerIrp returned %08x.\n", status) );\r
1265         }\r
1266 \r
1267         BUS_EXIT( BUS_DBG_PNP );\r
1268         return STATUS_MORE_PROCESSING_REQUIRED;\r
1269 }\r
1270 \r
1271 \r
1272 static NTSTATUS\r
1273 __fdo_set_power(\r
1274         IN                              DEVICE_OBJECT* const            p_dev_obj,\r
1275         IN                              IRP* const                                      p_irp,\r
1276                 OUT                     cl_irp_action_t* const          p_action )\r
1277 {\r
1278         NTSTATUS                        status;\r
1279         IO_STACK_LOCATION       *p_io_stack;\r
1280         bus_fdo_ext_t           *p_ext;\r
1281 \r
1282         BUS_ENTER( BUS_DBG_POWER );\r
1283 \r
1284         p_ext = p_dev_obj->DeviceExtension;\r
1285         p_io_stack = IoGetCurrentIrpStackLocation( p_irp );\r
1286 \r
1287         BUS_TRACE( BUS_DBG_POWER, \r
1288                 ("SET_POWER for FDO %p (ext %p): type %s, state %d, action %d \n",\r
1289                 p_dev_obj, p_ext,\r
1290                 (p_io_stack->Parameters.Power.Type)\r
1291                         ? "DevicePowerState" : "SystemPowerState",\r
1292                 p_io_stack->Parameters.Power.State.DeviceState, \r
1293                 p_io_stack->Parameters.Power.ShutdownType ));\r
1294 \r
1295         switch( p_io_stack->Parameters.Power.Type )\r
1296         {\r
1297         case SystemPowerState:\r
1298 #if 0\r
1299                 /*\r
1300                  * Process on the way up the stack.  We cannot block since the \r
1301                  * power dispatch function can be called at elevated IRQL if the\r
1302                  * device is in a paging/hibernation/crash dump path.\r
1303                  */\r
1304                 IoMarkIrpPending( p_irp );\r
1305                 IoCopyCurrentIrpStackLocationToNext( p_irp );\r
1306 #pragma warning( push, 3 )\r
1307                 IoSetCompletionRoutine( p_irp, __set_power_completion, NULL, \r
1308                         TRUE, TRUE, TRUE );\r
1309 #pragma warning( pop )\r
1310                 PoCallDriver( p_ext->cl_ext.p_next_do, p_irp );\r
1311 \r
1312                 *p_action = IrpDoNothing;\r
1313                 status = STATUS_PENDING;\r
1314                 break;\r
1315 #endif\r
1316         case DevicePowerState:\r
1317         default:\r
1318                 /* Pass down and let the PDO driver handle it. */\r
1319                 *p_action = IrpIgnore;\r
1320                 status = STATUS_SUCCESS;\r
1321                 break;\r
1322         }\r
1323 \r
1324         BUS_EXIT( BUS_DBG_POWER );\r
1325         return status;\r
1326 }\r
1327 \r
1328 \r
1329 /*\r
1330  * A CA GUID of zero means that all devices should be reported.\r
1331  */\r
1332 NTSTATUS\r
1333 bus_get_relations(\r
1334         IN                              cl_qlist_t*     const                   p_pdo_list,\r
1335         IN              const   net64_t                                         ca_guid,\r
1336         IN                              IRP* const                                      p_irp )\r
1337 {\r
1338         NTSTATUS                        status;\r
1339         DEVICE_RELATIONS        *p_rel;\r
1340         cl_list_item_t          *p_list_item;\r
1341         bus_pdo_ext_t           *p_pdo_ext;\r
1342         size_t                          n_devs = 0;\r
1343 \r
1344         BUS_ENTER( BUS_DBG_PNP );\r
1345 \r
1346         /* Count the number of child devices. */\r
1347         for( p_list_item = cl_qlist_head( p_pdo_list );\r
1348                 p_list_item != cl_qlist_end( p_pdo_list );\r
1349                 p_list_item = cl_qlist_next( p_list_item ) )\r
1350         {\r
1351                 p_pdo_ext = PARENT_STRUCT( p_list_item, bus_pdo_ext_t, list_item );\r
1352 \r
1353                 if( !p_pdo_ext->b_present )\r
1354                 {\r
1355                         // mark it missing to be removed in port_remove\r
1356                         p_pdo_ext->b_reported_missing = TRUE;\r
1357                         /*\r
1358                          * We don't report a PDO that is no longer present.  This is how\r
1359                          * the PDO will get cleaned up.\r
1360                          */\r
1361                         BUS_TRACE( BUS_DBG_PNP, ("Don't report PDO! %s: PDO %p, ext %p, "\r
1362                                 "present %d, missing %d .\n",\r
1363                                 p_pdo_ext->cl_ext.vfptr_pnp_po->identity,\r
1364                                 p_pdo_ext->cl_ext.p_self_do, p_pdo_ext, p_pdo_ext->b_present,\r
1365                                 p_pdo_ext->b_reported_missing ) );\r
1366                         continue;\r
1367                 }\r
1368                 \r
1369                 if( ca_guid && p_pdo_ext->ca_guid != ca_guid )\r
1370                         continue;\r
1371 \r
1372                 n_devs++;\r
1373         }\r
1374 \r
1375         if( !n_devs )\r
1376         {\r
1377                 BUS_TRACE_EXIT( BUS_DBG_PNP, ("Found 0 PDOs ca_guid %I64x\n", ca_guid));\r
1378                 return STATUS_NO_SUCH_DEVICE;\r
1379         }\r
1380 \r
1381         BUS_TRACE( BUS_DBG_PNP, ("Found %d PDOs ca_guid %I64x\n", n_devs, ca_guid));\r
1382 \r
1383         /* Add space for our child IOUs. */\r
1384         status = cl_alloc_relations( p_irp, n_devs );\r
1385         if( !NT_SUCCESS( status ) )\r
1386         {\r
1387                 BUS_TRACE_EXIT( BUS_DBG_ERROR,\r
1388                         ("cl_alloc_relations returned %08x.\n", status) );\r
1389                 return status;\r
1390         }\r
1391 \r
1392         p_rel = (DEVICE_RELATIONS*)p_irp->IoStatus.Information;\r
1393 \r
1394         for( p_list_item = cl_qlist_head( p_pdo_list );\r
1395                 p_list_item != cl_qlist_end( p_pdo_list );\r
1396                 p_list_item = cl_qlist_next( p_list_item ) )\r
1397         {\r
1398                 p_pdo_ext = PARENT_STRUCT( p_list_item, bus_pdo_ext_t, list_item );\r
1399 \r
1400                 if( !p_pdo_ext->b_present )\r
1401                         continue;\r
1402 \r
1403                 if( ca_guid && p_pdo_ext->ca_guid != ca_guid )\r
1404                         continue;\r
1405 \r
1406                 BUS_TRACE( BUS_DBG_PNP, ("Reported PDO %p(=%p), ext %p\n", \r
1407                         p_pdo_ext->cl_ext.p_self_do, p_pdo_ext->cl_ext.p_pdo, p_pdo_ext ));\r
1408                 \r
1409                 p_rel->Objects[p_rel->Count] = p_pdo_ext->cl_ext.p_pdo;\r
1410                 ObReferenceObject( p_rel->Objects[p_rel->Count++] );\r
1411         }\r
1412 \r
1413         BUS_EXIT( BUS_DBG_PNP );\r
1414         return STATUS_SUCCESS;\r
1415 }\r
1416 \r
1417 \r
1418 /*\r
1419  * find a bus filter instance (p_bfi) given an *cl_obj: port_mgr or iou_mgr. \r
1420  */\r
1421 \r
1422 bus_filter_t *\r
1423 get_bfi_by_obj(IN int obj_type, IN cl_obj_t *p_obj )\r
1424 {\r
1425         bus_filter_t    *p_bfi;\r
1426         bus_filter_t    *matched=NULL;\r
1427 \r
1428         CL_ASSERT((obj_type == BFI_PORT_MGR_OBJ) || (obj_type == BFI_IOU_MGR_OBJ));\r
1429 \r
1430         lock_control_event();\r
1431 \r
1432         for(p_bfi=g_bus_filters; p_bfi < &g_bus_filters[MAX_BUS_FILTERS]; p_bfi++) {\r
1433 \r
1434                 if ( !p_bfi->p_bus_ext )\r
1435                         continue;\r
1436 \r
1437                 if ( obj_type == BFI_PORT_MGR_OBJ ) {\r
1438                         if ( p_obj == p_bfi->p_port_mgr_obj ) {\r
1439                                 matched = p_bfi;\r
1440                                 break;\r
1441                         }\r
1442                 }\r
1443                 else {\r
1444                         if ( p_obj == p_bfi->p_iou_mgr_obj ) {\r
1445                                 matched = p_bfi;\r
1446                                 break;\r
1447                         }\r
1448                 }\r
1449         }\r
1450         unlock_control_event();\r
1451 \r
1452         BUS_TRACE( BUS_DBG_PNP,\r
1453                                 ("cl_obj %p type %s_MGR_OBJ --> bfi[%d] %p\n", p_obj,\r
1454                                 (obj_type == BFI_PORT_MGR_OBJ ? "PORT": "IOU"),\r
1455                                 (matched ? (matched - g_bus_filters) : (-1)), matched ) );\r
1456 \r
1457         return matched;\r
1458 }\r
1459 \r
1460 /*\r
1461  * find a bus filter instance given an HCA guid.\r
1462  * BFIs are bound to GUIDs in fdo_start().\r
1463  */\r
1464 \r
1465 bus_filter_t *\r
1466 get_bfi_by_ca_guid( IN net64_t ca_guid )\r
1467 {\r
1468         bus_filter_t    *p_bfi;\r
1469         bus_filter_t    *matched=NULL;\r
1470 \r
1471         if ( ca_guid == 0ULL )\r
1472         {\r
1473                 matched = g_bus_filters;\r
1474                 BUS_TRACE( BUS_DBG_PNP, ("ERR guid %I64x -> bfi[0] %p\n",\r
1475                                                                 ca_guid, matched) );\r
1476                 CL_ASSERT( ca_guid );\r
1477                 return matched;\r
1478         }\r
1479 \r
1480         lock_control_event();\r
1481 \r
1482         for(p_bfi=g_bus_filters; p_bfi < &g_bus_filters[MAX_BUS_FILTERS]; p_bfi++)\r
1483         {\r
1484                 if ( !p_bfi->p_bus_ext )\r
1485                         continue;\r
1486 \r
1487                 if ( ca_guid == p_bfi->ca_guid )\r
1488                 {\r
1489                         matched = p_bfi;\r
1490                         break;\r
1491                 }\r
1492         }\r
1493         unlock_control_event();\r
1494 \r
1495 #if DBG\r
1496         if ( !matched )\r
1497         {\r
1498                 BUS_PRINT( BUS_DBG_PNP, ("No Match ca_guid 0x%I64x -> bfi[%d] %p\n",\r
1499                                                                         ca_guid, -1, matched ) );\r
1500         }\r
1501 #endif\r
1502         return matched;\r
1503 }\r
1504 \r
1505 \r
1506 bus_filter_t *\r
1507 alloc_bfi( IN DRIVER_OBJECT  *p_driver_obj, OUT int *p_instance_count )\r
1508 {\r
1509         bus_filter_t    *p_bfi;\r
1510         bus_filter_t    *matched=NULL;\r
1511 \r
1512     /* Using unsafe function so that the IRQL remains at PASSIVE_LEVEL.\r
1513      * IoCreateDeviceSecure & IoCreateSymbolicLink must be called at\r
1514      * PASSIVE_LEVEL.\r
1515          */\r
1516         lock_control_event();\r
1517 \r
1518         // find 1st unused bfi slot.\r
1519         for(p_bfi=g_bus_filters; p_bfi < &g_bus_filters[MAX_BUS_FILTERS]; p_bfi++)\r
1520         {\r
1521                 if ( !p_bfi->p_bus_ext )\r
1522                 {\r
1523                         /* temp setting until 'real' p_bus_ext is alloc; see bus_add_device.\r
1524                          * If p_bus_ext is ! 0, then bfi slot is allocated, although it\r
1525                          * may not yet have a bound CA guid; set set_get_\r
1526                          */\r
1527                         p_bfi->p_bus_ext = (bus_fdo_ext_t*)p_driver_obj;\r
1528                         matched = p_bfi;\r
1529                         *p_instance_count = ++g_bfi_InstanceCount; // record in-use\r
1530                         break;\r
1531                 }\r
1532         }\r
1533         unlock_control_event();\r
1534 \r
1535 #if DBG\r
1536         RtlStringCbPrintfA ( p_bfi->whoami,\r
1537                                                  sizeof(p_bfi->whoami),\r
1538                                                  "bfi-%d",\r
1539                                                  (g_bfi_InstanceCount - 1) );\r
1540 \r
1541         p_bfi->magic = BFI_MAGIC;\r
1542 #endif\r
1543 \r
1544         BUS_TRACE( BUS_DBG_PNP, ("%s %p\n",\r
1545                                 (matched ? matched->whoami:"Nobody"), matched) );\r
1546 \r
1547         return matched;\r
1548 }\r
1549 \r
1550 \r
1551 int\r
1552 free_bfi( IN  bus_filter_t  *p_bfi )\r
1553 {\r
1554         int     remaining;\r
1555 \r
1556         lock_control_event();\r
1557         p_bfi->p_bus_ext = NULL;\r
1558         p_bfi->ca_guid = 0ULL;\r
1559         remaining = --g_bfi_InstanceCount; // one less bfi in-use\r
1560         unlock_control_event();\r
1561         \r
1562         return remaining;\r
1563 }\r
1564 \r
1565 int\r
1566 get_bfi_count( void )\r
1567 {\r
1568         int     ic;\r
1569 \r
1570         lock_control_event();\r
1571         ic = g_bfi_InstanceCount;\r
1572         unlock_control_event();\r
1573         \r
1574         return ic;\r
1575 }\r
1576 \r
1577 #if DBG\r
1578 char *get_obj_state_str(cl_state_t state)\r
1579 {\r
1580         switch( state ) {\r
1581           case CL_UNINITIALIZED:\r
1582                 return "UNINITIALIZED";\r
1583           case CL_INITIALIZED:\r
1584                 return "INITIALIZED";\r
1585           case CL_DESTROYING:\r
1586                 return "DESTROYING";\r
1587           case CL_DESTROYED:\r
1588                 return "DESTROYED";\r
1589           default:\r
1590                 break;\r
1591         }\r
1592         return "Err - Bad obj state";\r
1593 }\r
1594 #endif\r
1595 \r