[xhci] Avoid DMA during shutdown if firmware has disabled bus mastering

On some systems (observed with the Thunderbolt ports on a ThinkPad X1
Extreme Gen3 and a ThinkPad P53), the system firmware will disable bus
mastering on the xHCI controller and all PCI bridges at the point that
ExitBootServices() is called if the IOMMU is enabled.  This leaves the
xHCI controller unable to shut down cleanly since all commands will
fail with a timeout.

Commit 85eb961 ("[xhci] Allow for permanent failure of the command
mechanism") allows us to detect that this has happened and respond
cleanly.  However, some unidentified hardware component (either the
xHCI controller or one of the PCI bridges) seems to manage to enqueue
the attempted DMA operation and eventually complete it after the
operating system kernel has reenabled bus mastering.  This results in
a DMA operation to an area of memory that the hardware is no longer
permitted to access.  On Windows with the Driver Verifier enabled,
this will result in a STOP 0xE6 (DRIVER_VERIFIER_DMA_VIOLATION).

Work around this problem by detecting when bus mastering has been
disabled, and immediately failing the device to avoid initiating any
further DMA attempts.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/core/uri.c

@@ -79,12 +79,10 @@ size_t uri_decode ( const char *encoded, void *buf, size_t len ) {
 /**
  * Decode URI field in-place
  *
- * @v uri		URI
- * @v field		URI field index
+ * @v encoded		Encoded field, or NULL
  */
-static void uri_decode_inplace ( struct uri *uri, unsigned int field ) {
-	const char *encoded = uri_field ( uri, field );
-	char *decoded = ( ( char * ) encoded );
+static void uri_decode_inplace ( char *encoded ) {
+	char *decoded = encoded;
 	size_t len;
 
 	/* Do nothing if field is not present */
@@ -150,7 +148,7 @@ static int uri_character_escaped ( char c, unsigned int field ) {
 	 * parser but for any other URI parsers (e.g. HTTP query
 	 * string parsers, which care about '=' and '&').
 	 */
-	static const char *escaped[URI_FIELDS] = {
+	static const char *escaped[URI_EPATH] = {
 		/* Scheme or default: escape everything */
 		[URI_SCHEME]	= "/#:@?=&",
 		/* Opaque part: escape characters which would affect
@@ -172,20 +170,21 @@ static int uri_character_escaped ( char c, unsigned int field ) {
 		 * appears within paths.
 		 */
 		[URI_PATH]	= "#:@?",
-		/* Query: escape everything except '/', which
-		 * sometimes appears within queries.
-		 */
-		[URI_QUERY]	= "#:@?",
-		/* Fragment: escape everything */
-		[URI_FRAGMENT]	= "/#:@?",
 	};
 
-	return ( /* Always escape non-printing characters and whitespace */
-		 ( ! isprint ( c ) ) || ( c == ' ' ) ||
-		 /* Always escape '%' */
-		 ( c == '%' ) ||
-		 /* Escape field-specific characters */
-		 strchr ( escaped[field], c ) );
+	/* Always escape non-printing characters and whitespace */
+	if ( ( ! isprint ( c ) ) || ( c == ' ' ) )
+		return 1;
+
+	/* Escape nothing else in already-escaped fields */
+	if ( field >= URI_EPATH )
+		return 0;
+
+	/* Escape '%' and any field-specific characters */
+	if ( ( c == '%' ) || strchr ( escaped[field], c ) )
+		return 1;
+
+	return 0;
 }
 
 /**
@@ -262,10 +261,12 @@ static void uri_dump ( const struct uri *uri ) {
 		DBGC ( uri, " port \"%s\"", uri->port );
 	if ( uri->path )
 		DBGC ( uri, " path \"%s\"", uri->path );
-	if ( uri->query )
-		DBGC ( uri, " query \"%s\"", uri->query );
-	if ( uri->fragment )
-		DBGC ( uri, " fragment \"%s\"", uri->fragment );
+	if ( uri->epath )
+		DBGC ( uri, " epath \"%s\"", uri->epath );
+	if ( uri->equery )
+		DBGC ( uri, " equery \"%s\"", uri->equery );
+	if ( uri->efragment )
+		DBGC ( uri, " efragment \"%s\"", uri->efragment );
 	if ( uri->params )
 		DBGC ( uri, " params \"%s\"", uri->params->name );
 }
@@ -298,17 +299,19 @@ struct uri * parse_uri ( const char *uri_string ) {
 	char *raw;
 	char *tmp;
 	char *path;
+	char *epath;
 	char *authority;
 	size_t raw_len;
 	unsigned int field;
 
-	/* Allocate space for URI struct and a copy of the string */
+	/* Allocate space for URI struct and two copies of the string */
 	raw_len = ( strlen ( uri_string ) + 1 /* NUL */ );
-	uri = zalloc ( sizeof ( *uri ) + raw_len );
+	uri = zalloc ( sizeof ( *uri ) + ( 2 * raw_len ) );
 	if ( ! uri )
 		return NULL;
 	ref_init ( &uri->refcnt, uri_free );
 	raw = ( ( ( void * ) uri ) + sizeof ( *uri ) );
+	path = ( raw + raw_len );
 
 	/* Copy in the raw string */
 	memcpy ( raw, uri_string, raw_len );
@@ -328,7 +331,7 @@ struct uri * parse_uri ( const char *uri_string ) {
 	/* Chop off the fragment, if it exists */
 	if ( ( tmp = strchr ( raw, '#' ) ) ) {
 		*(tmp++) = '\0';
-		uri->fragment = tmp;
+		uri->efragment = tmp;
 	}
 
 	/* Identify absolute/relative URI */
@@ -338,47 +341,47 @@ struct uri * parse_uri ( const char *uri_string ) {
 		*(tmp++) = '\0';
 		if ( *tmp == '/' ) {
 			/* Absolute URI with hierarchical part */
-			path = tmp;
+			epath = tmp;
 		} else {
 			/* Absolute URI with opaque part */
 			uri->opaque = tmp;
-			path = NULL;
+			epath = NULL;
 		}
 	} else {
 		/* Relative URI */
-		path = raw;
+		epath = raw;
 	}
 
 	/* If we don't have a path (i.e. we have an absolute URI with
 	 * an opaque portion, we're already finished processing
 	 */
-	if ( ! path )
+	if ( ! epath )
 		goto done;
 
 	/* Chop off the query, if it exists */
-	if ( ( tmp = strchr ( path, '?' ) ) ) {
+	if ( ( tmp = strchr ( epath, '?' ) ) ) {
 		*(tmp++) = '\0';
-		uri->query = tmp;
+		uri->equery = tmp;
 	}
 
 	/* If we have no path remaining, then we're already finished
 	 * processing.
 	 */
-	if ( ! path[0] )
+	if ( ! epath[0] )
 		goto done;
 
 	/* Identify net/absolute/relative path */
-	if ( uri->scheme && ( strncmp ( path, "//", 2 ) == 0 ) ) {
+	if ( uri->scheme && ( strncmp ( epath, "//", 2 ) == 0 ) ) {
 		/* Net path.  If this is terminated by the first '/'
 		 * of an absolute path, then we have no space for a
 		 * terminator after the authority field, so shuffle
 		 * the authority down by one byte, overwriting one of
 		 * the two slashes.
 		 */
-		authority = ( path + 2 );
+		authority = ( epath + 2 );
 		if ( ( tmp = strchr ( authority, '/' ) ) ) {
 			/* Shuffle down */
-			uri->path = tmp;
+			uri->epath = tmp;
 			memmove ( ( authority - 1 ), authority,
 				  ( tmp - authority ) );
 			authority--;
@@ -386,10 +389,16 @@ struct uri * parse_uri ( const char *uri_string ) {
 		}
 	} else {
 		/* Absolute/relative path */
-		uri->path = path;
+		uri->epath = epath;
 		authority = NULL;
 	}
 
+	/* Create copy of path for decoding */
+	if ( uri->epath ) {
+		strcpy ( path, uri->epath );
+		uri->path = path;
+	}
+
 	/* If we don't have an authority (i.e. we have a non-net
 	 * path), we're already finished processing
 	 */
@@ -421,8 +430,8 @@ struct uri * parse_uri ( const char *uri_string ) {
 
  done:
 	/* Decode fields in-place */
-	for ( field = 0 ; field < URI_FIELDS ; field++ )
-		uri_decode_inplace ( uri, field );
+	for ( field = 0 ; field < URI_EPATH ; field++ )
+		uri_decode_inplace ( ( char * ) uri_field ( uri, field ) );
 
 	DBGC ( uri, "URI parsed \"%s\" to", uri_string );
 	uri_dump ( uri );
@@ -458,8 +467,8 @@ size_t format_uri ( const struct uri *uri, char *buf, size_t len ) {
 	static const char prefixes[URI_FIELDS] = {
 		[URI_PASSWORD] = ':',
 		[URI_PORT] = ':',
-		[URI_QUERY] = '?',
-		[URI_FRAGMENT] = '#',
+		[URI_EQUERY] = '?',
+		[URI_EFRAGMENT] = '#',
 	};
 	char prefix;
 	size_t used = 0;
@@ -480,6 +489,10 @@ size_t format_uri ( const struct uri *uri, char *buf, size_t len ) {
 		if ( ! uri_field ( uri, field ) )
 			continue;
 
+		/* Skip path field if encoded path is present */
+		if ( ( field == URI_PATH ) && uri->epath )
+			continue;
+
 		/* Prefix this field, if applicable */
 		prefix = prefixes[field];
 		if ( ( field == URI_HOST ) && ( uri->user != NULL ) )
@@ -676,6 +689,7 @@ char * resolve_path ( const char *base_path,
 struct uri * resolve_uri ( const struct uri *base_uri,
 			   struct uri *relative_uri ) {
 	struct uri tmp_uri;
+	char *tmp_epath = NULL;
 	char *tmp_path = NULL;
 	struct uri *new_uri;
 
@@ -685,20 +699,27 @@ struct uri * resolve_uri ( const struct uri *base_uri,
 
 	/* Mangle URI */
 	memcpy ( &tmp_uri, base_uri, sizeof ( tmp_uri ) );
-	if ( relative_uri->path ) {
-		tmp_path = resolve_path ( ( base_uri->path ?
-					    base_uri->path : "/" ),
-					  relative_uri->path );
+	if ( relative_uri->epath ) {
+		tmp_epath = resolve_path ( ( base_uri->epath ?
+					     base_uri->epath : "/" ),
+					   relative_uri->epath );
+		if ( ! tmp_epath )
+			goto err_epath;
+		tmp_path = strdup ( tmp_epath );
+		if ( ! tmp_path )
+			goto err_path;
+		uri_decode_inplace ( tmp_path );
+		tmp_uri.epath = tmp_epath;
 		tmp_uri.path = tmp_path;
-		tmp_uri.query = relative_uri->query;
-		tmp_uri.fragment = relative_uri->fragment;
+		tmp_uri.equery = relative_uri->equery;
+		tmp_uri.efragment = relative_uri->efragment;
 		tmp_uri.params = relative_uri->params;
-	} else if ( relative_uri->query ) {
-		tmp_uri.query = relative_uri->query;
-		tmp_uri.fragment = relative_uri->fragment;
+	} else if ( relative_uri->equery ) {
+		tmp_uri.equery = relative_uri->equery;
+		tmp_uri.efragment = relative_uri->efragment;
 		tmp_uri.params = relative_uri->params;
-	} else if ( relative_uri->fragment ) {
-		tmp_uri.fragment = relative_uri->fragment;
+	} else if ( relative_uri->efragment ) {
+		tmp_uri.efragment = relative_uri->efragment;
 		tmp_uri.params = relative_uri->params;
 	} else if ( relative_uri->params ) {
 		tmp_uri.params = relative_uri->params;
@@ -707,7 +728,14 @@ struct uri * resolve_uri ( const struct uri *base_uri,
 	/* Create demangled URI */
 	new_uri = uri_dup ( &tmp_uri );
 	free ( tmp_path );
+	free ( tmp_epath );
 	return new_uri;
+
+	free ( tmp_path );
+ err_path:
+	free ( tmp_epath );
+ err_epath:
+	return NULL;
 }
 
 /**
@@ -746,6 +774,7 @@ static struct uri * tftp_uri ( struct sockaddr *sa_server,
 	if ( asprintf ( &path, "/%s", filename ) < 0 )
 		goto err_path;
 	tmp.path = path;
+	tmp.epath = path;
 
 	/* Demangle URI */
 	uri = uri_dup ( &tmp );

src/drivers/bus/virtio-pci.c

@@ -17,37 +17,47 @@
 #include "ipxe/io.h"
 #include "ipxe/iomap.h"
 #include "ipxe/pci.h"
+#include "ipxe/dma.h"
 #include "ipxe/reboot.h"
 #include "ipxe/virtio-pci.h"
 #include "ipxe/virtio-ring.h"
 
-static int vp_alloc_vq(struct vring_virtqueue *vq, u16 num)
+static int vp_alloc_vq(struct vring_virtqueue *vq, u16 num, size_t header_size)
 {
-    size_t queue_size = PAGE_MASK + vring_size(num);
+    size_t ring_size = PAGE_MASK + vring_size(num);
     size_t vdata_size = num * sizeof(void *);
+    size_t queue_size = ring_size + vdata_size + header_size;
 
-    vq->queue = zalloc(queue_size + vdata_size);
+    vq->queue = dma_alloc(vq->dma, &vq->map, queue_size, queue_size);
     if (!vq->queue) {
         return -ENOMEM;
     }
 
+    memset ( vq->queue, 0, queue_size );
+    vq->queue_size = queue_size;
+
     /* vdata immediately follows the ring */
-    vq->vdata = (void **)(vq->queue + queue_size);
+    vq->vdata = (void **)(vq->queue + ring_size);
+
+    /* empty header immediately follows vdata */
+    vq->empty_header = (struct virtio_net_hdr_modern *)(vq->queue + ring_size + vdata_size);
 
     return 0;
 }
 
 void vp_free_vq(struct vring_virtqueue *vq)
 {
-    if (vq->queue) {
-        free(vq->queue);
+    if (vq->queue && vq->queue_size) {
+        dma_free(&vq->map, vq->queue, vq->queue_size);
         vq->queue = NULL;
         vq->vdata = NULL;
+        vq->queue_size = 0;
     }
 }
 
 int vp_find_vq(unsigned int ioaddr, int queue_index,
-               struct vring_virtqueue *vq)
+               struct vring_virtqueue *vq, struct dma_device *dma_dev,
+               size_t header_size)
 {
    struct vring * vr = &vq->vring;
    u16 num;
@@ -73,9 +83,10 @@ int vp_find_vq(unsigned int ioaddr, int queue_index,
    }
 
    vq->queue_index = queue_index;
+   vq->dma = dma_dev;
 
    /* initialize the queue */
-   rc = vp_alloc_vq(vq, num);
+   rc = vp_alloc_vq(vq, num, header_size);
    if (rc) {
            DBG("VIRTIO-PCI ERROR: failed to allocate queue memory\n");
            return rc;
@@ -87,8 +98,7 @@ int vp_find_vq(unsigned int ioaddr, int queue_index,
     * NOTE: vr->desc is initialized by vring_init()
     */
 
-   outl((unsigned long)virt_to_phys(vr->desc) >> PAGE_SHIFT,
-        ioaddr + VIRTIO_PCI_QUEUE_PFN);
+   outl(dma(&vq->map, vr->desc) >> PAGE_SHIFT, ioaddr + VIRTIO_PCI_QUEUE_PFN);
 
    return num;
 }
@@ -348,7 +358,8 @@ void vpm_notify(struct virtio_pci_modern_device *vdev,
 }
 
 int vpm_find_vqs(struct virtio_pci_modern_device *vdev,
-                 unsigned nvqs, struct vring_virtqueue *vqs)
+                 unsigned nvqs, struct vring_virtqueue *vqs,
+                 struct dma_device *dma_dev, size_t header_size)
 {
     unsigned i;
     struct vring_virtqueue *vq;
@@ -392,11 +403,12 @@ int vpm_find_vqs(struct virtio_pci_modern_device *vdev,
 
         vq = &vqs[i];
         vq->queue_index = i;
+        vq->dma = dma_dev;
 
         /* get offset of notification word for this vq */
         off = vpm_ioread16(vdev, &vdev->common, COMMON_OFFSET(queue_notify_off));
 
-        err = vp_alloc_vq(vq, size);
+        err = vp_alloc_vq(vq, size, header_size);
         if (err) {
             DBG("VIRTIO-PCI %p: failed to allocate queue memory\n", vdev);
             return err;
@@ -406,13 +418,16 @@ int vpm_find_vqs(struct virtio_pci_modern_device *vdev,
         /* activate the queue */
         vpm_iowrite16(vdev, &vdev->common, size, COMMON_OFFSET(queue_size));
 
-        vpm_iowrite64(vdev, &vdev->common, virt_to_phys(vq->vring.desc),
+        vpm_iowrite64(vdev, &vdev->common,
+                      dma(&vq->map, vq->vring.desc),
                       COMMON_OFFSET(queue_desc_lo),
                       COMMON_OFFSET(queue_desc_hi));
-        vpm_iowrite64(vdev, &vdev->common, virt_to_phys(vq->vring.avail),
+        vpm_iowrite64(vdev, &vdev->common,
+                      dma(&vq->map, vq->vring.avail),
                       COMMON_OFFSET(queue_avail_lo),
                       COMMON_OFFSET(queue_avail_hi));
-        vpm_iowrite64(vdev, &vdev->common, virt_to_phys(vq->vring.used),
+        vpm_iowrite64(vdev, &vdev->common,
+                      dma(&vq->map, vq->vring.used),
                       COMMON_OFFSET(queue_used_lo),
                       COMMON_OFFSET(queue_used_hi));
 

src/drivers/bus/virtio-ring.c

@@ -98,7 +98,7 @@ void vring_add_buf(struct vring_virtqueue *vq,
    for (i = head; out; i = vr->desc[i].next, out--) {
 
            vr->desc[i].flags = VRING_DESC_F_NEXT;
-           vr->desc[i].addr = (u64)virt_to_phys(list->addr);
+           vr->desc[i].addr = list->addr;
            vr->desc[i].len = list->length;
            prev = i;
            list++;
@@ -106,7 +106,7 @@ void vring_add_buf(struct vring_virtqueue *vq,
    for ( ; in; i = vr->desc[i].next, in--) {
 
            vr->desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
-           vr->desc[i].addr = (u64)virt_to_phys(list->addr);
+           vr->desc[i].addr = list->addr;
            vr->desc[i].len = list->length;
            prev = i;
            list++;

src/drivers/net/ecm.c

@@ -82,7 +82,7 @@ ecm_ethernet_descriptor ( struct usb_configuration_descriptor *config,
 /**
  * Get hardware MAC address
  *
- * @v usb		USB function
+ * @v func		USB function
  * @v desc		Ethernet functional descriptor
  * @v hw_addr		Hardware address to fill in
  * @ret rc		Return status code

src/drivers/net/virtio-net.c

@@ -29,6 +29,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/iobuf.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/pci.h>
+#include <ipxe/dma.h>
 #include <ipxe/if_ether.h>
 #include <ipxe/ethernet.h>
 #include <ipxe/virtio-pci.h>
@@ -99,8 +100,9 @@ struct virtnet_nic {
 	/** Pending rx packet count */
 	unsigned int rx_num_iobufs;
 
-	/** Virtio net dummy packet headers */
-	struct virtio_net_hdr_modern empty_header[QUEUE_NB];
+	/** DMA device */
+	struct dma_device *dma;
+
 };
 
 /** Add an iobuf to a virtqueue
@@ -115,7 +117,7 @@ static void virtnet_enqueue_iob ( struct net_device *netdev,
 				  int vq_idx, struct io_buffer *iobuf ) {
 	struct virtnet_nic *virtnet = netdev->priv;
 	struct vring_virtqueue *vq = &virtnet->virtqueue[vq_idx];
-	struct virtio_net_hdr_modern *header = &virtnet->empty_header[vq_idx];
+	struct virtio_net_hdr_modern *header = vq->empty_header;
 	unsigned int out = ( vq_idx == TX_INDEX ) ? 2 : 0;
 	unsigned int in = ( vq_idx == TX_INDEX ) ? 0 : 2;
 	size_t header_len = ( virtnet->virtio_version ?
@@ -132,11 +134,11 @@ static void virtnet_enqueue_iob ( struct net_device *netdev,
 			 * to header->flags for received packets.  Work around
 			 * this by using separate RX and TX headers.
 			 */
-			.addr = ( char* ) header,
+			.addr = dma ( &vq->map, header ),
 			.length = header_len,
 		},
 		{
-			.addr = ( char* ) iobuf->data,
+			.addr = iob_dma ( iobuf ),
 			.length = iob_len ( iobuf ),
 		},
 	};
@@ -161,7 +163,7 @@ static void virtnet_refill_rx_virtqueue ( struct net_device *netdev ) {
 		struct io_buffer *iobuf;
 
 		/* Try to allocate a buffer, stop for now if out of memory */
-		iobuf = alloc_iob ( len );
+		iobuf = alloc_rx_iob ( len, virtnet->dma );
 		if ( ! iobuf )
 			break;
 
@@ -215,7 +217,8 @@ static int virtnet_open_legacy ( struct net_device *netdev ) {
 
 	/* Initialize rx/tx virtqueues */
 	for ( i = 0; i < QUEUE_NB; i++ ) {
-		if ( vp_find_vq ( ioaddr, i, &virtnet->virtqueue[i] ) == -1 ) {
+		if ( vp_find_vq ( ioaddr, i, &virtnet->virtqueue[i], virtnet->dma,
+                                  sizeof ( struct virtio_net_hdr_modern ) ) == -1 ) {
 			DBGC ( virtnet, "VIRTIO-NET %p cannot register queue %d\n",
 			       virtnet, i );
 			virtnet_free_virtqueues ( netdev );
@@ -280,7 +283,8 @@ static int virtnet_open_modern ( struct net_device *netdev ) {
 	}
 
 	/* Initialize rx/tx virtqueues */
-	if ( vpm_find_vqs ( &virtnet->vdev, QUEUE_NB, virtnet->virtqueue ) ) {
+	if ( vpm_find_vqs ( &virtnet->vdev, QUEUE_NB, virtnet->virtqueue,
+                            virtnet->dma, sizeof ( struct virtio_net_hdr_modern ) ) ) {
 		DBGC ( virtnet, "VIRTIO-NET %p cannot register queues\n",
 		       virtnet );
 		virtnet_free_virtqueues ( netdev );
@@ -335,7 +339,7 @@ static void virtnet_close ( struct net_device *netdev ) {
 
 	/* Free rx iobufs */
 	list_for_each_entry_safe ( iobuf, next_iobuf, &virtnet->rx_iobufs, list ) {
-		free_iob ( iobuf );
+		free_rx_iob ( iobuf );
 	}
 	INIT_LIST_HEAD ( &virtnet->rx_iobufs );
 	virtnet->rx_num_iobufs = 0;
@@ -478,6 +482,12 @@ static int virtnet_probe_legacy ( struct pci_device *pci ) {
 
 	/* Enable PCI bus master and reset NIC */
 	adjust_pci_device ( pci );
+
+	/* Configure DMA */
+	virtnet->dma =  &pci->dma;
+	dma_set_mask_64bit ( virtnet->dma );
+	netdev->dma = virtnet->dma;
+
 	vp_reset ( ioaddr );
 
 	/* Load MAC address and MTU */
@@ -506,7 +516,7 @@ static int virtnet_probe_legacy ( struct pci_device *pci ) {
 	return 0;
 
 	unregister_netdev ( netdev );
- err_register_netdev:
+err_register_netdev:
 	vp_reset ( ioaddr );
 	netdev_nullify ( netdev );
 	netdev_put ( netdev );
@@ -586,6 +596,11 @@ static int virtnet_probe_modern ( struct pci_device *pci, int *found_dev ) {
 	/* Enable the PCI device */
 	adjust_pci_device ( pci );
 
+	/* Configure DMA */
+	virtnet->dma =  &pci->dma;
+	dma_set_mask_64bit ( virtnet->dma );
+	netdev->dma = virtnet->dma;
+
 	/* Reset the device and set initial status bits */
 	vpm_reset ( &virtnet->vdev );
 	vpm_add_status ( &virtnet->vdev, VIRTIO_CONFIG_S_ACKNOWLEDGE );
@@ -633,7 +648,6 @@ err_mac_address:
 	vpm_reset ( &virtnet->vdev );
 	netdev_nullify ( netdev );
 	netdev_put ( netdev );
-
 	virtio_pci_unmap_capability ( &virtnet->vdev.device );
 err_map_device:
 	virtio_pci_unmap_capability ( &virtnet->vdev.isr );

src/drivers/usb/xhci.c

@@ -1165,6 +1165,31 @@ static int xhci_reset ( struct xhci_device *xhci ) {
 	return -ETIMEDOUT;
 }
 
+/**
+ * Mark xHCI device as permanently failed
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_fail ( struct xhci_device *xhci ) {
+	size_t len;
+	int rc;
+
+	/* Mark command mechanism as permanently failed */
+	xhci->failed = 1;
+
+	/* Reset device */
+	if ( ( rc = xhci_reset ( xhci ) ) != 0 )
+		return rc;
+
+	/* Discard DCBAA entries since DCBAAP has been cleared */
+	assert ( xhci->dcbaa.context != NULL );
+	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
+	memset ( xhci->dcbaa.context, 0, len );
+
+	return 0;
+}
+
 /******************************************************************************
  *
  * Transfer request blocks
@@ -1720,6 +1745,10 @@ static void xhci_event_poll ( struct xhci_device *xhci ) {
 	unsigned int consumed;
 	unsigned int type;
 
+	/* Do nothing if device has permanently failed */
+	if ( xhci->failed )
+		return;
+
 	/* Poll for events */
 	profile_start ( &xhci_event_profiler );
 	for ( consumed = 0 ; ; consumed++ ) {
@@ -1778,6 +1807,7 @@ static void xhci_event_poll ( struct xhci_device *xhci ) {
  */
 static void xhci_abort ( struct xhci_device *xhci ) {
 	physaddr_t crp;
+	uint32_t crcr;
 
 	/* Abort the command */
 	DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name );
@@ -1786,8 +1816,18 @@ static void xhci_abort ( struct xhci_device *xhci ) {
 	/* Allow time for command to abort */
 	mdelay ( XHCI_COMMAND_ABORT_DELAY_MS );
 
-	/* Sanity check */
-	assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
+	/* Check for failure to abort */
+	crcr = readl ( xhci->op + XHCI_OP_CRCR );
+	if ( crcr & XHCI_CRCR_CRR ) {
+
+		/* Device has failed to abort a command and is almost
+		 * certainly beyond repair.  Reset device, abandoning
+		 * all state, and mark device as failed to avoid
+		 * delays on any future command attempts.
+		 */
+		DBGC ( xhci, "XHCI %s failed to abort command\n", xhci->name );
+		xhci_fail ( xhci );
+	}
 
 	/* Consume (and ignore) any final command status */
 	xhci_event_poll ( xhci );
@@ -1813,6 +1853,12 @@ static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
 	unsigned int i;
 	int rc;
 
+	/* Immediately fail all commands if command mechanism has failed */
+	if ( xhci->failed ) {
+		rc = -EPIPE;
+		goto err_failed;
+	}
+
 	/* Sanity check */
 	if ( xhci->pending ) {
 		DBGC ( xhci, "XHCI %s command ring busy\n", xhci->name );
@@ -1863,6 +1909,7 @@ static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
  err_enqueue:
 	xhci->pending = NULL;
  err_pending:
+ err_failed:
 	return rc;
 }
 
@@ -3412,14 +3459,36 @@ static int xhci_probe ( struct pci_device *pci ) {
 static void xhci_remove ( struct pci_device *pci ) {
 	struct xhci_device *xhci = pci_get_drvdata ( pci );
 	struct usb_bus *bus = xhci->bus;
+	uint16_t command;
 
+	/* Some systems are observed to disable bus mastering on
+	 * Thunderbolt controllers before we get a chance to shut
+	 * down.  Detect this and avoid attempting any DMA operations,
+	 * which are guaranteed to fail and may end up spuriously
+	 * completing after the operating system kernel starts up.
+	 */
+	pci_read_config_word ( pci, PCI_COMMAND, &command );
+	if ( ! ( command & PCI_COMMAND_MASTER ) ) {
+		DBGC ( xhci, "XHCI %s DMA was disabled\n", xhci->name );
+		xhci_fail ( xhci );
+	}
+
+	/* Unregister and free USB bus */
 	unregister_usb_bus ( bus );
 	free_usb_bus ( bus );
+
+	/* Reset device and undo any PCH-specific fixes */
 	xhci_reset ( xhci );
 	if ( xhci->quirks & XHCI_PCH )
 		xhci_pch_undo ( xhci, pci );
+
+	/* Release ownership back to BIOS */
 	xhci_legacy_release ( xhci );
+
+	/* Unmap registers */
 	iounmap ( xhci->regs );
+
+	/* Free device */
 	free ( xhci );
 }
 

src/drivers/usb/xhci.h

@@ -1115,6 +1115,8 @@ struct xhci_device {
 	struct xhci_event_ring event;
 	/** Current command (if any) */
 	union xhci_trb *pending;
+	/** Command mechanism has permanently failed */
+	int failed;
 
 	/** Device slots, indexed by slot ID */
 	struct xhci_slot **slot;

src/hci/readline.c

@@ -38,7 +38,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-#define READLINE_MAX 256
+#define READLINE_MAX 1024
 
 /**
  * Synchronise console with edited string
@@ -258,8 +258,8 @@ void history_free ( struct readline_history *history ) {
 int readline_history ( const char *prompt, const char *prefill,
 		       struct readline_history *history, unsigned long timeout,
 		       char **line ) {
-	char buf[READLINE_MAX];
 	struct edit_string string;
+	char *buf;
 	int key;
 	int move_by;
 	const char *new_string;
@@ -275,10 +275,14 @@ int readline_history ( const char *prompt, const char *prefill,
 	/* Ensure cursor is visible */
 	printf ( "\033[?25h" );
 
-	/* Initialise editable string */
+	/* Allocate buffer and initialise editable string */
+	buf = zalloc ( READLINE_MAX );
+	if ( ! buf ) {
+		rc = -ENOMEM;
+		goto done;
+	}
 	memset ( &string, 0, sizeof ( string ) );
-	init_editstring ( &string, buf, sizeof ( buf ) );
-	buf[0] = '\0';
+	init_editstring ( &string, buf, READLINE_MAX );
 
 	/* Prefill string, if applicable */
 	if ( prefill ) {
@@ -303,8 +307,13 @@ int readline_history ( const char *prompt, const char *prefill,
 		switch ( key ) {
 		case CR:
 		case LF:
-			*line = strdup ( buf );
-			rc = ( ( *line ) ? 0 : -ENOMEM );
+			/* Shrink string (ignoring failures) */
+			*line = realloc ( buf,
+					  ( strlen ( buf ) + 1 /* NUL */ ) );
+			if ( ! *line )
+				*line = buf;
+			buf = NULL;
+			rc = 0;
 			goto done;
 		case CTRL_C:
 			rc = -ECANCELED;
@@ -332,6 +341,7 @@ int readline_history ( const char *prompt, const char *prefill,
 
  done:
 	putchar ( '\n' );
+	free ( buf );
 	if ( history ) {
 		if ( *line && (*line)[0] )
 			history_append ( history, *line );

src/include/ipxe/uri.h

@@ -46,6 +46,20 @@ struct parameters;
  *   scheme = "ftp", user = "joe", password = "secret",
  *   host = "insecure.org", port = "8081", path = "/hidden/path/to",
  *   query = "what=is", fragment = "this"
+ *
+ * The URI syntax includes a percent-encoding mechanism that can be
+ * used to represent characters that would otherwise not be possible,
+ * such as a '/' character within the password field.  These encodings
+ * are decoded during the URI parsing stage, thereby allowing protocol
+ * implementations to consume the raw field values directly without
+ * further decoding.
+ *
+ * Some protocols (such as HTTP) communicate using URI-encoded values.
+ * For these protocols, the original encoded substring must be
+ * retained verbatim since the choice of whether or not to encode a
+ * particular character may have significance to the receiving
+ * application.  We therefore retain the originally-encoded substrings
+ * for the path, query, and fragment fields.
  */
 struct uri {
 	/** Reference count */
@@ -62,12 +76,14 @@ struct uri {
 	const char *host;
 	/** Port number */
 	const char *port;
-	/** Path */
+	/** Path (after URI decoding) */
 	const char *path;
-	/** Query */
-	const char *query;
-	/** Fragment */
-	const char *fragment;
+	/** Path (with original URI encoding) */
+	const char *epath;
+	/** Query (with original URI encoding) */
+	const char *equery;
+	/** Fragment (with original URI encoding) */
+	const char *efragment;
 	/** Form parameters */
 	struct parameters *params;
 } __attribute__ (( packed ));
@@ -100,8 +116,9 @@ enum uri_fields {
 	URI_HOST = URI_FIELD ( host ),
 	URI_PORT = URI_FIELD ( port ),
 	URI_PATH = URI_FIELD ( path ),
-	URI_QUERY = URI_FIELD ( query ),
-	URI_FRAGMENT = URI_FIELD ( fragment ),
+	URI_EPATH = URI_FIELD ( epath ),
+	URI_EQUERY = URI_FIELD ( equery ),
+	URI_EFRAGMENT = URI_FIELD ( efragment ),
 	URI_FIELDS
 };
 

src/include/ipxe/virtio-pci.h

@@ -1,6 +1,8 @@
 #ifndef _VIRTIO_PCI_H_
 # define _VIRTIO_PCI_H_
 
+#include <ipxe/dma.h>
+
 /* A 32-bit r/o bitmask of the features supported by the host */
 #define VIRTIO_PCI_HOST_FEATURES        0
 
@@ -198,7 +200,8 @@ struct vring_virtqueue;
 
 void vp_free_vq(struct vring_virtqueue *vq);
 int vp_find_vq(unsigned int ioaddr, int queue_index,
-               struct vring_virtqueue *vq);
+               struct vring_virtqueue *vq, struct dma_device *dma_dev,
+               size_t header_size);
 
 
 /* Virtio 1.0 I/O routines abstract away the three possible HW access
@@ -298,7 +301,8 @@ void vpm_notify(struct virtio_pci_modern_device *vdev,
                 struct vring_virtqueue *vq);
 
 int vpm_find_vqs(struct virtio_pci_modern_device *vdev,
-                 unsigned nvqs, struct vring_virtqueue *vqs);
+                 unsigned nvqs, struct vring_virtqueue *vqs,
+                 struct dma_device *dma_dev, size_t header_size);
 
 int virtio_pci_find_capability(struct pci_device *pci, uint8_t cfg_type);
 

src/include/ipxe/virtio-ring.h

@@ -2,6 +2,7 @@
 # define _VIRTIO_RING_H_
 
 #include <ipxe/virtio-pci.h>
+#include <ipxe/dma.h>
 
 /* Status byte for guest to report progress, and synchronize features. */
 /* We have seen device and processed generic fields (VIRTIO_CONFIG_F_VIRTIO) */
@@ -74,17 +75,21 @@ struct vring {
 
 struct vring_virtqueue {
    unsigned char *queue;
+   size_t queue_size;
+   struct dma_mapping map;
+   struct dma_device *dma;
    struct vring vring;
    u16 free_head;
    u16 last_used_idx;
    void **vdata;
+   struct virtio_net_hdr_modern *empty_header;
    /* PCI */
    int queue_index;
    struct virtio_pci_region notification;
 };
 
 struct vring_list {
-  char *addr;
+  physaddr_t addr;
   unsigned int length;
 };
 

src/net/tcp/httpcore.c

@@ -614,8 +614,8 @@ int http_open ( struct interface *xfer, struct http_method *method,
 
 	/* Calculate request URI length */
 	memset ( &request_uri, 0, sizeof ( request_uri ) );
-	request_uri.path = ( uri->path ? uri->path : "/" );
-	request_uri.query = uri->query;
+	request_uri.epath = ( uri->epath ? uri->epath : "/" );
+	request_uri.equery = uri->equery;
 	request_uri_len =
 		( format_uri ( &request_uri, NULL, 0 ) + 1 /* NUL */);
 

src/tests/uri_test.c

@@ -149,8 +149,10 @@ static void uri_okx ( struct uri *uri, struct uri *expected, const char *file,
 	okx ( uristrcmp ( uri->host, expected->host ) == 0, file, line );
 	okx ( uristrcmp ( uri->port, expected->port ) == 0, file, line );
 	okx ( uristrcmp ( uri->path, expected->path ) == 0, file, line );
-	okx ( uristrcmp ( uri->query, expected->query ) == 0, file, line );
-	okx ( uristrcmp ( uri->fragment, expected->fragment ) == 0, file, line);
+	okx ( uristrcmp ( uri->epath, expected->epath ) == 0, file, line );
+	okx ( uristrcmp ( uri->equery, expected->equery ) == 0, file, line );
+	okx ( uristrcmp ( uri->efragment, expected->efragment ) == 0,
+	      file, line);
 	okx ( uri->params == expected->params, file, line );
 }
 #define uri_ok( uri, expected ) uri_okx ( uri, expected, __FILE__, __LINE__ )
@@ -490,25 +492,33 @@ static struct uri_test uri_empty = {
 /** Basic HTTP URI */
 static struct uri_test uri_boot_ipxe_org = {
 	"http://boot.ipxe.org/demo/boot.php",
-	{ .scheme = "http", .host = "boot.ipxe.org", .path = "/demo/boot.php" }
+	{ .scheme = "http", .host = "boot.ipxe.org",
+	  .path = "/demo/boot.php", .epath = "/demo/boot.php" },
 };
 
 /** Basic opaque URI */
 static struct uri_test uri_mailto = {
 	"mailto:ipxe-devel@lists.ipxe.org",
-	{ .scheme = "mailto", .opaque = "ipxe-devel@lists.ipxe.org" }
+	{ .scheme = "mailto", .opaque = "ipxe-devel@lists.ipxe.org" },
+};
+
+/** Basic host-only URI */
+static struct uri_test uri_host = {
+	"http://boot.ipxe.org",
+	{ .scheme = "http", .host = "boot.ipxe.org" },
 };
 
 /** Basic path-only URI */
 static struct uri_test uri_path = {
 	"/var/lib/tftpboot/pxelinux.0",
-	{ .path = "/var/lib/tftpboot/pxelinux.0" },
+	{ .path = "/var/lib/tftpboot/pxelinux.0",
+	  .epath ="/var/lib/tftpboot/pxelinux.0" },
 };
 
 /** Path-only URI with escaped characters */
 static struct uri_test uri_path_escaped = {
 	"/hello%20world%3F",
-	{ .path = "/hello world?" },
+	{ .path = "/hello world?", .epath = "/hello%20world%3F" },
 };
 
 /** HTTP URI with all the trimmings */
@@ -521,8 +531,9 @@ static struct uri_test uri_http_all = {
 		.host = "example.com",
 		.port = "3001",
 		.path = "/~foo/cgi-bin/foo.pl",
-		.query = "a=b&c=d",
-		.fragment = "bit",
+		.epath = "/~foo/cgi-bin/foo.pl",
+		.equery = "a=b&c=d",
+		.efragment = "bit",
 	},
 };
 
@@ -533,8 +544,9 @@ static struct uri_test uri_http_escaped = {
 		.scheme = "https",
 		.host = "test.ipxe.org",
 		.path = "/wtf?\n",
-		.query = "kind#of/uri is",
-		.fragment = "this?",
+		.epath = "/wtf%3F%0A",
+		.equery = "kind%23of/uri%20is",
+		.efragment = "this%3F",
 	},
 };
 
@@ -550,8 +562,9 @@ static struct uri_test uri_http_escaped_improper = {
 		.scheme = "https",
 		.host = "test.ipxe.org",
 		.path = "/wtf?\n",
-		.query = "kind#of/uri is",
-		.fragment = "this?",
+		.epath = "/wt%66%3f\n",
+		.equery = "kind%23of/uri is",
+		.efragment = "this?",
 	},
 };
 
@@ -562,6 +575,7 @@ static struct uri_test uri_ipv6 = {
 		.scheme = "http",
 		.host = "[2001:ba8:0:1d4::6950:5845]",
 		.path = "/",
+		.epath = "/",
 	},
 };
 
@@ -573,6 +587,7 @@ static struct uri_test uri_ipv6_port = {
 		.host = "[2001:ba8:0:1d4::6950:5845]",
 		.port = "8001",
 		.path = "/boot",
+		.epath = "/boot",
 	},
 };
 
@@ -583,6 +598,7 @@ static struct uri_test uri_ipv6_local = {
 		.scheme = "http",
 		.host = "[fe80::69ff:fe50:5845%net0]",
 		.path = "/ipxe",
+		.epath = "/ipxe",
 	},
 };
 
@@ -598,6 +614,7 @@ static struct uri_test uri_ipv6_local_non_conforming = {
 		.scheme = "http",
 		.host = "[fe80::69ff:fe50:5845%net0]",
 		.path = "/ipxe",
+		.epath = "/ipxe",
 	},
 };
 
@@ -625,6 +642,7 @@ static struct uri_test uri_file_absolute = {
 	{
 		.scheme = "file",
 		.path = "/boot/script.ipxe",
+		.epath = "/boot/script.ipxe",
 	},
 };
 
@@ -635,6 +653,7 @@ static struct uri_test uri_file_volume = {
 		.scheme = "file",
 		.host = "hpilo",
 		.path = "/boot/script.ipxe",
+		.epath = "/boot/script.ipxe",
 	},
 };
 
@@ -736,6 +755,7 @@ static struct uri_pxe_test uri_pxe_absolute = {
 		.scheme = "http",
 		.host = "not.a.tftp",
 		.path = "/uri",
+		.epath = "/uri",
 	},
 	"http://not.a.tftp/uri",
 };
@@ -754,6 +774,7 @@ static struct uri_pxe_test uri_pxe_absolute_path = {
 		.scheme = "tftp",
 		.host = "192.168.0.2",
 		.path = "//absolute/path",
+		.epath = "//absolute/path",
 	},
 	"tftp://192.168.0.2//absolute/path",
 };
@@ -772,6 +793,7 @@ static struct uri_pxe_test uri_pxe_relative_path = {
 		.scheme = "tftp",
 		.host = "192.168.0.3",
 		.path = "/relative/path",
+		.epath = "/relative/path",
 	},
 	"tftp://192.168.0.3/relative/path",
 };
@@ -790,8 +812,9 @@ static struct uri_pxe_test uri_pxe_icky = {
 		.scheme = "tftp",
 		.host = "10.0.0.6",
 		.path = "/C:\\tftpboot\\icky#path",
+		.epath = "/C:\\tftpboot\\icky#path",
 	},
-	"tftp://10.0.0.6/C%3A\\tftpboot\\icky%23path",
+	"tftp://10.0.0.6/C:\\tftpboot\\icky#path",
 };
 
 /** PXE URI with custom port */
@@ -810,6 +833,7 @@ static struct uri_pxe_test uri_pxe_port = {
 		.host = "192.168.0.1",
 		.port = "4069",
 		.path = "//another/path",
+		.epath = "//another/path",
 	},
 	"tftp://192.168.0.1:4069//another/path",
 };
@@ -873,6 +897,7 @@ static struct uri_params_test uri_params = {
 		.scheme = "http",
 		.host = "boot.ipxe.org",
 		.path = "/demo/boot.php",
+		.epath = "/demo/boot.php",
 	},
 	NULL,
 	uri_params_list,
@@ -902,6 +927,7 @@ static struct uri_params_test uri_named_params = {
 		.host = "192.168.100.4",
 		.port = "3001",
 		.path = "/register",
+		.epath = "/register",
 	},
 	"foo",
 	uri_named_params_list,
@@ -917,6 +943,7 @@ static void uri_test_exec ( void ) {
 	uri_parse_format_dup_ok ( &uri_empty );
 	uri_parse_format_dup_ok ( &uri_boot_ipxe_org );
 	uri_parse_format_dup_ok ( &uri_mailto );
+	uri_parse_format_dup_ok ( &uri_host );
 	uri_parse_format_dup_ok ( &uri_path );
 	uri_parse_format_dup_ok ( &uri_path_escaped );
 	uri_parse_format_dup_ok ( &uri_http_all );

src/usr/imgmgmt.c

@@ -58,8 +58,8 @@ int imgdownload ( struct uri *uri, unsigned long timeout,
 	memcpy ( &uri_redacted, uri, sizeof ( uri_redacted ) );
 	uri_redacted.user = NULL;
 	uri_redacted.password = NULL;
-	uri_redacted.query = NULL;
-	uri_redacted.fragment = NULL;
+	uri_redacted.equery = NULL;
+	uri_redacted.efragment = NULL;
 	uri_string_redacted = format_uri_alloc ( &uri_redacted );
 	if ( ! uri_string_redacted ) {
 		rc = -ENOMEM;