lib/scatterlist: store pfn not page struct pointer

To be able to track page coming from GUP (get_user_pages*()) we will
need an extra bits as a flag. Storing pfn instead of pointer to page
give us the extra bits (all arch can store the maximum pfn in a long
minus 3bits).

Chain pointer are unmodified and should have the chain bit set. If
chain bit is set then we do not care about the third bits (ie it is
only valid for non chain pointer).

Signed-off-by: Jérôme Glisse <jglisse@redhat.com>
Cc: iommu@lists.linux-foundation.org
Cc: linux-fsdevel@vger.kernel.org
Cc: linux-block@vger.kernel.org
Cc: linux-mm@kvack.org
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Thomas Hellstrom <thellstrom@vmware.com>
Cc: Sakari Ailus <sakari.ailus@linux.intel.com>
Cc: Jason Gunthorpe <jgg@mellanox.com>

2 files changed, 46 insertions, 10 deletions

diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h
index c62c2af85a2e..8c70a668063c 100644
--- a/include/linux/scatterlist.h
+++ b/include/linux/scatterlist.h
@@ -49,8 +49,11 @@ struct sg_table {
  * Notes on SG table design.
  *
  * We use the unsigned long page_link field in the scatterlist struct to place
- * the page pointer AND encode information about the sg table as well. The two
- * lower bits are reserved for this information.
+ * the pfn AND encode information about the sg table as well. The three lower
+ * bits are reserved for this information. To keep sg_init_table() simple (ie
+ * just a memset) we have a valid pfn bits so that memset to zero of a sg array
+ * result in NULL page. The pfn valid bit is only meaningful if the chain bit
+ * is not set.
  *
  * If bit 0 is set, then the page_link contains a pointer to the next sg
  * table list. Otherwise the next entry is at sg + 1.
@@ -63,6 +66,10 @@ struct sg_table {
 
 #define SG_CHAIN	0x01UL
 #define SG_END		0x02UL
+#define SG_PFN_VALID	0x04UL
+#define SG_PFN_SHIFT	3
+
+#define SG_PFN_MASK	(~(SG_CHAIN | SG_END | SG_PFN_VALID))
 
 /*
  * We overload the LSB of the page pointer to indicate whether it's
@@ -74,6 +81,16 @@ struct sg_table {
 #define sg_chain_ptr(sg)	\
 	((struct scatterlist *) ((sg)->page_link & ~(SG_CHAIN | SG_END)))
 
+static unsigned long page_to_sg_pfn(const struct page *page)
+{
+	return page ? (page_to_pfn(page) << SG_PFN_SHIFT) | SG_PFN_VALID : 0UL;
+}
+
+static bool sg_is_page(const struct scatterlist *sg)
+{
+	return (sg->page_link & (SG_CHAIN | SG_PFN_VALID)) == SG_PFN_VALID;
+}
+
 /**
  * sg_assign_page - Assign a given page to an SG entry
  * @sg:		    SG entry
@@ -96,7 +113,7 @@ static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
 #ifdef CONFIG_DEBUG_SG
 	BUG_ON(sg_is_chain(sg));
 #endif
-	sg->page_link = page_link | (unsigned long) page;
+	sg->page_link = page_link | page_to_sg_pfn(page);
 }
 
 /**
@@ -148,7 +165,8 @@ static inline struct page *sg_page(struct scatterlist *sg)
 #ifdef CONFIG_DEBUG_SG
 	BUG_ON(sg_is_chain(sg));
 #endif
-	return (struct page *)((sg)->page_link & ~(SG_CHAIN | SG_END));
+	return sg_is_page(sg) ? pfn_to_page(sg->page_link >> SG_PFN_SHIFT) :
+				NULL;
 }
 
 /**
diff --git a/tools/virtio/linux/scatterlist.h b/tools/virtio/linux/scatterlist.h
index 369ee308b668..031baed18cf9 100644
--- a/tools/virtio/linux/scatterlist.h
+++ b/tools/virtio/linux/scatterlist.h
@@ -10,11 +10,28 @@ struct scatterlist {
 	dma_addr_t	dma_address;
 };
 
+#define SG_CHAIN	0x01UL
+#define SG_END		0x02UL
+#define SG_PFN_VALID	0x04UL
+#define SG_PFN_SHIFT	3
+
+#define SG_PFN_MASK	(~(SG_CHAIN | SG_END | SG_PFN_VALID))
+
 /* Scatterlist helpers, stolen from linux/scatterlist.h */
-#define sg_is_chain(sg)		((sg)->page_link & 0x01)
-#define sg_is_last(sg)		((sg)->page_link & 0x02)
+#define sg_is_chain(sg)		((sg)->page_link & SG_CHAIN)
+#define sg_is_last(sg)		((sg)->page_link & SG_END)
 #define sg_chain_ptr(sg)	\
-	((struct scatterlist *) ((sg)->page_link & ~0x03))
+	((struct scatterlist *) ((sg)->page_link & ~(SG_CHAIN | SG_END)))
+
+static unsigned long page_to_sg_pfn(const struct page *page)
+{
+	return page ? (page_to_pfn(page) << SG_PFN_SHIFT) | SG_PFN_VALID : 0UL;
+}
+
+static bool sg_is_page(const struct scatterlist *sg)
+{
+	return (sg->page_link & (SG_CHAIN | SG_PFN_VALID)) == SG_PFN_VALID;
+}
 
 /**
  * sg_assign_page - Assign a given page to an SG entry
@@ -28,7 +45,7 @@ struct scatterlist {
  **/
 static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
 {
-	unsigned long page_link = sg->page_link & 0x3;
+	unsigned long page_link = sg->page_link & (SG_CHAIN | SG_END);
 
 	/*
 	 * In order for the low bit stealing approach to work, pages
@@ -38,7 +55,7 @@ static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
 #ifdef CONFIG_DEBUG_SG
 	BUG_ON(sg_is_chain(sg));
 #endif
-	sg->page_link = page_link | (unsigned long) page;
+	sg->page_link = page_link | page_to_sg_pfn(page);
 }
 
 /**
@@ -68,7 +85,8 @@ static inline struct page *sg_page(struct scatterlist *sg)
 #ifdef CONFIG_DEBUG_SG
 	BUG_ON(sg_is_chain(sg));
 #endif
-	return (struct page *)((sg)->page_link & ~0x3);
+	return sg_is_page(sg) ? pfn_to_page(sg->page_link >> SG_PFN_SHIFT) :
+				NULL;
 }
 
 /*