summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJérôme Glisse <jglisse@redhat.com>2019-04-17 12:19:33 -0400
committerJérôme Glisse <jglisse@redhat.com>2019-05-22 17:19:11 -0400
commit9a2c97c07d4542fc3b6a89efb076fedde35e8948 (patch)
tree615979431ffb27b7a1040ad338f512f934ee466a
parent9950c15548a7adb07ae2a34e2c8ebdfa87887bc5 (diff)
block/bio: differentiate GUPed pages in bio_check_pages_dirty()
We want to handle differently pages that are coming from GUP (get_user_pages*()) as they will need to be release through new put_user_page() function and not with put_page(). Signed-off-by: Jérôme Glisse <jglisse@redhat.com> Cc: linux-fsdevel@vger.kernel.org Cc: linux-block@vger.kernel.org Cc: linux-mm@kvack.org Cc: John Hubbard <jhubbard@nvidia.com> Cc: Jan Kara <jack@suse.cz> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Johannes Thumshirn <jthumshirn@suse.de> Cc: Christoph Hellwig <hch@lst.de> Cc: Jens Axboe <axboe@kernel.dk> Cc: Ming Lei <ming.lei@redhat.com> Cc: Dave Chinner <david@fromorbit.com> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: Matthew Wilcox <willy@infradead.org> Cc: Boaz Harrosh <boaz@plexistor.com>
-rw-r--r--block/bio.c57
-rw-r--r--fs/block_dev.c2
-rw-r--r--fs/direct-io.c3
-rw-r--r--fs/iomap.c2
-rw-r--r--include/linux/bio.h2
5 files changed, 53 insertions, 13 deletions
diff --git a/block/bio.c b/block/bio.c
index 683cbb40f051..586e11dbf507 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -855,13 +855,18 @@ static void bio_get_pages(struct bio *bio)
get_page(bvec->bv_page);
}
-static void bio_release_pages(struct bio *bio)
+static void bio_release_pages(struct bio *bio, bool from_gup)
{
- struct bvec_iter_all iter_all;
struct bio_vec *bvec;
+ struct bvec_iter_all iter_all;
- bio_for_each_segment_all(bvec, bio, iter_all)
- put_page(bvec->bv_page);
+ if (from_gup) {
+ bio_for_each_segment_all(bvec, bio, iter_all)
+ put_user_page(bvec->bv_page);
+ } else {
+ bio_for_each_segment_all(bvec, bio, iter_all)
+ put_page(bvec->bv_page);
+ }
}
static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
@@ -1674,6 +1679,7 @@ static void bio_dirty_fn(struct work_struct *work);
static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
static DEFINE_SPINLOCK(bio_dirty_lock);
+static struct bio *bio_gup_dirty_list;
static struct bio *bio_dirty_list;
/*
@@ -1693,12 +1699,40 @@ static void bio_dirty_fn(struct work_struct *work)
bio_set_pages_dirty(bio);
if (!bio_flagged(bio, BIO_NO_PAGE_REF))
- bio_release_pages(bio);
+ bio_release_pages(bio, false);
+ bio_put(bio);
+ }
+
+ spin_lock_irq(&bio_dirty_lock);
+ next = bio_gup_dirty_list;
+ bio_gup_dirty_list = NULL;
+ spin_unlock_irq(&bio_dirty_lock);
+
+ while ((bio = next) != NULL) {
+ next = bio->bi_private;
+
+ bio_set_pages_dirty(bio);
+ if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+ bio_release_pages(bio, true);
bio_put(bio);
}
}
-void bio_check_pages_dirty(struct bio *bio)
+/**
+ * bio_check_pages_dirty() - queue up clean pages on workqueue to dirty them
+ * @bio: the bio struct containing the pages we should dirty
+ * @from_gup: did the pages in the bio came from GUP (get_user_pages*())
+ *
+ * This will go over all pages in the bio and if there is a non dirty page then
+ * the bio is added to a list of bio that need to get there page dirtied.
+ *
+ * We also need to know if the pages in the bio are coming from GUP or not as
+ * GUPed page need to be release through specific put_user_page(). Please see
+ * Documentation/vm/get_user_pages.rst for details on that. Rules of dumb is
+ * that if the bio has been populated with the help of iov_iter_get_pages*()
+ * then it might have GUPed pages (see iov_iter_get_pages_use_gup())
+ */
+void bio_check_pages_dirty(struct bio *bio, bool from_gup)
{
struct bio_vec *bvec;
unsigned long flags;
@@ -1710,13 +1744,18 @@ void bio_check_pages_dirty(struct bio *bio)
}
if (!bio_flagged(bio, BIO_NO_PAGE_REF))
- bio_release_pages(bio);
+ bio_release_pages(bio, from_gup);
bio_put(bio);
return;
defer:
spin_lock_irqsave(&bio_dirty_lock, flags);
- bio->bi_private = bio_dirty_list;
- bio_dirty_list = bio;
+ if (from_gup) {
+ bio->bi_private = bio_gup_dirty_list;
+ bio_gup_dirty_list = bio;
+ } else {
+ bio->bi_private = bio_dirty_list;
+ bio_dirty_list = bio;
+ }
spin_unlock_irqrestore(&bio_dirty_lock, flags);
schedule_work(&bio_dirty_work);
}
diff --git a/fs/block_dev.c b/fs/block_dev.c
index e6886c93c89d..7657b6924d44 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -333,7 +333,7 @@ static void blkdev_bio_end_io(struct bio *bio)
}
if (should_dirty) {
- bio_check_pages_dirty(bio);
+ bio_check_pages_dirty(bio, false);
} else {
if (!bio_flagged(bio, BIO_NO_PAGE_REF)) {
struct bvec_iter_all iter_all;
diff --git a/fs/direct-io.c b/fs/direct-io.c
index f2cfdacfbb7f..6334a26cfd73 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -549,7 +549,8 @@ static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)
}
if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) {
- bio_check_pages_dirty(bio); /* transfers ownership */
+ /* Transfers ownership. */
+ bio_check_pages_dirty(bio, false);
} else {
struct bvec_iter_all iter_all;
diff --git a/fs/iomap.c b/fs/iomap.c
index 23ef63fd1669..071880fee018 100644
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -1593,7 +1593,7 @@ static void iomap_dio_bio_end_io(struct bio *bio)
}
if (should_dirty) {
- bio_check_pages_dirty(bio);
+ bio_check_pages_dirty(bio, false);
} else {
if (!bio_flagged(bio, BIO_NO_PAGE_REF)) {
struct bvec_iter_all iter_all;
diff --git a/include/linux/bio.h b/include/linux/bio.h
index ea73df36529a..06d334fe38f9 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -436,7 +436,7 @@ extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
gfp_t, int);
extern void bio_set_pages_dirty(struct bio *bio);
-extern void bio_check_pages_dirty(struct bio *bio);
+extern void bio_check_pages_dirty(struct bio *bio, bool from_gup);
void generic_start_io_acct(struct request_queue *q, int op,
unsigned long sectors, struct hd_struct *part);