Sriram Chandramouli sriramch
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| the dataset is ~ 80gb | |
| external memory is turned on | |
| i do not think the cache files are created on the backing store. it is trying to make the entire dataset resident | |
| sample config: | |
| booster = gbtree | |
| objective = binary:logistic | |
| verbosity = 3 | |
| #single_precision_histogram = true |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/src/common/hist_util.cu b/src/common/hist_util.cu | |
| index b4cc797..1981264 100644 | |
| --- a/src/common/hist_util.cu | |
| +++ b/src/common/hist_util.cu | |
| @@ -96,9 +96,8 @@ struct GPUSketcher { | |
| bool has_weights_{false}; | |
| tree::TrainParam param_; | |
| - std::vector<WXQSketch> sketches_; | |
| + SketchContainer &sketch_container_; |
sriramch
/ xgboost_training_ext_mem
Created
April 25, 2019 16:57
xgboost external memory training patch
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/src/common/hist_util.cu b/src/common/hist_util.cu | |
| index b4cc797..5aab67d 100644 | |
| --- a/src/common/hist_util.cu | |
| +++ b/src/common/hist_util.cu | |
| @@ -349,7 +349,7 @@ struct GPUSketcher { | |
| }; | |
| void Sketch(const SparsePage& batch, const MetaInfo& info, | |
| - HistCutMatrix* hmat, int gpu_batch_nrows) { | |
| + std::vector<WXQSketch> &sketches, int gpu_batch_nrows) { |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/src/common/host_device_vector.h b/src/common/host_device_vector.h | |
| index 425cbff..4cb1677 100644 | |
| --- a/src/common/host_device_vector.h | |
| +++ b/src/common/host_device_vector.h | |
| @@ -132,7 +132,6 @@ class GPUDistribution { | |
| if (size == 0) { return 0; } | |
| if (offsets_.size() > 0) { | |
| // explicit offsets are provided | |
| - CHECK_EQ(offsets_.back(), size); | |
| return offsets_.at(index); |