Created
April 23, 2021 21:46
-
-
Save xavier83ar/2e3ae6f7de987f72c26bab1e58276b60 to your computer and use it in GitHub Desktop.
Export big data volumes. PHP + CakePHP (3-4) + CSV
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <?php | |
| /** | |
| * Created by javier | |
| * Date: 23/4/21 | |
| * Time: 18:35 | |
| */ | |
| namespace App\Controller; | |
| /** | |
| * Class ExampleController | |
| * @package App\Controller | |
| * | |
| * @property \Cake\ORM\Table Entities Ficticial table object | |
| */ | |
| class ExampleController extends AppController | |
| { | |
| /** | |
| * @return \Cake\Http\Response | |
| */ | |
| public function downloadCsv() | |
| { | |
| // Prepare query | |
| $query = $this->Entities->find(); | |
| // headers for the first line | |
| $headers = [ | |
| 'Field One', | |
| 'Field Two', | |
| 'Field Three', | |
| ]; | |
| $memFile = fopen('php://memory', 'w'); | |
| fputcsv($memFile, $headers); | |
| // set a page limit, a greater limit, then more memory usage | |
| $pageSize = 1000; | |
| $total = $query->count(); | |
| $pages = intval(floor($total/$pageSize)); | |
| $page = 1; | |
| $query | |
| ->limit($pageSize) | |
| ->page($page); | |
| $entities = $query->all(); | |
| while ($entities->count() > 0) { | |
| foreach ($entities as $entity) { | |
| $datos = [ | |
| $entity->field_one, | |
| $entity->field_two, | |
| $entity->field_three, | |
| ]; | |
| fputcsv($memFile, $datos); | |
| } | |
| if ($page > $pages) { | |
| break; | |
| } | |
| $query | |
| ->clearResult() // reset query results | |
| ->page($page++); // set next page | |
| $entities = $query->all(); | |
| } | |
| rewind($memFile); | |
| $content = stream_get_contents($memFile); | |
| fclose($memFile); | |
| return $this->response | |
| ->withStringBody($content) | |
| ->withDownload('exported-data.csv'); | |
| } | |
| } |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
A very basic example on how to deal with big data volumes (records from the database) and exporting it to a csv file, using CakePHP (version 3 and up).
This way I was able to export hundreds of thousands rows, with no more than 128 MB for PHP, wich was impossible looping over the main query without paging it.