Created
March 4, 2022 05:26
-
-
Save MolarFox/2b495de0f4a67df093e99a37ea203a6f to your computer and use it in GitHub Desktop.
FastAvro Basic writer + reader class for example file generation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python3 | |
| from typing import List | |
| from dataclasses import dataclass, field | |
| from fastavro import writer, reader, parse_schema | |
| # A simple base class to load in avros, anonymise the values of every record, and write back out | |
| # For generating non-identifying example files from live cust data, all conforming to schema | |
| @dataclass | |
| class AvroHandler: | |
| schema: dict = field(default_factory=dict) | |
| records: List[dict] = field(default_factory=list) | |
| def load_file(self, filepath: str) -> None: | |
| with open(filepath, 'rb') as in_file: | |
| rd = reader(in_file) | |
| self.schema = rd.writer_schema | |
| self.records = [rec for rec in rd] | |
| def mutate_records(self) -> None: | |
| raise NotImplementedError("Record mutator method must be implemented before use") | |
| def write_file(self, filepath:str) -> None: | |
| with open(filepath, 'wb') as out_file: | |
| writer(out_file, self.schema, self.records) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment