Last active
May 5, 2025 13:41
-
-
Save sunflower2333/daae8cc7c1f1c292aa8992e9018780c4 to your computer and use it in GitHub Desktop.
A new memory map generator for QC's new platforms (>8450)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The new qc platforms uses XBL Device Tree as its memory map source instead | |
| # if using uefi.cfg. | |
| # A dtb file is also easy to be parsed with libufdt. | |
| from pyfdt.pyfdt import FdtBlobParse, FdtNode, FdtProperty | |
| SIZE_2GB = 0x80000000 | |
| SIZE_4GB = 2 * SIZE_2GB | |
| RT_LIST = ["SYS_MEM", "MMAP_IO", | |
| "IO", "FIRMWARE_DEV", | |
| "MMAP_IO_PORT", "MEM_RES", | |
| "IO_RES", "EFI_RESOURCE_MEMORY_UNACCEPTED", | |
| "EFI_RESOURCE_MAX_MEMORY_TYPE"] | |
| HOB_LIST = ["AddMem", "NC", "NC", "NC", "NoHob", "AddDev", "HobOnlyNoCacheSetting", "AllocOnly", "NC", "NoMap", | |
| "MaxMem"] | |
| MT_LIST = ["Reserv", "LdCode", "LdData", "BsCode", "BsData", | |
| "RtCode", "RtData", "Conv", "Unusable", | |
| "ACPIReclaim", "ACPI_NVS ", "MmIO", "MmIOPort", | |
| "PalCode", "EfiMaxMemoryType"] | |
| CA_DICT = {9: "NS_DEVICE", | |
| 32: "WRITE_THROUGH_XN", | |
| 2: "WRITE_BACK", | |
| 34: "WRITE_BACK_XN", | |
| 37: "UNCACHED_UNBUFFERED_XN"} | |
| def mask_ra(ra) -> str: | |
| ra_string = "" | |
| # Most Common & Long attribute, just return short name | |
| if ra == 0x703C07: | |
| ra_string = "SYS_MEM_CAP" | |
| return ra_string | |
| if ra & 0x00000001: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_PRESENT" | |
| if ra & 0x00000002: | |
| ra_string += "|INITIALIZED" | |
| if ra & 0x00000004: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_TESTED" | |
| if ra & 0x00000008: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_SINGLE_BIT_ECC" | |
| if ra & 0x00000010: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_MULTIPLE_BIT_ECC" | |
| if ra & 0x00000020: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_ECC_RESERVED_1" | |
| if ra & 0x00000040: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_ECC_RESERVED_2" | |
| if ra & 0x00000080: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_READ_PROTECTED" | |
| if ra & 0x00000100: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_WRITE_PROTECTED" | |
| if ra & 0x00000200: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_EXECUTION_PROTECTED" | |
| if ra & 0x00000400: | |
| ra_string += "|UNCACHEABLE" | |
| if ra & 0x00000800: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_WRITE_COMBINEABLE" | |
| if ra & 0x00001000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_WRITE_THROUGH_CACHEABLE" | |
| if ra & 0x00002000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_WRITE_BACK_CACHEABLE" | |
| if ra & 0x00004000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_16_BIT_IO" | |
| if ra & 0x00008000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_32_BIT_IO" | |
| if ra & 0x00010000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_64_BIT_IO" | |
| if ra & 0x00020000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_UNCACHED_EXPORTED" | |
| if ra & 0x00040000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_READ_ONLY_PROTECTED" | |
| if ra & 0x00080000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_READ_ONLY_PROTECTABLE" | |
| if ra & 0x00100000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_READ_PROTECTABLE" | |
| if ra & 0x00200000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_WRITE_PROTECTABLE" | |
| if ra & 0x00400000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_EXECUTION_PROTECTABLE" | |
| if ra & 0x00800000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_PERSISTENT" | |
| if ra & 0x01000000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_PERSISTABLE" | |
| if ra & 0x02000000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_MORE_RELIABLE" | |
| if ra & 0x04000000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_ENCRYPTED" | |
| if ra & 0x08000000: | |
| ra_string += "|EFI_RESOURCE_ATTRIBUTE_SPECIAL_PURPOSE" | |
| # Remove the first '|' | |
| ra_string = ra_string[1:] | |
| return ra_string | |
| class Region: | |
| base = 0 | |
| size = 0 | |
| label = None | |
| resource_attr = "" | |
| build_hob = "" | |
| resource_type = 0 | |
| memory_type = 0 | |
| cache_attributes = 0 | |
| def __init__(self, base, size, label, ra, bh, rt, mt, ca): | |
| self.base = base | |
| self.size = size | |
| self.label = label.replace("_", " ") | |
| self.resource_attr = mask_ra(ra) | |
| self.build_hob = HOB_LIST[bh] if bh < len(HOB_LIST) else "0x" + str(bh) | |
| self.resource_type = RT_LIST[rt] if rt < len(RT_LIST) else "0x" + str(rt) | |
| self.memory_type = MT_LIST[mt] if mt < len(MT_LIST) else "0x" + str(mt) | |
| self.cache_attributes = CA_DICT[ca] | |
| def __str__(self): | |
| # Generate C style codes | |
| return ( | |
| f"\t{{ \"{self.label}\",{(16 - len(self.label)) * ' '}0x{self.base:08X}, 0x{self.size:08X}, {self.build_hob}, " | |
| f"{self.resource_type}, {self.resource_attr}, {self.memory_type}, {self.cache_attributes} }},\n") | |
| def find_cells_by_reg(path, cells_name, obj_fdt): | |
| cells_val = 0 | |
| folders = path.split('/') | |
| folders.pop(-1) | |
| for f in reversed(folders): | |
| if None is not obj_fdt.resolve_path(path + cells_name): | |
| cells_val = obj_fdt.resolve_path(path + cells_name)[0] | |
| return cells_val | |
| if f != '': | |
| path = path.removesuffix(f + '/') | |
| return cells_val # Not found | |
| def mem_node_to_memory_region_by_cells(this_addr_cells, this_size_cells, this_path, dtb) -> Region: | |
| # Get region base and size | |
| dt_reg = dtb.resolve_path(this_path + "reg") | |
| if dtb.resolve_path(this_path + "mem-label") is None: | |
| mem_label_prop_str = "MemLabel" | |
| resource_attribute_prop_str = "ResourceAttribute" | |
| build_hob_prop_str = "BuildHob" | |
| resource_type_prop_str = "ResourceType" | |
| memory_type_prop_str = "MemoryType" | |
| cache_attributes_prop_str = "CacheAttributes" | |
| else: | |
| mem_label_prop_str = "mem-label" | |
| resource_attribute_prop_str = "resource-attribute" | |
| build_hob_prop_str = "build-hob" | |
| resource_type_prop_str = "resource-type" | |
| memory_type_prop_str = "memory-type" | |
| cache_attributes_prop_str = "cache-attributes" | |
| # [addr, size] | |
| if this_addr_cells == 1: | |
| return Region(dt_reg[0], | |
| dt_reg[1] if this_size_cells == 1 else (dt_reg[1] << 32) + dt_reg[2], | |
| label=dtb.resolve_path(this_path + mem_label_prop_str)[0], | |
| ra=dtb.resolve_path(this_path + resource_attribute_prop_str)[0], | |
| bh=dtb.resolve_path(this_path + build_hob_prop_str)[0], | |
| rt=dtb.resolve_path(this_path + resource_type_prop_str)[0], | |
| mt=dtb.resolve_path(this_path + memory_type_prop_str)[0], | |
| ca=dtb.resolve_path(this_path + cache_attributes_prop_str)[0] | |
| ) | |
| if this_addr_cells == 2: | |
| return Region((dt_reg[0] << 32) + dt_reg[1], | |
| dt_reg[2] if this_size_cells == 1 else | |
| (dt_reg[2] << 32) + dt_reg[3], | |
| label=dtb.resolve_path(this_path + mem_label_prop_str)[0], | |
| ra=dtb.resolve_path(this_path + resource_attribute_prop_str)[0], | |
| bh=dtb.resolve_path(this_path + build_hob_prop_str)[0], | |
| rt=dtb.resolve_path(this_path + resource_type_prop_str)[0], | |
| mt=dtb.resolve_path(this_path + memory_type_prop_str)[0], | |
| ca=dtb.resolve_path(this_path + cache_attributes_prop_str)[0] | |
| ) | |
| class DDRBank: | |
| def __init__(self, base, size): | |
| self.base = base | |
| self.size = size | |
| def ddr_bank_to_msg(self): | |
| msg = "/* RAM Entry: Base 0x%016X\tSize 0x%016X" % (self.base, self.size) + " */" | |
| return msg | |
| def check_if_region_locates_in_bank(self, descriptor) -> bool: | |
| return (descriptor.base + descriptor.size <= self.base + self.size) and (descriptor.base >= self.base) | |
| def regs_to_ddr_banks_by_cells(this_addr_cells, this_size_cells, this_regs): | |
| this_ddr_banks = [] | |
| # [addr, size, addr, size, ...] | |
| for i in range(0, len(this_regs), this_addr_cells + this_size_cells): | |
| if this_addr_cells == 1: | |
| this_ddr_banks.append( | |
| DDRBank(this_regs[i], | |
| this_regs[i + 1] if this_size_cells == 1 else (this_regs[i + 1] << 32) + this_regs[i + 2])) | |
| if this_addr_cells == 2: | |
| this_ddr_banks.append(DDRBank((this_regs[i] << 32) + this_regs[i + 1], | |
| this_regs[i + 2] if this_size_cells == 1 else | |
| (this_regs[i + 2] << 32) + this_regs[i + 3])) | |
| return this_ddr_banks | |
| def generate_conv_region(base, size): | |
| # RA -> SYS_MEM_CAP | |
| # CA -> WRITE_BACK | |
| return Region(base, size, "RAM Partition", 0x703C07, HOB_LIST.index("AddMem"), | |
| RT_LIST.index("SYS_MEM"), MT_LIST.index("Conv"), 2) | |
| # Check Overlap in a sorted memory region descriptor array. | |
| def overlap_checker(in_descriptors): | |
| descriptors = in_descriptors.copy() | |
| # Firstly, sorted this array by region base | |
| descriptors.sort(key=lambda d: d.base) | |
| # Secondly, check overlap. | |
| overlap_flag = False | |
| last_desc = descriptors[0] | |
| descriptors.pop(0) | |
| for desc in descriptors: | |
| if last_desc.base + last_desc.size > desc.base: | |
| print("WARNING!!! Overlap Detected!") | |
| print("last_descriptor:\n\tbase: " + hex(last_desc.base) + ",\tsize: " + hex(last_desc.size)) | |
| print("current_descriptor:\n\tbase: " + hex(desc.base) + ",\tsize: " + hex(desc.size)) | |
| overlap_flag = True | |
| continue | |
| last_desc = desc | |
| continue | |
| if overlap_flag: | |
| raise MemoryError("Please Check Your Uefiplat.cfg !!!") | |
| # Check if there are memory gaps in given ddr bank and memory descriptors. | |
| def gap_checker(ddr_regions, in_descriptors): | |
| # backup descriptors | |
| result = in_descriptors.copy() | |
| descriptors = in_descriptors.copy() | |
| # Insure descriptors array is sorted. | |
| descriptors.sort(key=lambda d: d.base) | |
| # Firstly, check if all regions is valid. | |
| for desc in descriptors: | |
| available_flag = False | |
| for ddr_bank in ddr_regions: | |
| if ddr_bank.check_if_region_locates_in_bank(desc): | |
| available_flag = True | |
| if not available_flag: | |
| desc_info = "base: " + hex(desc.base) + ", \t size:" + hex(desc.size) | |
| print("// WARNING: gap_checker failed. Unknown error happen." + desc_info) | |
| # raise MemoryError("gap_checker failed. Unknown error happen.") | |
| # Secondly, check if there are gaps near regions | |
| last_desc = descriptors[0] | |
| descriptors.pop(0) | |
| for desc in descriptors: | |
| if last_desc.base + last_desc.size < desc.base: | |
| # Fill conventional regions here. | |
| # 1, Calculate conv region base and size. | |
| conv_region = generate_conv_region(last_desc.base + last_desc.size, | |
| desc.base - (last_desc.base + last_desc.size)) | |
| # 2. Check if conv region available in DDR banks. | |
| for ddr_bank in ddr_regions: | |
| if not ddr_bank.check_if_region_locates_in_bank(last_desc): | |
| continue | |
| if not ddr_bank.check_if_region_locates_in_bank(conv_region): | |
| conv_region.size = (ddr_bank.base + ddr_bank.size) - conv_region.base | |
| if conv_region.size: | |
| result.append(conv_region) | |
| last_desc = desc | |
| # Finally, sorted the array before returning. | |
| result.sort(key=lambda d: d.base) | |
| return result | |
| # This function will split a memory region which is greater than 4GB into 4GB pieces in a list. | |
| def split_region(descriptor): | |
| result = [] | |
| tmp_size = descriptor.size | |
| tmp_base = descriptor.base | |
| while tmp_size >= SIZE_4GB: | |
| result.append(generate_conv_region(tmp_base, SIZE_4GB)) | |
| tmp_size -= SIZE_4GB | |
| tmp_base += SIZE_4GB | |
| if tmp_size: | |
| result.append(generate_conv_region(tmp_base, tmp_size)) | |
| return result | |
| # In this function, all the unmapped memory in | |
| # ddr bank will be filled be conventional system memory. | |
| def ddr_filler(ddr_regions, in_descriptor): | |
| descriptor = in_descriptor.copy() | |
| # Make sure the array is sorted. | |
| descriptor.sort(key=lambda d: d.base) | |
| # Firstly, find the last region and all unmapped banks. | |
| passed_banks_flag = False | |
| for ddr_region in ddr_regions: | |
| if not passed_banks_flag: | |
| if not ddr_region.check_if_region_locates_in_bank(descriptor[-1]): | |
| continue | |
| # if (descriptor[-1].base + descriptor[-1].size) == (ddr_region.base + ddr_region.size): | |
| # continue | |
| passed_banks_flag = True | |
| descriptor += split_region(generate_conv_region(descriptor[-1].base + descriptor[-1].size, | |
| (ddr_region.base + ddr_region.size) - ( | |
| descriptor[-1].base + descriptor[-1].size))) | |
| else: | |
| # Secondly, split and fill the rest parts | |
| descriptor += split_region(generate_conv_region(ddr_region.base, ddr_region.size)) | |
| return descriptor | |
| class ConfigurationParameters: | |
| name = "" | |
| value = "" | |
| def __init__(self, name, value): | |
| self.name = name | |
| self.value = value | |
| def __str__(self): | |
| # EnableShell = 0x1 | |
| return f"\t{{\"{self.name}\", \"{self.value}\"}},\n" if type( | |
| self.value) == str else f"\t{{\"{self.name}\", 0x{self.value:04x}}},\n" | |
| if __name__ == '__main__': | |
| print("MemoryMap Generator V2") | |
| print("MIT License") | |
| print("Copyright (c) 2024-2025 Kancy Joe. All rights reserved.") | |
| # Open needed files | |
| xbl_dtb = open("post-ddr-pakala-1.0.dtb", 'rb') | |
| linux_dtb = open("linux_fdt", 'rb') | |
| mem_map_output = open("PlatformMemoryMapLib.c", 'wt') | |
| cfg_map_output = open("PlatformConfigurationMapLib.c", 'wt') | |
| """ | |
| Memory Map Operations | |
| """ | |
| # Init fdt object | |
| parsed_xbl_dtb = FdtBlobParse(xbl_dtb).to_fdt() | |
| mem_path = "/soc/memorymap/" # the last '/' is important | |
| register_path = "/soc/registermap/" # the last '/' is important | |
| tz_path = "/soc/memorymap/" # the last '/' is important | |
| # Get cells | |
| xbl_addr_cells = find_cells_by_reg(mem_path, "#address-cells", parsed_xbl_dtb) | |
| xbl_size_cells = find_cells_by_reg(mem_path, "#size-cells", parsed_xbl_dtb) | |
| # Memory Map template | |
| DDR_regions: list[Region] = [] | |
| Register_regions: list[Region] = [] | |
| # Parse Memory Sub-nodes | |
| # Get all ddr nodes | |
| ddr_nodes = parsed_xbl_dtb.resolve_path(mem_path) | |
| for node in ddr_nodes: | |
| if isinstance(node, FdtNode): | |
| DDR_regions.append( | |
| mem_node_to_memory_region_by_cells(xbl_addr_cells, xbl_size_cells, mem_path + node.name + "/", | |
| parsed_xbl_dtb)) | |
| # Sort regions by memory base. | |
| DDR_regions.sort(key=lambda r: r.base) | |
| # Init Linux DTB Object | |
| DDR_Banks = [] | |
| mem_bank_path = "/memory/reg/" | |
| parsed_linux_dtb = FdtBlobParse(linux_dtb).to_fdt() | |
| # Get cells | |
| linux_addr_cells = find_cells_by_reg(mem_bank_path, "#address-cells", parsed_linux_dtb) | |
| linux_size_cells = find_cells_by_reg(mem_bank_path, "#size-cells", parsed_linux_dtb) | |
| # Get banks | |
| ddr_banks = regs_to_ddr_banks_by_cells(linux_addr_cells, linux_size_cells, | |
| parsed_linux_dtb.resolve_path(mem_bank_path)) | |
| ddr_banks.sort(key=lambda d: d.base) | |
| # for bank in ddr_banks: | |
| # print(bank.ddr_bank_to_msg()) | |
| # Check if regions are in banks | |
| # Check for overlap. | |
| overlap_checker(DDR_regions) | |
| # Find and fill memory gaps between regions | |
| DDR_regions = gap_checker(ddr_banks, DDR_regions) | |
| # Fill the reset of spaces in ddr bank with conventional regions. | |
| DDR_regions = ddr_filler(ddr_banks, DDR_regions) | |
| # Parse Register Sub-nodes | |
| register_nodes = parsed_xbl_dtb.resolve_path(register_path) | |
| for node in register_nodes: | |
| if isinstance(node, FdtNode): | |
| Register_regions.append( | |
| mem_node_to_memory_region_by_cells(xbl_addr_cells, xbl_size_cells, register_path + node.name + "/", | |
| parsed_xbl_dtb)) | |
| map_temp = f"""// This file is auto-generated by MemoryMapGeneratorV2 | |
| #include <Library/BaseLib.h> | |
| #include <Library/PlatformMemoryMapLib.h> | |
| static ARM_MEMORY_REGION_DESCRIPTOR_EX gDeviceMemoryDescriptorEx[] = {{ | |
| /* Name Address Length HobOption ResourceAttribute ArmAttributes | |
| ResourceType MemoryType */ | |
| /* DDR Memory Regions */ | |
| {''.join(["\t" + bank.ddr_bank_to_msg() + "\n" for bank in ddr_banks])} | |
| {''.join([str(r) for r in DDR_regions])} | |
| /* Register regions */ | |
| {''.join([str(r) for r in Register_regions])} | |
| /* Terminator for MMU */ | |
| {{"Terminator", 0, 0, 0, 0, 0, 0, 0}}}}; | |
| ARM_MEMORY_REGION_DESCRIPTOR_EX *GetPlatformMemoryMap() | |
| {{ | |
| return gDeviceMemoryDescriptorEx; | |
| }} | |
| """ | |
| # print(map_temp) | |
| mem_map_output.write(map_temp) | |
| """ | |
| Configuration Map Operations | |
| """ | |
| # Set path | |
| config_path = "/sw/uefi/" # the last '/' is important | |
| # Var to store configurations pair | |
| int_params = [] | |
| str_params = [] | |
| # Walk the path and print | |
| configs = parsed_xbl_dtb.resolve_path(config_path + "int_param/") | |
| if configs is None: | |
| print("Processing xbl dtb without uefi configuration inside.") | |
| print("\tWARNING: PlatformConfigurationMapLib.c will not be generated.") | |
| print("\tPlease check uefiplat.cfg in your extracted uefi fw!") | |
| else: | |
| for c in configs: | |
| if isinstance(c, FdtProperty): | |
| # Hypothesis the cells = 2. | |
| int_params.append(ConfigurationParameters(c.name, (c[0] << 32 | c[1]))) | |
| # print(f"{c.get_name()}, 0x{(c[0] << 32 | c[1]):08x}") | |
| # Not need currently, reserved for future use. | |
| configs = parsed_xbl_dtb.resolve_path(config_path + "str_param/") | |
| for c in configs: | |
| if isinstance(c, FdtProperty): | |
| str_params.append(ConfigurationParameters(c.name, c[0])) | |
| # print(f"{c.get_name()}, {c[0]}") | |
| # Configuration map template | |
| config_temp = f"""// This file is auto-generated by MemoryMapGeneratorV2 | |
| #include <Library/BaseLib.h> | |
| #include <Library/PlatformConfigurationMapLib.h> | |
| static CONFIGURATION_DESCRIPTOR_EX gDeviceConfigurationDescriptorEx[] = {{ | |
| /* Int Parameters */ | |
| {''.join([str(p) for p in int_params])} | |
| /* String Parameters */ | |
| // Not needed at here. | |
| /* Terminator */ | |
| {{"Terminator", 0xFFFFFFFF}}}}; | |
| CONFIGURATION_DESCRIPTOR_EX *GetPlatformConfigurationMap() | |
| {{ | |
| return gDeviceConfigurationDescriptorEx; | |
| }} | |
| """ | |
| # {''.join([str(p) for p in str_params])} | |
| # print(config_temp) | |
| cfg_map_output.write(config_temp) | |
| # Clean up | |
| xbl_dtb.close() | |
| linux_dtb.close() | |
| mem_map_output.close() | |
| cfg_map_output.close() | |
| print("Done!") |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
How to use
/sys/firmware/fdtOutput
Output will be write into files, format like: