Skip to content

Instantly share code, notes, and snippets.

@pineapplemachine

pineapplemachine/pokeditor_csv_formatter.py

Last active October 9, 2022 17:47
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save pineapplemachine/b6ad5e9e94a502f5e6781273f256cb94 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save pineapplemachine/b6ad5e9e94a502f5e6781273f256cb94 to your computer and use it in GitHub Desktop.
Download ZIP
PokEditor human-readable CSV formatter
Raw
pokeditor_csv_formatter.py
"""
This is a script that can be used in conjunection with the PokEditor tool.
https://github.com/turtleisaac/PokEditor-v2
When using PokEditor locally, without online Google Sheets, game data is
represented in a set of CSVs in the `local/` subdirectory of the user's
chosen project directory.
These CSVs contain Excel-style Google Sheets formulas that are used to
refer to the value in another CSV. For example, the CSV which contains
mon evolution data will use the text `=Personal!$B$2` to refer to
the species name `BULBASAUR`, which is defined in the second column
of the second row of `Personal.csv`.
Unfortunately, this presents an issue to anyone who would like to modify
these CSVs locally. The data is not easily human-readable in this form.
This script can be used to take a PokEditor project's `local/` directory
and resolve all of these cell formulae to their final value, and write
the resulting CSVs to an output directory. The CSVs in the output will
have, for instance, the text `BULBASAUR` instead of the formula
`=Personal!$B$2`.
Use the `encode` command to do this.
$ python pokeditor_csv_formatter.py decode project/local project/local-readable
After making changes to the human-readable CSVs in this outputted
directory, the script can then be used to translate those human-readable
CSVs back into the more forumla-friendly format that PokEditor expects.
Use the `decode` command to do this.
$ python pokeditor_csv_formatter.py encode project/local project/local-readable
"""
import os
import sys
import csv
FORMATTING_CSV_NAME = "Formatting (DO NOT TOUCH)"
def print_usage():
print("Usage:")
print("$ python pokeditor_csv_formatter.py decode \"pokedit/project/local\" \"pokedit/project/local-human-readable\"")
print("$ python pokeditor_csv_formatter.py encode \"pokedit/project/local\" \"pokedit/project/local-human-readable\"")
def read_pokedit_csv(csv_path):
rows = []
with open(csv_path, "rt", encoding="utf-8") as csv_file:
for line in csv_file.readlines():
line_stripped = line[:-2] if line.endswith("\r\n") else line[:-1]
rows.append(line_stripped.split(", "))
return rows
def write_pokedit_csv(csv_path, rows):
with open(csv_path, "wt", newline="", encoding="utf-8") as csv_file:
for row in rows:
csv_file.write(", ".join(row))
csv_file.write("\r\n")
class ProjectIndirection:
def __init__(self, name, column_index):
self.name = name
self.column_index = column_index
def __eq__(self, other):
return self.name == other.name and self.column_index == other.column_index
def encode(self, row_number):
name = "'%s'" % self.name if " " in self.name else self.name
column_name = chr(ord("A") + self.column_index)
return "=%s!$%s$%s" % (name, column_name, row_number)
class ProjectFile:
def __init__(self, project, name, encoded_rows=None, decoded_rows=None, indirections=None):
self.project = project
self.name = name
self.encoded_rows = encoded_rows
self.decoded_rows = decoded_rows
self.indirections = indirections
self.column_maps = {}
@classmethod
def read_encoded(cls, project, csv_path):
name = os.path.splitext(os.path.basename(csv_path))[0]
return cls(project, name, encoded_rows=read_pokedit_csv(csv_path))
@classmethod
def read_decoded(cls, project, csv_path, ind_path, enc_path):
name = os.path.splitext(os.path.basename(csv_path))[0]
indirections = {}
encoded_rows = None
with open(ind_path, "rt", newline="", encoding="utf-8") as ind_file:
reader = csv.reader(ind_file, delimiter=",")
for row in reader:
i = int(row[0])
if i not in indirections:
indirections[i] = []
indirections[i].append(ProjectIndirection(row[1], int(row[2])))
with open(csv_path, "rt", newline="", encoding="utf-8") as csv_file:
reader = csv.reader(csv_file, delimiter=",")
decoded_rows = list(reader)
if os.path.exists(enc_path):
encoded_rows = read_pokedit_csv(enc_path)
return cls(
project,
name,
encoded_rows=encoded_rows,
decoded_rows=decoded_rows,
indirections=indirections,
)
def write_encoded(self, csv_path):
write_pokedit_csv(csv_path, self.encoded_rows)
def write_decoded(self, csv_path, ind_path):
with open(csv_path, "wt", newline="", encoding="utf-8") as csv_file:
writer = csv.writer(csv_file, delimiter=",")
writer.writerows(self.decoded_rows)
with open(ind_path, "wt", newline="", encoding="utf-8") as ind_file:
writer = csv.writer(ind_file, delimiter=",")
for column_index, indirection_list in self.indirections.items():
for indirection in indirection_list:
writer.writerow([
column_index,
indirection.name,
indirection.column_index,
])
def get_indirect_cell(self, column_index, row_number):
if self.decoded_rows is not None:
return self.decoded_rows[row_number - 1][column_index]
else:
return self.encoded_rows[row_number - 1][column_index]
def find_indirect_cell(self, column_index, value):
if column_index not in self.column_maps:
self.column_maps[column_index] = self.build_column_map(column_index)
return self.column_maps[column_index].get(value, None)
def build_column_map(self, column_index):
column_map = {}
for row_index, row in enumerate(self.decoded_rows):
column_map[row[column_index]] = 1 + row_index
return column_map
def decode_row(self, encoded_row):
decoded_row = list()
for column_index, encoded_cell in enumerate(encoded_row):
decoded_row.append(self.decode_cell(column_index, encoded_cell))
return decoded_row
def decode_cell(self, column_index, cell):
parts = None
if cell.startswith("='"):
parts = cell[2:].split("$")
if not parts[0].endswith("'!"):
raise ValueError("Malformed indirection in %s." % self.name)
parts[0] = parts[0][:-2]
elif cell.startswith("="):
parts = cell[1:].split("$")
if not parts[0].endswith("!"):
raise ValueError("Malformed indirection in %s." % self.name)
parts[0] = parts[0][:-1]
if parts is not None:
ind_name = parts[0]
ind_column_index = ord(parts[1]) - ord("A")
ind_row_number = int(parts[2])
ind_file = self.project.files_map[ind_name]
ind = ProjectIndirection(ind_name, ind_column_index)
if column_index not in self.indirections:
self.indirections[column_index] = [ind]
elif ind not in self.indirections[column_index]:
self.indirections[column_index].append(ind)
try:
return ind_file.get_indirect_cell(
ind_column_index, ind_row_number
)
except:
return cell
else:
return cell
def decode(self):
self.decoded_rows = []
self.indirections = {}
for encoded_row in self.encoded_rows:
self.decoded_rows.append(self.decode_row(encoded_row))
def encode_row(self, decoded_row):
encoded_row = list()
for column_index, decoded_cell in enumerate(decoded_row):
encoded_row.append(self.encode_cell(column_index, decoded_cell))
return encoded_row
def encode_cell(self, column_index, cell):
if column_index not in self.indirections:
return cell
else:
for ind in self.indirections[column_index]:
ind_file = self.project.files_map[ind.name]
row_number = ind_file.find_indirect_cell(ind.column_index, cell)
if self.name != "TM Learnsets" and ind.name == "Personal" and cell == " BULBASAUR":
raise ValueError("Wtf %s %s", self.name, row_number)
if row_number is not None:
return ind.encode(row_number)
return cell
def encode(self):
self.encoded_rows = []
for row_index, decoded_row in enumerate(self.decoded_rows):
if row_index == 0:
self.encoded_rows.append(decoded_row)
else:
self.encoded_rows.append(self.encode_row(decoded_row))
class Project:
def __init__(self):
self.files_map = {}
def read_encoded(self, project_path):
for root, dirs, files in os.walk(project_path):
for file_name in files:
if not file_name.endswith(".csv"):
continue
file_path = os.path.join(root, file_name)
project_file = ProjectFile.read_encoded(self, file_path)
self.files_map[project_file.name] = project_file
def read_decoded(self, project_path):
for root, dirs, files in os.walk(project_path):
for file_name in files:
if not file_name.endswith(".csv"):
continue
csv_path = os.path.join(root, file_name)
base_path = os.path.splitext(csv_path)[0]
ind_path = base_path + ".ind"
enc_path = base_path + ".enc"
project_file = ProjectFile.read_decoded(
self, csv_path, ind_path, enc_path
)
self.files_map[project_file.name] = project_file
def decode_files(self):
self.files_map[FORMATTING_CSV_NAME].decode()
for file in self.files_map.values():
if file.decoded_rows is None:
file.decode()
def encode_files(self):
for file in self.files_map.values():
if file.encoded_rows is None:
file.encode()
def write_encoded(self, project_path):
os.makedirs(project_path, exist_ok=True)
for file in self.files_map.values():
csv_path = os.path.join(project_path, file.name) + ".csv"
file.write_encoded(csv_path)
def write_decoded(self, project_path):
os.makedirs(project_path, exist_ok=True)
for file in self.files_map.values():
csv_path = os.path.join(project_path, file.name) + ".csv"
ind_path = os.path.join(project_path, file.name) + ".ind"
enc_path = os.path.join(project_path, file.name) + ".enc"
file.write_decoded(csv_path, ind_path)
if file.name == FORMATTING_CSV_NAME:
file.write_encoded(enc_path)
def test_project(encoded_input_path, decoded_output_path):
if encoded_input_path == decoded_output_path:
raise ValueError("Project input and output paths must be different.")
project_test_path = decoded_output_path + ".test"
decode_project(encoded_input_path, decoded_output_path)
encode_project(decoded_output_path, project_test_path)
any_not_matched = False
for root, dirs, files in os.walk(encoded_input_path):
for file_name in files:
if file_name.endswith(".csv"):
in_path = os.path.join(root, file_name)
out_path = os.path.join(project_test_path, file_name)
with open(in_path, "rt", encoding="utf-8") as in_file:
in_content = in_file.read()
with open(out_path, "rt", encoding="utf-8") as out_file:
out_content = out_file.read()
result = in_content == out_content
any_not_matched = any_not_matched or (not result)
print("%s: %s matches %s" % (result, in_path, out_path))
if any_not_matched:
print("TEST FAILED")
else:
print("TEST PASSED")
def encode_project(encoded_output_path, decoded_input_path):
if decoded_input_path == encoded_output_path:
raise ValueError("Project input and output paths must be different.")
project = Project()
project.read_decoded(decoded_input_path)
project.encode_files()
project.write_encoded(encoded_output_path)
def decode_project(encoded_input_path, decoded_output_path):
if encoded_input_path == decoded_output_path:
raise ValueError("Project input and output paths must be different.")
project = Project()
project.read_encoded(encoded_input_path)
project.decode_files()
project.write_decoded(decoded_output_path)
def __main__():
if len(sys.argv) != 4:
print_usage()
sys.exit(1)
elif sys.argv[1] == "decode":
decode_project(sys.argv[2], sys.argv[3])
elif sys.argv[1] == "encode":
encode_project(sys.argv[2], sys.argv[3])
elif sys.argv[1] == "test":
test_project(sys.argv[2], sys.argv[3])
else:
print_usage()
sys.exit(1)
if __name__ == "__main__":
__main__()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment