Skip to content

Instantly share code, notes, and snippets.

@rvtr
Last active June 24, 2025 23:08
Show Gist options
  • Select an option

  • Save rvtr/9cca361f404f07052d5870220748519e to your computer and use it in GitHub Desktop.

Select an option

Save rvtr/9cca361f404f07052d5870220748519e to your computer and use it in GitHub Desktop.
Scrapes planning applications from london.ca
#!/bin/bash
# Hey folks, please do not run this script more than necessary.
# Too many search requests will temporarily block searches for everyone, not just you.
# I do not want to DDOS London. I just want to allow for personal backups. Cheers!
echo -e "\n-========================================================================-"
echo -e "-=- -=-"
echo -e "-=- SCRAPE_PLANAPPS.SH: Downloads planning applications -=-"
echo -e "-=- -=-"
echo -e "-=- https://gist.github.com/rvtr/9cca361f404f07052d5870220748519e -=-"
echo -e "-=- Lillian Skinner (2025) -=-"
echo -e "-=- -=-"
echo -e "-========================================================================-"
echo "Starting job: SCRAPE_PLANAPPS: $(date)"
# Warning to all who read this script:
# It is badly written. I know it is bad, but I am tired okay, and sometimes sloppy just works.
# London seems to have recently blocked unusual user agents. Can't use wget or even ping. Thankfully pretend to be a real person!
WGET_UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Viewer/99.9.8782.87"
TEMP_DIR="./tmp/"
SEARCH_PAGE="./tmp/index_pa.html"
PROJECT_PAGE="./tmp/work_pa.html"
PROJECT_INFO="./tmp/info.txt"
PROJECT_ATTACH_NAMES="./tmp/names.txt"
PROJECT_ATTACH_URLS="./tmp/urls.txt"
PROJECT_IMAGE_NAMES="./tmp/image-names.txt"
PROJECT_IMAGE_URLS="./tmp/image-urls.txt"
#if [ -d "$TEMP_DIR" ]; then
# rm -r $TEMP_DIR
#fi
rm -f $SEARCH_PAGE
rm -f $PROJECT_PAGE
mkdir $TEMP_DIR
SEARCH_URL="https://london.ca/business-development/planning-development-applications/planning-applications"
j=0
SEARCH_END="FALSE"
while [[ $SEARCH_END == "FALSE" ]]; do
echo "SCRAPE_PLANAPPS: -========================================================================-"
echo "SCRAPE_PLANAPPS: Downloading search results... Page $j"
wget --user-agent="$WGET_UA" $SEARCH_URL"?page=$j" -O $SEARCH_PAGE --timestamping -q #--show-progress
if [ $? -ne 8 ]; then
PAGE_HAS_APPS=$(cat $SEARCH_PAGE | grep "teaser__title")
if [[ "$PAGE_HAS_APPS" != "" ]]; then
while IFS= read -r LINE; do
rm -f $PROJECT_INFO
PAGE_FOUND_APP=$(echo $LINE | grep "teaser__title")
if [[ "$PAGE_FOUND_APP" != "" ]]; then
echo "SCRAPE_PLANAPPS: -========================================================================-"
echo "SCRAPE_PLANAPPS: Task starting on: $(date)"
PROJECT_URL=$(echo $LINE | sed 's/.*href="\([^"]*\)".*/\1/p' | uniq)
PROJECT_URL=$(echo "https://london.ca"$PROJECT_URL)
echo "SCRAPE_PLANAPPS: Downloading page..."
wget --user-agent="$WGET_UA" $PROJECT_URL -O $PROJECT_PAGE --timestamping -q #--show-progress
# Removing COVID is due to the naming in the 2020s. Keeping it for revisiting wayback crawls.
PROJECT_NAME=$(cat $PROJECT_PAGE | grep "page-title" | grep "field--name-title" | sed 's/.*<span[^>]*>\([^<]*\)<[\/:-]span>.*/\1/p' | sed 's/&amp;/\&/g' | sed 's/&#039;/'\''/g' | sed 's/^COVID-19//p' | uniq | tr -d '\r' | tr -d '\n' | tr '/' '-')
echo "SCRAPE_PLANAPPS: Found project: $PROJECT_NAME"
echo "SCRAPE_PLANAPPS: Finding attachments..."
rm -f $PROJECT_ATTACH_URLS
rm -f $PROJECT_ATTACH_NAMES
rm -f $PROJECT_IMAGE_URLS
rm -f $PROJECT_IMAGE_NAMES
while IFS= read -r PLINE; do
if [[ "$NEXT_LINE_FITEM" == "TRUE" ]]; then
NEXT_LINE_FITEM="FALSE"
# Is this line an actual item?
PROJECT_INFO_IS_ITEMS=$(echo $PLINE | grep "field__items")
# Is this line bad data (usually scripts)?
PROJECT_INFO_IS_BAD=$(echo $PLINE | grep "</script>")
# Gotta add in the &s and 's.
PROJECT_INFO_ITEM=$(echo $PLINE | sed 's/.*<div class="field__item">\(<time[^>]*>\)\?\([^<]*\).*/\2/p' | sed 's/&amp;/\&/g' | sed 's/&#039;/'\''/g' | uniq)
if [[ $PROJECT_INFO_IS_ITEMS == "" ]] && [[ $PROJECT_INFO_IS_BAD == "" ]] && [[ $PROJECT_INFO_ITEM != "" ]]; then
# We'll check to see if a non-info item made it in. Sometimes attachments will get caught, but can be detected by "visually-hidden"
PROJECT_INFO_LABEL_BAD=$(echo $PROJECT_INFO_LABEL | grep "visually-hidden")
if [[ $PROJECT_INFO_LABEL_BAD == "" ]]; then
printf "%-17s: %s\n" "$PROJECT_INFO_LABEL" "$PROJECT_INFO_ITEM" >> $PROJECT_INFO
if [[ $PROJECT_INFO_LABEL == "File Number" ]]; then
PROJECT_FILE_NUM_2=""
PROJECT_FILE_NUM_TYPE_2=""
PROJECT_FILE_NUM_IS_MULTI=""
# Multiple file numbers may be listed. We will always use the first one as it is contained in PDF names.
# I think it takes priority. Anyways, here are the possible formats:
# XX-#####
# XX-#####/XX-#####
# XX-##### / XX-#####
# XX-##### and XX-#####
# XX-##### & XX-#####
#
# I think the city is allergic to standardization...
PROJECT_FILE_NUM_IS_MULTI=$(echo $PROJECT_INFO_ITEM | grep -e "and" -e "/" -e "&")
PROJECT_FILE_NUM=$(echo $PROJECT_INFO_ITEM | sed 's|/.*||' | sed 's| and .*||' | sed 's| & .*||' | sed 's/^[[:space:]]*//g' | sed 's/[[:space:]]*$//g' | uniq)
PROJECT_FILE_NUM_TYPE=$(echo "$PROJECT_FILE_NUM" | sed 's/^\([^-]*\)-.*$/\1/')
if [[ "$PROJECT_FILE_NUM_TYPE" == "Line of Sight" ]]; then
PROJECT_FILE_TYPE="Line of Sight"
elif [[ "$PROJECT_FILE_NUM_TYPE" == "O" ]]; then
PROJECT_FILE_TYPE="Official Plan Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE" == "Z" ]]; then
PROJECT_FILE_TYPE="Zoning By-law Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE" == "OZ" ]]; then
PROJECT_FILE_TYPE="Official Plan and Zoning By-law Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE" == "TZ" ]]; then
PROJECT_FILE_TYPE="Temporary Zoning By-law Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE" == "39T" ]]; then
PROJECT_FILE_TYPE="Draft Plan of Subdivision"
elif [[ "$PROJECT_FILE_NUM_TYPE" == "39CD" ]]; then
PROJECT_FILE_TYPE="Draft Plan of Condominium"
elif [[ "$PROJECT_FILE_NUM_TYPE" =~ ^SPA2[0-9]+$ ]]; then
PROJECT_FILE_TYPE="Site Plan Control Application"
else
PROJECT_FILE_TYPE="BAD RECORD TYPE"
fi
echo "SCRAPE_PLANAPPS: Found file# : $PROJECT_FILE_NUM ($PROJECT_FILE_TYPE)"
if [[ "$PROJECT_FILE_NUM_IS_MULTI" != "" ]]; then
# It isn't great, but if a project has 2 file numbers then we'll save it as both.
# I'm not sure how to get around this since I don't have a way to tag files.
PROJECT_FILE_NUM_2=$(echo $PROJECT_INFO_ITEM | sed 's|.*/||' | sed 's|.* and ||' | sed 's|.* & ||' | sed 's/^[[:space:]]*//g' | sed 's/[[:space:]]*$//g' | uniq)
PROJECT_FILE_NUM_TYPE_2=$(echo "$PROJECT_FILE_NUM_2" | sed 's/^\([^-]*\)-.*$/\1/')
if [[ "$PROJECT_FILE_NUM_TYPE_2" == "Line of Sight" ]]; then
PROJECT_FILE_TYPE_2="Line of Sight"
elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "O" ]]; then
PROJECT_FILE_TYPE_2="Official Plan Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "Z" ]]; then
PROJECT_FILE_TYPE_2="Zoning By-law Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "OZ" ]]; then
PROJECT_FILE_TYPE_2="Official Plan and Zoning By-law Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "TZ" ]]; then
PROJECT_FILE_TYPE_2="Temporary Zoning By-law Amendment"
elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "39T" ]]; then
PROJECT_FILE_TYPE_2="Draft Plan of Subdivision"
elif [[ "$PROJECT_FILE_NUM_TYPE_2" == "39CD" ]]; then
PROJECT_FILE_TYPE_2="Draft Plan of Condominium"
elif [[ "$PROJECT_FILE_NUM_TYPE_2" =~ ^SPA2[0-9]+$ ]]; then
PROJECT_FILE_TYPE_2="Site Plan Control Application"
else
PROJECT_FILE_TYPE_2="BAD RECORD TYPE"
fi
echo "SCRAPE_PLANAPPS: Also filed as: $PROJECT_FILE_NUM_2 ($PROJECT_FILE_TYPE_2)"
fi
fi
PROJECT_FOUND_TIME=$(echo $PLINE | grep "datetime")
if [[ $PROJECT_FOUND_TIME != "" ]]; then
PROJECT_TIME_YEAR=$(echo $PLINE | sed 's/.*\([0-9]\{4\}\).*/\1/p' | uniq)
PROJECT_TIME_MONTH_WORD=$(echo $PLINE | sed 's/.*,\s*\([A-Za-z]*\)\s[0-9]\{1,2\},.*/\1/p' | uniq)
PROJECT_TIME_DAY_SHORT=$(echo $PLINE | sed 's/.*,\s*[A-Za-z]*\s\([0-9]\{1,2\}\),.*/\1/p' | uniq)
PROJECT_TIME_DAY=$(printf "%02d" $PROJECT_TIME_DAY_SHORT)
case "$PROJECT_TIME_MONTH_WORD" in
January) PROJECT_TIME_MONTH="01" ;;
February) PROJECT_TIME_MONTH="02" ;;
March) PROJECT_TIME_MONTH="03" ;;
April) PROJECT_TIME_MONTH="04" ;;
May) PROJECT_TIME_MONTH="05" ;;
June) PROJECT_TIME_MONTH="06" ;;
July) PROJECT_TIME_MONTH="07" ;;
August) PROJECT_TIME_MONTH="08" ;;
September) PROJECT_TIME_MONTH="09" ;;
October) PROJECT_TIME_MONTH="10" ;;
November) PROJECT_TIME_MONTH="11" ;;
December) PROJECT_TIME_MONTH="12" ;;
*) PROJECT_TIME_MONTH="--" ;;
esac
echo "SCRAPE_PLANAPPS: Found date : $PROJECT_TIME_YEAR/$PROJECT_TIME_MONTH/$PROJECT_TIME_DAY"
fi
fi
fi
fi
if [[ "$NEXT_LINE_IMAGE" == "TRUE" ]]; then
NEXT_LINE_IMAGE="FALSE"
PROJECT_IMAGE_URL=$(echo $PLINE | sed 's/.*<img[^>]*src="\([^"]*\)".*/\1/p' | sed 's/?.*//' | uniq)
PROJECT_IMAGE_URL_SHORT=$(echo $PLINE | grep "https://london.ca")
if [[ $PROJECT_IMAGE_URL_SHORT == "" ]];then
PROJECT_IMAGE_URL=$(echo "https://london.ca"$PROJECT_IMAGE_URL)
fi
PROJECT_IMAGE_NAME=$(echo $PROJECT_IMAGE_URL | sed 's#.*/##p' | uniq)
echo $PROJECT_IMAGE_URL >> $PROJECT_IMAGE_URLS
echo $PROJECT_IMAGE_NAME >> $PROJECT_IMAGE_NAMES
fi
PROJECT_FOUND_FILE=$(echo $PLINE | grep "file--mime-application-")
if [[ $PROJECT_FOUND_FILE != "" ]]; then
PROJECT_ATTACH_URL=$(echo $PLINE | sed 's/.*href="\([^"]*\)".*/\1/p' | uniq)
# Newer links are relative paths, so we must add back the domain
PROJECT_ATTACH_URL_SHORT=$(echo $PLINE | grep "https://london.ca")
if [[ $PROJECT_ATTACH_URL_SHORT == "" ]];then
PROJECT_ATTACH_URL=$(echo "https://london.ca"$PROJECT_ATTACH_URL)
fi
PROJECT_ATTACH_NAME=$(echo $PLINE | sed 's/.*title="\([^"]*\)".*/\1/p' | sed 's/&amp;/\&/g' | sed 's/&#039;/'\''/g' | uniq)
echo $PROJECT_ATTACH_URL >> $PROJECT_ATTACH_URLS
echo $PROJECT_ATTACH_NAME >> $PROJECT_ATTACH_NAMES
fi
PROJECT_FOUND_FLABEL=$(echo $PLINE | grep "field__label")
if [[ "$PROJECT_FOUND_FLABEL" != "" ]]; then
PROJECT_INFO_LABEL=$(echo $PLINE | sed 's/.*<div class="field__label">\(<time[^>]*>\)\?\([^<]*\).*/\2/p' | uniq)
NEXT_LINE_FITEM="TRUE"
# Info boxes will always have a label on one line, then the contents in the next. (except for contact info lol)
# We're setting a flag to let the script know if an upcoming line is contents.
fi
PROJECT_FOUND_IMAGE=$(echo $PLINE | grep "field__label visually-hidden" | grep "Image")
if [[ "$PROJECT_FOUND_IMAGE" != "" ]]; then
NEXT_LINE_IMAGE="TRUE"
# Same idea as before but for the image shown on the main page.
fi
PROJECT_FOUND_EMAIL=$(echo $PLINE | grep "field--name-field-email" | sed 's/.*href="\([^"]*\)".*/\1/p' | sed 's|^mailto:||' | uniq)
if [[ "$PROJECT_FOUND_EMAIL" != "" ]]; then
printf "%-17s: %s\n" "Email" "$PROJECT_FOUND_EMAIL" >> $PROJECT_INFO
fi
PROJECT_FOUND_PLANNER=$(echo $PLINE | grep "field--name-name" | sed 's/.*<div[^>]*>\([^<]*\)<[\/:-]div>.*/\1/p' | uniq)
if [[ "$PROJECT_FOUND_PLANNER" != "" ]]; then
printf "\n%-17s: %s\n" "Planner" "$PROJECT_FOUND_PLANNER" >> $PROJECT_INFO
fi
done < $PROJECT_PAGE
echo "SCRAPE_PLANAPPS: Filing away all the datas..."
mkdir "./LondonArchive"
mkdir "./LondonArchive/Planning Applications"
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/"
fi
if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE == $PROJECT_FILE_TYPE_2 ]]; then
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME/"
fi
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments"
fi
PROJECT_SAVE_PATH="./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM & $PROJECT_FILE_NUM_2 - $PROJECT_NAME"
else
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME/"
fi
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME/Attachments" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME/Attachments"
fi
PROJECT_SAVE_PATH="./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE/$PROJECT_FILE_NUM - $PROJECT_NAME"
fi
if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/"
fi
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME/"
fi
if [ ! -d "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments" ]; then
mkdir "./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME/Attachments"
fi
PROJECT_SAVE_PATH_2="./LondonArchive/Planning Applications/$PROJECT_FILE_TYPE_2/$PROJECT_FILE_NUM_2 - $PROJECT_NAME"
fi
echo "SCRAPE_PLANAPPS: Saving attachments:"
while IFS= read -r LINEA1 && IFS= read -r LINEA2 <&3; do
#echo " - $LINEA1"
echo "SCRAPE_PLANAPPS: - $LINEA2"
wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH/Attachments/$LINEA2" --timestamping -q #--show-progress
if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then
wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH_2/Attachments/$LINEA2" --timestamping -q #--show-progress
fi
done < $PROJECT_ATTACH_URLS 3< $PROJECT_ATTACH_NAMES
echo "SCRAPE_PLANAPPS: All attachments saved."
if [[ "$PROJECT_IMAGE_URL" != "" ]]; then
PROJECT_IMAGE_URL=""
echo "SCRAPE_PLANAPPS: Saving photos:"
while IFS= read -r LINEA1 && IFS= read -r LINEA2 <&3; do
#echo " - $LINEA1"
echo "SCRAPE_PLANAPPS: - $LINEA2"
wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH/$LINEA2" --timestamping -q #--show-progress
if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then
wget --user-agent="$WGET_UA" "$LINEA1" -O "$PROJECT_SAVE_PATH_2/$LINEA2" --timestamping -q #--show-progress
fi
done < $PROJECT_IMAGE_URLS 3< $PROJECT_IMAGE_NAMES
echo "SCRAPE_PLANAPPS: All photos saved."
fi
echo "SCRAPE_PLANAPPS: Extracted info summary:"
cat $PROJECT_INFO > "$PROJECT_SAVE_PATH/Info.txt"
if [[ $PROJECT_FILE_NUM_2 != "" ]] && [[ $PROJECT_FILE_TYPE != $PROJECT_FILE_TYPE_2 ]]; then
cat $PROJECT_INFO > "$PROJECT_SAVE_PATH_2/Info.txt"
fi
cat $PROJECT_INFO
fi
done < $SEARCH_PAGE
else
SEARCH_END="TRUE"
echo "SCRAPE_PLANAPPS: No more pages!"
fi
else
SEARCH_END="TRUE"
echo "SCRAPE_PLANAPPS: No more pages!"
fi
((j++))
done
echo "Done job: SCRAPE_PLANAPPS: $(date)"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment