|
from nt import startfile |
|
from os import getenv |
|
from dotenv import load_dotenv |
|
from helium import set_driver,go_to, wait_until, click, Button, write |
|
from selenium import webdriver |
|
from selenium.webdriver.common.by import By |
|
from selenium.webdriver.firefox.options import Options |
|
from math import inf |
|
from datetime import datetime |
|
from pathlib import Path |
|
from json import loads, dumps |
|
from csv import DictWriter |
|
from time import sleep |
|
from selenium.webdriver.support.ui import WebDriverWait |
|
from selenium.webdriver.support import expected_conditions as EC |
|
from openpyxl import Workbook |
|
from openpyxl.worksheet.table import Table, TableStyleInfo |
|
|
|
|
|
load_dotenv() |
|
|
|
MED_WAIT = 15e3 |
|
|
|
def prep_table_in_sheet(sheet, sheet_headers, data, name): |
|
sheet.append(sheet_headers) |
|
|
|
for entry in data: |
|
sheet.append([entry.get(h, "") for h in sheet_headers]) |
|
|
|
end_col_letter = chr(64 + len(sheet_headers)) |
|
end_row = len(data) + 1 |
|
table_ref = f"A1:{end_col_letter}{end_row}" |
|
|
|
# Create and style the table |
|
tab = Table(displayName=name, ref=table_ref) |
|
style = TableStyleInfo( |
|
name="TableStyleMedium27", |
|
showFirstColumn=False, |
|
showLastColumn=False, |
|
showRowStripes=True, |
|
showColumnStripes=False |
|
) |
|
tab.tableStyleInfo = style |
|
sheet.add_table(tab) |
|
|
|
|
|
options = Options() |
|
if not getenv("HEADED"): |
|
options.add_argument("--headless") |
|
|
|
driver = webdriver.Firefox(options) |
|
json_file = Path(__file__).parent / "meta.json" |
|
meta = loads(json_file.read_text(encoding='utf-8-sig')) if json_file.exists() else {} |
|
fetched = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
|
summary = [] |
|
|
|
try: |
|
set_driver(driver) |
|
go_to("https://aarhusbolig.dk/min-side/boligsoegningsportal/mine-boligoensker/") |
|
|
|
|
|
sleep(.5) |
|
shadow_root = driver.find_element(By.CSS_SELECTOR, 'cookie-consent-element').shadow_root |
|
shadow_root.find_element(By.ID, "cookie-accept-all").click() |
|
|
|
|
|
sleep(.5) |
|
wait_until(Button("Log ind").exists) |
|
click(Button("Log ind")) |
|
|
|
WebDriverWait(driver, MED_WAIT).until( |
|
EC.presence_of_element_located((By.ID, "loginBoxUsername")) |
|
) |
|
write(getenv("EMAIL"), driver.find_element(By.ID, "loginBoxUsername")) |
|
write(getenv("PASSWORD"), driver.find_element(By.ID, "loginBoxPassword")) |
|
click(Button("Log ind", above="Glemt adgangskode?")) |
|
|
|
wait_until(Button("Min Side").exists) |
|
go_to("https://aarhusbolig.dk/min-side/boligsoegningsportal/mine-boligoensker/") |
|
|
|
while True: |
|
WebDriverWait(driver, MED_WAIT).until( |
|
EC.presence_of_element_located((By.CSS_SELECTOR, ".result")) |
|
) |
|
for result in driver.find_elements(By.CSS_SELECTOR, ".result"): |
|
Address = result.find_element(By.CSS_SELECTOR, ".title-name").text |
|
Dept = result.find_element(By.CSS_SELECTOR, ".show-hide-span").text |
|
grouped_under = result.find_element(By.CSS_SELECTOR, ".ten-group-text").text |
|
room = result.find_element(By.CSS_SELECTOR, ".result-rooms").text |
|
|
|
for options in result.find_elements(By.CSS_SELECTOR, "tr"): |
|
area = options.find_element(By.CSS_SELECTOR, ".result-sqm").text |
|
rent = options.find_element(By.CSS_SELECTOR, ".result-rent").text |
|
rank = options.find_element(By.CSS_SELECTOR, ".result-placement-external span.ng-binding").text |
|
|
|
key = f"{Address}{area}" |
|
option = meta.get(key, {}) |
|
option.update({"Address": Address, "Dept.": Dept, "Type": grouped_under, "Rooms": room, "Area": area, "Rent": rent}) |
|
|
|
option["Ranks"] = option.get("Ranks", []) |
|
option["Timestamps"] = option.get("Timestamps", []) |
|
rank = rank.strip() |
|
option["Ranks"].append(int(rank) if rank else inf) |
|
option["Timestamps"].append(fetched) |
|
|
|
meta[key] = option |
|
sum_up = {**option} |
|
ranks = sum_up.pop("Ranks") |
|
sum_up.pop("Timestamps") |
|
if len(ranks) > 1: |
|
if ranks[-1] != ranks[-2]: |
|
sum_up["Summary"] = "Improved" if ranks[-1] < ranks[-2] else "Worsened" |
|
else: |
|
sum_up["Summary"] = "No Change" |
|
sum_up["Prev"] = ranks[-2] |
|
sum_up["Next"] = ranks[-1] |
|
else: |
|
sum_up["Summary"] = "New" |
|
sum_up["Next"] = "N/A" |
|
sum_up["Prev"] = ranks[-1] if ranks else "N/A" |
|
summary.append(sum_up) |
|
|
|
|
|
note = driver.find_elements(By.CSS_SELECTOR, "button[aria-label='Næste side']") |
|
if Button("Næste side").exists(): |
|
driver.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});", note[-1]) |
|
sleep(0.5) |
|
wait_until(Button("Næste side").exists) |
|
click(Button("Næste side")) |
|
sleep(1) |
|
else: |
|
break |
|
except Exception as error: |
|
print(error) |
|
finally: |
|
driver.quit() |
|
|
|
json_file.write_text(dumps(meta, indent=4, ensure_ascii=False), encoding='utf-8-sig') |
|
writer = None |
|
|
|
xl = Path(__file__).parent / "prepared.xlsx" |
|
book = Workbook() |
|
summary_sheet = book.active |
|
|
|
# Write summary data to the sheet |
|
if summary: |
|
headers = list(summary[0].keys()) |
|
prep_table_in_sheet(summary_sheet, headers, summary, "Summary") |
|
|
|
sorted_rows = list(sorted(meta.values(), key=lambda x: x["Ranks"][-1])) |
|
choice_sheet = book.create_sheet("Total") |
|
rows = [] |
|
for row in sorted_rows: |
|
ranks = row.pop("Ranks") |
|
stamps = row.pop("Timestamps") |
|
for rank, stamp in reversed(list(zip(ranks, stamps))): |
|
prep = {"Timestamp": stamp, **row, "Rank": rank} |
|
rows.append(prep) |
|
if rows: |
|
prep_table_in_sheet(choice_sheet, list(rows[-1].keys()), rows, "Total") |
|
|
|
|
|
|
|
book.save(xl) |
|
startfile(str(xl)) |
Above is vbs script and in order to run it as soon I open laptop, run below in cmd in admin mode.