Skip to content

Instantly share code, notes, and snippets.

@abhitrueprogrammer
Forked from IshanJ25/webopac_scraping.py
Created December 7, 2025 07:02
Show Gist options
  • Select an option

  • Save abhitrueprogrammer/9f9f519c8d193cc676f00d5d685a5204 to your computer and use it in GitHub Desktop.

Select an option

Save abhitrueprogrammer/9f9f519c8d193cc676f00d5d685a5204 to your computer and use it in GitHub Desktop.
import time
import csv
import random
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from webdriver_manager.chrome import ChromeDriverManager
LOGIN_URL = "https://webopac.vit.ac.in/cgi-bin/koha/opac-user.pl"
DETAILS_URL = "https://webopac.vit.ac.in/cgi-bin/koha/opac-memberentry.pl"
LOGOUT_URL = "https://webopac.vit.ac.in/cgi-bin/koha/opac-main.pl?logout.x=1"
INPUT_FILE = "batch_1.csv"
OUTPUT_FILE = f"{INPUT_FILE.split('.')[0]}_output.csv"
def run_batch():
options = webdriver.ChromeOptions()
options.add_argument("--headless")
options.add_argument("--disable-gpu")
options.add_argument("--no-sandbox")
options.add_argument("--log-level=3")
options.add_argument("--window-size=1920,1080")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
wait = WebDriverWait(driver, 8)
ppl_list = []
try:
with open(INPUT_FILE, 'r', newline='') as f:
reader = csv.reader(f)
next(reader)
for row in reader:
if len(row) >= 3:
ppl_list.append(row)
except FileNotFoundError:
print(f"Error: {INPUT_FILE} not found")
return
with open(OUTPUT_FILE, 'w', newline='') as f:
csv.writer(f).writerow(["name", "regno", "phone"])
total = len(ppl_list)
for i, (name, userid, password) in enumerate(ppl_list, 1):
name = name.strip()
userid = userid.strip()
password = password.strip()
print(f"({i}/{total}) Processing {userid}...", end=' ', flush=True)
try:
driver.get(LOGIN_URL)
wait.until(EC.presence_of_element_located((By.ID, "userid"))).clear()
driver.find_element(By.ID, "userid").send_keys(userid)
driver.find_element(By.ID, "password").clear()
driver.find_element(By.ID, "password").send_keys(password)
driver.find_element(By.CSS_SELECTOR, "#auth input[type='submit']").click()
try:
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.ID, "loggedinuser-menu")))
driver.get(DETAILS_URL)
wait.until(EC.presence_of_element_located((By.ID, "memberentry-form")))
phone = driver.find_element(By.ID, "borrower_phone").get_attribute("value")
with open(OUTPUT_FILE, 'a', newline='') as f:
csv.writer(f).writerow([name, userid, phone])
print("Success")
driver.get(LOGOUT_URL)
except TimeoutException:
error_msg = "CREDENTIALS MISMATCH"
with open(OUTPUT_FILE, 'a', newline='') as f:
csv.writer(f).writerow([name, userid, error_msg])
print("Failure (Auth)")
if i < total:
time.sleep(random.randint(1, 5))
except Exception as e:
print("Failure (System Error)" + str(e))
driver.delete_all_cookies()
driver.quit()
if __name__ == "__main__":
run_batch()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment