ettorerizza
8/7/2017 - 7:02 PM

scrape patronym in the database http://patrom.fltr.ucl.ac.be

scrape patronym in the database http://patrom.fltr.ucl.ac.be

#! python3
import requests
from bs4 import BeautifulSoup
import string
import pandas as pd

url = "http://patrom.fltr.ucl.ac.be/contemporain/query.cfm"

letters = list(string.ascii_lowercase)

data = []

for i in letters:
    print(i)
    try:
        params = {
                "nom1":i,
                "bol1":"and",
                "nom2":"",
                "bol2":"and",
                "nom3":"",
                "nom1_required":"Vous devez mettre une valeur dans la clef 1"}
        
        r = requests.post(url, data = params, timeout = 30)
        
        soup = BeautifulSoup(r.content, "lxml")
        
        table = soup.find("table")
        
        rows = table.find_all('tr')
        for row in rows:
            cols = row.find_all('td')
            cols = [ele.text.strip() for ele in cols]
            data.append([ele for ele in cols if ele])
    except Exception as e: 
        print(e)
        continue

print(len(data))

colnames = ["NOM",	"Nom",	"Total", "Anv",	"Bxl",	"BFl",	
            "BW",	"FlOc",	"FlOr",	"Ht",	"Lg",	"Lb",	"Lx",	"Nr",	"INS"]

pd = pd.DataFrame(data)
pd.to_csv("mylist.csv", header=colnames, encoding = "utf-8" )