import requests
from bs4 import BeautifulSoup
page = requests.get('http://url')
soup = BeautifulSoup(page.text)
def url_list():
with open('URL-LIST.txt', 'r') as r:
urls = r.read() #may need to experiment with readline() or trying this outside of a function
return urls
def htmls(urls, html=[]):
"""
Find URLs in each line,
Get Page
Yield 1-by-1 or return list of each
"""
page = ""
for line in urls:
page = requests.get(line)
html.insert(0, page)
yield html
# can also do
# yield line
# yield will create generator and return one page at a time for you to parse, instead of a full list, more memory efficient and may make more sense for this
def table_scraps(html, res=[]):
"""
- Assuming the data is a basic HTML table
<div id="heh" >
- adjust soup to search based on attrs
- Another example
<td align="right">
- soup.find_all('td', attrs={"align":"right"})
- soup.find_all('div', attrs={"id" : "gay"})
- etc.
"""
soup = BeautifulSoup(html.text)
content = soup.find_all('div', attrs={"id":"heh"})
# what you want to do with the data
# using patterns, stripping tags, etc.
# insert data into res, a list
res.insert(0, wanted_data)
return res
if __name__ == "__main__":
urls = url_list()
htmls(urls)
table_scraps(soup)
Haven't tested any of it, but should get you started if you're doing the Python route, BeautifulSoup is what I would use for basic HTML scraping.