1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
| import requests
from bs4 import BeautifulSoup
import csv
import xmltodict
import pandas as pd
header = ['url','title','description']
# write the header to our (fresh) csv
with open('meta-data.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
writer.writerow(header)
#get the parent sitemap
r = requests.get("http://localhost/sitemaps/sitemap.xml")
xml = r.text
soup = BeautifulSoup(xml,features="lxml")
sitemapTags = soup.find_all("sitemap")
print("The number of sitemaps are {0}".format(len(sitemapTags)))
#init array of all links
links_arr = []
#get nested sitemaps
for sitemap in sitemapTags:
siteMapUrl = sitemap.findNext("loc").text
print(siteMapUrl)
r = requests.get(siteMapUrl)
xml = r.text
soup = BeautifulSoup(xml,features="lxml")
for link in soup.findAll('loc'):
linkstr = str(link)
linkstr = linkstr.replace("<loc>", "")
linkstr = linkstr.replace("</loc>", "")
#add to our list of links to scrape
links_arr.append(linkstr)
#scrape all our pages from the combined sitemaps
for url in links_arr:
#temp var
data = []
print(url)
response = requests.get(url)
soup = BeautifulSoup(response.text,features="lxml")
metas = soup.find_all('meta')
title = soup.find('title').text
description = ''
for meta in metas:
if 'name' in meta.attrs and meta.attrs['name'] == 'description':
description = (meta.attrs['content'])
data.append([url,title,description])
# append the csv with each result
with open('meta-data.csv', 'a', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
writer.writerows(data)
|