Update sweebot.py
This commit is contained in:
parent
e3426ef2ae
commit
0af5f1b9a1
1 changed files with 3 additions and 0 deletions
|
@ -609,6 +609,9 @@ while True:
|
||||||
if i[:8] == "https://":
|
if i[:8] == "https://":
|
||||||
try:
|
try:
|
||||||
e = get(i, headers=headers, timeout=10)
|
e = get(i, headers=headers, timeout=10)
|
||||||
|
header = e.headers
|
||||||
|
content_type = header.get('content-type')
|
||||||
|
content_len = header.get('Content-length')
|
||||||
if content_type in allowedparse:
|
if content_type in allowedparse:
|
||||||
if e.ok:
|
if e.ok:
|
||||||
soup = BeautifulSoup(e.text, 'html.parser')
|
soup = BeautifulSoup(e.text, 'html.parser')
|
||||||
|
|
Loading…
Reference in a new issue