Update sweebot.py

2024-11-11 16:53:43 -08:00 · 2024-11-11 16:53:43 -08:00 · 0af5f1b9a1
commit 0af5f1b9a1
parent e3426ef2ae
1 changed files with 3 additions and 0 deletions
--- a/sweebot.py
+++ b/sweebot.py
@ -609,6 +609,9 @@ while True:
                            if i[:8] == "https://":
                                try:
                                    e = get(i, headers=headers, timeout=10)
+                                    header = e.headers
+                                    content_type = header.get('content-type')
+                                    content_len = header.get('Content-length')
                                    if content_type in allowedparse:
                                        if e.ok:
                                            soup = BeautifulSoup(e.text, 'html.parser')