From 0af5f1b9a18a90a0d071c15f135fce4e6b0a63c6 Mon Sep 17 00:00:00 2001 From: Swee Date: Mon, 11 Nov 2024 16:53:43 -0800 Subject: [PATCH] Update sweebot.py --- sweebot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sweebot.py b/sweebot.py index e94a56e..b50471b 100644 --- a/sweebot.py +++ b/sweebot.py @@ -609,6 +609,9 @@ while True: if i[:8] == "https://": try: e = get(i, headers=headers, timeout=10) + header = e.headers + content_type = header.get('content-type') + content_len = header.get('Content-length') if content_type in allowedparse: if e.ok: soup = BeautifulSoup(e.text, 'html.parser')