hkc/nfuck
hkc
/
nfuck
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Increased retries count

This commit is contained in:
Casey 2024-11-15 11:05:45 +03:00
parent 07d8259efc
commit 892cdf5478
Signed by: hkc
GPG Key ID: F0F6CFE11CDB0960
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nfuck/link_verifier.py
View File

@ -1,5 +1,5 @@
from typing import NamedTuple, Optional from typing import NamedTuple, Optional
from httpx import AsyncClient from httpx import AsyncClient, AsyncHTTPTransport
from re import Match, Pattern, compile as regexp, IGNORECASE from re import Match, Pattern, compile as regexp, IGNORECASE
from random import choice from random import choice
from logging import DEBUG, getLogger from logging import DEBUG, getLogger
@ -46,6 +46,7 @@ REGEX_PATTERNS: list[tuple[float, Pattern, str]] = [
MAX_SCORE = 30 # sum(t[0] for t in REGEX_PATTERNS) MAX_SCORE = 30 # sum(t[0] for t in REGEX_PATTERNS)
transport = AsyncHTTPTransport(retries=5)
def explain_verification(content: str) -> list[tuple[float, str, Match]]: def explain_verification(content: str) -> list[tuple[float, str, Match]]:
result: list[tuple[float, str, Match]] = [] result: list[tuple[float, str, Match]] = []
@ -74,7 +75,8 @@ async def verify_link(url: str) -> float:
async with AsyncClient( async with AsyncClient(
headers={"User-Agent": get_random_useragent()}, headers={"User-Agent": get_random_useragent()},
follow_redirects=True, follow_redirects=True,
max_redirects=32 max_redirects=32,
transport=transport
) as client: ) as client:
data = await client.get(url) data = await client.get(url)
for score, explanation, match in explain_verification(data.text): for score, explanation, match in explain_verification(data.text):