refac
This commit is contained in:
1 parent
d16d62d1f1
commit
886248de36
1 file changed
+12
-2
@@ -197,14 +197,24 @@ def _extract_text_from_binary_response(
|
|||||||
os.remove(tmp_path)
|
os.remove(tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
TEXT_APPLICATION_CONTENT_TYPES = {
|
||||||
|
'application/javascript',
|
||||||
|
'application/json',
|
||||||
|
'application/xml',
|
||||||
|
'application/x-javascript',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _is_text_content_type(content_type: str) -> bool:
|
def _is_text_content_type(content_type: str) -> bool:
|
||||||
"""Return True if the content type should be handled by the web loader."""
|
"""Return True if the content type should be handled by the web loader."""
|
||||||
ct = content_type.split(';')[0].strip().lower()
|
ct = content_type.split(';')[0].strip().lower()
|
||||||
|
if not ct:
|
||||||
|
return True
|
||||||
if ct.startswith('text/'):
|
if ct.startswith('text/'):
|
||||||
return True
|
return True
|
||||||
if any(t in ct for t in ['xml', 'json', 'javascript']):
|
if ct in TEXT_APPLICATION_CONTENT_TYPES:
|
||||||
return True
|
return True
|
||||||
return not ct # empty / missing → assume HTML
|
return ct.endswith(('+xml', '+json'))
|
||||||
|
|
||||||
|
|
||||||
async def get_content_from_url(request, url: str) -> str:
|
async def get_content_from_url(request, url: str) -> str:
|
||||||
|
|||||||
Reference in new issue
Block a user