airkjw/blog-writer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: /idea 명령 타임아웃 — 첫 기사만 크롤링으로 속도 최적화

Browse Source 
5개 기사 모두 리다이렉트 추적하면 50초+ 걸려서 Telegram 타임아웃 발생.
첫 번째 기사만 URL 변환+크롤링하고 나머지는 RSS 제목만 저장.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
JOUNGWOOK KWON
2026-03-30 15:26:15 +09:00
parent 3d6736503f
commit c836c720da
1 changed files with 15 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
bots/scheduler.py
View File

@@ -801,39 +801,34 @@ def _search_and_build_topic(keyword: str, corner: str = '') -> dict:
link = entry.get('link', '')
pub_date = entry.get('published', '')
# Google 뉴스 URL → 실제 기사 URL 변환
real_url = link
if 'news.google.com' in link:
try:
resp = requests.head(link, timeout=8, allow_redirects=True,
headers={'User-Agent': 'Mozilla/5.0'})
if resp.url and 'news.google.com' not in resp.url:
real_url = resp.url
except Exception:
pass
# Google 뉴스 RSS 제목에서 "- 매체명" 분리
sources.append({
'url': real_url,
'url': link,
'title': title,
'date': pub_date,
})
# 첫 번째 기사에서 설명/이미지 크롤링
if not best_description and real_url != link:
try:
# 첫 번째 기사만 리다이렉트 추적 + 크롤링 (속도 최적화)
if sources and 'news.google.com' in sources[0]['url']:
try:
resp = requests.head(sources[0]['url'], timeout=5, allow_redirects=True,
headers={'User-Agent': 'Mozilla/5.0'})
if resp.url and 'news.google.com' not in resp.url:
sources[0]['url'] = resp.url
# og:description, og:image 크롤링
from bs4 import BeautifulSoup
resp = requests.get(real_url, timeout=10,
page = requests.get(resp.url, timeout=5,
headers={'User-Agent': 'Mozilla/5.0'})
if resp.status_code == 200:
soup = BeautifulSoup(resp.text, 'lxml')
if page.status_code == 200:
soup = BeautifulSoup(page.text, 'lxml')
og_desc = soup.find('meta', property='og:description')
if og_desc and og_desc.get('content'):
best_description = og_desc['content'].strip()[:300]
og_img = soup.find('meta', property='og:image')
if og_img and og_img.get('content', '').startswith('http'):
best_image = og_img['content']
except Exception:
pass
except Exception:
pass
except Exception:
pass