diff --git a/crawlers/crawl_today.py b/crawlers/crawl_today.py index e95cebc..edc1dcf 100644 --- a/crawlers/crawl_today.py +++ b/crawlers/crawl_today.py @@ -49,11 +49,47 @@ def extract_post_content(html_bytes): text = re.sub(r'\n\d{4}[/\-]\d{1,2}[/\-]\d{1,2}\s+\d{1,2}:\d{2}:\d{2}\s*$', '', text) # 截取免责声明及之前的内容 - disclaimer = '免责声明及风险提示: 所有交易人员,凡未采用本站中介交易的,被骗后果自负。' + disclaimer = '免责声明及风险提示:所有交易人员,凡未采用本站中介交易的,被骗后果自负。' discl_pos = text.find(disclaimer) if discl_pos != -1: text = text[:discl_pos + len(disclaimer)] - + + # 清理尾部网站版权和导航信息 + footer_patterns = [ + 'BoardJumpListSelect', + 'Powered By Dvbbs', + '京ICP备', + '页面执行时间', + '发短信', + '购买论坛点券', + '我能做什么', + '我发表的主题', + '我参与的主题', + '基本资料修改', + '用户密码修改', + '联系资料修改', + '用户短信服务', + '编辑好友列表', + '个人文件管理', + '通行证设置', + '今日贴数图例', + '主题数图例', + '总帖数图例', + '在线图例', + '在线情况', + '用户组在线图例', + '文件集浏览', + '图片集浏览', + 'Flash浏览', + '音乐集浏览', + '电影集浏览', + '贺卡发送', + ] + for pat in footer_patterns: + pos = text.find(pat) + if pos != -1: + text = text[:pos] + return text if text else None @@ -361,7 +397,7 @@ class YichensTodaySpider(PaginationSpider): "updated_at = NOW()", "crawled_at = NOW()", "url = EXCLUDED.url", - "instance_id = EXCLUDED.instance_id", + ] query = sql.SQL("INSERT INTO yichens_posts ({}) VALUES ({}) ON CONFLICT (post_id) DO UPDATE SET {}").format(