From 8a100a9a955828700e2748834495eef7fc2a247f Mon Sep 17 00:00:00 2001
From: caibotmini <caibotmi@coolbot.com>
Date: Tue, 7 Apr 2026 14:38:25 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=B8=96=E5=AD=90?=
 =?UTF-8?q?=E5=86=85=E5=AE=B9=E8=A7=A3=E6=9E=90=20-=20=E6=B8=85=E7=90=86?=
 =?UTF-8?q?=E7=BD=91=E7=AB=99=E5=85=8D=E8=B4=A3=E5=A3=B0=E6=98=8E=E5=92=8C?=
 =?UTF-8?q?=E5=BA=95=E9=83=A8=E5=AF=BC=E8=88=AA?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- 修复免责声明空格匹配问题
- 增加 BoardJumpListSelect/Powered By Dvbbs/京ICP备等底部清理
- 清理历史污染数据10条
---
 crawlers/crawl_today.py | 42 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/crawlers/crawl_today.py b/crawlers/crawl_today.py
index e95cebc..edc1dcf 100644
--- a/crawlers/crawl_today.py
+++ b/crawlers/crawl_today.py
@@ -49,11 +49,47 @@ def extract_post_content(html_bytes):
     text = re.sub(r'\n\d{4}[/\-]\d{1,2}[/\-]\d{1,2}\s+\d{1,2}:\d{2}:\d{2}\s*$', '', text)
 
     # 截取免责声明及之前的内容
-    disclaimer = '免责声明及风险提示： 所有交易人员，凡未采用本站中介交易的，被骗后果自负。'
+    disclaimer = '免责声明及风险提示：所有交易人员，凡未采用本站中介交易的，被骗后果自负。'
     discl_pos = text.find(disclaimer)
     if discl_pos != -1:
         text = text[:discl_pos + len(disclaimer)]
-
+    
+    # 清理尾部网站版权和导航信息
+    footer_patterns = [
+        'BoardJumpListSelect',
+        'Powered By Dvbbs',
+        '京ICP备',
+        '页面执行时间',
+        '发短信',
+        '购买论坛点券',
+        '我能做什么',
+        '我发表的主题',
+        '我参与的主题',
+        '基本资料修改',
+        '用户密码修改',
+        '联系资料修改',
+        '用户短信服务',
+        '编辑好友列表',
+        '个人文件管理',
+        '通行证设置',
+        '今日贴数图例',
+        '主题数图例',
+        '总帖数图例',
+        '在线图例',
+        '在线情况',
+        '用户组在线图例',
+        '文件集浏览',
+        '图片集浏览',
+        'Flash浏览',
+        '音乐集浏览',
+        '电影集浏览',
+        '贺卡发送',
+    ]
+    for pat in footer_patterns:
+        pos = text.find(pat)
+        if pos != -1:
+            text = text[:pos]
+    
     return text if text else None
 
 
@@ -361,7 +397,7 @@ class YichensTodaySpider(PaginationSpider):
             "updated_at = NOW()",
             "crawled_at = NOW()",
             "url = EXCLUDED.url",
-            "instance_id = EXCLUDED.instance_id",
+            
         ]
         
         query = sql.SQL("INSERT INTO yichens_posts ({}) VALUES ({}) ON CONFLICT (post_id) DO UPDATE SET {}").format(