feat: 增量脚本去掉LIMIT - 每次跑完所有待处理帖子

scripts/crown_extract_incremental.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-冠字号增量提取脚本 V3 - 修复版
+冠字号全量提取脚本 - 跑完所有待处理帖子
 """
 import sys
 sys.path.insert(0, '/root/coolbot-data')
@@ -106,12 +106,12 @@ def get_category(title):
     return '其他'
 
 def main():
-    print(f'[{datetime.now()}] 冠字号增量提取开始...')
+    print(f'[{datetime.now()}] 冠字号全量提取开始...')
 
     conn = psycopg2.connect(**DB_CONFIG)
     cur = conn.cursor()
 
-    # 用 NOT EXISTS 替代 NOT IN，避免 TEXT/INTEGER 类型转换问题
+    # 无 LIMIT，全量跑完所有待处理帖子
     cur.execute("""
         SELECT p.id, p.post_id, p.title, p.content, p.post_type,
                p.author_username, p.price, p.price_unit, p.url, p.crawled_at
@@ -122,14 +122,15 @@ def main():
               WHERE c.post_id = CAST(p.id AS TEXT)
           )
         ORDER BY p.id
-        LIMIT 1000
     """)
     posts = cur.fetchall()
-    print(f'待处理新帖子: {len(posts)} 条')
+    total = len(posts)
+    print(f'待处理新帖子: {total} 条')
 
     new_count = 0
     skip_count = 0
     total_codes = 0
+    processed = 0
 
     for (pid, post_id, title, content, post_type,
          author, price, price_unit, url, crawled_at) in posts:
@@ -137,8 +138,12 @@ def main():
         text = f'{title or ""} {content or ""}'
         codes = extract_codes(text)
         total_codes += len(codes)
+        processed += 1
+
         if not codes:
             skip_count += 1
+            if processed % 500 == 0:
+                print(f'  已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
             continue
 
         features = extract_features(text)
@@ -167,8 +172,12 @@ def main():
             except Exception as e:
                 print(f'  插入失败 post_id={post_id}, code={code}: {e}')
 
+        if processed % 500 == 0:
+            print(f'  已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
+            conn.commit()
+
     conn.commit()
-    print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条')
+    print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条, 总处理: {total} 条')
 
     cur.execute('SELECT COUNT(*) FROM collections')
     print(f'collections表当前总量: {cur.fetchone()[0]} 条')