caibotmi
/
CoolBotDataSys
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 增量脚本去掉LIMIT - 每次跑完所有待处理帖子

This commit is contained in:
caibotmini 2026-04-07 19:43:16 +08:00
parent 25129bfc07
commit c20a2f50ce
1 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scripts/crown_extract_incremental.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
冠字号增量提取脚本 V3 - 修复版 冠字号全量提取脚本 - 跑完所有待处理帖子
""" """
import sys import sys
sys.path.insert(0, '/root/coolbot-data') sys.path.insert(0, '/root/coolbot-data')
@ -106,12 +106,12 @@ def get_category(title):
return '其他' return '其他'
def main(): def main():
print(f'[{datetime.now()}] 冠字号增量提取开始...') print(f'[{datetime.now()}] 冠字号全量提取开始...')
conn = psycopg2.connect(**DB_CONFIG) conn = psycopg2.connect(**DB_CONFIG)
cur = conn.cursor() cur = conn.cursor()
# 用 NOT EXISTS 替代 NOT IN，避免 TEXT/INTEGER 类型转换问题 # 无 LIMIT，全量跑完所有待处理帖子
cur.execute(""" cur.execute("""
SELECT p.id, p.post_id, p.title, p.content, p.post_type, SELECT p.id, p.post_id, p.title, p.content, p.post_type,
p.author_username, p.price, p.price_unit, p.url, p.crawled_at p.author_username, p.price, p.price_unit, p.url, p.crawled_at
@ -122,14 +122,15 @@ def main():
WHERE c.post_id = CAST(p.id AS TEXT) WHERE c.post_id = CAST(p.id AS TEXT)
) )
ORDER BY p.id ORDER BY p.id
LIMIT 1000
""") """)
posts = cur.fetchall() posts = cur.fetchall()
print(f'待处理新帖子: {len(posts)} 条') total = len(posts)
print(f'待处理新帖子: {total} 条')
new_count = 0 new_count = 0
skip_count = 0 skip_count = 0
total_codes = 0 total_codes = 0
processed = 0
for (pid, post_id, title, content, post_type, for (pid, post_id, title, content, post_type,
author, price, price_unit, url, crawled_at) in posts: author, price, price_unit, url, crawled_at) in posts:
@ -137,8 +138,12 @@ def main():
text = f'{title or ""} {content or ""}' text = f'{title or ""} {content or ""}'
codes = extract_codes(text) codes = extract_codes(text)
total_codes += len(codes) total_codes += len(codes)
processed += 1
if not codes: if not codes:
skip_count += 1 skip_count += 1
if processed % 500 == 0:
print(f' 已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
continue continue
features = extract_features(text) features = extract_features(text)
@ -167,8 +172,12 @@ def main():
except Exception as e: except Exception as e:
print(f' 插入失败 post_id={post_id}, code={code}: {e}') print(f' 插入失败 post_id={post_id}, code={code}: {e}')
if processed % 500 == 0:
print(f' 已处理 {processed}/{total} 条，当前新增 {new_count} 条...')
conn.commit()
conn.commit() conn.commit()
print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条') print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条, 总处理: {total} 条')
cur.execute('SELECT COUNT(*) FROM collections') cur.execute('SELECT COUNT(*) FROM collections')
print(f'collections表当前总量: {cur.fetchone()[0]} 条') print(f'collections表当前总量: {cur.fetchone()[0]} 条')