caibotmi
/
CoolBotDataSys
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 修复增量提取NOT IN子查询bug - 改用NOT EXISTS避免类型转换问题

This commit is contained in:
caibotmini 2026-04-07 19:33:58 +08:00
parent b9554562de
commit 25129bfc07
1 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
scripts/crown_extract_incremental.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
冠字号增量提取脚本 - 只提取新增帖子的藏品数据 冠字号增量提取脚本 V3 - 修复版
""" """
import sys import sys
sys.path.insert(0, '/root/coolbot-data') sys.path.insert(0, '/root/coolbot-data')
@ -111,31 +111,32 @@ def main():
conn = psycopg2.connect(**DB_CONFIG) conn = psycopg2.connect(**DB_CONFIG)
cur = conn.cursor() cur = conn.cursor()
# 增量：只选还没有在 collections 表中的帖子 # 用 NOT EXISTS 替代 NOT IN，避免 TEXT/INTEGER 类型转换问题
cur.execute(""" cur.execute("""
SELECT id, post_id, title, content, post_type, SELECT p.id, p.post_id, p.title, p.content, p.post_type,
author_username, price, price_unit, url, crawled_at p.author_username, p.price, p.price_unit, p.url, p.crawled_at
FROM yichens_posts FROM yichens_posts p
WHERE category IN ('龙钞', '马钞', '蛇钞', '其他') WHERE p.category IN ('龙钞', '马钞', '蛇钞', '其他')
AND id NOT IN ( AND NOT EXISTS (
SELECT DISTINCT CAST(post_id AS INTEGER) SELECT 1 FROM collections c
FROM collections WHERE c.post_id = CAST(p.id AS TEXT)
WHERE post_id IS NOT NULL
) )
ORDER BY id ORDER BY p.id
LIMIT 500 LIMIT 1000
""") """)
posts = cur.fetchall() posts = cur.fetchall()
print(f'待处理新帖子: {len(posts)} 条') print(f'待处理新帖子: {len(posts)} 条')
new_count = 0 new_count = 0
skip_count = 0 skip_count = 0
total_codes = 0
for (pid, post_id, title, content, post_type, for (pid, post_id, title, content, post_type,
author, price, price_unit, url, crawled_at) in posts: author, price, price_unit, url, crawled_at) in posts:
text = f'{title or ""} {content or ""}' text = f'{title or ""} {content or ""}'
codes = extract_codes(text) codes = extract_codes(text)
total_codes += len(codes)
if not codes: if not codes:
skip_count += 1 skip_count += 1
continue continue
@ -167,8 +168,7 @@ def main():
print(f' 插入失败 post_id={post_id}, code={code}: {e}') print(f' 插入失败 post_id={post_id}, code={code}: {e}')
conn.commit() conn.commit()
total = new_count + skip_count print(f'完成！新增: {new_count} 条, 总冠号: {total_codes}, 无冠号跳过: {skip_count} 条')
print(f'完成！新增: {new_count} 条, 无冠号跳过: {skip_count} 条, 总处理: {total} 条')
cur.execute('SELECT COUNT(*) FROM collections') cur.execute('SELECT COUNT(*) FROM collections')
print(f'collections表当前总量: {cur.fetchone()[0]} 条') print(f'collections表当前总量: {cur.fetchone()[0]} 条')