From b9554562dee464b75ab51675c98790f7d86bce16 Mon Sep 17 00:00:00 2001 From: caibotmini Date: Tue, 7 Apr 2026 15:29:52 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E5=86=A0=E5=AD=97?= =?UTF-8?q?=E5=8F=B7=E5=A2=9E=E9=87=8F=E6=8F=90=E5=8F=96=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=20-=20=E6=AF=8F=E5=A4=A93=E7=82=B9/11=E7=82=B9/16=E7=82=B9?= =?UTF-8?q?=E5=A2=9E=E9=87=8F=E5=90=8C=E6=AD=A5=E8=97=8F=E5=93=81=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 只提取新帖子的冠字号，INSERT ... ON CONFLICT DO NOTHING - 删掉原来的 DELETE + 全量 INSERT 逻辑 - 加唯一索引 idx_collections_post_crown (post_id, crown_code) --- scripts/crown_extract_incremental.py | 178 +++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 scripts/crown_extract_incremental.py diff --git a/scripts/crown_extract_incremental.py b/scripts/crown_extract_incremental.py new file mode 100644 index 0000000..6f157a4 --- /dev/null +++ b/scripts/crown_extract_incremental.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +""" +冠字号增量提取脚本 - 只提取新增帖子的藏品数据 +""" +import sys +sys.path.insert(0, '/root/coolbot-data') + +import re, psycopg2 +from datetime import datetime + +DB_CONFIG = { + 'host': 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com', + 'port': 5432, + 'database': 'coolbot_data', + 'user': 'coolbot', + 'password': 'Coolbot123' +} + +FEATURE_KEYWORDS = [ + '无47', '无34', '无347', '无4', '无3', '无2', + '标十', '标百', '标九', '标准十', '标准百', + '首日', '生日', '金马', '银马', + '金钩', '倒置', '满号', '圆圆', + '豹子', '顺子', '恐龙', '天龙', + 'PMG', '爱藏', '尾号', + '大象', '麒麟', '老虎', '狮子', + '金马王', '天马', '龙马精神', + '连号', '散号', '一刀', '龙凤', '熊猫', +] + +def is_date_code(code): + if len(code) != 8: + return False + try: + year = int(code[:4]) + month = int(code[4:6]) + day = int(code[6:8]) + if 2000 <= year <= 2030 and 1 <= month <= 12 and 1 <= day <= 31: + return True + except: + pass + return False + +def is_phone(code): + return len(code) == 11 and code.startswith('1') + +def is_valid_crown(code): + if not code: + return False + if code.startswith('J0'): + return len(code) in (10, 11) + if len(code) not in (8, 9): + return False + if is_phone(code): + return False + if len(code) == 8 and is_date_code(code): + return False + return True + +def extract_codes(text): + if not text: + return set() + codes = set() + for p in [r'J0\d{8}', r'J0\d{9}']: + codes.update(re.findall(p, text)) + for c in re.findall(r'\b\d{8}\b', text): + if is_valid_crown(c): + codes.add(c) + for c in re.findall(r'\b\d{9}\b', text): + if is_valid_crown(c): + codes.add(c) + return codes + +def extract_features(text): + if not text: + return None + feat = [kw for kw in FEATURE_KEYWORDS if kw in text] + return '|'.join(feat) if feat else None + +def extract_price(text): + if not text: + return None, '元' + prices = [] + for p in [r'(\d+(?:\.\d+)?)\s*元', r'(\d+(?:\.\d+)?)\s*/\s*[张件组百]']: + for x in re.findall(p, text): + try: + v = float(x) + if 1 <= v < 100000: + prices.append(v) + except: + pass + return (min(prices), '元') if prices else (None, '元') + +def get_category(title): + if not title: + return '其他' + for kw in ['龙', '龙钞', '小龙钞', '钞王']: + if kw in title: + return '龙钞' + for kw in ['马', '马钞']: + if kw in title: + return '马钞' + for kw in ['蛇', '蛇钞']: + if kw in title: + return '蛇钞' + return '其他' + +def main(): + print(f'[{datetime.now()}] 冠字号增量提取开始...') + + conn = psycopg2.connect(**DB_CONFIG) + cur = conn.cursor() + + # 增量：只选还没有在 collections 表中的帖子 + cur.execute(""" + SELECT id, post_id, title, content, post_type, + author_username, price, price_unit, url, crawled_at + FROM yichens_posts + WHERE category IN ('龙钞', '马钞', '蛇钞', '其他') + AND id NOT IN ( + SELECT DISTINCT CAST(post_id AS INTEGER) + FROM collections + WHERE post_id IS NOT NULL + ) + ORDER BY id + LIMIT 500 + """) + posts = cur.fetchall() + print(f'待处理新帖子: {len(posts)} 条') + + new_count = 0 + skip_count = 0 + + for (pid, post_id, title, content, post_type, + author, price, price_unit, url, crawled_at) in posts: + + text = f'{title or ""} {content or ""}' + codes = extract_codes(text) + if not codes: + skip_count += 1 + continue + + features = extract_features(text) + price_val, _ = extract_price(text) + price_val = price_val if price_val else price + category = get_category(title) + + for code in codes: + try: + cur.execute(""" + INSERT INTO collections ( + name, crown_code, category, post_id, post_url, + author, price, price_unit, + number_feature, post_title, post_type, + post_crawled_at, created_at, updated_at + ) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,NOW(),NOW()) + ON CONFLICT (post_id, crown_code) DO NOTHING + """, ( + code, code, category, str(post_id), url, + author, price_val, price_unit or '元', + features, title, post_type, + crawled_at + )) + if cur.rowcount > 0: + new_count += 1 + except Exception as e: + print(f' 插入失败 post_id={post_id}, code={code}: {e}') + + conn.commit() + total = new_count + skip_count + print(f'完成！新增: {new_count} 条, 无冠号跳过: {skip_count} 条, 总处理: {total} 条') + + cur.execute('SELECT COUNT(*) FROM collections') + print(f'collections表当前总量: {cur.fetchone()[0]} 条') + conn.close() + +if __name__ == '__main__': + main()