468 lines
17 KiB
Python
468 lines
17 KiB
Python
"""只采集今天的新帖子 - 优化版"""
|
||
import re
|
||
import sys
|
||
import os
|
||
sys.path.insert(0, '/root/coolbot-data')
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
from datetime import datetime
|
||
from crawlers.base import PaginationSpider
|
||
from database import get_db
|
||
|
||
def extract_post_content(html_bytes):
|
||
"""从详情页原始HTML提取帖子正文（GBK编码）。
|
||
定位：找到含 min-height:200px 的 div，向下到 <hr />（签名）为止。
|
||
"""
|
||
if isinstance(html_bytes, str):
|
||
page = html_bytes
|
||
else:
|
||
page = html_bytes.decode('gbk', errors='replace')
|
||
|
||
idx = page.find('min-height:200px')
|
||
if idx == -1:
|
||
return None
|
||
|
||
div_start = page.rfind('<div', 0, idx)
|
||
if div_start == -1:
|
||
return None
|
||
|
||
hr_idx = page.find('<hr />', idx)
|
||
discl_idx = page.find('\u514d\u8d23\u5371\u58f0\u660e', idx)
|
||
|
||
candidates = [x for x in [hr_idx, discl_idx] if x != -1]
|
||
end = min(candidates) if candidates else (hr_idx if hr_idx != -1 else len(page))
|
||
|
||
chunk = page[div_start:end]
|
||
|
||
text = re.sub(r'<div[^>]*>', '\n', chunk)
|
||
text = re.sub(r'</div>', '', text)
|
||
text = re.sub(r'<p[^>]*>', '\n', text)
|
||
text = re.sub(r'</p>', '', text)
|
||
text = re.sub(r'<br\s*/?>', '\n', text)
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
text = text.replace(' ', ' ').replace(' ', ' ')
|
||
text = text.replace('&', '&').replace('<', '<').replace('>', '>')
|
||
|
||
lines = [l.strip() for l in text.split('\n') if l.strip()]
|
||
text = '\n'.join(lines)
|
||
text = re.sub(r'\n\d{4}[/\-]\d{1,2}[/\-]\d{1,2}\s+\d{1,2}:\d{2}:\d{2}\s*$', '', text)
|
||
|
||
# 截取免责声明及之前的内容
|
||
disclaimer = '免责声明及风险提示：所有交易人员，凡未采用本站中介交易的，被骗后果自负。'
|
||
discl_pos = text.find(disclaimer)
|
||
if discl_pos != -1:
|
||
text = text[:discl_pos + len(disclaimer)]
|
||
|
||
# 清理尾部网站版权和导航信息
|
||
footer_patterns = [
|
||
'BoardJumpListSelect',
|
||
'Powered By Dvbbs',
|
||
'京ICP备',
|
||
'页面执行时间',
|
||
'发短信',
|
||
'购买论坛点券',
|
||
'我能做什么',
|
||
'我发表的主题',
|
||
'我参与的主题',
|
||
'基本资料修改',
|
||
'用户密码修改',
|
||
'联系资料修改',
|
||
'用户短信服务',
|
||
'编辑好友列表',
|
||
'个人文件管理',
|
||
'通行证设置',
|
||
'今日贴数图例',
|
||
'主题数图例',
|
||
'总帖数图例',
|
||
'在线图例',
|
||
'在线情况',
|
||
'用户组在线图例',
|
||
'文件集浏览',
|
||
'图片集浏览',
|
||
'Flash浏览',
|
||
'音乐集浏览',
|
||
'电影集浏览',
|
||
'贺卡发送',
|
||
]
|
||
for pat in footer_patterns:
|
||
pos = text.find(pat)
|
||
if pos != -1:
|
||
text = text[:pos]
|
||
|
||
return text if text else None
|
||
|
||
|
||
|
||
import logging
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
class YichensTodaySpider(PaginationSpider):
|
||
def __init__(self, instance_id=None):
|
||
super().__init__("一尘网今日采集", "pm001")
|
||
self.instance_id = instance_id or os.environ.get('CRAWLER_INSTANCE_ID', 'unknown')
|
||
self.base_url = "http://www.pm001.net"
|
||
self.board_id = "151"
|
||
self.forum_url = f"{self.base_url}/index.asp?boardid={self.board_id}"
|
||
self.max_pages = 2
|
||
self.encoding = "gbk"
|
||
self.min_delay = 1.5
|
||
self.max_delay = 3.0
|
||
|
||
def get(self, url, **kwargs):
|
||
try:
|
||
self._random_delay()
|
||
headers = {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
}
|
||
response = requests.get(url, timeout=30, headers=headers, **kwargs)
|
||
try:
|
||
response.encoding = self.encoding
|
||
except:
|
||
response.encoding = "gb2312"
|
||
return response
|
||
except Exception as e:
|
||
print(f"请求失败: {e}")
|
||
return None
|
||
|
||
def extract_posts_with_dates(self, html):
|
||
"""从列表页提取所有帖子ID和"发表于"时间，返回 [(post_id, date_str), ...]
|
||
使用BeautifulSoup解析listtitle div，比正则更准确
|
||
"""
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
posts = []
|
||
listtitle_divs = soup.find_all('div', class_='listtitle')
|
||
|
||
for div in listtitle_divs:
|
||
link = div.find('a', href=re.compile(r'dispbbs\.asp\?boardID=151&ID=\d+'))
|
||
if not link:
|
||
continue
|
||
|
||
href = link.get('href', '')
|
||
id_match = re.search(r'\&ID=(\d+)', href)
|
||
if not id_match:
|
||
continue
|
||
post_id = id_match.group(1)
|
||
|
||
title_attr = link.get('title', '')
|
||
# title格式: 《标题》\n作者：xxx\n发表于：2026/4/5 9:35:00
|
||
date_match = re.search(r'发表于：(\d{4}/\d{1,2}/\d{1,2})', title_attr)
|
||
if date_match:
|
||
date_str = date_match.group(1)
|
||
posts.append((post_id, date_str))
|
||
|
||
return posts
|
||
|
||
def parse_post_detail(self, html, url):
|
||
"""解析详情页，提取真实发帖时间"""
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
page_text = soup.get_text()
|
||
|
||
match = re.search(r"\&ID=(\d+)", url)
|
||
post_id = match.group(1) if match else None
|
||
|
||
title = None
|
||
title_elem = soup.find("title")
|
||
if title_elem:
|
||
title_text = title_elem.get_text(strip=True)
|
||
if "[" in title_text:
|
||
title = title_text.split("[")[0].strip()
|
||
|
||
author = None
|
||
m = re.search(r">>>\s*([^\s<]+)", page_text)
|
||
if m:
|
||
author = m.group(1).strip()
|
||
|
||
created_at = None
|
||
m = re.search(r"(\d{4}/\d{1,2}/\d{1,2}\s+\d{1,2}:\d{2}:\d{2})", page_text)
|
||
if m:
|
||
created_at = m.group(1).replace("/", "-")
|
||
|
||
phones = re.findall(r'1[3-9]\d{9,10}', page_text)
|
||
contact = ",".join(dict.fromkeys(phones[:5])) if phones else None
|
||
|
||
price = None
|
||
price_unit = None
|
||
if title:
|
||
price, price_unit = self.extract_price(title)
|
||
if not price:
|
||
price, price_unit = self.extract_price(page_text)
|
||
|
||
content = extract_post_content(html)
|
||
|
||
special_types = []
|
||
if title:
|
||
if "救生圈" in title or "88" in title or "888" in title:
|
||
special_types.append("救生圈")
|
||
if "标十" in title:
|
||
special_types.append("标十")
|
||
if "标百" in title:
|
||
special_types.append("标百")
|
||
|
||
# post_type 分类逻辑：出售 > 求购 > 其他 > 默认出售
|
||
if title:
|
||
if any(c in title for c in ["出", "售", "卖"]):
|
||
post_type = "deal"
|
||
elif any(c in title for c in ["收", "求", "购", "要"]):
|
||
post_type = "want"
|
||
elif any(c in title for c in ["确认", "朋友", "投诉"]):
|
||
post_type = "normal"
|
||
else:
|
||
post_type = "deal" # 默认出售
|
||
else:
|
||
post_type = "normal"
|
||
|
||
# category 分类逻辑（优先级：龙钞 > 马钞 > 蛇钞 > 其他）
|
||
category = "其他"
|
||
if title:
|
||
if any(c in title for c in ["龙", "龙钞", "小龙钞", "钞王"]):
|
||
category = "龙钞"
|
||
elif any(c in title for c in ["马", "马钞"]):
|
||
category = "马钞"
|
||
elif any(c in title for c in ["蛇", "蛇钞"]):
|
||
category = "蛇钞"
|
||
|
||
# 从 postuserinfo div 的 <b> 标签提取用户名（跳过电话号码）
|
||
if not author:
|
||
userinfos = soup.find_all('div', class_='postuserinfo')
|
||
for ui in userinfos:
|
||
for b in ui.find_all('b'):
|
||
text = b.get_text(strip=True)
|
||
if text and not re.match(r'^1\d{10}$', text) and len(text) > 1:
|
||
author = text
|
||
break
|
||
if author:
|
||
break
|
||
|
||
return {
|
||
"post_id": post_id,
|
||
"title": title,
|
||
"content": content,
|
||
"author_username": author,
|
||
"price": price,
|
||
"price_unit": price_unit,
|
||
"contact": contact,
|
||
"post_time": created_at,
|
||
"special_types": "|".join(special_types) if special_types else None,
|
||
"category": category,
|
||
"post_type": post_type,
|
||
"url": f"http://www.pm001.net/dispbbs.asp?boardID={self.board_id}&ID={post_id}&page=1",
|
||
}
|
||
|
||
def extract_price(self, text):
|
||
if not text:
|
||
return None, None
|
||
patterns = [
|
||
r"(\d+)\s*元\s*(?:一张|一刀|一条|单张)?",
|
||
r"[收售出求兑买]\s*(\d+)\s*元?",
|
||
]
|
||
for pattern in patterns:
|
||
m = re.search(pattern, text)
|
||
if m:
|
||
try:
|
||
val = float(m.group(1))
|
||
if val < 0 or val > 999999999:
|
||
continue
|
||
price_unit = "元"
|
||
if "张" in text:
|
||
price_unit = "元/张"
|
||
elif "刀" in text:
|
||
price_unit = "元/刀"
|
||
elif "条" in text:
|
||
price_unit = "元/条"
|
||
return val, price_unit
|
||
except:
|
||
pass
|
||
return None, None
|
||
|
||
def crawl_today(self):
|
||
"""全量采集：爬取第1-max_pages页所有帖子，不过滤日期
|
||
用于一次性补全所有帖子的post_time等字段
|
||
"""
|
||
today = datetime.now().date()
|
||
today_str = today.strftime("%Y/%m/%d").lstrip('0').replace('/0', '/')
|
||
|
||
print(f"开始全量采集第1-{self.max_pages}页 (今天: {today_str})")
|
||
|
||
saved = 0
|
||
checked = 0
|
||
|
||
for page in range(1, self.max_pages + 1):
|
||
page_url = f"{self.forum_url}&page={page}"
|
||
print(f"\n扫描第 {page} 页...")
|
||
|
||
response = self.get(page_url)
|
||
if not response:
|
||
continue
|
||
|
||
posts = self.extract_posts_with_dates(response.text)
|
||
print(f" 该页共 {len(posts)} 条帖子")
|
||
|
||
for post_id, date_str in posts:
|
||
checked += 1
|
||
|
||
# 标准化日期
|
||
normalized_date = date_str.lstrip('0').replace('/0', '/')
|
||
|
||
# 访问详情页
|
||
detail_url = f"{self.base_url}/dispbbs.asp?boardID={self.board_id}&ID={post_id}&page=1"
|
||
detail_response = self.get(detail_url)
|
||
|
||
if not detail_response:
|
||
continue
|
||
|
||
detail = self.parse_post_detail(detail_response.text, detail_url)
|
||
|
||
# 验证详情页的真实发帖时间
|
||
post_time_str = detail.get('post_time', '')
|
||
if not post_time_str:
|
||
continue
|
||
|
||
try:
|
||
post_time = datetime.strptime(post_time_str, '%Y-%m-%d %H:%M:%S').date()
|
||
is_today = (post_time == today)
|
||
label = f"今日{normalized_date}" if is_today else f"历史{post_time}"
|
||
except:
|
||
label = "时间异常"
|
||
|
||
# 保存（ON CONFLICT会更新post_time等字段）
|
||
if self.save_full_post(detail):
|
||
saved += 1
|
||
print(f" [保存] {post_id} - {detail.get('title', '')[:25]}... ({label}中已存{saved}条)")
|
||
else:
|
||
print(f" [失败] {post_id}")
|
||
|
||
print(f" 页码 {page} 完成: 检查{len(posts)}条")
|
||
|
||
print(f"\n===== 采集完成 =====")
|
||
print(f"总计检查: {checked} 条")
|
||
print(f"成功保存: {saved} 条")
|
||
return saved
|
||
|
||
def save_full_post(self, post):
|
||
if not post or not post.get("post_id"):
|
||
return False
|
||
|
||
import psycopg2.sql as sql
|
||
|
||
has_lifebuoy = post.get("special_types") and "救生圈" in post.get("special_types", "")
|
||
|
||
cols = ['post_id', 'title', 'content', 'category', 'post_type', 'price',
|
||
'price_unit', 'special_types', 'number_features', 'author_username',
|
||
'author_id', 'contact', 'has_lifebuoy', 'reply_count', 'view_count',
|
||
'post_time', 'crawled_at', 'updated_at', 'url', 'instance_id']
|
||
|
||
vals = [
|
||
sql.Literal(post.get("post_id")),
|
||
sql.Literal(post.get("title")),
|
||
sql.Literal(post.get("content")),
|
||
sql.Literal(post.get("category")),
|
||
sql.Literal(post.get("post_type")),
|
||
sql.Literal(post.get("price")),
|
||
sql.Literal(post.get("price_unit")),
|
||
sql.Literal(post.get("special_types")),
|
||
sql.Literal(post.get("number_features")),
|
||
sql.Literal(post.get("author_username")),
|
||
sql.Literal(f"user_{post.get('post_id')}"),
|
||
sql.Literal(post.get("contact")),
|
||
sql.Literal(has_lifebuoy),
|
||
sql.Literal(0),
|
||
sql.Literal(0),
|
||
sql.Literal(post.get("post_time")),
|
||
sql.SQL('NOW()'),
|
||
sql.SQL('NOW()'),
|
||
sql.Literal(post.get("url")),
|
||
sql.Literal(self.instance_id),
|
||
]
|
||
|
||
update_assigns = [
|
||
"title = EXCLUDED.title",
|
||
"content = EXCLUDED.content",
|
||
"category = EXCLUDED.category",
|
||
"post_type = EXCLUDED.post_type",
|
||
"price = EXCLUDED.price",
|
||
"price_unit = EXCLUDED.price_unit",
|
||
"special_types = EXCLUDED.special_types",
|
||
"number_features = EXCLUDED.number_features",
|
||
"author_username = EXCLUDED.author_username",
|
||
"author_id = EXCLUDED.author_id",
|
||
"contact = EXCLUDED.contact",
|
||
"has_lifebuoy = EXCLUDED.has_lifebuoy",
|
||
"reply_count = EXCLUDED.reply_count",
|
||
"view_count = EXCLUDED.view_count",
|
||
"post_time = EXCLUDED.post_time",
|
||
"updated_at = NOW()",
|
||
"crawled_at = NOW()",
|
||
"url = EXCLUDED.url",
|
||
|
||
]
|
||
|
||
query = sql.SQL("INSERT INTO yichens_posts ({}) VALUES ({}) ON CONFLICT (post_id) DO UPDATE SET {}").format(
|
||
sql.SQL(', ').join(sql.Identifier(c) for c in cols),
|
||
sql.SQL(', ').join(vals),
|
||
sql.SQL(', ').join(sql.SQL(a) for a in update_assigns),
|
||
)
|
||
|
||
try:
|
||
with get_db() as conn:
|
||
if not conn:
|
||
print("数据库连接失败")
|
||
return False
|
||
cur = conn.cursor()
|
||
cur.execute(query)
|
||
conn.commit()
|
||
cur.close()
|
||
return True
|
||
except Exception as e:
|
||
print(f"保存失败: {e}")
|
||
return False
|
||
|
||
def run(self):
|
||
log_id = self._log_start()
|
||
try:
|
||
result = self.crawl_today()
|
||
self._log_finish(log_id, "success", result)
|
||
return result
|
||
except Exception as e:
|
||
print(f"爬虫异常: {e}")
|
||
self._log_finish(log_id, "failed", 0, str(e))
|
||
return 0
|
||
|
||
def _log_start(self):
|
||
try:
|
||
with get_db() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"INSERT INTO crawl_logs (spider_name, status, started_at, instance_id) VALUES (%s, %s, NOW(), %s) RETURNING id",
|
||
(self.source, "running", self.instance_id)
|
||
)
|
||
log_id = cur.fetchone()[0]
|
||
conn.commit()
|
||
cur.close()
|
||
return log_id
|
||
except Exception as e:
|
||
print(f"记录日志失败: {e}")
|
||
return None
|
||
|
||
def _log_finish(self, log_id, status, items_count, error=""):
|
||
if log_id is None:
|
||
return
|
||
try:
|
||
with get_db() as conn:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"UPDATE crawl_logs SET status=%s, items_count=%s, error_message=%s, finished_at=NOW() WHERE id=%s",
|
||
(status, items_count, error, log_id)
|
||
)
|
||
conn.commit()
|
||
cur.close()
|
||
except Exception as e:
|
||
print(f"更新日志失败: {e}")
|
||
|
||
if __name__ == "__main__":
|
||
spider = YichensTodaySpider()
|
||
spider.run()
|