74 lines
2.3 KiB
Python
74 lines
2.3 KiB
Python
|
"""
|
||
|
|
定时任务调度器 - 修复版
|
||
|
|
集成 APScheduler 实现爬虫自动化
|
||
|
|
"""
|
||
|
|
import logging
|
||
|
|
from apscheduler.schedulers.blocking import BlockingScheduler
|
||
|
|
from apscheduler.triggers.cron import CronTrigger
|
||
|
|
from apscheduler.triggers.interval import IntervalTrigger
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
# 全局调度器实例
|
||
|
|
scheduler = BlockingScheduler(timezone="Asia/Shanghai")
|
||
|
|
|
||
|
|
def crawl_today_job():
|
||
|
|
"""每日采集任务"""
|
||
|
|
from crawlers.crawl_today import YichensTodaySpider
|
||
|
|
logger.info("[定时任务] 开始执行一尘网今日采集")
|
||
|
|
try:
|
||
|
|
spider = YichensTodaySpider()
|
||
|
|
result = spider.run()
|
||
|
|
logger.info(f"[定时任务] 采集完成，新增 {result} 条")
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"[定时任务] 采集失败: {e}")
|
||
|
|
|
||
|
|
def crawl_incremental_job():
|
||
|
|
"""增量采集任务（每小时）"""
|
||
|
|
from crawlers.crawl_today import YichensTodaySpider
|
||
|
|
logger.info("[定时任务] 开始执行增量采集")
|
||
|
|
try:
|
||
|
|
spider = YichensTodaySpider()
|
||
|
|
spider.max_pages = 2 # 增量只扫2页
|
||
|
|
result = spider.crawl_today()
|
||
|
|
logger.info(f"[定时任务] 增量采集完成，新增 {result} 条")
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"[定时任务] 增量采集失败: {e}")
|
||
|
|
|
||
|
|
def init_scheduler():
|
||
|
|
"""初始化定时任务"""
|
||
|
|
# 每天早上 8 点采集
|
||
|
|
scheduler.add_job(
|
||
|
|
crawl_today_job,
|
||
|
|
CronTrigger(hour=8, minute=0, timezone="Asia/Shanghai"),
|
||
|
|
id='yichens_daily_crawl',
|
||
|
|
name='一尘网每日采集',
|
||
|
|
replace_existing=True
|
||
|
|
)
|
||
|
|
|
||
|
|
# 每小时增量采集（如果需要）
|
||
|
|
scheduler.add_job(
|
||
|
|
crawl_incremental_job,
|
||
|
|
IntervalTrigger(hours=1, timezone="Asia/Shanghai"),
|
||
|
|
id='yichens_incremental',
|
||
|
|
name='一尘网增量采集',
|
||
|
|
replace_existing=True
|
||
|
|
)
|
||
|
|
|
||
|
|
logger.info("定时任务已注册: 每日8点采集 + 每小时增量")
|
||
|
|
return scheduler
|
||
|
|
|
||
|
|
def start_scheduler():
|
||
|
|
"""启动调度器"""
|
||
|
|
init_scheduler()
|
||
|
|
logger.info("调度器启动")
|
||
|
|
try:
|
||
|
|
scheduler.start()
|
||
|
|
except (KeyboardInterrupt, SystemExit):
|
||
|
|
logger.info("调度器停止")
|
||
|
|
scheduler.shutdown()
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
logging.basicConfig(level=logging.INFO)
|
||
|
|
start_scheduler()
|