30 lines
956 B
Bash
30 lines
956 B
Bash
|
#!/bin/bash
|
||
|
|
cd /root/coolbot-data
|
||
|
|
export PYTHONPATH=/usr/lib64/python3.6/site-packages:$PYTHONPATH
|
||
|
|
export CRAWLER_INSTANCE_ID=server_3
|
||
|
|
python3 -u -c "
|
||
|
|
import sys
|
||
|
|
sys.path.insert(0, '/usr/lib64/python3.6/site-packages')
|
||
|
|
sys.path.insert(0, '/root/coolbot-data')
|
||
|
|
import os
|
||
|
|
os.environ['DB_HOST'] = 'pgm-bp1t1008h019ez6c.pg.rds.aliyuncs.com'
|
||
|
|
os.environ['DB_PORT'] = '5432'
|
||
|
|
os.environ['DB_USER'] = 'coolbot'
|
||
|
|
os.environ['DB_PASSWORD'] = 'Coolbot123'
|
||
|
|
os.environ['DB_NAME'] = 'coolbot_data'
|
||
|
|
os.environ['CRAWLER_INSTANCE_ID'] = 'server_3'
|
||
|
|
from crawlers.crawl_today import YichensTodaySpider
|
||
|
|
spider = YichensTodaySpider()
|
||
|
|
spider.max_pages = 2
|
||
|
|
spider.min_delay = 0.2
|
||
|
|
spider.max_delay = 0.5
|
||
|
|
log_id = spider._log_start()
|
||
|
|
try:
|
||
|
|
result = spider.crawl_today()
|
||
|
|
spider._log_finish(log_id, 'success', result)
|
||
|
|
print('[server_3] 采集完成: ' + str(result) + '条', flush=True)
|
||
|
|
except Exception as e:
|
||
|
|
spider._log_finish(log_id, 'failed', 0, str(e))
|
||
|
|
raise
|
||
|
|
"
|